Compare commits

...

239 Commits

Author SHA1 Message Date
Georgios Konstantopoulos
7840143e6d feat(trie): add batch node fetch API to reduce O(n²) proof walks
Amp-Thread-ID: https://ampcode.com/threads/T-019bfe25-43f3-75ac-98f7-32bf937b69e1
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 07:40:11 +00:00
YK
11d9f38077 test(e2e): comprehensive RocksDB storage E2E tests (#21423) 2026-01-27 07:08:57 +00:00
Matthias Seitz
226ce14ca1 perf(trie): use is_zero() check to avoid copy in is_storage_empty (#21459)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 00:42:26 +00:00
Dan Cline
a6e1dea2d7 chore: add logging for internal fcu errors (#21456) 2026-01-26 23:24:48 +00:00
Georgios Konstantopoulos
71ed68e944 perf(db): flatten HashedPostState before persisting (#21422) 2026-01-26 22:49:01 +00:00
DaniPopes
adecbd7814 chore: log docker sccache stats (#21455) 2026-01-26 22:30:20 +00:00
Matthias Seitz
26a37f3c00 chore: use Default::default() for TransactionInfo for forward compatibility (#21454)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 22:15:41 +00:00
DaniPopes
0bfa7fa5fa ci: typorino (#21453) 2026-01-26 21:39:35 +00:00
Georgios Konstantopoulos
18bec10a0b perf(docker): use shared cache mounts for parallel builds (#21451)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:00:45 +00:00
DaniPopes
1e33821e19 ci: use depot cache in Dockerfile.depot (#21450) 2026-01-26 20:37:33 +00:00
ethfanWilliam
da92733be8 fix: use unwrap_or_else for lazy evaluation of BlobParams::cancun (#21442) 2026-01-26 20:19:28 +00:00
DaniPopes
c41c8e6cae chore: reduce number of nightly builds (#21446) 2026-01-26 20:06:09 +00:00
DaniPopes
1ccc174e7b chore: remove unused docker from makefile (#21445) 2026-01-26 19:53:55 +00:00
ethfanWilliam
f1459fcf91 fix(stages): retain RocksDB TempDir in TestStageDB to prevent premature deletion (#21444) 2026-01-26 19:43:11 +00:00
Dan Cline
94235d64a8 fix(pruner): prune account and storage changeset static files (#21346) 2026-01-26 19:28:18 +00:00
Dan Cline
7fe60017cf chore(metrics): add a gas_last metric similar to new_payload_last (#21437) 2026-01-26 17:54:20 +00:00
Brian Picciano
f9ec2fafa0 refactor(trie): always use ParallelSparseTrie, deprecate config flags (#21435) 2026-01-26 17:02:06 +00:00
Arsenii Kulikov
768a687189 perf: use shared channel for prewarm workers (#21429) 2026-01-26 15:49:44 +00:00
Rez
b87cde5479 feat: configurable EVM execution limits (#21088)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-26 15:27:09 +00:00
figtracer
ab685579f0 feat(rpc): add transaction hash caching to EthStateCache (#21180)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:37:53 +00:00
Matthias Seitz
c7faafd183 fix(rpc): add block timestamp validation in eth_simulateV1 (#21397)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:12:28 +00:00
Matthias Seitz
935a2cc056 fix(rpc): use correct error codes for eth_simulateV1 reverts and halts (#21412)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:06:38 +00:00
Matthias Seitz
507cf58db0 fix(rpc): add block number validation in eth_simulateV1 (#21396)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 13:47:20 +00:00
Matthias Seitz
6cfd369d17 fix(rpc): populate block_hash in eth_simulateV1 logs (#21413)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:41:19 +00:00
Matthias Seitz
934f462d01 feat(cli): make stopping on invalid block the default for reth import (#21403)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:41:06 +00:00
Matthias Seitz
d4f28b02ff feat(rpc): implement movePrecompileToAddress for eth_simulateV1 (#21414)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:40:12 +00:00
Matthias Seitz
963bfeeeed fix(rpc): set prevrandao to zero for eth_simulateV1 simulated blocks (#21399)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:39:37 +00:00
Matthias Seitz
adbe6d9da0 fix(rpc): cap simulate_v1 default gas limit to RPC gas cap (#21402)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:39:15 +00:00
Matthias Seitz
6d19c0ed8e fix(engine): only warn for critical capability mismatches (#21398)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:36:49 +00:00
Andrey Kolishchak
4baf2baec4 fix(net): FetchFullBlockRangeFuture can get stuck forever after partial body fetch + error (#21411)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:34:07 +00:00
emmmm
0b5f79e8c9 docs(rpc): add reth_subscribePersistedBlock method (#21420) 2026-01-26 10:48:35 +00:00
Georgios Konstantopoulos
afe164baca test: add E2E test for RocksDB provider functionality (#21419)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-26 10:24:10 +00:00
Hwangjae Lee
31fdbe914c docs(tracing): fix incorrect example description in lib.rs (#21417)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-26 10:19:36 +00:00
Ahsen Kamal
6870747246 feat(payload): add fn for system transaction check (#21407)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-25 14:47:22 +00:00
Fallengirl
0ad8c772e1 fix(era-utils): export correct era1 CompressedBody payload (#21409) 2026-01-25 14:36:24 +00:00
github-actions[bot]
5440d0d89a chore(deps): weekly cargo update (#21406)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-25 10:39:48 +00:00
Matthias Seitz
0eea4d76e9 chore: remove unused imports in storage-api (#21400)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-24 15:49:21 +00:00
YK
8a1702cd74 fix(rocksdb): filter history writes to only changed accounts/storage (#21339)
Co-authored-by: Tempo AI <ai@tempo.xyz>
2026-01-24 13:07:16 +00:00
cui
7feb56d5f6 feat: prealloc vec (#21391) 2026-01-24 11:30:34 +00:00
cui
0aa922c4e8 feat: change from stable sort to unstable sort (#21387) 2026-01-24 11:21:47 +00:00
Matthias Seitz
ccff9a08f0 chore: fix clippy unnecessary_sort_by lint (#21385) 2026-01-24 03:13:49 +00:00
Matthias Seitz
eb788cc7cf fix(docker): pass vergen git vars as build args (#21384)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-24 03:21:43 +01:00
Dan Cline
fb05a0654f fix(engine): use LazyTrieData::deferred for chain notification (#21383)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-23 22:32:21 +00:00
ethfanWilliam
d5a36dcc00 perf(trie): parallelize merge_ancestors_into_overlay extend ops (#21379) 2026-01-23 22:26:07 +00:00
iPLAY888
ffbef9e3cd chore: removed needless collect (#21381) 2026-01-23 21:59:19 +00:00
Dan Cline
820c112e8e feat(engine): add metric for forkchoiceUpdated response -> newPayload (#21380) 2026-01-23 21:57:15 +00:00
Alexey Shekhirin
9285f7eafc ci: use depot for docker (#20380)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-23 21:14:55 +00:00
joshieDo
9a4c6d8a11 feat(rocksdb): static file based healing for TransactionHashNumbers (#21343) 2026-01-23 20:11:47 +00:00
Dan Cline
963c26550a fix(trie): only clone required keys in on_prefetch_proofs (#21378) 2026-01-23 21:13:01 +01:00
joshieDo
3648483512 feat(rocksdb): add WAL size tracking metric and Grafana dashboard (#21295)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 19:59:10 +00:00
joshieDo
ab418642b4 fix(stages): commit RocksDB batches before flush and configure immediate WAL cleanup (#21374)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-23 19:28:52 +00:00
joshieDo
decb56fae1 feat(rocksdb): changeset-based crash recovery healing for history indices (#21341) 2026-01-23 19:28:10 +00:00
Matthias Seitz
ee1ec8f9f0 perf(trie): parallelize COW extend operations with rayon (#21375) 2026-01-23 19:31:04 +01:00
Georgios Konstantopoulos
d7bf87da52 feat(engine): add metric for state root task fallback success (#21371) 2026-01-23 18:21:44 +00:00
Georgios Konstantopoulos
dd0c6d279f revert: perf(trie): parallelize merge_ancestors_into_overlay (#21202) (#21370) 2026-01-23 19:09:19 +01:00
Alexey Shekhirin
c137ed836f perf(engine): fixed-cache for execution cache (#21128)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Tempo AI <ai@tempo.xyz>
2026-01-23 17:57:42 +00:00
Dan Cline
a543752f7d chore(reth-bench): make from-block a required flag (#21372) 2026-01-23 17:52:33 +00:00
joshieDo
b814893221 feat(stages): flush RocksDB at end of history and tx_lookup stages (#21367) 2026-01-23 17:02:53 +00:00
Georgios Konstantopoulos
fcef82261d fix(libmdbx): handle errors gracefully in TransactionInner::drop (#21368) 2026-01-23 16:37:15 +00:00
iPLAY888
d3846d98a9 refactor: refactor get_idle_peer_for to use Iterator::find (#21321) 2026-01-23 15:56:09 +00:00
Alexey Shekhirin
1f536cce65 test(e2e): selfdestruct pre- and post-Dencun (#21363) 2026-01-23 15:41:08 +00:00
Matthias Seitz
0ddaf1b26c feat(engine): add BAL metrics type for EIP-7928 (#21356) 2026-01-23 15:17:33 +00:00
Gigi
830cd5e355 chore: update snmalloc upstream repository link (#21360) 2026-01-23 14:57:46 +00:00
Georgios Konstantopoulos
f77d7d5983 feat(reth-bench): support human-readable gas format in generate-big-block (#21361) 2026-01-23 14:24:34 +00:00
Georgios Konstantopoulos
a2237c534e feat(p2p): add reth p2p enode command (#21357)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 13:23:44 +00:00
Arsenii Kulikov
1bd8fab887 feat(txpool): add Block associated type to TransactionValidator trait (#21359) 2026-01-23 13:16:05 +00:00
Matthias Seitz
22a68756c7 fix(tree): evict changeset cache even when finalized block is unset (#21354) 2026-01-23 11:26:57 +00:00
Hwangjae Lee
d99c0ffd62 chore(etc): update ethereum-metrics-exporter GitHub URL (#21348)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-23 10:59:53 +00:00
Georgios Konstantopoulos
ad476e2b5c chore: add yongkangc as codeowner for crates/storage/provider (#21349) 2026-01-23 07:18:18 +00:00
Matthias Seitz
6df249c1f1 feat(engine): stub Amsterdam engine API endpoints (newPayloadV5, getPayloadV6, BALs) (#21344)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
Co-authored-by: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com>
2026-01-22 20:48:11 +00:00
Arsenii Kulikov
5a076df09a feat: allow setting custom debug block provider (#21345)
Co-authored-by: Karl <yh975593284@gmail.com>
2026-01-22 20:40:26 +00:00
YK
f07629eac0 perf: avoid creating RocksDB transactions for legacy MDBX-only nodes (#21325) 2026-01-22 20:30:52 +00:00
Dan Cline
f643e93c35 feat(reth-bench): send-invalid-payload command (#21335) 2026-01-22 19:42:19 +00:00
Matthias Seitz
653362a436 ci: align check-alloy workflow with main clippy job (#21329) 2026-01-22 20:48:53 +01:00
Seola Oh
a02508600c chore(txpool): explicitly deref RwLockReadGuard in PartialEq impl (#21336) 2026-01-22 19:35:00 +00:00
Georgios Konstantopoulos
937a7f226d fix(rpc): use Default for SimulateError to prepare for alloy breaking change (#21319)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 19:14:58 +00:00
joshieDo
a0df561117 fix(rocksdb): periodic batch commits in stages to prevent OOM (#21334)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 19:04:56 +00:00
Arsenii Kulikov
be5a4ac7a6 feat: bump alloy and alloy-evm (#21337) 2026-01-22 18:43:24 +00:00
Georgios Konstantopoulos
0c854b6f14 fix(net): limit pending pool imports for broadcast transactions (#21254)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-22 18:32:07 +00:00
Georgios Konstantopoulos
28a31cd579 fix: use unwrap_or_else for lazy evaluation of StorageSettings::legacy (#21332) 2026-01-22 17:02:15 +00:00
Brian Picciano
da12451c9c chore(trie): Cleanup unused trie changesets code (#21323) 2026-01-22 16:57:46 +00:00
Georgios Konstantopoulos
247ce3c4e9 feat(storage): warn storage settings diff at startup (#21320)
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-22 16:40:10 +00:00
iPLAY888
bf43ebaa29 fix(cli): handle invalid hex in db list --search (#21315) 2026-01-22 16:18:36 +00:00
Matthias Seitz
a01ecce73f test: add e2e tests for invalid payload handling via Engine API (#21288) 2026-01-22 15:55:36 +00:00
Arsenii Kulikov
3e55c6ca6e fix: always check upper subtrie for keys (#21276)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-22 15:47:50 +00:00
Brian Picciano
2ac7d719f3 feat(trie): add V2 account proof computation and refactor proof types (reapply) (#21316)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:46:01 +00:00
andrewshab
965705ff88 fix: remove collect (#21318)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:24:51 +00:00
Dan Cline
ebe2ca1366 feat: add StaticFileSegment::StorageChangeSets (#20896) 2026-01-22 15:03:47 +00:00
Matthias Seitz
cc242f83fd feat(rpc): respect history expiry in eth_getLogs per EIP-4444 (#21304) 2026-01-22 14:55:50 +00:00
joshieDo
12cf3d685b fix(provider): add CommitOrder for RocksDB/MDBX unwind atomicity (#21311) 2026-01-22 14:54:47 +00:00
Matthias Seitz
ad5b533ad1 chore: rm patches (#21317) 2026-01-22 15:48:53 +01:00
joshieDo
118f15f345 feat(rocksdb): disable bloom filter for default column family (#21312) 2026-01-22 13:47:34 +00:00
joshieDo
97481f69e5 perf(rocksdb): disable compression and bloom filters for TransactionHashNumbers CF (#21310) 2026-01-22 13:31:16 +00:00
Georgios Konstantopoulos
f692ac7d1e perf(prune): use bulk table clear for PruneMode::Full (#21302) 2026-01-22 13:01:17 +00:00
andrewshab
4b1c341ced fix: remove redundant clone (#21300) 2026-01-22 12:43:19 +00:00
Georgios Konstantopoulos
865f8f8951 perf(prune): sort tx hashes for efficient TransactionLookup pruning (#21297) 2026-01-22 12:10:07 +00:00
joshieDo
492fc20fd1 fix(cli): clear rocksdb tables in drop-stage command (#21299)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 12:09:36 +00:00
Sergei Shulepov
ad9886abb8 fix(mdbx): mark reserve as unsafe (#21263) 2026-01-22 12:03:12 +00:00
Matthias Seitz
5c3e45cd6b fix: handle incomplete receipts gracefully in receipt root task (#21285) 2026-01-22 10:52:56 +00:00
Emma Jamieson-Hoare
68fdba32d2 chore(release): prep v1.10.2 release (#21287)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-22 10:50:10 +00:00
Matthias Seitz
8f6a0a2992 ci: add on-demand workflow to check alloy breaking changes (#21267) 2026-01-22 10:47:38 +00:00
Matthias Seitz
ec9c7f8d3e perf(db): use ArrayVec for StoredNibbles key encoding (#21279) 2026-01-22 02:05:50 +00:00
Matthias Seitz
dbdaf068f0 fix(engine): clear execution cache when block validation fails (#21282) 2026-01-22 01:01:22 +00:00
Matthias Seitz
055bf63ee9 refactor: use Default::default() for Header in tests (#21277) 2026-01-21 22:50:10 +00:00
Georgios Konstantopoulos
2305c3ebeb feat(rpc): respect history expiry in block() and map to PrunedHistoryUnavailable (#21270) 2026-01-21 22:22:05 +00:00
joshieDo
eb55c3c3da feat(grafana): add RocksDB metrics dashboard (#21243)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 22:09:42 +00:00
Alexey Shekhirin
72e1467ba3 fix(prune): avoid panic in tx lookup (#21275) 2026-01-21 21:21:53 +00:00
Alexey Shekhirin
74edce0089 revert: feat(trie): add V2 account proof computation and refactor proof types (#21214) (#21274) 2026-01-21 21:07:13 +00:00
Georgios Konstantopoulos
8c645d5762 feat(reth-bench): accept short notation for --target-gas-limit (#21273) 2026-01-21 21:04:10 +00:00
Georgios Konstantopoulos
b7d2ee2566 feat(engine): add metric for execution cache unavailability due to concurrent use (#21265)
Co-authored-by: Tempo AI <ai@tempo.xyz>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-21 20:17:45 +00:00
Matthias Seitz
7609deddda perf(trie): parallelize merge_ancestors_into_overlay (#21202) 2026-01-21 20:08:03 +00:00
Matthias Seitz
ec50fd40b3 chore(chainspec): use ..Default::default() in create_chain_config (#21266) 2026-01-21 19:19:24 +00:00
YK
624ddc5779 feat(stages): add RocksDB support for IndexStorageHistoryStage (#21175) 2026-01-21 17:05:19 +00:00
Georgios Konstantopoulos
dd72cfe23e refactor: remove static_files.to_settings() and add edge feature to RocksDB flags (#21225) 2026-01-21 16:52:24 +00:00
joshieDo
ff8ac97e33 fix(stages): clear ETL collectors on HeaderStage error paths (#21258) 2026-01-21 16:27:30 +00:00
Alexey Shekhirin
0974485863 feat(reth-bench): add --target-gas-limit option to gas-limit-ramp (#21262) 2026-01-21 16:19:22 +00:00
かりんとう
274394e777 fix: fix payload file filter prefix in replay-payloads (#21255) 2026-01-21 16:11:03 +00:00
Emma Jamieson-Hoare
1954c91a60 chore: update CODEOWNERS (#21223)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-21 14:40:54 +00:00
Sergei Shulepov
9cf82c8403 fix: supply a real ptr to mdbx_dbi_flags_ex (#21230) 2026-01-21 14:23:26 +00:00
Brian Picciano
f85fcba872 feat(trie): add V2 account proof computation and refactor proof types (#21214)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 14:18:44 +00:00
joshieDo
ebaa4bda3a feat(rocksdb): add missing observability (#21253) 2026-01-21 14:14:34 +00:00
joshieDo
04d4c9a02f fix(rocksdb): flush all column families on drop and show SST/memtable sizes (#21251)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 12:44:08 +00:00
Arsenii Kulikov
3065a328f9 fix: clear overlay_cache in with_extended_hashed_state_overlay (#21233) 2026-01-21 12:08:24 +00:00
Sergei Shulepov
43a84f1231 refactor(engine): move execution logic from metrics to payload_validator (#21226) 2026-01-21 11:17:30 +00:00
Matthias Seitz
5a5c21cc1b feat(txpool): add IntoIterator for AllPoolTransactions (#21241) 2026-01-21 10:01:32 +00:00
Matthias Seitz
8a8a9126d6 feat(execution-types): add receipts_iter and logs_iter helpers to Chain (#21240) 2026-01-21 09:59:15 +00:00
Emilia Hane
6f73c2447d feat(trie): Add serde-bincode-compat feature to reth-trie (#21235) 2026-01-21 09:42:52 +00:00
Sergei Shulepov
2cae438642 fix: sigsegv handler (#21231)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-21 09:42:36 +00:00
Georgios Konstantopoulos
37b5db0d47 feat(cli): add RocksDB table stats to reth db stats command (#21221)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-21 08:45:17 +00:00
joshieDo
238433e146 fix(rocksdb): flush memtables before dropping (#21234) 2026-01-21 02:19:36 +00:00
Georgios Konstantopoulos
660964a0f5 feat(node): log storage settings after genesis init (#21229) 2026-01-21 00:58:23 +00:00
Matthias Seitz
22b465dd64 chore(trie): remove unnecessary clone in into_sorted_ref (#21232) 2026-01-20 22:57:08 +00:00
Georgios Konstantopoulos
3ff575b877 feat(engine): add --engine.disable-cache-metrics flag (#21228)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-20 22:03:12 +00:00
かりんとう
d12752dc8a feat(engine): add time_between_forkchoice_updated metric (#21227) 2026-01-20 21:06:11 +00:00
Georgios Konstantopoulos
869b5d0851 feat(edge): enable transaction_hash_numbers_in_rocksdb for edge builds (#21224)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 20:02:02 +00:00
Georgios Konstantopoulos
78de3d8f61 perf(db): use Cow::Borrowed in walk_dup to avoid allocation (#21220) 2026-01-20 19:31:50 +00:00
YK
bc79cc44c9 feat(cli): add --rocksdb.* flags for RocksDB table routing (#21191) 2026-01-20 19:29:05 +00:00
Georgios Konstantopoulos
ff8f434dcd feat(cli): add reth db checksum rocksdb command (#21217)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 19:10:34 +00:00
Arsenii Kulikov
9662dc5271 fix: properly save history indices in pipeline (#21222) 2026-01-20 18:20:28 +00:00
Alexey Shekhirin
3ba37082dc fix(reth-bench): replay-payloads prefix (#21219) 2026-01-20 18:36:35 +01:00
Ahsen Kamal
7934294988 perf(trie): dispatch storage proofs in lexicographical order (#21213)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
2026-01-20 17:09:20 +00:00
Georgios Konstantopoulos
7371bd3f29 chore(db-api): remove sharded_key_encode benchmark (#21215)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 17:01:12 +00:00
Georgios Konstantopoulos
80980b8e4d feat(pruning): add DefaultPruningValues for overridable pruning defaults (#21207)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-20 16:58:29 +00:00
Matthias Seitz
2e2cd67663 perf(chain-state): parallelize into_sorted with rayon (#21193) 2026-01-20 16:42:16 +00:00
Georgios Konstantopoulos
4f009728e2 feat(cli): add reth db checksum mdbx/static-file command (#21211)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:11:51 +00:00
Georgios Konstantopoulos
39d5ae73e8 feat(storage): add read-only mode for RocksDB provider (#21210)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:09:51 +00:00
Georgios Konstantopoulos
5ef200eaad perf(db): stack-allocate ShardedKey and StorageShardedKey encoding (#21200)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 15:58:43 +00:00
ethfanWilliam
d002dacc13 chore: remove deprecated and unused ExecuteOutput struct (#20887)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 15:06:26 +00:00
Alexey Shekhirin
bb39cba504 ci: partition bench codspeed job (#20332)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 14:29:48 +00:00
YK
bd144a4c42 feat(stages): add RocksDB support for IndexAccountHistoryStage (#21165)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 14:23:29 +00:00
tonis
a0845bab18 feat: Check CL/Reth capability compatibility (#20348)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 14:19:31 +00:00
Brian Picciano
346cc0da71 feat(trie): add AsyncAccountValueEncoder for V2 proof computation (#21197)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:50:29 +00:00
Matthias Seitz
ea3d4663ae perf(trie): use HashMap reserve heuristic in MultiProof::extend (#21199) 2026-01-20 13:34:41 +00:00
Hwangjae Lee
3667d3b5aa perf(trie): defer child RLP conversion in proof_v2 for async encoder support (#20873)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-20 13:33:08 +00:00
Brian Picciano
7cfb19c98e feat(trie): Add V2 reveal method and target types (#21196)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:25:54 +00:00
joshieDo
5a38871489 fix: set StaticFileArgs defaults for edge (#21208) 2026-01-20 12:39:36 +00:00
Brian Picciano
c825c8c187 chore(trie): Move hybrid check for trie input merges into common code (#21198) 2026-01-20 12:38:46 +00:00
Matthias Seitz
8f37cd08fc feat(engine-api): add EIP-7928 BAL stub methods (#21204) 2026-01-20 11:33:27 +00:00
Matthias Seitz
c9dad4765d chore: bump version to 1.10.1 (#21188) 2026-01-19 14:04:08 +00:00
Dan Cline
1d55abeef3 chore: rename extend_ref methods on sorted data structures (#21043) 2026-01-19 13:04:57 +00:00
Niven
f7460e219c fix(flashblocks): Add flashblock ws connection retry period (#20510) 2026-01-19 12:01:33 +00:00
Georgios Konstantopoulos
0c66315f20 chore(bench): add --disable-tx-gossip to benchmark node args (#21171) 2026-01-19 11:45:56 +00:00
MozirDmitriy
6a2010e595 refactor(stages): reuse history index cache buffers in collect_history_indices (#21017) 2026-01-19 11:39:52 +00:00
Georgios Konstantopoulos
c2435ff6f8 feat(download): resumable snapshot downloads with auto-retry (#21161) 2026-01-19 10:26:24 +00:00
DaniPopes
52ec8e9491 ci: update to tempoxyz (#21176) 2026-01-19 10:21:37 +00:00
Georgios Konstantopoulos
a901d80ee6 chore: apply spelling and typo fixes (#21182) 2026-01-19 10:21:25 +00:00
MoNyAvA
915164078f docs: document minimal storage mode in pruning FAQ (#21025)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-19 10:27:45 +01:00
github-actions[bot]
be3234d848 chore(deps): weekly cargo update (#21167)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-18 14:57:20 +00:00
Matthias Seitz
f624372334 feat(execution-types): add receipts_iter helper (#21162)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-17 19:20:28 +01:00
Matthias Seitz
40bc9d3860 revert: undo Chain crate, add LazyTrieData to trie-common (#21155) 2026-01-17 15:57:09 +00:00
Georgios Konstantopoulos
1ea574417f feat(engine): add new_payload_interval metric (start-to-start) (#21159) 2026-01-17 12:15:45 +00:00
Georgios Konstantopoulos
27e055f790 feat(engine): add time_between_new_payloads metric (#21158) 2026-01-17 10:20:22 +00:00
Georgios Konstantopoulos
d5dc0b27eb fix(storage-api): gate reth-chain dependency behind std feature
The reth-chain crate is inherently std-only (uses BTreeMap, Arc, etc.)
and was breaking the riscv32imac no_std builds by pulling in serde_core
which doesn't support no_std properly.

This makes reth-chain optional and only enables it when std feature is
active, gating the block_writer module that uses Chain behind std.
2026-01-17 08:32:10 +00:00
Georgios Konstantopoulos
c11c13000f perf(storage): batch trie updates across blocks in save_blocks (#21142)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-17 07:15:40 +00:00
Matthias Seitz
6bf43ab24a refactor: use ExecutionOutcome::single instead of tuple From (#21152) 2026-01-17 01:51:26 +00:00
Matthias Seitz
574bde0d6f chore(chain-state): reorganize deferred_trie.rs impl blocks (#21151) 2026-01-17 01:39:29 +00:00
Matthias Seitz
79b8ffb828 feat(primitives-traits): add try_recover_signers for parallel batch recovery (#21103) 2026-01-17 01:24:53 +00:00
Dan Cline
c617d25c36 perf: make Chain use DeferredTrieData (#21137)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-17 01:05:35 +00:00
Georgios Konstantopoulos
b96a30821f fix(engine): request head block download when not buffered after backfill (#21150) 2026-01-17 00:33:27 +00:00
Mablr
012fbf5110 fix(docs/cli): update help.rs to use nightly toolchain (#21149) 2026-01-16 23:35:26 +00:00
Arsenii Kulikov
d7a5d1f872 fix: properly record span fields (#21148) 2026-01-16 23:25:54 +00:00
Matthias Seitz
3a39251f79 fix: release mutex before dropping ancestors in wait_cloned (#21146) 2026-01-16 22:32:23 +00:00
Julian Meyer
f6dbf2d82d feat(db): implement extra dup methods (#20964)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 21:31:52 +00:00
Brian Picciano
13707faf1a feat(consensus): incremental receipt root computation in background task (#21131) 2026-01-16 19:53:59 +00:00
Arsenii Kulikov
6e6415690c perf: start saving cache sooner (#21130) 2026-01-16 18:55:18 +00:00
Matthias Seitz
b81e373d78 chore(deps): bump vergen and vergen-git2 to 9.1.0 (#21141) 2026-01-16 20:00:43 +01:00
Arun Dhyani
a164654145 fix(exex): prevent ExExManager deadlock when buffer clears after being full (#21135) 2026-01-16 18:42:23 +00:00
Matthias Seitz
905bb95f8b perf(engine): defer trie overlay computation with LazyOverlay (#21133) 2026-01-16 18:25:04 +00:00
YK
13c32625bc feat(storage): add EitherReader for routing history queries to MDBX or RocksDB (#21063)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-16 17:44:43 +00:00
YK
1be9fab5bf perf: Optimize multiproof sequencer add_proof (#21129) 2026-01-16 17:33:48 +00:00
Arsenii Kulikov
80eb0d0fb6 refactor: use BlockExecutionOutcome in ExecutedBlock (#21123) 2026-01-16 17:07:19 +00:00
Matthias Seitz
5e178f6ac6 chore(deps): update alloy-evm and alloy-op-evm to 0.26.3 (#21126) 2026-01-16 17:24:45 +01:00
Matthias Seitz
b4b64096c8 perf(cli): use available_parallelism as default for re-execute (#21010) 2026-01-16 16:08:30 +00:00
figtracer
e313de818b chore(provider): pre alloc tx hashes (#21114) 2026-01-16 15:40:47 +00:00
rakita
86c414081a feat: stagging revm v34.0.0 (#20627)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 14:56:27 +00:00
Brian Picciano
a74cb9cbc3 feat(trie): in-memory trie changesets (#20997) 2026-01-16 01:06:31 +00:00
YK
e25411c32b perf(trie): fix extend_sorted_vec O(n log n) → O(n+m) merge (#21098) 2026-01-16 00:17:22 +00:00
Matthias Seitz
ec3323bba0 refactor(chain-state): extract blocks_to_chain helper (#21110) 2026-01-15 23:27:11 +00:00
Dan Cline
26cd132631 fix(reth-bench): use requests hash (#21111) 2026-01-15 19:19:16 +00:00
DaniPopes
079f59c2be perf: reserve in extend_sorted_vec (#21109) 2026-01-15 19:10:20 +00:00
joshieDo
e9b079ad62 feat: add rocksdb to save_blocks (#21003)
Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com>
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-15 18:33:19 +00:00
Dan Cline
b1f107b171 feat(reth-bench): add generate-big-block command (#21082) 2026-01-15 15:30:04 +00:00
YK
7d0e7e72de perf(trie): add k-way merge batch optimization for merge_overlay_trie_input (#21080) 2026-01-15 15:22:15 +00:00
joshieDo
f012b3391e feat: parallelize save_blocks (#20993)
Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com>
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-15 14:58:06 +00:00
joshieDo
d225fc1d7f feat: add get/set db settings for rocksdb (#21095) 2026-01-15 14:48:05 +00:00
Dan Cline
d469b7f1d0 feat(rpc): add flag to skip invalid transactions in testing_buildBlockV1 (#21094)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-15 12:05:30 +00:00
YK
9bcd3712c8 test(storage): add parametrized MDBX/RocksDB history lookup equivalence tests (#20871) 2026-01-15 11:16:40 +00:00
Emma Jamieson-Hoare
b25f32a977 chore(release): set version v1.10.0 (#21091)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-15 10:50:35 +00:00
Emma Jamieson-Hoare
905de96944 chore: release 1.9.4 (#21048)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-15 09:41:54 +00:00
Sergei Shulepov
27fbd9a7de fix(db): change commit return type from Result<bool> to Result<()> (#21077)
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
2026-01-14 23:56:27 +00:00
DaniPopes
26a99ac5a3 perf: small improvement to extend_sorted_vec (#21032) 2026-01-14 23:46:58 +00:00
James Prestwich
1265a89c21 refactor: make use of dbi consistent across mdbx interface (#21079) 2026-01-14 23:42:42 +00:00
Matthias Seitz
b9ff5941eb feat(primitives): add SealedBlock::decode_sealed for efficient RLP decoding (#21030) 2026-01-14 22:49:55 +00:00
Sergei Shulepov
a75a0a5db7 feat(cli): support file:// URLs in reth download (#21026)
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
2026-01-14 22:30:42 +00:00
Matthias Seitz
0a4bac77d0 feat(primitives): add From<Sealed<B>> for SealedBlock<B> (#21078) 2026-01-14 22:19:09 +00:00
Kamil Szczygieł
1fbd5a95f8 feat: Support for sending logs through OTLP (#21039)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-14 21:29:00 +00:00
Arsenii Kulikov
1bc07fad8e perf: use binary search in ForwardInMemoryCursor (#21049) 2026-01-14 19:31:11 +00:00
Arsenii Kulikov
8cb506c4d3 perf: don't clone entire keys set (#21042) 2026-01-14 19:26:23 +00:00
ethfanWilliam
15f16a5a2e fix: propagate keccak-cache-global feature to reth-optimism-cli (#21051)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-14 19:22:22 +00:00
Brian Picciano
5cf1d2a0b0 fix(trie): Update branch masks when revealing blinded nodes (#20937) 2026-01-14 19:12:15 +00:00
Matthias Seitz
59fb25d892 feat(bench-compare): add --skip-wait-syncing flag (#21035) 2026-01-14 16:24:19 +01:00
Alexey Shekhirin
665a0a8553 feat(cli): parse URL path and display ETA in reth download (#21014) 2026-01-14 10:01:01 +00:00
DaniPopes
54735ce0f4 perf: use fixed-map for StaticFileSegment maps (#21001)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-14 00:52:54 +00:00
joshieDo
a73e73adef feat(storage): split static file commit into sync_all and finalize (#20984) 2026-01-13 16:27:55 +00:00
github-actions[bot]
4f3bd3eac1 chore(deps): weekly cargo update (#20924)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-13 14:42:54 +00:00
YK
ae41823be6 fix: propagate edge feature to reth-node-core for version output (#20998) 2026-01-13 14:35:24 +00:00
Matthias Seitz
1fa71f893c test: add testing_buildBlockV1 RPC method and Osaka test (#20990) 2026-01-13 15:18:52 +01:00
ANtutov
c6b17848dd fix(trie): remove redundant storage trie root calculation in witness (#20965) 2026-01-13 13:12:39 +00:00
Alexey Shekhirin
a5dd7d0106 feat(node): --minimal flag (#20960) 2026-01-13 12:54:26 +00:00
Emilia Hane
61354e6c21 chore(test): use reth_optimism_chainspec::BASE_SEPOLIA in tests (#20988) 2026-01-13 12:07:47 +00:00
DaniPopes
2444533a04 perf: use in-memory length for static files metrics (#20987) 2026-01-13 11:37:00 +00:00
kurahin
8fa01eb62e fix: use global default for rpc_proof_permits CLI flag (#20967) 2026-01-12 23:03:51 +00:00
DaniPopes
c5e00e4aeb perf(db): throttle metrics reporting (#20974)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-12 22:44:24 +00:00
joshieDo
98a35cc870 fix: propagate FEATURES to sub-makes (#20975) 2026-01-12 20:03:34 +00:00
YK
46d670eca5 fix(stages): use static files for unwind in SenderRecovery stage (#20972)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-12 19:22:49 +00:00
DaniPopes
25906b7b3e fix(libmdbx): use correct size for freelist u32 values (#20970) 2026-01-12 18:52:03 +00:00
Matthias Seitz
1b3d815cb8 fix(rpc): validate eth_feeHistory newest_block against chain head (#20969)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-12 18:48:46 +00:00
DaniPopes
23f3f8e820 feat: add tracing-tracy (#20958)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-12 18:37:37 +00:00
DaniPopes
2663942b50 chore(deps): bump metrics (#20968) 2026-01-12 18:13:38 +00:00
546 changed files with 32376 additions and 9941 deletions

View File

@@ -12,7 +12,7 @@ workflows:
# Check that `A` activates the features of `B`.
"propagate-feature",
# These are the features to check:
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer,portable,keccak-cache-global",
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global",
# Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
"--left-side-feature-missing=ignore",
# Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.

43
.github/CODEOWNERS vendored
View File

@@ -1,45 +1,52 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
crates/era/ @mattsse
crates/era-downloader/ @mattsse
crates/era-utils/ @mattsse
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
crates/ethereum/ @mattsse @Rjected
crates/etl/ @joshieDo @shekhirin
crates/evm/ @rakita @mattsse @Rjected
crates/evm/ @mattsse @Rjected @klkvr
crates/exex/ @shekhirin
crates/fs-util/ @mattsse
crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected @fgimenez
crates/optimism/ @mattsse @Rjected
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/ress/ @shekhirin @Rjected
crates/revm/ @mattsse
crates/rpc/ @mattsse @Rjected
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/stateless/ @mattsse
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
crates/storage/db-api/ @joshieDo
crates/storage/db-common/ @Rjected
crates/storage/db/ @joshieDo @rakita
crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @fgimenez
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
bin/reth/ @mattsse @shekhirin @Rjected
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

View File

@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
cmd=(cargo codspeed build --profile profiling)
crates=(
-p reth-primitives
-p reth-trie
-p reth-trie-common
-p reth-trie-sparse
)
"${cmd[@]}" --features test-utils "${crates[@]}"

View File

@@ -17,6 +17,16 @@ name: bench
jobs:
codspeed:
runs-on: depot-ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
include:
- partition: 1
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
- partition: 2
crates: "-p reth-trie"
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
steps:
- uses: actions/checkout@v6
with:
@@ -32,10 +42,10 @@ jobs:
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: ./.github/scripts/codspeed-build.sh
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run --workspace
run: cargo codspeed run ${{ matrix.crates }}
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}

65
.github/workflows/check-alloy.yml vendored Normal file
View File

@@ -0,0 +1,65 @@
# Checks reth compilation against alloy branches to detect breaking changes.
# Run on-demand via workflow_dispatch.
name: Check Alloy Breaking Changes
on:
workflow_dispatch:
inputs:
alloy_branch:
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
required: false
type: string
alloy_evm_branch:
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
required: false
type: string
op_alloy_branch:
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
required: false
type: string
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Apply alloy patches
run: |
ARGS=""
if [ -n "${{ inputs.alloy_branch }}" ]; then
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
fi
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
fi
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
fi
if [ -z "$ARGS" ]; then
echo "No branches specified, nothing to patch"
exit 1
fi
./scripts/patch-alloy.sh $ARGS
echo "=== Final patch section ==="
tail -50 Cargo.toml
- name: Check workspace
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings

View File

@@ -15,6 +15,6 @@ permissions:
jobs:
update:
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
secrets:
token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,54 +0,0 @@
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
# images from this workflow will be tagged with the git sha of the branch used
# and will NOT tag it as `latest`.
name: docker-git
on:
workflow_dispatch: {}
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
GIT_SHA: ${{ github.sha }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the git-sha-tagged reth image'
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
- name: 'Build and push the git-sha-tagged op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,65 +0,0 @@
# Publishes the nightly Docker image.
name: docker-nightly
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the nightly reth image'
command: 'make PROFILE=maxperf docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
- name: 'Build and push the nightly op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
steps:
- uses: actions/checkout@v6
- name: Remove bloatware
uses: laverdet/remove-bloatware@v1.0.0
with:
docker: true
lang: rust
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,4 +1,9 @@
# Publishes the Docker image.
# Publishes Docker images.
#
# Triggers:
# - Push tag v*: builds release (RC or latest)
# - Schedule: builds nightly + profiling
# - Manual: builds git-sha or nightly
name: docker
@@ -6,84 +11,94 @@ on:
push:
tags:
- v*
env:
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
build_type:
description: "Build type"
required: true
type: choice
options:
- git-sha
- nightly
default: git-sha
dry_run:
description: "Skip pushing images (dry run)"
required: false
type: boolean
default: false
jobs:
build-rc:
if: contains(github.ref, '-rc')
name: build and push as release candidate
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
build:
if: ${{ !contains(github.ref, '-rc') }}
name: build and push as latest
name: Build Docker images
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
id-token: write
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Set up Depot CLI
uses: depot/setup-action@v1
- name: Log in to GHCR
uses: docker/login-action@v3
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get git info for vergen
id: git
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
echo "dirty=false" >> "$GITHUB_OUTPUT"
- name: Determine build parameters
id: params
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
REGISTRY="ghcr.io/${{ github.repository_owner }}"
if [[ "${{ github.event_name }}" == "push" ]]; then
VERSION="${GITHUB_REF#refs/tags/}"
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION},${REGISTRY}/op-reth:latest" >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
uses: depot/bake-action@v1
env:
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
with:
project: ${{ vars.DEPOT_PROJECT_ID }}
files: docker-bake.hcl
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
optimism.tags=${{ steps.params.outputs.optimism_tags }}

View File

@@ -285,7 +285,7 @@ jobs:
- run: zepter run check
deny:
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
uses: tempoxyz/ci/.github/workflows/deny.yml@main
lint-success:
name: lint success

View File

@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
unsafe impl GlobalAlloc for LimitedAllocator { ... }
// Binary search requires sorted input. Panics on unsorted slices.
fn find_index(items: &[Item], target: &Item) -> Option
fn find_index(items: &[Item], target: &Item) -> Option<usize>
// Timeout set to 5s to match EVM block processing limits
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);

1483
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.9.3"
version = "1.10.2"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
@@ -473,22 +473,22 @@ reth-ress-protocol = { path = "crates/ress/protocol" }
reth-ress-provider = { path = "crates/ress/provider" }
# revm
revm = { version = "33.1.0", default-features = false }
revm-bytecode = { version = "7.1.1", default-features = false }
revm-database = { version = "9.0.5", default-features = false }
revm-state = { version = "8.1.1", default-features = false }
revm-primitives = { version = "21.0.2", default-features = false }
revm-interpreter = { version = "31.1.0", default-features = false }
revm-database-interface = { version = "8.0.5", default-features = false }
op-revm = { version = "14.1.0", default-features = false }
revm-inspectors = "0.33.2"
revm = { version = "34.0.0", default-features = false }
revm-bytecode = { version = "8.0.0", default-features = false }
revm-database = { version = "10.0.0", default-features = false }
revm-state = { version = "9.0.0", default-features = false }
revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.1"
# eth
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.4.1"
alloy-dyn-abi = "1.5.2"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.1.0", default-features = false }
alloy-evm = { version = "0.25.1", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
@@ -497,36 +497,36 @@ alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.2.1", default-features = false }
alloy-contract = { version = "1.2.1", default-features = false }
alloy-eips = { version = "1.2.1", default-features = false }
alloy-genesis = { version = "1.2.1", default-features = false }
alloy-json-rpc = { version = "1.2.1", default-features = false }
alloy-network = { version = "1.2.1", default-features = false }
alloy-network-primitives = { version = "1.2.1", default-features = false }
alloy-provider = { version = "1.2.1", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.2.1", default-features = false }
alloy-rpc-client = { version = "1.2.1", default-features = false }
alloy-rpc-types = { version = "1.2.1", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.2.1", default-features = false }
alloy-rpc-types-anvil = { version = "1.2.1", default-features = false }
alloy-rpc-types-beacon = { version = "1.2.1", default-features = false }
alloy-rpc-types-debug = { version = "1.2.1", default-features = false }
alloy-rpc-types-engine = { version = "1.2.1", default-features = false }
alloy-rpc-types-eth = { version = "1.2.1", default-features = false }
alloy-rpc-types-mev = { version = "1.2.1", default-features = false }
alloy-rpc-types-trace = { version = "1.2.1", default-features = false }
alloy-rpc-types-txpool = { version = "1.2.1", default-features = false }
alloy-serde = { version = "1.2.1", default-features = false }
alloy-signer = { version = "1.2.1", default-features = false }
alloy-signer-local = { version = "1.2.1", default-features = false }
alloy-transport = { version = "1.2.1" }
alloy-transport-http = { version = "1.2.1", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.2.1", default-features = false }
alloy-transport-ws = { version = "1.2.1", default-features = false }
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
# op
alloy-op-evm = { version = "0.25.0", default-features = false }
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -555,6 +555,7 @@ dirs-next = "2.0.0"
dyn-clone = "1.0.17"
eyre = "0.6"
fdlimit = "0.3.0"
fixed-map = { version = "0.9", default-features = false }
humantime = "2.1"
humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
@@ -587,7 +588,7 @@ tracing-appender = "0.2"
url = { version = "2.3", default-features = false }
zstd = "0.13"
byteorder = "1"
mini-moka = "0.10"
fixed-cache = { version = "0.1.7", features = ["stats"] }
moka = "0.12"
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
@@ -596,9 +597,9 @@ chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-exporter-prometheus = { version = "0.16.0", default-features = false }
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.19.0" }
metrics-util = { default-features = false, version = "0.20.0" }
# proc-macros
proc-macro2 = "1.0"
@@ -664,6 +665,7 @@ opentelemetry_sdk = "0.31"
opentelemetry = "0.31"
opentelemetry-otlp = "0.31"
opentelemetry-semantic-conventions = "0.31"
opentelemetry-appender-tracing = "0.31"
tracing-opentelemetry = "0.32"
# misc-testing
@@ -734,17 +736,18 @@ tracing-journald = "0.3"
tracing-logfmt = "0.3.3"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
triehash = "0.8"
typenum = "1.15.0"
vergen = "9.0.4"
vergen = "9.1.0"
visibility = "0.1.1"
walkdir = "2.3.3"
vergen-git2 = "1.0.5"
vergen-git2 = "9.1.0"
# networking
ipnet = "2.11"
# [patch.crates-io]
[patch.crates-io]
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
@@ -787,5 +790,10 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }

View File

@@ -1,15 +0,0 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth"]

99
Dockerfile.depot Normal file
View File

@@ -0,0 +1,99 @@
# syntax=docker/dockerfile:1
# Unified Dockerfile for reth and op-reth, optimized for Depot builds
# Usage:
# reth: --build-arg BINARY=reth
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
RUN apt-get update && apt-get install -y libclang-dev pkg-config
# Install sccache for compilation caching
RUN cargo install sccache --locked
ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
# Builds a cargo-chef plan
FROM chef AS planner
COPY --exclude=.git . .
RUN cargo chef prepare --recipe-path recipe.json
FROM chef AS builder
COPY --from=planner /app/recipe.json recipe.json
# Binary to build (reth or op-reth)
ARG BINARY=reth
# Manifest path for the binary
ARG MANIFEST_PATH=bin/reth
# Build profile, release by default
ARG BUILD_PROFILE=release
ENV BUILD_PROFILE=$BUILD_PROFILE
# Extra Cargo flags
ARG RUSTFLAGS=""
ENV RUSTFLAGS="$RUSTFLAGS"
# Extra Cargo features
ARG FEATURES=""
ENV FEATURES=$FEATURES
# Git info for vergen (since .git is excluded from Docker context)
ARG VERGEN_GIT_SHA=""
ARG VERGEN_GIT_DESCRIBE=""
ARG VERGEN_GIT_DIRTY="false"
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Build dependencies
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --locked --recipe-path recipe.json --manifest-path $MANIFEST_PATH/Cargo.toml
# Build application
COPY --exclude=.git . .
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml
RUN sccache --show-stats || true
# Copy binary to a known location (ARG not resolved in COPY)
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
cp /app/target/release/$BINARY /app/binary
FROM ubuntu:24.04 AS runtime
WORKDIR /app
# Binary name for entrypoint
ARG BINARY=reth
# Install runtime dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Copy binary from build stage and create canonical symlink for entrypoint
COPY --from=builder /app/binary /usr/local/bin/
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
chmod +x /usr/local/bin/$BINARY
# Copy licenses
COPY LICENSE-* ./
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth-binary"]

View File

@@ -1,15 +0,0 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/op-reth"]

134
Makefile
View File

@@ -35,9 +35,6 @@ EEST_TESTS_TAG := v4.5.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
# The docker image name
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
##@ Help
.PHONY: help
@@ -242,137 +239,6 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
--features "$(FEATURES)" \
--profile "$(PROFILE)"
##@ Docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-git-sha
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-latest
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call docker_build_push,nightly,nightly)
.PHONY: docker-build-push-nightly-edge-profiling
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Create a cross-arch Docker image with the given tags and push it
define docker_build_push
$(MAKE) build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
$(MAKE) build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
docker buildx build --file ./Dockerfile.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Optimism docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-git-sha
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-latest
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call op_docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly,nightly)
.PHONY: op-docker-build-push-nightly-edge-profiling
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly-profiling
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
$(call docker_build_push,nightly-profiling,nightly-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly-profiling
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
# Create a cross-arch Docker image with the given tags and push it
define op_docker_build_push
$(MAKE) op-build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
$(MAKE) op-build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
docker buildx build --file ./DockerfileOp.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Other
.PHONY: clean

View File

@@ -71,7 +71,11 @@ jemalloc = [
"reth-node-core/jemalloc",
]
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
tracy-allocator = ["reth-cli-util/tracy-allocator"]
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
tracy = [
"reth-node-core/tracy",
"reth-tracing/tracy",
]
min-error-logs = [
"tracing/release_max_level_error",

View File

@@ -147,6 +147,11 @@ pub(crate) struct Args {
#[arg(long)]
pub no_clear_cache: bool,
/// Skip waiting for the node to sync before starting benchmarks.
/// When enabled, assumes the node is already synced and skips the initial tip check.
#[arg(long)]
pub skip_wait_syncing: bool,
#[command(flatten)]
pub logs: LogArgs,
@@ -578,7 +583,11 @@ async fn run_warmup_phase(
node_manager.start_node(&binary_path, warmup_ref, "warmup", &additional_args).await?;
// Wait for node to be ready and get its current tip
let current_tip = node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?;
let current_tip = if args.skip_wait_syncing {
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
} else {
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
};
info!("Warmup node is ready at tip: {}", current_tip);
// Clear filesystem caches before warmup run only (unless disabled)
@@ -632,7 +641,11 @@ async fn run_benchmark_workflow(
let (mut node_process, _) = node_manager
.start_node(&binary_path, &args.baseline_ref, "baseline", &additional_args)
.await?;
let starting_tip = node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?;
let starting_tip = if args.skip_wait_syncing {
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
} else {
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
};
info!("Node starting tip: {}", starting_tip);
node_manager.stop_node(&mut node_process).await?;
@@ -699,7 +712,11 @@ async fn run_benchmark_workflow(
node_manager.start_node(&binary_path, git_ref, ref_type, &additional_args).await?;
// Wait for node to be ready and get its current tip (wherever it is)
let current_tip = node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?;
let current_tip = if args.skip_wait_syncing {
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
} else {
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
};
info!("Node is ready at tip: {}", current_tip);
// Calculate benchmark range

View File

@@ -163,6 +163,7 @@ impl NodeManager {
"eth,reth".to_string(),
"--disable-discovery".to_string(),
"--trusted-only".to_string(),
"--disable-tx-gossip".to_string(),
]);
// Add tracing arguments if OTLP endpoint is configured
@@ -458,6 +459,76 @@ impl NodeManager {
.wrap_err("Timed out waiting for node to be ready and synced")?
}
/// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
///
/// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
/// the node is still syncing.
pub(crate) async fn wait_for_rpc_and_get_tip(
&self,
child: &mut tokio::process::Child,
) -> Result<u64> {
info!("Waiting for node RPC to be ready (skipping sync wait)...");
let max_wait = Duration::from_secs(60);
let check_interval = Duration::from_secs(2);
let rpc_url = "http://localhost:8545";
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
let provider = ProviderBuilder::new().connect_http(url);
let start_time = tokio::time::Instant::now();
let mut iteration = 0;
timeout(max_wait, async {
loop {
iteration += 1;
debug!(
"RPC readiness check iteration {} (elapsed: {:?})",
iteration,
start_time.elapsed()
);
if let Some(status) = child.try_wait()? {
return Err(eyre!("Node process exited unexpectedly with {status}"));
}
match provider.get_block_number().await {
Ok(tip) => {
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
let ws_connect = WsConnect::new(&ws_url);
match RpcClient::connect_pubsub(ws_connect).await {
Ok(_) => {
info!(
"Node RPC is ready at block: {} (took {:?}, {} iterations)",
tip,
start_time.elapsed(),
iteration
);
return Ok(tip);
}
Err(e) => {
debug!(
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
iteration, e
);
}
}
}
Err(e) => {
debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
}
}
sleep(check_interval).await;
}
})
.await
.wrap_err("Timed out waiting for node RPC to be ready")?
}
/// Stop the reth node gracefully
pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
let pid = child.id().ok_or_eyre("Child process ID should be available")?;

View File

@@ -17,21 +17,26 @@ workspace = true
reth-cli-runner.workspace = true
reth-cli-util.workspace = true
reth-engine-primitives.workspace = true
reth-ethereum-primitives.workspace = true
reth-fs-util.workspace = true
reth-node-api.workspace = true
reth-node-core.workspace = true
reth-primitives-traits.workspace = true
reth-rpc-api.workspace = true
reth-tracing.workspace = true
reth-chainspec.workspace = true
# alloy
alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
alloy-rpc-types-engine.workspace = true
alloy-rpc-types-engine = { workspace = true, features = ["kzg"] }
alloy-transport-http.workspace = true
alloy-transport-ipc.workspace = true
alloy-transport-ws.workspace = true
@@ -85,7 +90,11 @@ jemalloc = [
"reth-node-core/jemalloc",
]
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
tracy-allocator = ["reth-cli-util/tracy-allocator"]
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
tracy = [
"reth-node-core/tracy",
"reth-tracing/tracy",
]
min-error-logs = [
"tracing/release_max_level_error",

View File

@@ -0,0 +1,216 @@
//! Benchmarks empty block processing by ramping the block gas limit.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
};
use alloy_eips::BlockNumberOrTag;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
use clap::Parser;
use reqwest::Url;
use reth_chainspec::ChainSpec;
use reth_cli_runner::CliContext;
use reth_ethereum_primitives::TransactionSigned;
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
engine_rpc_url: String,
/// Path to the JWT secret for Engine API authentication.
#[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
jwt_secret: PathBuf,
/// Output directory for benchmark results and generated payloads.
#[arg(long, value_name = "OUTPUT")]
output: PathBuf,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
// Ensure output directory exists
if self.output.is_file() {
return Err(eyre::eyre!("Output path must be a directory"));
}
if !self.output.exists() {
std::fs::create_dir_all(&self.output)?;
info!("Created output directory: {:?}", self.output);
}
// Set up authenticated provider (used for both Engine API and eth_ methods)
let jwt = std::fs::read_to_string(&self.jwt_secret)?;
let jwt = JwtSecret::from_hex(jwt)?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
let client = ClientBuilder::default().connect_with(auth_transport).await?;
let provider = RootProvider::<AnyNetwork>::new(client);
// Get chain spec - required for fork detection
let chain_id = provider.get_chain_id().await?;
let chain_spec = ChainSpec::from_chain_id(chain_id)
.ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
// Fetch the current head block as parent
let parent_block = provider
.get_block_by_number(BlockNumberOrTag::Latest)
.full()
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
match mode {
RampMode::Blocks(blocks) => {
info!(
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
let new_payload_version = request.new_payload_version;
let (payload, sidecar) = build_payload(&provider, request).await?;
let mut block =
payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
let max_increase = max_gas_limit_increase(parent_header.gas_limit);
let gas_limit =
parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
block.header.gas_limit = gas_limit;
let block_hash = block.header.hash_slow();
// Regenerate the payload from the modified block, but keep the original sidecar
// which contains the actual execution requests data (not just the hash)
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
let (version, params) = payload_to_new_payload(
payload,
sidecar,
false,
block.header.withdrawals_root,
Some(new_payload_version),
)?;
// Save payload to file with version info for replay
let payload_path =
self.output.join(format!("payload_block_{}.json", block.header.number));
let file =
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
let payload_json = serde_json::to_string_pretty(&file)?;
std::fs::write(&payload_path, &payload_json)?;
info!(block_number = block.header.number, path = %payload_path.display(), "Saved payload");
call_new_payload(&provider, version, params).await?;
let forkchoice_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: block_hash,
finalized_block_hash: block_hash,
};
call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
parent_header = block.header;
parent_hash = block_hash;
blocks_processed += 1;
}
let final_gas_limit = parent_header.gas_limit;
info!(
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed,
final_gas_limit,
"Benchmark complete"
);
Ok(())
}
}
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}

View File

@@ -0,0 +1,632 @@
//! Command for generating large blocks by packing transactions from real blocks.
//!
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ForkchoiceState, JwtSecret,
PayloadAttributes,
};
use alloy_transport::layers::RetryBackoffLayer;
use clap::Parser;
use eyre::Context;
use reqwest::Url;
use reth_cli_runner::CliContext;
use reth_rpc_api::TestingBuildBlockRequestV1;
use std::future::Future;
use tokio::sync::mpsc;
use tracing::{info, warn};
/// A single transaction with its gas used and raw encoded bytes.
#[derive(Debug, Clone)]
pub struct RawTransaction {
/// The actual gas used by the transaction (from receipt).
pub gas_used: u64,
/// The transaction type (e.g., 3 for EIP-4844 blob txs).
pub tx_type: u8,
/// The raw RLP-encoded transaction bytes.
pub raw: Bytes,
}
/// Abstraction over sources of transactions for big block generation.
///
/// Implementors provide transactions from different sources (RPC, database, files, etc.)
pub trait TransactionSource {
/// Fetch transactions from a specific block number.
///
/// Returns `Ok(None)` if the block doesn't exist.
/// Returns `Ok(Some((transactions, gas_used)))` with the block's transactions and total gas.
fn fetch_block_transactions(
&self,
block_number: u64,
) -> impl Future<Output = eyre::Result<Option<(Vec<RawTransaction>, u64)>>> + Send;
}
/// RPC-based transaction source that fetches from a remote node.
#[derive(Debug)]
pub struct RpcTransactionSource {
provider: RootProvider<AnyNetwork>,
}
impl RpcTransactionSource {
/// Create a new RPC transaction source.
pub const fn new(provider: RootProvider<AnyNetwork>) -> Self {
Self { provider }
}
/// Create from an RPC URL with retry backoff.
pub fn from_url(rpc_url: &str) -> eyre::Result<Self> {
let client = ClientBuilder::default()
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let provider = RootProvider::<AnyNetwork>::new(client);
Ok(Self { provider })
}
}
impl TransactionSource for RpcTransactionSource {
async fn fetch_block_transactions(
&self,
block_number: u64,
) -> eyre::Result<Option<(Vec<RawTransaction>, u64)>> {
// Fetch block and receipts in parallel
let (block, receipts) = tokio::try_join!(
self.provider.get_block_by_number(block_number.into()).full(),
self.provider.get_block_receipts(block_number.into())
)?;
let Some(block) = block else {
return Ok(None);
};
let Some(receipts) = receipts else {
return Err(eyre::eyre!("Receipts not found for block {}", block_number));
};
let block_gas_used = block.header.gas_used;
// Convert cumulative gas from receipts to per-tx gas_used
let mut prev_cumulative = 0u64;
let transactions: Vec<RawTransaction> = block
.transactions
.txns()
.zip(receipts.iter())
.map(|(tx, receipt)| {
let cumulative = receipt.inner.inner.inner.receipt.cumulative_gas_used;
let gas_used = cumulative - prev_cumulative;
prev_cumulative = cumulative;
let with_encoded = tx.inner.inner.clone().into_encoded();
RawTransaction {
gas_used,
tx_type: tx.inner.ty(),
raw: with_encoded.encoded_bytes().clone(),
}
})
.collect();
Ok(Some((transactions, block_gas_used)))
}
}
/// Collects transactions from a source up to a target gas usage.
#[derive(Debug)]
pub struct TransactionCollector<S> {
source: S,
target_gas: u64,
}
impl<S: TransactionSource> TransactionCollector<S> {
/// Create a new transaction collector.
pub const fn new(source: S, target_gas: u64) -> Self {
Self { source, target_gas }
}
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < self.target_gas {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(block = current_block, "Block not found, stopping");
break;
};
for tx in block_txs {
// Skip blob transactions (EIP-4844, type 3)
if tx.tx_type == 3 {
continue;
}
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
total_gas += tx.gas_used;
}
if total_gas >= self.target_gas {
break;
}
}
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = self.target_gas.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
}
info!(
total_txs = transactions.len(),
total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok((transactions, total_gas, current_block))
}
}
/// `reth bench generate-big-block` command
///
/// Generates a large block by fetching transactions from existing blocks and packing them
/// into a single block using the `testing_buildBlockV1` RPC endpoint.
#[derive(Debug, Parser)]
pub struct Command {
/// The RPC URL to use for fetching blocks (can be an external archive node).
#[arg(long, value_name = "RPC_URL")]
rpc_url: String,
/// The engine RPC URL (with JWT authentication).
#[arg(long, value_name = "ENGINE_RPC_URL", default_value = "http://localhost:8551")]
engine_rpc_url: String,
/// The RPC URL for `testing_buildBlockV1` calls (same node as engine, regular RPC port).
#[arg(long, value_name = "TESTING_RPC_URL", default_value = "http://localhost:8545")]
testing_rpc_url: String,
/// Path to the JWT secret file for engine API authentication.
#[arg(long, value_name = "JWT_SECRET")]
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
target_gas: u64,
/// Block number to start fetching transactions from (required).
///
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
/// The command collects transactions from historical blocks starting at this number
/// to pack into large blocks.
///
/// How to determine this value:
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
/// - If gas limit ramping has already been performed: use the block number that was the chain
/// tip BEFORE ramping began (you must track this yourself)
///
/// Using a block after ramping started will cause transaction collection to fail
/// because those blocks contain synthetic transactions that cannot be replayed.
#[arg(long, value_name = "FROM_BLOCK")]
from_block: u64,
/// Execute the payload (call newPayload + forkchoiceUpdated).
/// If false, only builds the payload and prints it.
#[arg(long, default_value = "false")]
execute: bool,
/// Number of payloads to generate. Each payload uses the previous as parent.
/// When count == 1, the payload is only generated and saved, not executed.
/// When count > 1, each payload is executed before building the next.
#[arg(long, default_value = "1")]
count: u64,
/// Number of transaction batches to prefetch in background when count > 1.
/// Higher values reduce latency but use more memory.
#[arg(long, default_value = "4")]
prefetch_buffer: usize,
/// Output directory for generated payloads. Each payload is saved as `payload_block_N.json`.
#[arg(long, value_name = "OUTPUT_DIR")]
output_dir: std::path::PathBuf,
}
/// A built payload ready for execution.
struct BuiltPayload {
block_number: u64,
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
}
impl Command {
/// Execute the `generate-big-block` command
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(target_gas = self.target_gas, count = self.count, "Generating big block(s)");
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
let jwt = JwtSecret::from_hex(jwt.trim())?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
// Set up testing RPC provider (for testing_buildBlockV1)
info!("Connecting to Testing RPC at {}", self.testing_rpc_url);
let testing_client = ClientBuilder::default()
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(self.testing_rpc_url.parse()?);
let testing_provider = RootProvider::<AnyNetwork>::new(testing_client);
// Get the parent block (latest canonical block)
info!(endpoint = "engine", method = "eth_getBlockByNumber", block = "latest", "RPC call");
let parent_block = auth_provider
.get_block_by_number(BlockNumberOrTag::Latest)
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let parent_hash = parent_block.header.hash;
let parent_number = parent_block.header.number;
let parent_timestamp = parent_block.header.timestamp;
info!(
parent_hash = %parent_hash,
parent_number = parent_number,
"Using initial parent block"
);
// Create output directory
std::fs::create_dir_all(&self.output_dir).wrap_err_with(|| {
format!("Failed to create output directory: {:?}", self.output_dir)
})?;
let start_block = self.from_block;
// Use pipelined execution when generating multiple payloads
if self.count > 1 {
self.execute_pipelined(
&auth_provider,
&testing_provider,
start_block,
parent_hash,
parent_timestamp,
)
.await?;
} else {
// Single payload - collect transactions and build
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential(
&auth_provider,
&testing_provider,
transactions,
parent_hash,
parent_timestamp,
)
.await?;
}
info!(count = self.count, output_dir = %self.output_dir.display(), "All payloads generated");
Ok(())
}
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
transactions: Vec<Bytes>,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let built = self
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
}
parent_hash = built.block_hash;
parent_timestamp = built.timestamp;
}
Ok(())
}
/// Pipelined execution - fetches transactions and builds payloads in background.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
start_block: u64,
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(error = %e, "Failed to create transaction source");
return;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
});
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
let is_last = i == self.count - 1;
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
);
self.build_payload(
testing_provider,
&transactions,
i,
parent_hash,
parent_timestamp,
)
.await?
};
self.save_payload(&current_payload)?;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_receiver);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
&self,
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
}
/// Static version for use in spawned tasks.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
let request = TestingBuildBlockRequestV1 {
parent_block_hash: parent_hash,
payload_attributes: PayloadAttributes {
timestamp: parent_timestamp + 12,
prev_randao: B256::ZERO,
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
},
transactions: transactions.to_vec(),
extra_data: None,
};
let total_tx_bytes: usize = transactions.iter().map(|tx| tx.len()).sum();
info!(
payload = index + 1,
tx_count = transactions.len(),
total_tx_bytes = total_tx_bytes,
parent_hash = %parent_hash,
"Sending to testing_buildBlockV1"
);
let envelope: ExecutionPayloadEnvelopeV5 =
testing_provider.client().request("testing_buildBlockV1", [request]).await?;
let v4_envelope = envelope.try_into_v4()?;
let inner = &v4_envelope.envelope_inner.execution_payload.payload_inner.payload_inner;
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
}
/// Save a payload to disk.
fn save_payload(&self, payload: &BuiltPayload) -> eyre::Result<()> {
let filename = format!("payload_block_{}.json", payload.block_number);
let filepath = self.output_dir.join(&filename);
let json = serde_json::to_string_pretty(&payload.envelope)?;
std::fs::write(&filepath, &json)
.wrap_err_with(|| format!("Failed to write payload to {:?}", filepath))?;
info!(block_number = payload.block_number, block_hash = %payload.block_hash, path = %filepath.display(), "Payload saved");
Ok(())
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload,
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
if !fcu_result.is_valid() {
return Err(eyre::eyre!("FCU rejected: {:?}", fcu_result));
}
Ok(())
}
}

View File

@@ -0,0 +1,266 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
PayloadAttributes, PayloadId, PraguePayloadFields,
};
use eyre::OptionExt;
use reth_chainspec::{ChainSpec, EthereumHardforks};
use reth_node_api::EngineApiMessageVersion;
use tracing::debug;
/// Prepared payload request data for triggering block building.
pub(crate) struct PayloadRequest {
/// The payload attributes for the new block.
pub(crate) attributes: PayloadAttributes,
/// The forkchoice state pointing to the parent block.
pub(crate) forkchoice_state: ForkchoiceState,
/// The engine API version for FCU calls.
pub(crate) fcu_version: EngineApiMessageVersion,
/// The getPayload version to use (1-5).
pub(crate) get_payload_version: u8,
/// The newPayload version to use.
pub(crate) new_payload_version: EngineApiMessageVersion,
}
/// Prepare payload attributes and forkchoice state for a new block.
pub(crate) fn prepare_payload_request(
chain_spec: &ChainSpec,
timestamp: u64,
parent_hash: B256,
) -> PayloadRequest {
let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
// FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
let fcu_version = if cancun_active {
EngineApiMessageVersion::V3
} else if shanghai_active {
EngineApiMessageVersion::V2
} else {
EngineApiMessageVersion::V1
};
// getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
// newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
let (get_payload_version, new_payload_version) = if osaka_active {
(5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
} else if prague_active {
(4, EngineApiMessageVersion::V4)
} else if cancun_active {
(3, EngineApiMessageVersion::V3)
} else if shanghai_active {
(2, EngineApiMessageVersion::V2)
} else {
(1, EngineApiMessageVersion::V1)
};
PayloadRequest {
attributes: PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: Address::ZERO,
withdrawals: shanghai_active.then(Vec::new),
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
},
forkchoice_state: ForkchoiceState {
head_block_hash: parent_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
},
fcu_version,
get_payload_version,
new_payload_version,
}
}
/// Trigger payload building via FCU and retrieve the built payload.
///
/// This sends a forkchoiceUpdated with payload attributes to start building,
/// then calls getPayload to retrieve the result.
pub(crate) async fn build_payload(
provider: &RootProvider<AnyNetwork>,
request: PayloadRequest,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
let fcu_result = call_forkchoice_updated(
provider,
request.fcu_version,
request.forkchoice_state,
Some(request.attributes.clone()),
)
.await?;
let payload_id =
fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
get_payload_with_sidecar(
provider,
request.get_payload_version,
payload_id,
request.attributes.parent_beacon_block_root,
)
.await
}
/// Convert an RPC block to a consensus header and block hash.
pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
let block_hash = block.header.hash;
let header = block.header.inner.clone().into_header_with_defaults();
(header, block_hash)
}
/// Compute versioned hashes from KZG commitments.
fn versioned_hashes_from_commitments(
commitments: &[alloy_primitives::FixedBytes<48>],
) -> Vec<B256> {
commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
}
/// Fetch an execution payload using the appropriate engine API version.
pub(crate) async fn get_payload_with_sidecar(
provider: &RootProvider<AnyNetwork>,
version: u8,
payload_id: PayloadId,
parent_beacon_block_root: Option<B256>,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
debug!(get_payload_version = ?version, ?payload_id, "Sending getPayload");
match version {
1 => {
let payload = provider.get_payload_v1(payload_id).await?;
Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
}
2 => {
let envelope = provider.get_payload_v2(payload_id).await?;
let payload = match envelope.execution_payload {
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
};
Ok((payload, ExecutionPayloadSidecar::none()))
}
3 => {
let envelope = provider.get_payload_v3(payload_id).await?;
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V3")?,
versioned_hashes,
};
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v3(cancun_fields),
))
}
4 => {
let envelope = provider.get_payload_v4(payload_id).await?;
let versioned_hashes = versioned_hashes_from_commitments(
&envelope.envelope_inner.blobs_bundle.commitments,
);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V4")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.envelope_inner.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
5 => {
// V5 (Osaka) - use raw request since alloy doesn't have get_payload_v5 yet
let envelope = provider.get_payload_v5(payload_id).await?;
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V5")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -6,9 +6,17 @@ use reth_node_core::args::LogArgs;
use reth_tracing::FileWorkerGuard;
mod context;
mod gas_limit_ramp;
mod generate_big_block;
pub(crate) mod helpers;
pub use generate_big_block::{
RawTransaction, RpcTransactionSource, TransactionCollector, TransactionSource,
};
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -27,6 +35,9 @@ pub enum Subcommands {
/// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
NewPayloadFcu(new_payload_fcu::Command),
/// Benchmark which builds empty blocks with a ramped gas limit.
GasLimitRamp(gas_limit_ramp::Command),
/// Benchmark which only calls subsequent `newPayload` calls.
NewPayloadOnly(new_payload_only::Command),
@@ -41,6 +52,41 @@ pub enum Subcommands {
/// `cast block latest --full --json | reth-bench send-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex)`
SendPayload(send_payload::Command),
/// Generate a large block by packing transactions from existing blocks.
///
/// This command fetches transactions from real blocks and packs them into a single
/// block using the `testing_buildBlockV1` RPC endpoint.
///
/// Example:
///
/// `reth-bench generate-big-block --rpc-url http://localhost:8545 --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex --target-gas
/// 30000000`
GenerateBigBlock(generate_big_block::Command),
/// Replay pre-generated payloads from a directory.
///
/// This command reads payload files from a previous `generate-big-block` run and replays
/// them in sequence using `newPayload` followed by `forkchoiceUpdated`.
///
/// Example:
///
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -51,8 +97,12 @@ impl BenchmarkCommand {
match self.command {
Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}

View File

@@ -13,8 +13,7 @@ use crate::{
bench::{
context::BenchContext,
output::{
CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow, COMBINED_OUTPUT_SUFFIX,
GAS_OUTPUT_SUFFIX,
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
@@ -27,7 +26,6 @@ use alloy_rpc_client::RpcClient;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use csv::Writer;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
@@ -123,6 +121,7 @@ impl Command {
auth_provider,
mut next_block,
is_optimism,
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let buffer_size = self.rpc_block_buffer_size;
@@ -188,6 +187,7 @@ impl Command {
result
} {
let gas_used = block.header.gas_used;
let gas_limit = block.header.gas_limit;
let block_number = block.header.number;
let transaction_count = block.transactions.len() as u64;
@@ -211,6 +211,7 @@ impl Command {
let fcu_latency = total_latency - new_payload_result.latency;
let combined_result = CombinedResult {
block_number,
gas_limit,
transaction_count,
new_payload_result,
fcu_latency,
@@ -240,28 +241,11 @@ impl Command {
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (_, Vec<CombinedResult>) =
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
// Write CSV output files
if let Some(ref path) = self.benchmark.output {
let output_path = path.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;
for result in combined_results {
writer.serialize(result)?;
}
writer.flush()?;
let output_path = path.join(GAS_OUTPUT_SUFFIX);
info!("Writing total gas output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;
for row in &gas_output_results {
writer.serialize(row)?;
}
writer.flush()?;
info!("Finished writing benchmark output files to {:?}.", path);
write_benchmark_results(path, &gas_output_results, combined_results)?;
}
let gas_output = TotalGasOutput::new(gas_output_results)?;

View File

@@ -49,6 +49,7 @@ impl Command {
auth_provider,
mut next_block,
is_optimism,
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let buffer_size = self.rpc_block_buffer_size;
@@ -96,11 +97,7 @@ impl Command {
let transaction_count = block.transactions.len() as u64;
let gas_used = block.header.gas_used;
debug!(
target: "reth-bench",
number=?block.header.number,
"Sending payload to engine",
);
debug!(number=?block.header.number, "Sending payload to engine");
let (version, params) = block_to_new_payload(block, is_optimism)?;

View File

@@ -1,10 +1,13 @@
//! Contains various benchmark output formats, either for logging or for
//! serialization to / from files.
use alloy_primitives::B256;
use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Serialize};
use std::time::Duration;
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{path::Path, time::Duration};
use tracing::info;
/// This is the suffix for gas output csv files.
pub(crate) const GAS_OUTPUT_SUFFIX: &str = "total_gas.csv";
@@ -15,6 +18,17 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
/// This is the suffix for new payload output csv files.
pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
/// Serialized format for gas ramp payloads on disk.
#[derive(Debug, Serialize, Deserialize)]
pub(crate) struct GasRampPayloadFile {
/// Engine API version (1-5).
pub(crate) version: u8,
/// The block hash for FCU.
pub(crate) block_hash: B256,
/// The params to pass to newPayload.
pub(crate) params: serde_json::Value,
}
/// This represents the results of a single `newPayload` call in the benchmark, containing the gas
/// used and the `newPayload` latency.
#[derive(Debug)]
@@ -67,6 +81,8 @@ impl Serialize for NewPayloadResult {
pub(crate) struct CombinedResult {
/// The block number of the block being processed.
pub(crate) block_number: u64,
/// The gas limit of the block.
pub(crate) gas_limit: u64,
/// The number of transactions in the block.
pub(crate) transaction_count: u64,
/// The `newPayload` result.
@@ -88,7 +104,7 @@ impl std::fmt::Display for CombinedResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Payload {} processed at {:.4} Ggas/s, used {} total gas. Combined gas per second: {:.4} Ggas/s. fcu latency: {:?}, newPayload latency: {:?}",
"Block {} processed at {:.4} Ggas/s, used {} total gas. Combined: {:.4} Ggas/s. fcu: {:?}, newPayload: {:?}",
self.block_number,
self.new_payload_result.gas_per_second() / GIGAGAS as f64,
self.new_payload_result.gas_used,
@@ -110,10 +126,11 @@ impl Serialize for CombinedResult {
let fcu_latency = self.fcu_latency.as_micros();
let new_payload_latency = self.new_payload_result.latency.as_micros();
let total_latency = self.total_latency.as_micros();
let mut state = serializer.serialize_struct("CombinedResult", 6)?;
let mut state = serializer.serialize_struct("CombinedResult", 7)?;
// flatten the new payload result because this is meant for CSV writing
state.serialize_field("block_number", &self.block_number)?;
state.serialize_field("gas_limit", &self.gas_limit)?;
state.serialize_field("transaction_count", &self.transaction_count)?;
state.serialize_field("gas_used", &self.new_payload_result.gas_used)?;
state.serialize_field("new_payload_latency", &new_payload_latency)?;
@@ -167,6 +184,36 @@ impl TotalGasOutput {
}
}
/// Write benchmark results to CSV files.
///
/// Writes two files to the output directory:
/// - `combined_latency.csv`: Per-block latency results
/// - `total_gas.csv`: Per-block gas usage over time
pub(crate) fn write_benchmark_results(
output_dir: &Path,
gas_results: &[TotalGasRow],
combined_results: Vec<CombinedResult>,
) -> eyre::Result<()> {
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;
for result in combined_results {
writer.serialize(result)?;
}
writer.flush()?;
let output_path = output_dir.join(GAS_OUTPUT_SUFFIX);
info!("Writing total gas output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;
for row in gas_results {
writer.serialize(row)?;
}
writer.flush()?;
info!("Finished writing benchmark output files to {:?}.", output_dir);
Ok(())
}
/// This serializes the `time` field of the [`TotalGasRow`] to microseconds.
///
/// This is essentially just for the csv writer, which would have headers

View File

@@ -0,0 +1,332 @@
//! Command for replaying pre-generated payloads from disk.
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::output::GasRampPayloadFile,
valid_payload::{call_forkchoice_updated, call_new_payload},
};
use alloy_primitives::B256;
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use clap::Parser;
use eyre::Context;
use reqwest::Url;
use reth_cli_runner::CliContext;
use reth_node_api::EngineApiMessageVersion;
use std::path::PathBuf;
use tracing::{debug, info};
/// `reth bench replay-payloads` command
///
/// Replays pre-generated payloads from a directory by calling `newPayload` followed by
/// `forkchoiceUpdated` for each payload in sequence.
#[derive(Debug, Parser)]
pub struct Command {
/// The engine RPC URL (with JWT authentication).
#[arg(long, value_name = "ENGINE_RPC_URL", default_value = "http://localhost:8551")]
engine_rpc_url: String,
/// Path to the JWT secret file for engine API authentication.
#[arg(long, value_name = "JWT_SECRET")]
jwt_secret: PathBuf,
/// Directory containing payload files (`payload_block_N.json`).
#[arg(long, value_name = "PAYLOAD_DIR")]
payload_dir: PathBuf,
/// Optional limit on the number of payloads to replay.
/// If not specified, replays all payloads in the directory.
#[arg(long, value_name = "COUNT")]
count: Option<usize>,
/// Skip the first N payloads.
#[arg(long, value_name = "SKIP", default_value = "0")]
skip: usize,
/// Optional directory containing gas ramp payloads to replay first.
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
}
/// A loaded payload ready for execution.
struct LoadedPayload {
/// The index (from filename).
index: u64,
/// The payload envelope.
envelope: ExecutionPayloadEnvelopeV4,
/// The block hash.
block_hash: B256,
}
/// A gas ramp payload loaded from disk.
struct GasRampPayload {
/// Block number from filename.
block_number: u64,
/// Engine API version for newPayload.
version: EngineApiMessageVersion,
/// The file contents.
file: GasRampPayloadFile,
}
impl Command {
/// Execute the `replay-payloads` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
let jwt = JwtSecret::from_hex(jwt.trim())?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
// Get parent block (latest canonical block) - we need this for the first FCU
let parent_block = auth_provider
.get_block_by_number(alloy_eips::BlockNumberOrTag::Latest)
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let initial_parent_hash = parent_block.header.hash;
let initial_parent_number = parent_block.header.number;
info!(
parent_hash = %initial_parent_hash,
parent_number = initial_parent_number,
"Using initial parent block"
);
// Load all payloads upfront to avoid I/O delays between phases
let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
if payloads.is_empty() {
return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
}
info!(count = payloads.len(), "Loaded gas ramp payloads from disk");
payloads
} else {
Vec::new()
};
let payloads = self.load_payloads()?;
if payloads.is_empty() {
return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
}
info!(count = payloads.len(), "Loaded main payloads from disk");
let mut parent_hash = initial_parent_hash;
// Replay gas ramp payloads first
for (i, payload) in gas_ramp_payloads.iter().enumerate() {
info!(
gas_ramp_payload = i + 1,
total = gas_ramp_payloads.len(),
block_number = payload.block_number,
block_hash = %payload.file.block_hash,
"Executing gas ramp payload (newPayload + FCU)"
);
call_new_payload(&auth_provider, payload.version, payload.file.params.clone()).await?;
let fcu_state = ForkchoiceState {
head_block_hash: payload.file.block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
parent_hash = payload.file.block_hash;
}
if !gas_ramp_payloads.is_empty() {
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
for (i, payload) in payloads.iter().enumerate() {
info!(
payload = i + 1,
total = payloads.len(),
index = payload.index,
block_hash = %payload.block_hash,
"Executing payload (newPayload + FCU)"
);
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
parent_hash = payload.block_hash;
}
info!(count = payloads.len(), "All payloads replayed successfully");
Ok(())
}
/// Load and parse all payload files from the directory.
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
let mut payloads = Vec::new();
// Read directory entries
let entries: Vec<_> = std::fs::read_dir(&self.payload_dir)
.wrap_err_with(|| format!("Failed to read directory {:?}", self.payload_dir))?
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
// Parse filenames to get indices and sort
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
.into_iter()
.filter_map(|e| {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract index from "payload_NNN.json"
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let index: u64 = index_str.parse().ok()?;
Some((index, e.path()))
})
.collect();
indexed_paths.sort_by_key(|(idx, _)| *idx);
// Apply skip and count
let indexed_paths: Vec<_> = indexed_paths.into_iter().skip(self.skip).collect();
let indexed_paths: Vec<_> = match self.count {
Some(count) => indexed_paths.into_iter().take(count).collect(),
None => indexed_paths,
};
// Load each payload
for (index, path) in indexed_paths {
let content = std::fs::read_to_string(&path)
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
info!(
index = index,
block_hash = %block_hash,
path = %path.display(),
"Loaded payload"
);
payloads.push(LoadedPayload { index, envelope, block_hash });
}
Ok(payloads)
}
/// Load and parse gas ramp payload files from a directory.
fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
let mut payloads = Vec::new();
let entries: Vec<_> = std::fs::read_dir(dir)
.wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
// Parse filenames to get block numbers and sort
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
.into_iter()
.filter_map(|e| {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract block number from "payload_block_NNN.json"
let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let block_number: u64 = block_str.parse().ok()?;
Some((block_number, e.path()))
})
.collect();
indexed_paths.sort_by_key(|(num, _)| *num);
for (block_number, path) in indexed_paths {
let content = std::fs::read_to_string(&path)
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
let file: GasRampPayloadFile = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let version = match file.version {
1 => EngineApiMessageVersion::V1,
2 => EngineApiMessageVersion::V2,
3 => EngineApiMessageVersion::V3,
4 => EngineApiMessageVersion::V4,
5 => EngineApiMessageVersion::V5,
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
};
info!(
block_number,
block_hash = %file.block_hash,
path = %path.display(),
"Loaded gas ramp payload"
);
payloads.push(GasRampPayload { block_number, version, file });
}
Ok(payloads)
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: &ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
info!(?status, "newPayloadV4 response");
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
info!(?fcu_result, "forkchoiceUpdatedV3 response");
Ok(())
}
}

View File

@@ -0,0 +1,219 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
pub(super) struct InvalidationConfig {
// Explicit value overrides (Option<T>)
pub(super) parent_hash: Option<B256>,
pub(super) fee_recipient: Option<Address>,
pub(super) state_root: Option<B256>,
pub(super) receipts_root: Option<B256>,
pub(super) logs_bloom: Option<Bloom>,
pub(super) prev_randao: Option<B256>,
pub(super) block_number: Option<u64>,
pub(super) gas_limit: Option<u64>,
pub(super) gas_used: Option<u64>,
pub(super) timestamp: Option<u64>,
pub(super) extra_data: Option<Bytes>,
pub(super) base_fee_per_gas: Option<u64>,
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
pub(super) invalidate_state_root: bool,
pub(super) invalidate_receipts_root: bool,
pub(super) invalidate_gas_used: bool,
pub(super) invalidate_block_number: bool,
pub(super) invalidate_timestamp: bool,
pub(super) invalidate_base_fee: bool,
pub(super) invalidate_transactions: bool,
pub(super) invalidate_block_hash: bool,
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
}
impl InvalidationConfig {
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
self.block_hash.is_some() || self.invalidate_block_hash
}
/// Applies invalidations to a V1 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
let mut changes = Vec::new();
// Explicit value overrides
if let Some(parent_hash) = self.parent_hash {
payload.parent_hash = parent_hash;
changes.push(format!("parent_hash = {parent_hash}"));
}
if let Some(fee_recipient) = self.fee_recipient {
payload.fee_recipient = fee_recipient;
changes.push(format!("fee_recipient = {fee_recipient}"));
}
if let Some(state_root) = self.state_root {
payload.state_root = state_root;
changes.push(format!("state_root = {state_root}"));
}
if let Some(receipts_root) = self.receipts_root {
payload.receipts_root = receipts_root;
changes.push(format!("receipts_root = {receipts_root}"));
}
if let Some(logs_bloom) = self.logs_bloom {
payload.logs_bloom = logs_bloom;
changes.push("logs_bloom = <custom>".to_string());
}
if let Some(prev_randao) = self.prev_randao {
payload.prev_randao = prev_randao;
changes.push(format!("prev_randao = {prev_randao}"));
}
if let Some(block_number) = self.block_number {
payload.block_number = block_number;
changes.push(format!("block_number = {block_number}"));
}
if let Some(gas_limit) = self.gas_limit {
payload.gas_limit = gas_limit;
changes.push(format!("gas_limit = {gas_limit}"));
}
if let Some(gas_used) = self.gas_used {
payload.gas_used = gas_used;
changes.push(format!("gas_used = {gas_used}"));
}
if let Some(timestamp) = self.timestamp {
payload.timestamp = timestamp;
changes.push(format!("timestamp = {timestamp}"));
}
if let Some(ref extra_data) = self.extra_data {
payload.extra_data = extra_data.clone();
changes.push(format!("extra_data = {} bytes", extra_data.len()));
}
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
}
if let Some(block_hash) = self.block_hash {
payload.block_hash = block_hash;
changes.push(format!("block_hash = {block_hash}"));
}
// Auto-invalidation flags
if self.invalidate_parent_hash {
let random_hash = B256::random();
payload.parent_hash = random_hash;
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
}
if self.invalidate_state_root {
payload.state_root = B256::ZERO;
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
}
if self.invalidate_receipts_root {
payload.receipts_root = B256::ZERO;
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
}
if self.invalidate_gas_used {
let invalid_gas = payload.gas_limit + 1;
payload.gas_used = invalid_gas;
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
}
if self.invalidate_block_number {
let invalid_number = payload.block_number + 999;
payload.block_number = invalid_number;
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
}
if self.invalidate_timestamp {
payload.timestamp = 0;
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
}
if self.invalidate_base_fee {
payload.base_fee_per_gas = U256::ZERO;
changes
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
}
if self.invalidate_transactions {
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
payload.transactions.insert(0, invalid_tx);
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
}
if self.invalidate_block_hash {
let random_hash = B256::random();
payload.block_hash = random_hash;
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
}
changes
}
/// Applies invalidations to a V2 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
// Handle withdrawals invalidation (V2+)
if self.invalidate_withdrawals {
let fake_withdrawal = Withdrawal {
index: u64::MAX,
validator_index: u64::MAX,
address: Address::ZERO,
amount: u64::MAX,
};
payload.withdrawals.push(fake_withdrawal);
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
}
changes
}
/// Applies invalidations to a V3 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
// Explicit overrides for V3 fields
if let Some(blob_gas_used) = self.blob_gas_used {
payload.blob_gas_used = blob_gas_used;
changes.push(format!("blob_gas_used = {blob_gas_used}"));
}
if let Some(excess_blob_gas) = self.excess_blob_gas {
payload.excess_blob_gas = excess_blob_gas;
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
}
// Auto-invalidation for V3 fields
if self.invalidate_blob_gas_used {
payload.blob_gas_used = u64::MAX;
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
}
if self.invalidate_excess_blob_gas {
payload.excess_blob_gas = u64::MAX;
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -0,0 +1,367 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use invalidation::InvalidationConfig;
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::{BufReader, Read, Write};
/// Command for generating and sending an invalid `engine_newPayload` request.
///
/// Takes a valid block and modifies fields to make it invalid for testing
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
#[derive(Debug, Parser)]
pub struct Command {
// ==================== Input Options ====================
/// Path to the JSON file containing the block. If not specified, stdin will be used.
#[arg(short, long, help_heading = "Input Options")]
path: Option<String>,
/// The engine RPC URL to use.
#[arg(
short,
long,
help_heading = "Input Options",
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
required_unless_present("mode")
)]
rpc_url: Option<String>,
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
/// itself.
#[arg(short, long, help_heading = "Input Options")]
jwt_secret: Option<String>,
/// The newPayload version to use (3 or 4).
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
new_payload_version: u8,
/// The output mode to use.
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
mode: Mode,
// ==================== Explicit Value Overrides ====================
/// Override the parent hash with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_hash: Option<B256>,
/// Override the fee recipient (coinbase) with a specific address.
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
fee_recipient: Option<Address>,
/// Override the state root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
state_root: Option<B256>,
/// Override the receipts root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
receipts_root: Option<B256>,
/// Override the block number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
block_number: Option<u64>,
/// Override the gas limit with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_limit: Option<u64>,
/// Override the gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_used: Option<u64>,
/// Override the timestamp with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
timestamp: Option<u64>,
/// Override the base fee per gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
base_fee_per_gas: Option<u64>,
/// Override the block hash with a specific value (skips hash recalculation).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
block_hash: Option<B256>,
/// Override the blob gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
blob_gas_used: Option<u64>,
/// Override the excess blob gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
excess_blob_gas: Option<u64>,
/// Override the parent beacon block root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_beacon_block_root: Option<B256>,
/// Override the requests hash with a specific value (EIP-7685).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_parent_hash: bool,
/// Invalidate the state root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_state_root: bool,
/// Invalidate the receipts root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_receipts_root: bool,
/// Invalidate the gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_gas_used: bool,
/// Invalidate the block number by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_number: bool,
/// Invalidate the timestamp by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_timestamp: bool,
/// Invalidate the base fee by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_base_fee: bool,
/// Invalidate the transactions by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_transactions: bool,
/// Invalidate the block hash by not recalculating it after modifications.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_hash: bool,
/// Invalidate the withdrawals by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_withdrawals: bool,
/// Invalidate the blob gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_blob_gas_used: bool,
/// Invalidate the excess blob gas by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_excess_blob_gas: bool,
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
skip_hash_recalc: bool,
/// Print what would be done without actually sending the payload.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
dry_run: bool,
}
#[derive(Debug, Clone, clap::ValueEnum)]
enum Mode {
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
/// payload into the `cast` command.
Execute,
/// Print the `cast` command. Caution: this may not work with large blocks because of the
/// command length limit.
Cast,
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
Json,
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
Err(_) => Ok(Some(secret.clone())),
},
None => Ok(None),
}
}
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
parent_hash: self.parent_hash,
fee_recipient: self.fee_recipient,
state_root: self.state_root,
receipts_root: self.receipts_root,
logs_bloom: None,
prev_randao: None,
block_number: self.block_number,
gas_limit: self.gas_limit,
gas_used: self.gas_used,
timestamp: self.timestamp,
extra_data: None,
base_fee_per_gas: self.base_fee_per_gas,
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
invalidate_gas_used: self.invalidate_gas_used,
invalidate_block_number: self.invalidate_block_number,
invalidate_timestamp: self.invalidate_timestamp,
invalidate_base_fee: self.invalidate_base_fee,
invalidate_transactions: self.invalidate_transactions,
invalidate_block_hash: self.invalidate_block_hash,
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
}
}
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = self.read_input()?;
let jwt_secret = self.load_jwt_secret()?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
.into_consensus();
let config = self.build_invalidation_config();
let parent_beacon_block_root =
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
if !skip_recalc {
let new_hash = match execution_payload.clone().into_block_raw() {
Ok(block) => block.header.hash_slow(),
Err(e) => {
eprintln!(
"Warning: Could not recalculate block hash: {e}. Using original hash."
);
match &execution_payload {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
}
}
};
match &mut execution_payload {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
}
}
if self.dry_run {
println!("=== Dry Run ===");
println!("Changes that would be applied:");
for change in &changes {
println!(" - {}", change);
}
if changes.is_empty() {
println!(" (no changes)");
}
if skip_recalc {
println!(" - Block hash recalculation: SKIPPED");
} else {
println!(" - Block hash recalculation: PERFORMED");
}
println!("\nResulting payload JSON:");
let json = serde_json::to_string_pretty(&execution_payload)?;
println!("{}", json);
return Ok(());
}
let json_request = if use_v4 {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
requests_hash.unwrap_or_default(),
))?
} else {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
))?
};
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
}
if let Some(secret) = &jwt_secret {
command.arg("--jwt-secret").arg(secret);
}
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
process
.stdin
.take()
.ok_or_eyre("stdin not available")?
.write_all(json_request.as_bytes())?;
process.wait()?;
}
Mode::Cast => {
let mut cmd = format!(
"cast rpc engine_newPayloadV{} --raw '{}'",
self.new_payload_version, json_request
);
if let Some(rpc_url) = self.rpc_url {
cmd += &format!(" --rpc-url {rpc_url}");
}
if let Some(secret) = &jwt_secret {
cmd += &format!(" --jwt-secret {secret}");
}
println!("{cmd}");
}
Mode::Json => {
println!("{json_request}");
}
}
Ok(())
}
}

View File

@@ -3,15 +3,16 @@
//! before sending additional calls.
use alloy_eips::eip7685::Requests;
use alloy_primitives::B256;
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
use alloy_rpc_types_engine::{
ExecutionPayload, ExecutionPayloadInputV2, ForkchoiceState, ForkchoiceUpdated,
PayloadAttributes, PayloadStatus,
ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ForkchoiceState,
ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
};
use alloy_transport::TransportResult;
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
use reth_node_api::EngineApiMessageVersion;
use tracing::error;
use tracing::{debug, error};
/// An extension trait for providers that implement the engine API, to wait for a VALID response.
#[async_trait::async_trait]
@@ -52,6 +53,13 @@ where
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
method = "engine_forkchoiceUpdatedV1",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v1(fork_choice_state, payload_attributes.clone()).await?;
@@ -82,6 +90,13 @@ where
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
method = "engine_forkchoiceUpdatedV2",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v2(fork_choice_state, payload_attributes.clone()).await?;
@@ -112,6 +127,13 @@ where
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
method = "engine_forkchoiceUpdatedV3",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v3(fork_choice_state, payload_attributes.clone()).await?;
@@ -148,33 +170,47 @@ pub(crate) fn block_to_new_payload(
// Convert to execution payload
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
}
pub(crate) fn payload_to_new_payload(
payload: ExecutionPayload,
sidecar: ExecutionPayloadSidecar,
is_optimism: bool,
withdrawals_root: Option<B256>,
target_version: Option<EngineApiMessageVersion>,
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
let (version, params) = match payload {
ExecutionPayload::V3(payload) => {
let cancun = sidecar.cancun().unwrap();
if let Some(prague) = sidecar.prague() {
// Use target version if provided (for Osaka), otherwise default to V4
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
if is_optimism {
let withdrawals_root = withdrawals_root.ok_or_else(|| {
eyre::eyre!("Missing withdrawals root for Optimism payload")
})?;
(
EngineApiMessageVersion::V4,
version,
serde_json::to_value((
OpExecutionPayloadV4 {
payload_inner: payload,
withdrawals_root: block.withdrawals_root.unwrap(),
},
OpExecutionPayloadV4 { payload_inner: payload, withdrawals_root },
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
Requests::default(),
))?,
)
} else {
// Extract actual Requests from RequestsOrHash
let requests = prague.requests.requests_hash();
(
EngineApiMessageVersion::V4,
version,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
prague.requests.requests_hash(),
requests,
))?,
)
}
@@ -217,6 +253,8 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
) -> TransportResult<()> {
let method = version.method_name();
debug!(method, "Sending newPayload");
let mut status: PayloadStatus = provider.client().request(method, &params).await?;
while !status.is_valid() {
@@ -237,12 +275,15 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
/// `EngineApiMessageVersion`, using the provided forkchoice state and payload attributes for the
/// actual engine api message call.
///
/// Note: For Prague (V4), we still use forkchoiceUpdatedV3 as there is no V4.
pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
provider: P,
message_version: EngineApiMessageVersion,
forkchoice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await

View File

@@ -81,12 +81,16 @@ backon.workspace = true
tempfile.workspace = true
[features]
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
default = ["jemalloc", "otlp", "otlp-logs", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
otlp = [
"reth-ethereum-cli/otlp",
"reth-node-core/otlp",
]
otlp-logs = [
"reth-ethereum-cli/otlp-logs",
"reth-node-core/otlp-logs",
]
js-tracer = [
"reth-node-builder/js-tracer",
"reth-node-ethereum/js-tracer",
@@ -131,6 +135,11 @@ jemalloc-unprefixed = [
tracy-allocator = [
"reth-cli-util/tracy-allocator",
"reth-ethereum-cli/tracy-allocator",
"tracy",
]
tracy = [
"reth-ethereum-cli/tracy",
"reth-node-core/tracy",
]
# Because jemalloc is default and preferred over snmalloc when both features are
@@ -171,7 +180,7 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
edge = ["reth-ethereum-cli/edge"]
edge = ["reth-ethereum-cli/edge", "reth-node-core/edge"]
[[bin]]
name = "reth"

View File

@@ -25,8 +25,8 @@
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!

View File

@@ -41,6 +41,7 @@ derive_more.workspace = true
metrics.workspace = true
parking_lot.workspace = true
pin-project.workspace = true
rayon = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
# optional deps for test-utils
@@ -84,6 +85,7 @@ test-utils = [
"reth-trie/test-utils",
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"

View File

@@ -163,14 +163,29 @@ impl DeferredTrieData {
anchor_hash: B256,
ancestors: &[Self],
) -> ComputedTrieData {
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
#[cfg(feature = "rayon")]
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|| match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
|| match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
#[cfg(not(feature = "rayon"))]
let (sorted_hashed_state, sorted_trie_updates) = (
match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
// Reuse parent's overlay if available and anchors match.
// We can only reuse the parent's overlay if it was built on top of the same
@@ -191,11 +206,33 @@ impl DeferredTrieData {
Default::default(), // prefix_sets are per-block, not cumulative
);
// Only trigger COW clone if there's actually data to add.
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
#[cfg(feature = "rayon")]
{
rayon::join(
|| {
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
},
|| {
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
},
);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
#[cfg(not(feature = "rayon"))]
{
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
}
overlay
}
@@ -242,13 +279,22 @@ impl DeferredTrieData {
for ancestor in ancestors {
let ancestor_data = ancestor.wait_cloned();
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
}
// Extend with current block's sorted data last (takes precedence)
state_mut.extend_ref(sorted_hashed_state);
nodes_mut.extend_ref(sorted_trie_updates);
#[cfg(feature = "rayon")]
rayon::join(
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
);
#[cfg(not(feature = "rayon"))]
{
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
}
overlay
}
@@ -287,6 +333,11 @@ impl DeferredTrieData {
&inputs.ancestors,
);
*state = DeferredState::Ready(computed.clone());
// Release lock before inputs (and its ancestors) drop to avoid holding it
// while their potential last Arc refs drop (which could trigger recursive locking)
drop(state);
computed
}
}
@@ -516,7 +567,7 @@ mod tests {
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
let trie_updates = Arc::default();
let mut overlay = TrieInputSorted::default();
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
DeferredTrieData::ready(ComputedTrieData {
hashed_state,

View File

@@ -10,15 +10,18 @@ use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256};
use parking_lot::RwLock;
use reth_chainspec::ChainInfo;
use reth_ethereum_primitives::EthPrimitives;
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_metrics::{metrics::Gauge, Metrics};
use reth_primitives_traits::{
BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
SignedTransaction,
};
use reth_storage_api::StateProviderBox;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::{collections::BTreeMap, ops::Deref, sync::Arc, time::Instant};
use reth_trie::{
updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, SortedTrieData,
TrieInputSorted,
};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tokio::sync::{broadcast, watch};
/// Size of the broadcast channel used to notify canonical state events.
@@ -648,7 +651,7 @@ impl<N: NodePrimitives> BlockState<N> {
}
/// Returns the `Receipts` of executed block that determines the state.
pub fn receipts(&self) -> &Vec<Vec<N::Receipt>> {
pub fn receipts(&self) -> &Vec<N::Receipt> {
&self.block.execution_outcome().receipts
}
@@ -659,15 +662,7 @@ impl<N: NodePrimitives> BlockState<N> {
///
/// This clones the vector of receipts. To avoid it, use [`Self::executed_block_receipts_ref`].
pub fn executed_block_receipts(&self) -> Vec<N::Receipt> {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().cloned().unwrap_or_default()
self.receipts().clone()
}
/// Returns a slice of `Receipt` of executed block that determines the state.
@@ -675,15 +670,7 @@ impl<N: NodePrimitives> BlockState<N> {
/// has only one element corresponding to the executed block associated to
/// the state.
pub fn executed_block_receipts_ref(&self) -> &[N::Receipt] {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().map(|receipts| receipts.deref()).unwrap_or_default()
self.receipts()
}
/// Returns an iterator over __parent__ `BlockStates`.
@@ -767,7 +754,7 @@ pub struct ExecutedBlock<N: NodePrimitives = EthPrimitives> {
/// Recovered Block
pub recovered_block: Arc<RecoveredBlock<N::Block>>,
/// Block's execution outcome.
pub execution_output: Arc<ExecutionOutcome<N::Receipt>>,
pub execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
/// Deferred trie data produced by execution.
///
/// This allows deferring the computation of the trie data which can be expensive.
@@ -779,7 +766,15 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
fn default() -> Self {
Self {
recovered_block: Default::default(),
execution_output: Default::default(),
execution_output: Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: Default::default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: Default::default(),
}),
trie_data: DeferredTrieData::ready(ComputedTrieData::default()),
}
}
@@ -800,7 +795,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// payload builders). This is the safe default path.
pub fn new(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: ComputedTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data: DeferredTrieData::ready(trie_data) }
@@ -822,7 +817,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Use [`Self::new()`] instead when trie data is already computed and available immediately.
pub const fn with_deferred_trie_data(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: DeferredTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data }
@@ -842,7 +837,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Returns a reference to the block's execution outcome
#[inline]
pub fn execution_outcome(&self) -> &ExecutionOutcome<N::Receipt> {
pub fn execution_outcome(&self) -> &BlockExecutionOutput<N::Receipt> {
&self.execution_output
}
@@ -942,37 +937,53 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
pub fn to_chain_notification(&self) -> CanonStateNotification<N> {
match self {
Self::Commit { new } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
);
chain
}));
CanonStateNotification::Commit { new }
CanonStateNotification::Commit { new: Arc::new(Self::blocks_to_chain(new)) }
}
Self::Reorg { new, old } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
Self::Reorg { new, old } => CanonStateNotification::Reorg {
new: Arc::new(Self::blocks_to_chain(new)),
old: Arc::new(Self::blocks_to_chain(old)),
},
}
}
/// Converts a slice of executed blocks into a [`Chain`].
fn blocks_to_chain(blocks: &[ExecutedBlock<N>]) -> Chain<N> {
match blocks {
[] => Chain::default(),
[first, rest @ ..] => {
let trie_data_handle = first.trie_data_handle();
let mut chain = Chain::from_block(
first.recovered_block().clone(),
ExecutionOutcome::from((
first.execution_outcome().clone(),
first.block_number(),
)),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
);
for exec in rest {
let trie_data_handle = exec.trie_data_handle();
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
ExecutionOutcome::from((
exec.execution_outcome().clone(),
exec.block_number(),
)),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
);
chain
}));
let old = Arc::new(old.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
);
chain
}));
CanonStateNotification::Reorg { new, old }
}
chain
}
}
}
@@ -1266,7 +1277,7 @@ mod tests {
let state = BlockState::new(block);
assert_eq!(state.receipts(), &receipts);
assert_eq!(state.receipts(), receipts.first().unwrap());
}
#[test]
@@ -1543,33 +1554,31 @@ mod tests {
let block2a =
test_block_builder.get_executed_block_with_number(2, block1.recovered_block.hash());
let sample_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
..Default::default()
};
// Test commit notification
let chain_commit = NewCanonicalChain::Commit { new: vec![block0.clone(), block1.clone()] };
// Build expected trie updates map
let mut expected_trie_updates = BTreeMap::new();
expected_trie_updates.insert(0, block0.trie_updates());
expected_trie_updates.insert(1, block1.trie_updates());
// Build expected trie data map
let mut expected_trie_data = BTreeMap::new();
expected_trie_data
.insert(0, LazyTrieData::ready(block0.hashed_state(), block0.trie_updates()));
expected_trie_data
.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
// Build expected hashed state map
let mut expected_hashed_state = BTreeMap::new();
expected_hashed_state.insert(0, block0.hashed_state());
expected_hashed_state.insert(1, block1.hashed_state());
// Build expected execution outcome (first_block matches first block number)
let commit_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 0,
..Default::default()
};
assert_eq!(
chain_commit.to_chain_notification(),
CanonStateNotification::Commit {
new: Arc::new(Chain::new(
vec![block0.recovered_block().clone(), block1.recovered_block().clone()],
sample_execution_outcome.clone(),
expected_trie_updates,
expected_hashed_state
commit_execution_outcome,
expected_trie_data,
))
}
);
@@ -1580,40 +1589,39 @@ mod tests {
old: vec![block1.clone(), block2.clone()],
};
// Build expected trie updates for old chain
let mut old_trie_updates = BTreeMap::new();
old_trie_updates.insert(1, block1.trie_updates());
old_trie_updates.insert(2, block2.trie_updates());
// Build expected trie data for old chain
let mut old_trie_data = BTreeMap::new();
old_trie_data.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
old_trie_data.insert(2, LazyTrieData::ready(block2.hashed_state(), block2.trie_updates()));
// Build expected trie updates for new chain
let mut new_trie_updates = BTreeMap::new();
new_trie_updates.insert(1, block1a.trie_updates());
new_trie_updates.insert(2, block2a.trie_updates());
// Build expected trie data for new chain
let mut new_trie_data = BTreeMap::new();
new_trie_data
.insert(1, LazyTrieData::ready(block1a.hashed_state(), block1a.trie_updates()));
new_trie_data
.insert(2, LazyTrieData::ready(block2a.hashed_state(), block2a.trie_updates()));
// Build expected hashed state for old chain
let mut old_hashed_state = BTreeMap::new();
old_hashed_state.insert(1, block1.hashed_state());
old_hashed_state.insert(2, block2.hashed_state());
// Build expected hashed state for new chain
let mut new_hashed_state = BTreeMap::new();
new_hashed_state.insert(1, block1a.hashed_state());
new_hashed_state.insert(2, block2a.hashed_state());
// Build expected execution outcome for reorg chains (first_block matches first block
// number)
let reorg_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 1,
..Default::default()
};
assert_eq!(
chain_reorg.to_chain_notification(),
CanonStateNotification::Reorg {
old: Arc::new(Chain::new(
vec![block1.recovered_block().clone(), block2.recovered_block().clone()],
sample_execution_outcome.clone(),
old_trie_updates,
old_hashed_state
reorg_execution_outcome.clone(),
old_trie_data,
)),
new: Arc::new(Chain::new(
vec![block1a.recovered_block().clone(), block2a.recovered_block().clone()],
sample_execution_outcome,
new_trie_updates,
new_hashed_state
reorg_execution_outcome,
new_trie_data,
))
}
);

View File

@@ -0,0 +1,186 @@
//! Lazy overlay computation for trie input.
//!
//! This module provides [`LazyOverlay`], a type that computes the [`TrieInputSorted`]
//! lazily on first access. This allows execution to start before the trie overlay
//! is fully computed.
use crate::DeferredTrieData;
use alloy_primitives::B256;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::sync::{Arc, OnceLock};
use tracing::{debug, trace};
/// Inputs captured for lazy overlay computation.
#[derive(Clone)]
struct LazyOverlayInputs {
/// The persisted ancestor hash (anchor) this overlay should be built on.
anchor_hash: B256,
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
blocks: Vec<DeferredTrieData>,
}
/// Lazily computed trie overlay.
///
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
/// but for overlay computation.
///
/// # Fast Path vs Slow Path
///
/// - **Fast path**: If the tip block's cached `anchored_trie_input` is ready and its `anchor_hash`
/// matches our expected anchor, we can reuse it directly (O(1)).
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
#[derive(Clone)]
pub struct LazyOverlay {
/// Computed result, cached after first access.
inner: Arc<OnceLock<TrieInputSorted>>,
/// Inputs for lazy computation.
inputs: LazyOverlayInputs,
}
impl std::fmt::Debug for LazyOverlay {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LazyOverlay")
.field("anchor_hash", &self.inputs.anchor_hash)
.field("num_blocks", &self.inputs.blocks.len())
.field("computed", &self.inner.get().is_some())
.finish()
}
}
impl LazyOverlay {
/// Create a new lazy overlay with the given anchor hash and block handles.
///
/// # Arguments
///
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
}
/// Returns the anchor hash this overlay is built on.
pub const fn anchor_hash(&self) -> B256 {
self.inputs.anchor_hash
}
/// Returns the number of in-memory blocks this overlay covers.
pub const fn num_blocks(&self) -> usize {
self.inputs.blocks.len()
}
/// Returns true if the overlay has already been computed.
pub fn is_computed(&self) -> bool {
self.inner.get().is_some()
}
/// Returns the computed trie input, computing it if necessary.
///
/// The first call triggers computation (which may block waiting for deferred data).
/// Subsequent calls return the cached result immediately.
pub fn get(&self) -> &TrieInputSorted {
self.inner.get_or_init(|| self.compute())
}
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
let input = self.get();
(Arc::clone(&input.nodes), Arc::clone(&input.state))
}
/// Compute the trie input overlay.
fn compute(&self) -> TrieInputSorted {
let anchor_hash = self.inputs.anchor_hash;
let blocks = &self.inputs.blocks;
if blocks.is_empty() {
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
return TrieInputSorted::default();
}
// Fast path: Check if tip block's overlay is ready and anchor matches.
// The tip block (first in list) has the cumulative overlay from all ancestors.
if let Some(tip) = blocks.first() {
let data = tip.wait_cloned();
if let Some(anchored) = &data.anchored_trie_input {
if anchored.anchor_hash == anchor_hash {
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
return (*anchored.trie_input).clone();
}
debug!(
target: "chain_state::lazy_overlay",
computed_anchor = %anchored.anchor_hash,
%anchor_hash,
"Anchor mismatch, falling back to merge"
);
}
}
// Slow path: Merge all blocks' trie data into a new overlay.
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
Self::merge_blocks(blocks)
}
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
///
/// Blocks are ordered newest to oldest.
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
if blocks.is_empty() {
return TrieInputSorted::default();
}
let state =
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
let nodes =
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
}
}
#[cfg(test)]
mod tests {
use super::*;
use reth_trie::{updates::TrieUpdates, HashedPostState};
fn empty_deferred(anchor: B256) -> DeferredTrieData {
DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
anchor,
Vec::new(),
)
}
#[test]
fn empty_blocks_returns_default() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
let result = overlay.get();
assert!(result.state.is_empty());
assert!(result.nodes.is_empty());
}
#[test]
fn single_block_uses_data_directly() {
let anchor = B256::random();
let deferred = empty_deferred(anchor);
let overlay = LazyOverlay::new(anchor, vec![deferred]);
assert!(!overlay.is_computed());
let _ = overlay.get();
assert!(overlay.is_computed());
}
#[test]
fn cached_after_first_access() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
// First access computes
let _ = overlay.get();
assert!(overlay.is_computed());
// Second access uses cache
let _ = overlay.get();
assert!(overlay.is_computed());
}
}

View File

@@ -14,6 +14,9 @@ pub use in_memory::*;
mod deferred_trie;
pub use deferred_trie::*;
mod lazy_overlay;
pub use lazy_overlay::*;
mod noop;
mod chain_info;

View File

@@ -280,7 +280,6 @@ mod tests {
vec![block1.clone(), block2.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
// Create a commit notification
@@ -319,13 +318,11 @@ mod tests {
vec![block1.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
let new_chain = Arc::new(Chain::new(
vec![block2.clone(), block3.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
// Create a reorg notification
@@ -391,7 +388,6 @@ mod tests {
vec![block1.clone(), block2.clone()],
execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
// Create a commit notification containing the new chain segment.
@@ -449,12 +445,8 @@ mod tests {
ExecutionOutcome { receipts: old_receipts, ..Default::default() };
// Create an old chain segment to be reverted, containing `old_block1`.
let old_chain: Arc<Chain> = Arc::new(Chain::new(
vec![old_block1.clone()],
old_execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
let old_chain: Arc<Chain> =
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, BTreeMap::new()));
// Define block2 for the new chain segment, which will be committed.
let mut body = BlockBody::<TransactionSigned>::default();
@@ -482,12 +474,8 @@ mod tests {
ExecutionOutcome { receipts: new_receipts, ..Default::default() };
// Create a new chain segment to be committed, containing `new_block1`.
let new_chain = Arc::new(Chain::new(
vec![new_block1.clone()],
new_execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
let new_chain =
Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, BTreeMap::new()));
// Create a reorg notification with both reverted (old) and committed (new) chain segments.
let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain };

View File

@@ -3,10 +3,7 @@ use crate::{
CanonStateSubscriptions, ComputedTrieData,
};
use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH};
use alloy_eips::{
eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE},
eip7685::Requests,
};
use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE};
use alloy_primitives::{Address, BlockNumber, B256, U256};
use alloy_signer::SignerSync;
use alloy_signer_local::PrivateKeySigner;
@@ -16,7 +13,7 @@ use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS};
use reth_ethereum_primitives::{
Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned,
};
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_primitives_traits::{
proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root},
Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader,
@@ -201,7 +198,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
fn get_executed_block(
&mut self,
block_number: BlockNumber,
receipts: Vec<Vec<Receipt>>,
mut receipts: Vec<Vec<Receipt>>,
parent_hash: B256,
) -> ExecutedBlock {
let block = self.generate_random_block(block_number, parent_hash);
@@ -209,12 +206,15 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
let trie_data = ComputedTrieData::default();
ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(ExecutionOutcome::new(
BundleState::default(),
receipts,
block_number,
vec![Requests::default()],
)),
Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: receipts.pop().unwrap_or_default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: BundleState::default(),
}),
trie_data,
)
}

View File

@@ -25,6 +25,7 @@ pub use alloy_chains::{Chain, ChainKind, NamedChain};
/// Re-export for convenience
pub use reth_ethereum_forks::*;
pub use alloy_evm::EvmLimitParams;
pub use api::EthChainSpec;
pub use info::ChainInfo;
#[cfg(any(test, feature = "test-utils"))]

View File

@@ -278,6 +278,7 @@ pub fn create_chain_config(
// Check if DAO fork is supported (it has an activation block)
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
#[allow(clippy::needless_update)]
ChainConfig {
chain_id: chain.map(|c| c.id()).unwrap_or(0),
homestead_block: block_num(EthereumHardfork::Homestead),
@@ -313,6 +314,7 @@ pub fn create_chain_config(
extra_fields: Default::default(),
deposit_contract_address,
blob_schedule,
..Default::default()
}
}
@@ -460,6 +462,18 @@ impl ChainSpec {
pub fn builder() -> ChainSpecBuilder {
ChainSpecBuilder::default()
}
/// Map a chain ID to a known chain spec, if available.
pub fn from_chain_id(chain_id: u64) -> Option<Arc<Self>> {
match NamedChain::try_from(chain_id).ok()? {
NamedChain::Mainnet => Some(MAINNET.clone()),
NamedChain::Sepolia => Some(SEPOLIA.clone()),
NamedChain::Holesky => Some(HOLESKY.clone()),
NamedChain::Hoodi => Some(HOODI.clone()),
NamedChain::Dev => Some(DEV.clone()),
_ => None,
}
}
}
impl<H: BlockHeader> ChainSpec<H> {

View File

@@ -50,6 +50,7 @@ reth-stages-types = { workspace = true, optional = true }
reth-static-file-types = { workspace = true, features = ["clap"] }
reth-static-file.workspace = true
reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true
@@ -83,6 +84,7 @@ backon.workspace = true
secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] }
tokio-stream.workspace = true
reqwest.workspace = true
url.workspace = true
metrics.workspace = true
# io
@@ -130,4 +132,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -27,7 +27,7 @@ use reth_provider::{
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
ProviderFactory, StaticFileProviderFactory,
ProviderFactory, StaticFileProviderFactory, StorageSettings,
};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
/// All static files related arguments
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
// TransactionDB only support read-write mode
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
if access.is_read_write() {
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
}
Ok(Environment { config, provider_factory, data_dir })

View File

@@ -1,145 +0,0 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use reth_chainspec::EthereumHardforks;
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
self.table.view(&ChecksumViewer {
tool,
start_key: self.start_key,
end_key: self.end_key,
limit: self.limit,
})?;
Ok(())
}
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(100_000) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,288 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use itertools::Itertools;
use reth_chainspec::EthereumHardforks;
use reth_db::{static_file::iter_static_files, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[cfg(all(unix, feature = "edge"))]
mod rocksdb;
/// Interval for logging progress during checksum computation.
const PROGRESS_LOG_INTERVAL: usize = 100_000;
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
#[command(subcommand)]
subcommand: Subcommand,
}
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Calculates the checksum of a database table
Mdbx {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a static file segment
StaticFile {
/// The static file segment
#[arg(value_enum)]
segment: StaticFileSegment,
/// The block number to start from (inclusive).
#[arg(long)]
start_block: Option<u64>,
/// The block number to end at (inclusive).
#[arg(long)]
end_block: Option<u64>,
/// The maximum number of rows to checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a RocksDB table
#[cfg(all(unix, feature = "edge"))]
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: rocksdb::RocksDbTable,
/// The maximum number of records to checksum.
#[arg(long)]
limit: Option<usize>,
},
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
match self.subcommand {
Subcommand::Mdbx { table, start_key, end_key, limit } => {
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
}
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
checksum_static_file(tool, segment, start_block, end_block, limit)?;
}
#[cfg(all(unix, feature = "edge"))]
Subcommand::Rocksdb { table, limit } => {
rocksdb::checksum_rocksdb(tool, table, limit)?;
}
}
Ok(())
}
}
/// Creates a new hasher with the standard seed used for checksum computation.
fn checksum_hasher() -> impl Hasher {
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,
limit: Option<usize>,
) -> eyre::Result<()> {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
warn!("Error checking consistency of static files: {err}");
}
let static_files = iter_static_files(static_file_provider.directory())?;
let ranges = static_files
.get(segment)
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
segment,
start_block,
end_block,
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
let jar_provider = static_file_provider
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
.ok_or_else(|| {
eyre::eyre!(
"Failed to get segment provider for segment {} at range {}",
segment,
block_range
)
})?;
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!(
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
segment, checksum, total, elapsed
);
Ok(())
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,106 @@
//! RocksDB checksum implementation.
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
use crate::common::CliNodeTypes;
use clap::ValueEnum;
use reth_chainspec::EthereumHardforks;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl RocksDbTable {
/// Returns the table name as a string
const fn name(&self) -> &'static str {
match self {
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
Self::AccountsHistory => tables::AccountsHistory::NAME,
Self::StoragesHistory => tables::StoragesHistory::NAME,
}
}
}
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
let start_time = Instant::now();
let limit = limit.unwrap_or(usize::MAX);
info!(
"Computing checksum for RocksDB table `{}`, limit={:?}",
table.name(),
if limit == usize::MAX { None } else { Some(limit) }
);
let (checksum, total) = match table {
RocksDbTable::TransactionHashNumbers => {
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
}
RocksDbTable::AccountsHistory => {
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
}
RocksDbTable::StoragesHistory => {
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
}
};
let elapsed = start_time.elapsed();
info!(
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
table.name(),
checksum,
total,
elapsed
);
Ok(())
}
/// Computes checksum for a specific RocksDB table by iterating over rows.
fn checksum_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
limit: usize,
) -> eyre::Result<(u64, usize)> {
let iter = rocksdb.raw_iter::<T>()?;
let mut hasher = checksum_hasher();
let mut total = 0usize;
for entry in iter {
let (key_bytes, value_bytes) = entry?;
hasher.write(&key_bytes);
hasher.write(&value_bytes);
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break;
}
}
Ok((hasher.finish(), total))
}

View File

@@ -29,7 +29,7 @@ impl Command {
let static_file_provider = tool.provider_factory.static_file_provider();
let static_files = iter_static_files(static_file_provider.directory())?;
if let Some(segment_static_files) = static_files.get(&segment) {
if let Some(segment_static_files) = static_files.get(segment) {
for (block_range, _) in segment_static_files {
static_file_provider.delete_jar(segment, block_range.start())?;
}

View File

@@ -21,6 +21,7 @@ use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
/// The arguments for the `reth db get` command
@@ -82,6 +83,33 @@ impl Command {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
let key = table_key::<tables::StorageChangeSets>(&key)?;
let provider = tool.provider_factory.static_file_provider();
if let Some(storage_key) = storage_key {
let entry = provider.get_storage_before_block(
key.block_number(),
key.address(),
storage_key,
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(());
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
@@ -112,6 +140,9 @@ impl Command {
AccountChangesetMask::MASK,
)
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changesets handled above");
}
};
// handle account changesets differently if a subkey is provided.
@@ -190,6 +221,9 @@ impl Command {
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changeset static files are special cased before this match")
}
}
}
}

View File

@@ -61,19 +61,21 @@ impl Command {
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
let search = match self.search.as_deref() {
Some(search) => {
if let Some(search) = search.strip_prefix("0x") {
return hex::decode(search).unwrap()
hex::decode(search).wrap_err(
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
)?
} else {
search.as_bytes().to_vec()
}
search.as_bytes().to_vec()
})
.unwrap_or_default();
}
None => Vec::new(),
};
ListFilter {
Ok(ListFilter {
skip: self.skip,
len: self.len,
search,
@@ -82,7 +84,7 @@ impl Command {
min_value_size: self.min_value_size,
reverse: self.reverse,
only_count: self.count,
}
})
}
}
@@ -100,7 +102,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
tx.disable_long_read_transaction_safety();
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let total_entries = stats.entries();
let final_entry_idx = total_entries.saturating_sub(1);
if self.args.skip > final_entry_idx {
@@ -115,7 +117,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
}
let list_filter = self.args.list_filter();
let list_filter = self.args.list_filter()?;
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;

View File

@@ -39,7 +39,7 @@ pub enum Subcommands {
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Calculates the content checksum of a table
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),
@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
command.execute(&tool, ctx.task_executor, &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {

View File

@@ -54,6 +54,26 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage history in rocksdb instead of MDBX
StoragesHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction hash to number mapping in rocksdb instead of MDBX
TransactionHashNumbers {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account history in rocksdb instead of MDBX
AccountHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -100,6 +120,7 @@ impl Command {
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
@@ -128,6 +149,38 @@ impl Command {
settings.account_changesets_in_static_files = value;
println!("Set account_changesets_in_static_files = {}", value);
}
SetCommand::StoragesHistory { value } => {
if settings.storages_history_in_rocksdb == value {
println!("storages_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.storages_history_in_rocksdb = value;
println!("Set storages_history_in_rocksdb = {}", value);
}
SetCommand::TransactionHashNumbers { value } => {
if settings.transaction_hash_numbers_in_rocksdb == value {
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.transaction_hash_numbers_in_rocksdb = value;
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
}
SetCommand::AccountHistory { value } => {
if settings.account_history_in_rocksdb == value {
println!("account_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
}
// Write updated settings

View File

@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
use reth_provider::{
providers::{ProviderNodeTypes, StaticFileProvider},
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
@@ -61,6 +64,11 @@ impl Command {
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
println!("\n");
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
println!("{rocksdb_stats_table}");
Ok(())
}
@@ -88,7 +96,7 @@ impl Command {
let stats = tx
.inner
.db_stat(&table_db)
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {db_table}"))?;
// Defaults to 16KB right now but we should
@@ -129,7 +137,8 @@ impl Command {
table.add_row(row);
let freelist = tx.inner.env().freelist()?;
let pagesize = tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;
let pagesize =
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
let freelist_size = freelist * pagesize;
let mut row = Row::new();
@@ -147,6 +156,70 @@ impl Command {
Ok(table)
}
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"RocksDB Table Name",
"# Entries",
"SST Size",
"Memtable Size",
"Total Size",
"Pending Compaction",
]);
let stats = tool.provider_factory.rocksdb_provider().table_stats();
let mut total_sst: u64 = 0;
let mut total_memtable: u64 = 0;
let mut total_size: u64 = 0;
let mut total_pending: u64 = 0;
for stat in &stats {
total_sst += stat.sst_size_bytes;
total_memtable += stat.memtable_size_bytes;
total_size += stat.estimated_size_bytes;
total_pending += stat.pending_compaction_bytes;
let mut row = Row::new();
row.add_cell(Cell::new(&stat.name))
.add_cell(Cell::new(stat.estimated_num_keys))
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
table.add_row(row);
}
if !stats.is_empty() {
let max_widths = table.column_max_content_widths();
let mut separator = Row::new();
for width in max_widths {
separator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(separator);
let mut row = Row::new();
row.add_cell(Cell::new("RocksDB Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_sst as f64)))
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
let wal_size = tool.provider_factory.rocksdb_provider().wal_size_bytes();
let mut row = Row::new();
row.add_cell(Cell::new("WAL"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(wal_size as f64)))
.add_cell(Cell::new(""));
table.add_row(row);
}
table
}
fn static_files_stats_table<N: NodePrimitives>(
&self,
data_dir: ChainPath<DataDirPath>,

View File

@@ -2,20 +2,22 @@ use crate::common::EnvironmentArgs;
use clap::Parser;
use eyre::Result;
use lz4::Decoder;
use reqwest::Client;
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_fs_util as fs;
use std::{
borrow::Cow,
io::{self, Read, Write},
path::Path,
fs::OpenOptions,
io::{self, BufWriter, Read, Write},
path::{Path, PathBuf},
sync::{Arc, OnceLock},
time::{Duration, Instant},
};
use tar::Archive;
use tokio::task;
use tracing::info;
use url::Url;
use zstd::stream::read::Decoder as ZstdDecoder;
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
@@ -85,6 +87,9 @@ impl DownloadDefaults {
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
);
help.push_str(self.default_base_url.as_ref());
help.push_str(
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
);
help
}
@@ -170,12 +175,14 @@ struct DownloadProgress {
downloaded: u64,
total_size: u64,
last_displayed: Instant,
started_at: Instant,
}
impl DownloadProgress {
/// Creates new progress tracker with given total size
fn new(total_size: u64) -> Self {
Self { downloaded: 0, total_size, last_displayed: Instant::now() }
let now = Instant::now();
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
}
/// Converts bytes to human readable format (B, KB, MB, GB)
@@ -191,6 +198,18 @@ impl DownloadProgress {
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
}
/// Format duration as human readable string
fn format_duration(duration: Duration) -> String {
let secs = duration.as_secs();
if secs < 60 {
format!("{secs}s")
} else if secs < 3600 {
format!("{}m {}s", secs / 60, secs % 60)
} else {
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
}
}
/// Updates progress bar
fn update(&mut self, chunk_size: u64) -> Result<()> {
self.downloaded += chunk_size;
@@ -201,8 +220,24 @@ impl DownloadProgress {
let formatted_total = Self::format_size(self.total_size);
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
// Calculate ETA based on current speed
let elapsed = self.started_at.elapsed();
let eta = if self.downloaded > 0 {
let remaining = self.total_size.saturating_sub(self.downloaded);
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
if speed > 0.0 {
Duration::from_secs_f64(remaining as f64 / speed)
} else {
Duration::ZERO
}
} else {
Duration::ZERO
};
let eta_str = Self::format_duration(eta);
// Pad with spaces to clear any previous longer line
print!(
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total})",
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
);
io::stdout().flush()?;
self.last_displayed = Instant::now();
@@ -246,29 +281,30 @@ enum CompressionFormat {
impl CompressionFormat {
/// Detect compression format from file extension
fn from_url(url: &str) -> Result<Self> {
if url.ends_with(EXTENSION_TAR_LZ4) {
let path =
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
if path.ends_with(EXTENSION_TAR_LZ4) {
Ok(Self::Lz4)
} else if url.ends_with(EXTENSION_TAR_ZSTD) {
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
Ok(Self::Zstd)
} else {
Err(eyre::eyre!("Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}", url))
Err(eyre::eyre!(
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
path
))
}
}
}
/// Downloads and extracts a snapshot, blocking until finished.
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
let client = reqwest::blocking::Client::builder().build()?;
let response = client.get(url).send()?.error_for_status()?;
let total_size = response.content_length().ok_or_else(|| {
eyre::eyre!(
"Server did not provide Content-Length header. This is required for snapshot downloads"
)
})?;
let progress_reader = ProgressReader::new(response, total_size);
let format = CompressionFormat::from_url(url)?;
/// Extracts a compressed tar archive to the target directory with progress tracking.
fn extract_archive<R: Read>(
reader: R,
total_size: u64,
format: CompressionFormat,
target_dir: &Path,
) -> Result<()> {
let progress_reader = ProgressReader::new(reader, total_size);
match format {
CompressionFormat::Lz4 => {
@@ -285,6 +321,185 @@ fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
Ok(())
}
/// Extracts a snapshot from a local file.
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let file = std::fs::File::open(path)?;
let total_size = file.metadata()?.len();
extract_archive(file, total_size, format, target_dir)
}
const MAX_DOWNLOAD_RETRIES: u32 = 10;
const RETRY_BACKOFF_SECS: u64 = 5;
/// Wrapper that tracks download progress while writing data.
/// Used with [`io::copy`] to display progress during downloads.
struct ProgressWriter<W> {
inner: W,
progress: DownloadProgress,
}
impl<W: Write> Write for ProgressWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
let _ = self.progress.update(n as u64);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Downloads a file with resume support using HTTP Range requests.
/// Automatically retries on failure, resuming from where it left off.
/// Returns the path to the downloaded file and its total size.
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
let file_name = Url::parse(url)
.ok()
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
.unwrap_or_else(|| "snapshot.tar".to_string());
let final_path = target_dir.join(&file_name);
let part_path = target_dir.join(format!("{file_name}.part"));
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
let mut total_size: Option<u64> = None;
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
if let Some(total) = total_size &&
existing_size >= total
{
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, total));
}
if attempt > 1 {
info!(target: "reth::cli",
"Retry attempt {}/{} - resuming from {} bytes",
attempt, MAX_DOWNLOAD_RETRIES, existing_size
);
}
let mut request = client.get(url);
if existing_size > 0 {
request = request.header(RANGE, format!("bytes={existing_size}-"));
if attempt == 1 {
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
}
}
let response = match request.send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(e) => {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
let size = if is_partial {
response
.headers()
.get("Content-Range")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.split('/').next_back())
.and_then(|v| v.parse().ok())
} else {
response.content_length()
};
if total_size.is_none() {
total_size = size;
}
let current_total = total_size.ok_or_else(|| {
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
})?;
let file = if is_partial && existing_size > 0 {
OpenOptions::new()
.append(true)
.open(&part_path)
.map_err(|e| fs::FsPathError::open(e, &part_path))?
} else {
fs::create_file(&part_path)?
};
let start_offset = if is_partial { existing_size } else { 0 };
let mut progress = DownloadProgress::new(current_total);
progress.downloaded = start_offset;
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
let mut reader = response;
let copy_result = io::copy(&mut reader, &mut writer);
let flush_result = writer.inner.flush();
println!();
if let Err(e) = copy_result.and(flush_result) {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, current_total));
}
Err(last_error
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
info!(target: "reth::cli", "Extracting snapshot...");
let file = fs::open(&downloaded_path)?;
extract_archive(file, total_size, format, target_dir)?;
fs::remove_file(&downloaded_path)?;
info!(target: "reth::cli", "Removed downloaded archive");
Ok(())
}
/// Downloads and extracts a snapshot, blocking until finished.
///
/// Supports both `file://` URLs for local files and HTTP(S) URLs for remote downloads.
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
let format = CompressionFormat::from_url(url)?;
if let Ok(parsed_url) = Url::parse(url) &&
parsed_url.scheme() == "file"
{
let file_path = parsed_url
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
extract_from_file(&file_path, format, target_dir)
} else {
download_and_extract(url, format, target_dir)
}
}
async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
let target_dir = target_dir.to_path_buf();
let url = url.to_string();
@@ -343,6 +558,7 @@ mod tests {
assert!(help.contains("Available snapshot sources:"));
assert!(help.contains("merkle.io"));
assert!(help.contains("publicnode.com"));
assert!(help.contains("file://"));
}
#[test]
@@ -367,4 +583,25 @@ mod tests {
assert_eq!(defaults.available_snapshots.len(), 4); // 2 defaults + 2 added
assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string()));
}
#[test]
fn test_compression_format_detection() {
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
}
}

View File

@@ -26,6 +26,14 @@ pub struct ImportCommand<C: ChainSpecParser> {
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
/// Fail immediately when an invalid block is encountered.
///
/// By default, the import will stop at the last valid block if an invalid block is
/// encountered during execution or validation, leaving the database at the last valid
/// block state. When this flag is set, the import will instead fail with an error.
#[arg(long, verbatim_doc_comment)]
fail_on_invalid_block: bool,
/// The path(s) to block file(s) for import.
///
/// The online stages (headers and bodies) are replaced by a file import, after which the
@@ -52,7 +60,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len());
let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len };
let import_config = ImportConfig {
no_state: self.no_state,
chunk_len: self.chunk_len,
fail_on_invalid_block: self.fail_on_invalid_block,
};
let executor = components.evm_config().clone();
let consensus = Arc::new(components.consensus().clone());
@@ -81,7 +93,20 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
total_decoded_blocks += result.total_decoded_blocks;
total_decoded_txns += result.total_decoded_txns;
if !result.is_complete() {
// Check if we stopped due to an invalid block
if result.stopped_on_invalid_block {
info!(target: "reth::cli",
"Stopped at last valid block {} due to invalid block {} in file: {}. Imported {} blocks, {} transactions",
result.last_valid_block.unwrap_or(0),
result.bad_block.unwrap_or(0),
path.display(),
result.total_imported_blocks,
result.total_imported_txns);
// Stop importing further files and exit successfully
break;
}
if !result.is_successful() {
return Err(eyre::eyre!(
"Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions",
path.display(),
@@ -98,7 +123,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
}
info!(target: "reth::cli",
"All files imported successfully. Total: {}/{} blocks, {}/{} transactions",
"Import complete. Total: {}/{} blocks, {}/{} transactions",
total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns);
Ok(())
@@ -139,4 +164,20 @@ mod tests {
assert_eq!(args.paths[1], PathBuf::from("file2.rlp"));
assert_eq!(args.paths[2], PathBuf::from("file3.rlp"));
}
#[test]
fn parse_import_command_with_fail_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "--fail-on-invalid-block", "chain.rlp"]);
assert!(args.fail_on_invalid_block);
assert_eq!(args.paths.len(), 1);
assert_eq!(args.paths[0], PathBuf::from("chain.rlp"));
}
#[test]
fn parse_import_command_default_stops_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "chain.rlp"]);
assert!(!args.fail_on_invalid_block);
}
}

View File

@@ -22,11 +22,11 @@ use reth_provider::{
StageCheckpointReader,
};
use reth_prune::PruneModes;
use reth_stages::{prelude::*, Pipeline, StageId, StageSet};
use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
use reth_static_file::StaticFileProducer;
use std::{path::Path, sync::Arc};
use tokio::sync::watch;
use tracing::{debug, error, info};
use tracing::{debug, error, info, warn};
/// Configuration for importing blocks from RLP files.
#[derive(Debug, Clone, Default)]
@@ -35,6 +35,9 @@ pub struct ImportConfig {
pub no_state: bool,
/// Chunk byte length to read from file.
pub chunk_len: Option<u64>,
/// If true, fail immediately when an invalid block is encountered.
/// By default (false), the import stops at the last valid block and exits successfully.
pub fail_on_invalid_block: bool,
}
/// Result of an import operation.
@@ -48,6 +51,12 @@ pub struct ImportResult {
pub total_imported_blocks: usize,
/// Total number of transactions imported into the database.
pub total_imported_txns: usize,
/// Whether the import was stopped due to an invalid block.
pub stopped_on_invalid_block: bool,
/// The block number that was invalid, if any.
pub bad_block: Option<u64>,
/// The last valid block number when stopped due to invalid block.
pub last_valid_block: Option<u64>,
}
impl ImportResult {
@@ -56,6 +65,14 @@ impl ImportResult {
self.total_decoded_blocks == self.total_imported_blocks &&
self.total_decoded_txns == self.total_imported_txns
}
/// Returns true if the import was successful, considering stop-on-invalid-block mode.
///
/// In stop-on-invalid-block mode, a partial import is considered successful if we
/// stopped due to an invalid block (leaving the DB at the last valid block).
pub fn is_successful(&self) -> bool {
self.is_complete() || self.stopped_on_invalid_block
}
}
/// Imports blocks from an RLP-encoded file into the database.
@@ -103,6 +120,11 @@ where
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
// Track if we stopped due to an invalid block
let mut stopped_on_invalid_block = false;
let mut bad_block_number: Option<u64> = None;
let mut last_valid_block_number: Option<u64> = None;
while let Some(file_client) =
reader.next_chunk::<BlockTy<N>>(consensus.clone(), Some(sealed_header)).await?
{
@@ -137,12 +159,51 @@ where
// Run pipeline
info!(target: "reth::import", "Starting sync pipeline");
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
if import_config.fail_on_invalid_block {
// Original behavior: fail on unwind
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
}
} else {
// Default behavior: Use run_loop() to handle unwinds gracefully
let result = tokio::select! {
res = pipeline.run_loop() => res,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
};
match result {
Ok(ControlFlow::Unwind { target, bad_block }) => {
// An invalid block was encountered; stop at last valid block
let bad = bad_block.block.number;
warn!(
target: "reth::import",
bad_block = bad,
last_valid_block = target,
"Invalid block encountered during import; stopping at last valid block"
);
stopped_on_invalid_block = true;
bad_block_number = Some(bad);
last_valid_block_number = Some(target);
break;
}
Ok(ControlFlow::Continue { block_number }) => {
debug!(target: "reth::import", block_number, "Pipeline chunk completed");
}
Ok(ControlFlow::NoProgress { block_number }) => {
debug!(target: "reth::import", ?block_number, "Pipeline made no progress");
}
Err(e) => {
// Propagate other pipeline errors
return Err(e.into());
}
}
}
sealed_header = provider_factory
@@ -160,9 +221,20 @@ where
total_decoded_txns,
total_imported_blocks,
total_imported_txns,
stopped_on_invalid_block,
bad_block: bad_block_number,
last_valid_block: last_valid_block_number,
};
if !result.is_complete() {
if result.stopped_on_invalid_block {
info!(target: "reth::import",
total_imported_blocks,
total_imported_txns,
bad_block = ?result.bad_block,
last_valid_block = ?result.last_valid_block,
"Import stopped at last valid block due to invalid block"
);
} else if !result.is_complete() {
error!(target: "reth::import",
total_decoded_blocks,
total_imported_blocks,

View File

@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -166,12 +171,16 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -187,6 +196,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,

View File

@@ -0,0 +1,34 @@
//! Enode identifier command
use clap::Parser;
use reth_cli_util::get_secret_key;
use reth_network_peers::NodeRecord;
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf,
};
/// Print the enode identifier for a given secret key.
#[derive(Parser, Debug)]
pub struct Command {
/// Path to the secret key file for discovery.
pub discovery_secret: PathBuf,
/// Optional IP address to include in the enode URL.
///
/// If not provided, defaults to 0.0.0.0.
#[arg(long)]
pub ip: Option<IpAddr>,
}
impl Command {
/// Execute the enode command.
pub fn execute(self) -> eyre::Result<()> {
let sk = get_secret_key(&self.discovery_secret)?;
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
let addr = SocketAddr::new(ip, 30303);
let enr = NodeRecord::from_secret_key(addr, &sk);
println!("{enr}");
Ok(())
}
}

View File

@@ -18,6 +18,7 @@ use reth_node_core::{
};
pub mod bootnode;
pub mod enode;
pub mod rlpx;
/// `reth p2p` command
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Subcommands::Bootnode(command) => {
command.execute().await?;
}
Subcommands::Enode(command) => {
command.execute()?;
}
}
Ok(())
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
Subcommands::Body { args, .. } => Some(&args.chain),
Subcommands::Rlpx(_) => None,
Subcommands::Bootnode(_) => None,
Subcommands::Enode(_) => None,
}
}
}
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
Rlpx(rlpx::Command),
/// Bootnode command
Bootnode(bootnode::Command),
/// Print enode identifier
Enode(enode::Command),
}
#[derive(Debug, Clone, Parser)]
@@ -225,4 +232,16 @@ mod tests {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
}
#[test]
fn parse_enode_cmd() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret"]);
}
#[test]
fn parse_enode_cmd_with_ip() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
}
}

View File

@@ -42,9 +42,9 @@ pub struct Command<C: ChainSpecParser> {
#[arg(long)]
to: Option<u64>,
/// Number of tasks to run in parallel
#[arg(long, default_value = "10")]
num_tasks: u64,
/// Number of tasks to run in parallel. Defaults to the number of available CPUs.
#[arg(long)]
num_tasks: Option<u64>,
/// Continues with execution when an invalid block is encountered and collects these blocks.
#[arg(long)]
@@ -84,12 +84,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
}
};
let num_tasks = self.num_tasks.unwrap_or_else(|| {
std::thread::available_parallelism().map(|n| n.get() as u64).unwrap_or(10)
});
let total_blocks = max_block - min_block;
let total_gas = calculate_gas_used_from_headers(
&provider_factory.static_file_provider(),
min_block..=max_block,
)?;
let blocks_per_task = total_blocks / self.num_tasks;
let blocks_per_task = total_blocks / num_tasks;
let db_at = {
let provider_factory = provider_factory.clone();
@@ -107,10 +111,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let _guard = cancellation.drop_guard();
let mut tasks = JoinSet::new();
for i in 0..self.num_tasks {
for i in 0..num_tasks {
let start_block = min_block + i * blocks_per_task;
let end_block =
if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task };
if i == num_tasks - 1 { max_block } else { start_block + blocks_per_task };
// Spawn thread executing blocks
let provider_factory = provider_factory.clone();
@@ -148,7 +152,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
};
if let Err(err) = consensus
.validate_block_post_execution(&block, &result)
.validate_block_post_execution(&block, &result, None)
.wrap_err_with(|| {
format!("Failed to validate block {} {}", block.number(), block.hash())
})

View File

@@ -15,7 +15,8 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, TrieWriter,
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
StorageSettingsCache,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -90,11 +91,14 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(highest_block)?;
}
}
}
}
let provider_rw = tool.provider_factory.database_provider_rw()?;
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
let tx = provider_rw.tx_ref();
match self.stage {
@@ -167,13 +171,21 @@ impl<C: ChainSpecParser> Command<C> {
None,
)?;
}
StageEnum::MerkleChangeSets => {
provider_rw.clear_trie_changesets()?;
reset_stage_checkpoint(tx, StageId::MerkleChangeSets)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
}
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
}
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
@@ -181,7 +193,14 @@ impl<C: ChainSpecParser> Command<C> {
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
}
StageEnum::TxLookup => {
tx.clear::<tables::TransactionHashNumbers>()?;
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;
} else {
tx.clear::<tables::TransactionHashNumbers>()?;
}
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;

View File

@@ -11,7 +11,6 @@ use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_cli_util::get_secret_key;
use reth_config::config::{HashingConfig, SenderRecoveryConfig, TransactionLookupConfig};
use reth_db_api::database_metrics::DatabaseMetrics;
use reth_downloaders::{
bodies::bodies::BodiesDownloaderBuilder,
headers::reverse_headers::ReverseHeadersDownloaderBuilder,
@@ -19,19 +18,19 @@ use reth_downloaders::{
use reth_exex::ExExManagerHandle;
use reth_network::BlockDownloaderProvider;
use reth_network_p2p::HeadersClient;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{
args::{NetworkArgs, StageEnum},
version::version_metadata,
};
use reth_node_metrics::{
chain::ChainSpecInfo,
hooks::Hooks,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
use reth_provider::{
ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
StageCheckpointWriter, StaticFileProviderFactory,
StageCheckpointWriter,
};
use reth_stages::{
stages::{
@@ -139,20 +138,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
Hooks::builder()
.with_hook({
let db = provider_factory.db_ref().clone();
move || db.report_metrics()
})
.with_hook({
let sfp = provider_factory.static_file_provider();
move || {
if let Err(error) = sfp.report_metrics() {
error!(%error, "Failed to report metrics from static file provider");
}
}
})
.build(),
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);

View File

@@ -26,7 +26,8 @@ rand_08.workspace = true
thiserror.workspace = true
serde.workspace = true
tracy-client = { workspace = true, optional = true, features = ["demangle"] }
tracy-client = { workspace = true, optional = true }
reth-tracing = { workspace = true, optional = true }
[dev-dependencies]
rand.workspace = true
@@ -46,7 +47,7 @@ jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"]
# Wraps the selected allocator in the tracy profiling allocator
tracy-allocator = ["dep:tracy-client"]
tracy-allocator = ["dep:tracy-client", "dep:reth-tracing"]
snmalloc = ["dep:snmalloc-rs"]

View File

@@ -25,7 +25,6 @@ cfg_if::cfg_if! {
cfg_if::cfg_if! {
if #[cfg(feature = "tracy-allocator")] {
type AllocatorWrapper = tracy_client::ProfiledAllocator<AllocatorInner>;
tracy_client::register_demangler!();
const fn new_allocator_wrapper() -> AllocatorWrapper {
AllocatorWrapper::new(AllocatorInner {}, 100)
}

View File

@@ -8,6 +8,9 @@
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
#![cfg_attr(docsrs, feature(doc_cfg))]
#[cfg(feature = "tracy-allocator")]
use reth_tracing as _;
pub mod allocator;
pub mod cancellation;

View File

@@ -121,7 +121,16 @@ pub fn install() {
unsafe {
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
let mut alt_stack: libc::stack_t = mem::zeroed();
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
// generous here and just use a size of a page.
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
let page_sz = if raw_page_sz == -1 {
// Fallback alignment in case sysconf fails.
4096_usize
} else {
raw_page_sz as usize
};
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
alt_stack.ss_size = alt_stack_size;
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());

View File

@@ -2,9 +2,8 @@
use reth_network_types::{PeersConfig, SessionsConfig};
use reth_prune_types::PruneModes;
use reth_stages_types::ExecutionStageThresholds;
use reth_static_file_types::StaticFileSegment;
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
use std::{
collections::HashMap,
path::{Path, PathBuf},
time::Duration,
};
@@ -439,6 +438,8 @@ pub struct BlocksPerFileConfig {
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
pub storage_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -452,6 +453,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
@@ -470,20 +472,25 @@ impl StaticFilesConfig {
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
eyre::ensure!(
storage_change_sets != Some(0),
"Storage changesets segment blocks per file must be greater than 0"
);
Ok(())
}
/// Converts the blocks per file configuration into a [`HashMap`] per segment.
pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
/// Converts the blocks per file configuration into a [`StaticFileMap`].
pub fn as_blocks_per_file_map(&self) -> StaticFileMap<u64> {
let BlocksPerFileConfig {
headers,
transactions,
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
let mut map = HashMap::new();
let mut map = StaticFileMap::default();
// Iterating over all possible segments allows us to do an exhaustive match here,
// to not forget to configure new segments in the future.
for segment in StaticFileSegment::iter() {
@@ -493,6 +500,7 @@ impl StaticFilesConfig {
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
StaticFileSegment::StorageChangeSets => storage_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {
@@ -551,7 +559,6 @@ impl PruneConfig {
/// - `Option<PruneMode>` fields: set from `other` only if `self` is `None`.
/// - `block_interval`: set from `other` only if `self.block_interval ==
/// DEFAULT_BLOCK_INTERVAL`.
/// - `merkle_changesets`: always set from `other`.
/// - `receipts_log_filter`: set from `other` only if `self` is empty and `other` is non-empty.
pub fn merge(&mut self, other: Self) {
let Self {
@@ -564,7 +571,6 @@ impl PruneConfig {
account_history,
storage_history,
bodies_history,
merkle_changesets,
receipts_log_filter,
},
} = other;
@@ -581,8 +587,6 @@ impl PruneConfig {
self.segments.account_history = self.segments.account_history.or(account_history);
self.segments.storage_history = self.segments.storage_history.or(storage_history);
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
// Merkle changesets is not optional; always take the value from `other`
self.segments.merkle_changesets = merkle_changesets;
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
self.segments.receipts_log_filter = receipts_log_filter;
@@ -1079,18 +1083,6 @@ transaction_lookup = 'full'
receipts = { distance = 16384 }
#";
let _conf: Config = toml::from_str(s).unwrap();
let s = r"#
[prune]
block_interval = 5
[prune.segments]
sender_recovery = { distance = 16384 }
transaction_lookup = 'full'
receipts = 'full'
#";
let err = toml::from_str::<Config>(s).unwrap_err().to_string();
assert!(err.contains("invalid value: string \"full\""), "{}", err);
}
#[test]
@@ -1104,7 +1096,6 @@ receipts = 'full'
account_history: None,
storage_history: Some(PruneMode::Before(5000)),
bodies_history: None,
merkle_changesets: PruneMode::Before(0),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Full,
@@ -1121,7 +1112,6 @@ receipts = 'full'
account_history: Some(PruneMode::Distance(2000)),
storage_history: Some(PruneMode::Distance(3000)),
bodies_history: None,
merkle_changesets: PruneMode::Distance(10000),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
(Address::random(), PruneMode::Distance(1000)),
(Address::random(), PruneMode::Before(2000)),
@@ -1140,7 +1130,6 @@ receipts = 'full'
assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000));
assert_eq!(config1.segments.receipts_log_filter, original_filter);
}

View File

@@ -1,14 +1,11 @@
//! Collection of methods for block validation.
use alloy_consensus::{BlockHeader as _, Transaction, EMPTY_OMMER_ROOT_HASH};
use alloy_consensus::{BlockHeader as _, EMPTY_OMMER_ROOT_HASH};
use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7840::BlobParams};
use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks};
use reth_consensus::{ConsensusError, TxGasLimitTooHighErr};
use reth_consensus::ConsensusError;
use reth_primitives_traits::{
constants::{
GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MAX_TX_GAS_LIMIT_OSAKA, MINIMUM_GAS_LIMIT,
},
transaction::TxHashRef,
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
Block, BlockBody, BlockHeader, GotExpected, SealedBlock, SealedHeader,
};
@@ -146,7 +143,7 @@ pub fn validate_block_pre_execution<B, ChainSpec>(
) -> Result<(), ConsensusError>
where
B: Block,
ChainSpec: EthereumHardforks,
ChainSpec: EthChainSpec + EthereumHardforks,
{
post_merge_hardfork_fields(block, chain_spec)?;
@@ -154,19 +151,6 @@ where
if let Err(error) = block.ensure_transaction_root_valid() {
return Err(ConsensusError::BodyTransactionRootDiff(error.into()))
}
// EIP-7825 validation
if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) {
for tx in block.body().transactions() {
if tx.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA {
return Err(TxGasLimitTooHighErr {
tx_hash: *tx.tx_hash(),
gas_limit: tx.gas_limit(),
max_allowed: MAX_TX_GAS_LIMIT_OSAKA,
}
.into());
}
}
}
Ok(())
}

View File

@@ -15,6 +15,12 @@ use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
use alloy_consensus::Header;
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
use core::error::Error;
/// Pre-computed receipt root and logs bloom.
///
/// When provided to [`FullConsensus::validate_block_post_execution`], this allows skipping
/// the receipt root computation and using the pre-computed values instead.
pub type ReceiptRootBloom = (B256, Bloom);
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
@@ -39,11 +45,15 @@ pub trait FullConsensus<N: NodePrimitives>: Consensus<N::Block> {
///
/// See the Yellow Paper sections 4.3.2 "Holistic Validity".
///
/// If `receipt_root_bloom` is provided, the implementation should use the pre-computed
/// receipt root and logs bloom instead of computing them from the receipts.
///
/// Note: validating blocks does not include other validations of the Consensus
fn validate_block_post_execution(
&self,
block: &RecoveredBlock<N::Block>,
result: &BlockExecutionResult<N::Receipt>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError>;
}

View File

@@ -18,7 +18,7 @@
//!
//! **Not for production use** - provides no security guarantees or consensus validation.
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use alloc::sync::Arc;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
@@ -76,6 +76,7 @@ impl<N: NodePrimitives> FullConsensus<N> for NoopConsensus {
&self,
_block: &RecoveredBlock<N::Block>,
_result: &BlockExecutionResult<N::Receipt>,
_receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
Ok(())
}

View File

@@ -1,4 +1,4 @@
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use core::sync::atomic::{AtomicBool, Ordering};
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
@@ -51,6 +51,7 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
&self,
_block: &RecoveredBlock<N::Block>,
_result: &BlockExecutionResult<N::Receipt>,
_receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
if self.fail_validation() {
Err(ConsensusError::BaseFeeMissing)

View File

@@ -72,3 +72,11 @@ derive_more.workspace = true
[[test]]
name = "e2e_testsuite"
path = "tests/e2e-testsuite/main.rs"
[[test]]
name = "rocksdb"
path = "tests/rocksdb/main.rs"
required-features = ["edge"]
[features]
edge = ["reth-node-core/edge", "reth-provider/rocksdb", "reth-cli-commands/edge"]

View File

@@ -103,7 +103,10 @@ where
N: NodeBuilderHelper,
{
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_tree_config_modifier(move |_| tree_config.clone())
.with_tree_config_modifier(move |base| {
// Apply caller's tree_config but preserve the small cache size from base
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
})
.with_node_config_modifier(move |config| config.set_dev(is_dev))
.with_connect_nodes(connect_nodes)
.build()

View File

@@ -2,11 +2,11 @@ use crate::{network::NetworkTestContext, payload::PayloadTestContext, rpc::RpcTe
use alloy_consensus::{transaction::TxHashRef, BlockHeader};
use alloy_eips::BlockId;
use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B256};
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState};
use alloy_rpc_types_eth::BlockNumberOrTag;
use eyre::Ok;
use futures_util::Future;
use jsonrpsee::http_client::HttpClient;
use jsonrpsee::{core::client::ClientT, http_client::HttpClient};
use reth_chainspec::EthereumHardforks;
use reth_network_api::test_utils::PeersHandleProvider;
use reth_node_api::{
@@ -20,6 +20,7 @@ use reth_provider::{
BlockReader, BlockReaderIdExt, CanonStateNotificationStream, CanonStateSubscriptions,
HeaderProvider, StageCheckpointReader,
};
use reth_rpc_api::TestingBuildBlockRequestV1;
use reth_rpc_builder::auth::AuthServerHandle;
use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt};
use reth_stages_types::StageId;
@@ -319,4 +320,20 @@ where
Ok(crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle))
}
/// Calls the `testing_buildBlockV1` RPC on this node.
///
/// This endpoint builds a block using the provided parent, payload attributes, and
/// transactions. Requires the `Testing` RPC module to be enabled.
pub async fn testing_build_block_v1(
&self,
request: TestingBuildBlockRequestV1,
) -> eyre::Result<ExecutionPayloadEnvelopeV5> {
let client =
self.rpc_client().ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?;
let res: ExecutionPayloadEnvelopeV5 =
client.request("testing_buildBlockV1", [request]).await?;
eyre::Ok(res)
}
}

View File

@@ -112,11 +112,13 @@ where
..NetworkArgs::default()
};
// Apply tree config modifier if present
// Apply tree config modifier if present, with test-appropriate defaults
let base_tree_config =
reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024);
let tree_config = if let Some(modifier) = self.tree_config_modifier {
modifier(reth_node_api::TreeConfig::default())
modifier(base_tree_config)
} else {
reth_node_api::TreeConfig::default()
base_tree_config
};
let mut nodes = (0..self.num_nodes)

View File

@@ -38,6 +38,18 @@ impl TransactionTestContext {
signed.encoded_2718().into()
}
/// Creates a transfer with a specific nonce and signs it, returning bytes.
/// Uses high `max_fee_per_gas` (1000 gwei) to ensure tx acceptance regardless of basefee.
pub async fn transfer_tx_bytes_with_nonce(
chain_id: u64,
wallet: PrivateKeySigner,
nonce: u64,
) -> Bytes {
let tx = tx(chain_id, 21000, None, None, nonce, Some(1000e9 as u128));
let signed = Self::sign_tx(wallet, tx).await;
signed.encoded_2718().into()
}
/// Creates a deployment transaction and signs it, returning an envelope.
pub async fn deploy_tx(
chain_id: u64,

View File

@@ -0,0 +1,470 @@
//! E2E tests for `RocksDB` provider functionality.
#![cfg(all(feature = "edge", unix))]
use alloy_consensus::BlockHeader;
use alloy_primitives::B256;
use alloy_rpc_types_eth::{Transaction, TransactionReceipt};
use eyre::Result;
use jsonrpsee::core::client::ClientT;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
use reth_db::tables;
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
use reth_node_builder::NodeConfig;
use reth_node_core::args::RocksDbArgs;
use reth_node_ethereum::EthereumNode;
use reth_payload_builder::EthPayloadBuilderAttributes;
use reth_provider::RocksDBProviderFactory;
use std::{sync::Arc, time::Duration};
const ROCKSDB_POLL_TIMEOUT: Duration = Duration::from_secs(60);
const ROCKSDB_POLL_INTERVAL: Duration = Duration::from_millis(50);
/// Polls RPC until the given `tx_hash` is visible as pending (not yet mined).
/// Prevents race conditions where `advance_block` is called before txs are in the pool.
/// Returns the pending transaction.
async fn wait_for_pending_tx<C: ClientT>(client: &C, tx_hash: B256) -> Transaction {
let start = std::time::Instant::now();
loop {
let tx: Option<Transaction> = client
.request("eth_getTransactionByHash", [tx_hash])
.await
.expect("RPC request failed");
if let Some(tx) = tx {
assert!(
tx.block_number.is_none(),
"Expected pending tx but tx_hash={tx_hash:?} is already mined in block {:?}",
tx.block_number
);
return tx;
}
assert!(
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
"Timed out after {:?} waiting for tx_hash={tx_hash:?} to appear in pending pool",
start.elapsed()
);
tokio::time::sleep(ROCKSDB_POLL_INTERVAL).await;
}
}
/// Polls `RocksDB` until the given `tx_hash` appears in `TransactionHashNumbers`.
/// Returns the `tx_number` on success, or panics on timeout.
async fn poll_tx_in_rocksdb<P: RocksDBProviderFactory>(provider: &P, tx_hash: B256) -> u64 {
let start = std::time::Instant::now();
let mut interval = ROCKSDB_POLL_INTERVAL;
loop {
// Re-acquire handle each iteration to avoid stale snapshot reads
let rocksdb = provider.rocksdb_provider();
let tx_number: Option<u64> =
rocksdb.get::<tables::TransactionHashNumbers>(tx_hash).expect("RocksDB get failed");
if let Some(n) = tx_number {
return n;
}
assert!(
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
"Timed out after {:?} waiting for tx_hash={tx_hash:?} in RocksDB",
start.elapsed()
);
tokio::time::sleep(interval).await;
// Simple backoff: 50ms -> 100ms -> 200ms (capped)
interval = std::cmp::min(interval * 2, Duration::from_millis(200));
}
}
/// Returns the test chain spec for `RocksDB` tests.
fn test_chain_spec() -> Arc<ChainSpec> {
Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(
serde_json::from_str(include_str!("../../src/testsuite/assets/genesis.json"))
.expect("failed to parse genesis.json"),
)
.cancun_activated()
.build(),
)
}
/// Returns test payload attributes for the given timestamp.
fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
let attributes = alloy_rpc_types_engine::PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
};
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Enables `RocksDB` for `TransactionHashNumbers` table.
///
/// Note: Static file changesets are disabled because `persistence_threshold(0)` causes
/// a race where the static file writer expects sequential block numbers but receives
/// them out of order, resulting in `UnexpectedStaticFileBlockNumber` errors.
fn with_rocksdb_enabled<C>(mut config: NodeConfig<C>) -> NodeConfig<C> {
config.rocksdb = RocksDbArgs { tx_hash: true, ..Default::default() };
config.static_files.storage_changesets = false;
config.static_files.account_changesets = false;
config
}
/// Smoke test: node boots with `RocksDB` routing enabled.
#[tokio::test]
async fn test_rocksdb_node_startup() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let (nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.with_node_config_modifier(with_rocksdb_enabled)
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Verify RocksDB provider is functional (can query without error)
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let missing_hash = B256::from([0xab; 32]);
let result: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
assert!(result.is_none(), "Missing hash should return None");
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
Ok(())
}
/// Block mining works with `RocksDB` storage.
#[tokio::test]
async fn test_rocksdb_block_mining() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.with_node_config_modifier(with_rocksdb_enabled)
.build()
.await?;
assert_eq!(nodes.len(), 1);
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
// Mine 3 blocks with transactions
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client should be available");
for i in 1..=3u64 {
let raw_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), i - 1)
.await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Wait for tx to enter pending pool before mining
wait_for_pending_tx(&client, tx_hash).await;
let payload = nodes[0].advance_block().await?;
let block = payload.block();
assert_eq!(block.number(), i);
assert_ne!(block.hash(), B256::ZERO);
// Verify tx was actually included in the block
let receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
let receipt = receipt.expect("Receipt should exist after mining");
assert_eq!(receipt.block_number, Some(i), "Tx should be in block {i}");
}
// Verify all blocks are stored
for i in 0..=3 {
let block_hash = nodes[0].block_hash(i);
assert_ne!(block_hash, B256::ZERO);
}
Ok(())
}
/// Tx hash lookup exercises `TransactionHashNumbers` table.
#[tokio::test]
async fn test_rocksdb_transaction_queries() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Inject and mine a transaction
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client should be available");
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Wait for tx to enter pending pool before mining
wait_for_pending_tx(&client, tx_hash).await;
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Query each transaction by hash
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
let tx = tx.expect("Transaction should be found");
assert_eq!(tx.block_number, Some(1));
let receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
let receipt = receipt.expect("Receipt should be found");
assert_eq!(receipt.block_number, Some(1));
assert!(receipt.status());
// Direct RocksDB assertion - poll with timeout since persistence is async
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
assert_eq!(tx_number, 0, "First tx should have TxNumber 0");
// Verify missing hash returns None
let missing_hash = B256::from([0xde; 32]);
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let missing_tx_number: Option<u64> =
rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
assert!(missing_tx_number.is_none());
let missing_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [missing_hash]).await?;
assert!(missing_tx.is_none(), "expected no transaction for missing hash");
let missing_receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [missing_hash]).await?;
assert!(missing_receipt.is_none(), "expected no receipt for missing hash");
Ok(())
}
/// Multiple transactions in the same block are correctly persisted to `RocksDB`.
#[tokio::test]
async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
// Create 3 txs from the same wallet with sequential nonces
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client");
let mut tx_hashes = Vec::new();
for nonce in 0..3 {
let raw_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce)
.await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
tx_hashes.push(tx_hash);
}
// Wait for all txs to appear in pending pool before mining
for tx_hash in &tx_hashes {
wait_for_pending_tx(&client, *tx_hash).await;
}
// Mine one block containing all 3 txs
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Verify block contains all 3 txs
let block: Option<alloy_rpc_types_eth::Block> =
client.request("eth_getBlockByNumber", ("0x1", true)).await?;
let block = block.expect("Block 1 should exist");
assert_eq!(block.transactions.len(), 3, "Block should contain 3 txs");
// Verify each tx via RPC
for tx_hash in &tx_hashes {
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
let tx = tx.expect("Transaction should be found");
assert_eq!(tx.block_number, Some(1), "All txs should be in block 1");
}
// Poll RocksDB for all tx hashes and collect tx_numbers
let mut tx_numbers = Vec::new();
for tx_hash in &tx_hashes {
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
tx_numbers.push(n);
}
// Verify tx_numbers form the set {0, 1, 2}
tx_numbers.sort();
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be 0, 1, 2");
Ok(())
}
/// Transactions across multiple blocks have globally continuous `tx_numbers`.
#[tokio::test]
async fn test_rocksdb_txs_across_blocks() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client");
// Block 1: 2 transactions
let tx_hash_0 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await,
)
.await?;
let tx_hash_1 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await,
)
.await?;
// Wait for both txs to appear in pending pool
wait_for_pending_tx(&client, tx_hash_0).await;
wait_for_pending_tx(&client, tx_hash_1).await;
let payload1 = nodes[0].advance_block().await?;
assert_eq!(payload1.block().number(), 1);
// Block 2: 1 transaction
let tx_hash_2 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await,
)
.await?;
wait_for_pending_tx(&client, tx_hash_2).await;
let payload2 = nodes[0].advance_block().await?;
assert_eq!(payload2.block().number(), 2);
// Verify block contents via RPC
let tx0: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_1]).await?;
let tx2: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_2]).await?;
assert_eq!(tx0.expect("tx0").block_number, Some(1));
assert_eq!(tx1.expect("tx1").block_number, Some(1));
assert_eq!(tx2.expect("tx2").block_number, Some(2));
// Poll RocksDB and verify global tx_number continuity
let all_tx_hashes = [tx_hash_0, tx_hash_1, tx_hash_2];
let mut tx_numbers = Vec::new();
for tx_hash in &all_tx_hashes {
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
tx_numbers.push(n);
}
// Verify they form a continuous sequence {0, 1, 2}
tx_numbers.sort();
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be globally continuous: 0, 1, 2");
// Re-query block 1 txs after block 2 is mined (regression guard)
let tx0_again: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
assert!(tx0_again.is_some(), "Block 1 tx should still be queryable after block 2");
Ok(())
}
/// Pending transactions should NOT appear in `RocksDB` until mined.
#[tokio::test]
async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_node_config_modifier(with_rocksdb_enabled)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
// Inject tx but do NOT mine
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Verify tx is in pending pool via RPC
let client = nodes[0].rpc_client().expect("RPC client");
wait_for_pending_tx(&client, tx_hash).await;
let pending_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash]).await?;
assert!(pending_tx.is_some(), "Pending tx should be visible via RPC");
assert!(pending_tx.unwrap().block_number.is_none(), "Pending tx should have no block_number");
// Assert tx is NOT in RocksDB before mining (single check - tx is confirmed pending)
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let tx_number: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(tx_hash)?;
assert!(
tx_number.is_none(),
"Pending tx should NOT be in RocksDB before mining, but found tx_number={:?}",
tx_number
);
// Now mine the block
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Poll until tx appears in RocksDB
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
assert_eq!(tx_number, 0, "First tx should have tx_number 0");
// Verify tx is now mined via RPC
let mined_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash]).await?;
assert_eq!(mined_tx.expect("mined tx").block_number, Some(1));
Ok(())
}

View File

@@ -448,12 +448,14 @@ mod tests {
nonce: account.nonce,
code_hash: account.bytecode_hash.unwrap_or_default(),
code: None,
account_id: None,
}),
original_info: (i == 0).then(|| AccountInfo {
balance: account.balance.checked_div(U256::from(2)).unwrap_or(U256::ZERO),
nonce: 0,
code_hash: account.bytecode_hash.unwrap_or_default(),
code: None,
account_id: None,
}),
storage,
status: AccountStatus::default(),

View File

@@ -34,6 +34,11 @@ fn default_account_worker_count() -> usize {
/// The size of proof targets chunk to spawn in one multiproof calculation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
/// computation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
/// Default number of reserved CPU cores for non-reth processes.
///
/// This will be deducted from the thread count of main reth global threadpool.
@@ -45,7 +50,17 @@ pub const DEFAULT_PREWARM_MAX_CONCURRENCY: usize = 16;
const DEFAULT_BLOCK_BUFFER_LIMIT: u32 = EPOCH_SLOTS as u32 * 2;
const DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH: u32 = 256;
const DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE: usize = 4;
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: u64 = 4 * 1024 * 1024 * 1024;
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: usize = default_cross_block_cache_size();
const fn default_cross_block_cache_size() -> usize {
if cfg!(test) {
1024 * 1024 // 1 MB in tests
} else if cfg!(target_pointer_width = "32") {
usize::MAX // max possible on wasm32 / 32-bit
} else {
4 * 1024 * 1024 * 1024 // 4 GB on 64-bit
}
}
/// Determines if the host has enough parallelism to run the payload processor.
///
@@ -95,12 +110,10 @@ pub struct TreeConfig {
disable_state_cache: bool,
/// Whether to disable parallel prewarming.
disable_prewarming: bool,
/// Whether to disable the parallel sparse trie state root algorithm.
disable_parallel_sparse_trie: bool,
/// Whether to enable state provider metrics.
state_provider_metrics: bool,
/// Cross-block cache size in bytes.
cross_block_cache_size: u64,
cross_block_cache_size: usize,
/// Whether the host has enough parallelism to run state root task.
has_enough_parallelism: bool,
/// Whether multiproof task should chunk proof targets.
@@ -137,6 +150,8 @@ pub struct TreeConfig {
account_worker_count: usize,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl Default for TreeConfig {
@@ -151,7 +166,6 @@ impl Default for TreeConfig {
always_compare_trie_updates: false,
disable_state_cache: false,
disable_prewarming: false,
disable_parallel_sparse_trie: false,
state_provider_metrics: false,
cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE,
has_enough_parallelism: has_enough_parallelism(),
@@ -166,6 +180,7 @@ impl Default for TreeConfig {
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
enable_proof_v2: false,
disable_cache_metrics: false,
}
}
}
@@ -183,9 +198,8 @@ impl TreeConfig {
always_compare_trie_updates: bool,
disable_state_cache: bool,
disable_prewarming: bool,
disable_parallel_sparse_trie: bool,
state_provider_metrics: bool,
cross_block_cache_size: u64,
cross_block_cache_size: usize,
has_enough_parallelism: bool,
multiproof_chunking_enabled: bool,
multiproof_chunk_size: usize,
@@ -198,6 +212,7 @@ impl TreeConfig {
storage_worker_count: usize,
account_worker_count: usize,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
persistence_threshold,
@@ -209,7 +224,6 @@ impl TreeConfig {
always_compare_trie_updates,
disable_state_cache,
disable_prewarming,
disable_parallel_sparse_trie,
state_provider_metrics,
cross_block_cache_size,
has_enough_parallelism,
@@ -224,6 +238,7 @@ impl TreeConfig {
storage_worker_count,
account_worker_count,
enable_proof_v2,
disable_cache_metrics,
}
}
@@ -262,6 +277,17 @@ impl TreeConfig {
self.multiproof_chunk_size
}
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
/// and the chunk size is at the default value.
pub const fn effective_multiproof_chunk_size(&self) -> usize {
if self.enable_proof_v2 && self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
{
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
} else {
self.multiproof_chunk_size
}
}
/// Return the number of reserved CPU cores for non-reth processes
pub const fn reserved_cpu_cores(&self) -> usize {
self.reserved_cpu_cores
@@ -278,11 +304,6 @@ impl TreeConfig {
self.state_provider_metrics
}
/// Returns whether or not the parallel sparse trie is disabled.
pub const fn disable_parallel_sparse_trie(&self) -> bool {
self.disable_parallel_sparse_trie
}
/// Returns whether or not state cache is disabled.
pub const fn disable_state_cache(&self) -> bool {
self.disable_state_cache
@@ -300,7 +321,7 @@ impl TreeConfig {
}
/// Returns the cross-block cache size.
pub const fn cross_block_cache_size(&self) -> u64 {
pub const fn cross_block_cache_size(&self) -> usize {
self.cross_block_cache_size
}
@@ -403,7 +424,7 @@ impl TreeConfig {
}
/// Setter for cross block cache size.
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: u64) -> Self {
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: usize) -> Self {
self.cross_block_cache_size = cross_block_cache_size;
self
}
@@ -420,15 +441,6 @@ impl TreeConfig {
self
}
/// Setter for whether to disable the parallel sparse trie
pub const fn with_disable_parallel_sparse_trie(
mut self,
disable_parallel_sparse_trie: bool,
) -> Self {
self.disable_parallel_sparse_trie = disable_parallel_sparse_trie;
self
}
/// Setter for whether multiproof task should chunk proof targets.
pub const fn with_multiproof_chunking_enabled(
mut self,
@@ -516,4 +528,15 @@ impl TreeConfig {
self.enable_proof_v2 = enable_proof_v2;
self
}
/// Returns whether cache metrics recording is disabled.
pub const fn disable_cache_metrics(&self) -> bool {
self.disable_cache_metrics
}
/// Setter for whether to disable cache metrics recording.
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
self.disable_cache_metrics = disable_cache_metrics;
self
}
}

View File

@@ -62,7 +62,8 @@ pub trait EngineTypes:
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
> + DeserializeOwned
+ Serialize
{
@@ -106,6 +107,14 @@ pub trait EngineTypes:
+ Send
+ Sync
+ 'static;
/// Execution Payload V6 envelope type.
type ExecutionPayloadEnvelopeV6: DeserializeOwned
+ Serialize
+ Clone
+ Unpin
+ Send
+ Sync
+ 'static;
}
/// Type that validates the payloads processed by the engine API.

View File

@@ -25,6 +25,7 @@ reth-tasks.workspace = true
reth-node-types.workspace = true
reth-chainspec.workspace = true
reth-engine-primitives.workspace = true
reth-trie-db.workspace = true
# async
futures.workspace = true
@@ -40,6 +41,8 @@ reth-evm-ethereum.workspace = true
reth-exex-types.workspace = true
reth-primitives-traits.workspace = true
reth-node-ethereum.workspace = true
reth-trie-db.workspace = true
alloy-eips.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio-stream.workspace = true

View File

@@ -26,6 +26,7 @@ use reth_provider::{
use reth_prune::PrunerWithFactory;
use reth_stages_api::{MetricEventsSender, Pipeline};
use reth_tasks::TaskSpawner;
use reth_trie_db::ChangesetCache;
use std::{
pin::Pin,
sync::Arc,
@@ -84,6 +85,7 @@ where
tree_config: TreeConfig,
sync_metrics_tx: MetricEventsSender,
evm_config: C,
changeset_cache: ChangesetCache,
) -> Self
where
V: EngineValidator<N::Payload>,
@@ -109,6 +111,7 @@ where
tree_config,
engine_kind,
evm_config,
changeset_cache,
);
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
@@ -156,6 +159,7 @@ mod tests {
};
use reth_prune::Pruner;
use reth_tasks::TokioTaskExecutor;
use reth_trie_db::ChangesetCache;
use std::sync::Arc;
use tokio::sync::{mpsc::unbounded_channel, watch};
use tokio_stream::wrappers::UnboundedReceiverStream;
@@ -188,6 +192,8 @@ mod tests {
let pruner = Pruner::new_with_factory(provider_factory.clone(), vec![], 0, 0, None, rx);
let evm_config = EthEvmConfig::new(chain_spec.clone());
let changeset_cache = ChangesetCache::new();
let engine_validator = BasicEngineValidator::new(
blockchain_db.clone(),
consensus.clone(),
@@ -195,6 +201,7 @@ mod tests {
engine_payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache.clone(),
);
let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel();
@@ -214,6 +221,7 @@ mod tests {
TreeConfig::default(),
sync_metrics_tx,
evm_config,
changeset_cache,
);
}
}

View File

@@ -12,7 +12,7 @@ workspace = true
[dependencies]
# reth
reth-chain-state.workspace = true
reth-chain-state = { workspace = true, features = ["rayon"] }
reth-chainspec = { workspace = true, optional = true }
reth-consensus.workspace = true
reth-db.workspace = true
@@ -34,6 +34,8 @@ reth-trie-parallel.workspace = true
reth-trie-sparse = { workspace = true, features = ["std", "metrics"] }
reth-trie-sparse-parallel = { workspace = true, features = ["std"] }
reth-trie.workspace = true
reth-trie-common.workspace = true
reth-trie-db.workspace = true
# alloy
alloy-evm.workspace = true
@@ -51,7 +53,7 @@ revm-primitives.workspace = true
futures.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] }
mini-moka = { workspace = true, features = ["sync"] }
fixed-cache.workspace = true
moka = { workspace = true, features = ["sync"] }
smallvec.workspace = true
@@ -94,7 +96,7 @@ reth-tracing.workspace = true
reth-node-ethereum.workspace = true
reth-e2e-test-utils.workspace = true
# alloy
# revm
revm-state.workspace = true
assert_matches.workspace = true
@@ -133,6 +135,8 @@ test-utils = [
"reth-static-file",
"reth-tracing",
"reth-trie/test-utils",
"reth-trie-common/test-utils",
"reth-trie-db/test-utils",
"reth-trie-sparse/test-utils",
"reth-prune-types?/test-utils",
"reth-trie-parallel/test-utils",

View File

@@ -26,7 +26,9 @@ fn create_bench_state(num_accounts: usize) -> EvmState {
nonce: 10,
code_hash: B256::from_slice(&rng.random::<[u8; 32]>()),
code: Default::default(),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::empty(),
transaction_id: 0,

View File

@@ -62,6 +62,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
storage: HashMap::default(),
status: AccountStatus::SelfDestructed,
transaction_id: 0,
original_info: Box::new(AccountInfo::default()),
}
} else {
RevmAccount {
@@ -70,6 +71,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
nonce: rng.random::<u64>(),
code_hash: KECCAK_EMPTY,
code: Some(Default::default()),
account_id: None,
},
storage: (0..rng.random_range(0..=params.storage_slots_per_account))
.map(|_| {
@@ -84,6 +86,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
})
.collect(),
status: AccountStatus::Touched,
original_info: Box::new(AccountInfo::default()),
transaction_id: 0,
}
};
@@ -239,7 +242,10 @@ fn bench_state_root(c: &mut Criterion) {
std::convert::identity,
),
StateProviderBuilder::new(provider.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(provider),
OverlayStateProviderFactory::new(
provider,
reth_trie_db::ChangesetCache::new(),
),
&TreeConfig::default(),
None,
);

View File

@@ -7,7 +7,7 @@ use reth_ethereum_primitives::EthPrimitives;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::ProviderNodeTypes, BlockExecutionWriter, BlockHashReader, ChainStateBlockWriter,
DBProvider, DatabaseProviderFactory, ProviderFactory,
DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksMode,
};
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
use reth_stages_api::{MetricEvent, MetricEventsSender};
@@ -151,7 +151,7 @@ where
if last_block.is_some() {
let provider_rw = self.provider.database_provider_rw()?;
provider_rw.save_blocks(blocks)?;
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
provider_rw.commit()?;
}
@@ -159,6 +159,7 @@ where
self.metrics.save_blocks_block_count.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
Ok(last_block)
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1,25 +1,15 @@
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
use alloy_consensus::transaction::TxHashRef;
use alloy_evm::{
block::{BlockExecutor, ExecutableTx},
Evm,
};
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
use core::borrow::BorrowMut;
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
use reth_errors::{BlockExecutionError, ProviderError};
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
use reth_errors::ProviderError;
use reth_evm::metrics::ExecutorMetrics;
use reth_execution_types::BlockExecutionOutput;
use reth_metrics::{
metrics::{Counter, Gauge, Histogram},
Metrics,
};
use reth_primitives_traits::SignedTransaction;
use reth_trie::updates::TrieUpdates;
use revm::database::{states::bundle_state::BundleRetention, State};
use revm_primitives::Address;
use std::time::Instant;
use tracing::{debug_span, trace};
use std::time::{Duration, Instant};
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
@@ -32,104 +22,30 @@ pub(crate) struct EngineApiMetrics {
pub(crate) block_validation: BlockValidationMetrics,
/// Canonical chain and reorg related metrics
pub tree: TreeMetrics,
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
#[allow(dead_code)]
pub(crate) bal: BalMetrics,
}
impl EngineApiMetrics {
/// Helper function for metered execution
fn metered<F, R>(&self, f: F) -> R
where
F: FnOnce() -> (u64, R),
{
// Execute the block and record the elapsed time.
let execute_start = Instant::now();
let (gas_used, output) = f();
let execution_duration = execute_start.elapsed().as_secs_f64();
// Update gas metrics.
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_duration);
self.executor.execution_duration.set(execution_duration);
output
}
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
/// execution.
/// Records metrics for block execution.
///
/// This method updates metrics for execution time, gas usage, and the number
/// of accounts, storage slots and bytecodes loaded and updated.
pub(crate) fn execute_metered<E, DB>(
/// of accounts, storage slots and bytecodes updated.
pub(crate) fn record_block_execution<R>(
&self,
executor: E,
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
transaction_count: usize,
state_hook: Box<dyn OnStateHook>,
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
where
DB: alloy_evm::Database,
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
{
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
// they are globally registered so that the data recorded in the hook will
// be accessible.
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
output: &BlockExecutionOutput<R>,
execution_duration: Duration,
) {
let execution_secs = execution_duration.as_secs_f64();
let gas_used = output.result.gas_used;
let mut senders = Vec::with_capacity(transaction_count);
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
let f = || {
let start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.entered()
.in_scope(|| executor.apply_pre_execution_changes())?;
self.executor.pre_execution_histogram.record(start.elapsed());
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
loop {
let start = Instant::now();
let Some(tx) = transactions.next() else { break };
self.executor.transaction_wait_histogram.record(start.elapsed());
let tx = tx?;
senders.push(*tx.signer());
let span =
debug_span!(target: "engine::tree", "execute tx", tx_hash=?tx.tx().tx_hash());
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.executor.transaction_execution_histogram.record(start.elapsed());
// record the tx gas used
enter.record("gas_used", gas_used);
}
drop(exec_span);
let start = Instant::now();
let result = debug_span!(target: "engine::tree", "finish")
.entered()
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result));
self.executor.post_execution_histogram.record(start.elapsed());
result
};
// Use metered to execute and track timing/gas metrics
let (mut db, result) = self.metered(|| {
let res = f();
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
(gas_used, res)
})?;
// merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.entered()
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
// Update gas metrics
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_secs);
self.executor.execution_duration.set(execution_secs);
// Update the metrics for the number of accounts, storage slots and bytecodes updated
let accounts = output.state.state.len();
@@ -140,8 +56,31 @@ impl EngineApiMetrics {
self.executor.accounts_updated_histogram.record(accounts as f64);
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
}
Ok((output, senders))
/// Returns a reference to the executor metrics for use in state hooks.
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
&self.executor
}
/// Records the duration of block pre-execution changes (e.g., beacon root update).
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
self.executor.pre_execution_histogram.record(elapsed);
}
/// Records the duration of block post-execution changes (e.g., finalization).
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
self.executor.post_execution_histogram.record(elapsed);
}
/// Records the time spent waiting for the next transaction from the iterator.
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
self.executor.transaction_wait_histogram.record(elapsed);
}
/// Records the duration of a single transaction execution.
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
self.executor.transaction_execution_histogram.record(elapsed);
}
}
@@ -197,6 +136,12 @@ pub(crate) struct EngineMetrics {
#[derive(Metrics)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ForkchoiceUpdatedMetrics {
/// Finish time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of forkchoice updated messages received.
pub(crate) forkchoice_updated_messages: Counter,
/// The total count of forkchoice updated messages with payload received.
@@ -219,18 +164,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
pub(crate) forkchoice_updated_last: Gauge,
/// Time diff between new payload call response and the next forkchoice updated call request.
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
/// time).
pub(crate) time_between_forkchoice_updated: Histogram,
/// Time from previous forkchoice updated start to current forkchoice updated start (total
/// interval).
pub(crate) forkchoice_updated_interval: Histogram,
}
impl ForkchoiceUpdatedMetrics {
/// Increment the forkchoiceUpdated counter based on the given result
pub(crate) fn update_response_metrics(
&self,
&mut self,
start: Instant,
latest_new_payload_at: &mut Option<Instant>,
has_attrs: bool,
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
) {
let elapsed = start.elapsed();
let finish = Instant::now();
let elapsed = finish - start;
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_forkchoice_updated.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.forkchoice_updated_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.forkchoice_status() {
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
@@ -257,7 +219,10 @@ impl ForkchoiceUpdatedMetrics {
pub(crate) struct NewPayloadStatusMetrics {
/// Finish time of the latest new payload call.
#[metric(skip)]
pub(crate) latest_at: Option<Instant>,
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest new payload call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of new payload messages received.
pub(crate) new_payload_messages: Counter,
/// The total count of new payload messages that we responded to with
@@ -277,6 +242,8 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) new_payload_error: Counter,
/// The total gas of valid new payload messages received.
pub(crate) new_payload_total_gas: Histogram,
/// The gas used for the last valid new payload.
pub(crate) new_payload_total_gas_last: Gauge,
/// The gas per second of valid new payload messages received.
pub(crate) new_payload_gas_per_second: Histogram,
/// The gas per second for the last new payload call.
@@ -285,6 +252,12 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) new_payload_latency: Histogram,
/// Latency for the last new payload call.
pub(crate) new_payload_last: Gauge,
/// Time from previous payload finish to current payload start (idle time).
pub(crate) time_between_new_payloads: Histogram,
/// Time from previous payload start to current payload start (total interval).
pub(crate) new_payload_interval: Histogram,
/// Time diff between forkchoice updated call response and the next new payload call request.
pub(crate) forkchoice_updated_new_payload_time_diff: Histogram,
}
impl NewPayloadStatusMetrics {
@@ -292,18 +265,27 @@ impl NewPayloadStatusMetrics {
pub(crate) fn update_response_metrics(
&mut self,
start: Instant,
latest_forkchoice_updated_at: &mut Option<Instant>,
result: &Result<TreeOutcome<PayloadStatus>, InsertBlockFatalError>,
gas_used: u64,
) {
let finish = Instant::now();
let elapsed = finish - start;
self.latest_at = Some(finish);
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_new_payloads.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.new_payload_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.status {
PayloadStatusEnum::Valid => {
self.new_payload_valid.increment(1);
self.new_payload_total_gas.record(gas_used as f64);
self.new_payload_total_gas_last.set(gas_used as f64);
let gas_per_second = gas_used as f64 / elapsed.as_secs_f64();
self.new_payload_gas_per_second.record(gas_per_second);
self.new_payload_gas_per_second_last.set(gas_per_second);
@@ -317,9 +299,40 @@ impl NewPayloadStatusMetrics {
self.new_payload_messages.increment(1);
self.new_payload_latency.record(elapsed);
self.new_payload_last.set(elapsed);
if let Some(latest_forkchoice_updated_at) = latest_forkchoice_updated_at.take() {
self.forkchoice_updated_new_payload_time_diff
.record(start - latest_forkchoice_updated_at);
}
}
}
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
///
/// See also <https://github.com/ethereum/execution-metrics/issues/5>
#[allow(dead_code)]
#[derive(Metrics, Clone)]
#[metrics(scope = "execution.block_access_list")]
pub(crate) struct BalMetrics {
/// Size of the BAL in bytes for the current block.
pub(crate) size_bytes: Gauge,
/// Total number of blocks with valid BALs.
pub(crate) valid_total: Counter,
/// Total number of blocks with invalid BALs.
pub(crate) invalid_total: Counter,
/// Time taken to validate the BAL against actual execution.
pub(crate) validation_time_seconds: Histogram,
/// Number of account changes in the BAL.
pub(crate) account_changes: Gauge,
/// Number of storage changes in the BAL.
pub(crate) storage_changes: Gauge,
/// Number of balance changes in the BAL.
pub(crate) balance_changes: Gauge,
/// Number of nonce changes in the BAL.
pub(crate) nonce_changes: Gauge,
/// Number of code changes in the BAL.
pub(crate) code_changes: Gauge,
}
/// Metrics for non-execution related block validation.
#[derive(Metrics, Clone)]
#[metrics(scope = "sync.block_validation")]
@@ -328,16 +341,14 @@ pub(crate) struct BlockValidationMetrics {
pub(crate) state_root_storage_tries_updated_total: Counter,
/// Total number of times the parallel state root computation fell back to regular.
pub(crate) state_root_parallel_fallback_total: Counter,
/// Total number of times the state root task failed but the fallback succeeded.
pub(crate) state_root_task_fallback_success_total: Counter,
/// Latest state root duration, ie the time spent blocked waiting for the state root.
pub(crate) state_root_duration: Gauge,
/// Histogram for state root duration ie the time spent blocked waiting for the state root
pub(crate) state_root_histogram: Histogram,
/// Histogram of deferred trie computation duration.
pub(crate) deferred_trie_compute_duration: Histogram,
/// Histogram of time spent waiting for deferred trie data to become available.
pub(crate) deferred_trie_wait_duration: Histogram,
/// Trie input computation duration
pub(crate) trie_input_duration: Histogram,
/// Payload conversion and validation latency
pub(crate) payload_validation_duration: Gauge,
/// Histogram of payload validation latency
@@ -387,133 +398,10 @@ pub(crate) struct BlockBufferMetrics {
mod tests {
use super::*;
use alloy_eips::eip7685::Requests;
use alloy_evm::block::StateChangeSource;
use alloy_primitives::{B256, U256};
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm_ethereum::EthEvm;
use reth_ethereum_primitives::Receipt;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::RecoveredBlock;
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
database_interface::EmptyDB,
inspector::NoOpInspector,
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
Context, MainBuilder, MainContext,
};
use revm_primitives::Bytes;
use std::sync::mpsc;
/// A simple mock executor for testing that doesn't require complex EVM setup
struct MockExecutor {
state: EvmState,
hook: Option<Box<dyn OnStateHook>>,
}
impl MockExecutor {
fn new(state: EvmState) -> Self {
Self { state, hook: None }
}
}
// Mock Evm type for testing
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
impl BlockExecutor for MockExecutor {
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Evm = MockEvm;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
}
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
// Call hook with our mock state for each transaction
if let Some(hook) = self.hook.as_mut() {
hook.on_state(StateChangeSource::Transaction(0), &self.state);
}
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 1000, // Mock gas used
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(1000)
}
fn finish(
self,
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
let Self { hook, state, .. } = self;
// Call hook with our mock state
if let Some(mut hook) = hook {
hook.on_state(StateChangeSource::Transaction(0), &state);
}
// Create a mock EVM
let db = State::builder()
.with_database(EmptyDB::default())
.with_bundle_update()
.without_state_clear()
.build();
let evm = EthEvm::new(
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
false,
);
// Return successful result like the original tests
Ok((
evm,
BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 1000,
blob_gas_used: 0,
},
))
}
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
self.hook = hook;
}
fn evm(&self) -> &Self::Evm {
panic!("Mock executor evm() not implemented")
}
fn evm_mut(&mut self) -> &mut Self::Evm {
panic!("Mock executor evm_mut() not implemented")
}
}
struct ChannelStateHook {
output: i32,
sender: mpsc::Sender<i32>,
}
impl OnStateHook for ChannelStateHook {
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
let _ = self.sender.send(self.output);
}
}
use reth_revm::db::BundleState;
fn setup_test_recorder() -> Snapshotter {
let recorder = DebuggingRecorder::new();
@@ -523,37 +411,7 @@ mod tests {
}
#[test]
fn test_executor_metrics_hook_called() {
let metrics = EngineApiMetrics::default();
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, rx) = mpsc::channel();
let expected_output = 42;
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
let state = EvmState::default();
let executor = MockExecutor::new(state);
// This will fail to create the EVM but should still call the hook
let _result = metrics.execute_metered::<_, EmptyDB>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
);
// Check if hook was called (it might not be if finish() fails early)
match rx.try_recv() {
Ok(actual_output) => assert_eq!(actual_output, expected_output),
Err(_) => {
// Hook wasn't called, which is expected if the mock fails early
// The test still validates that the code compiles and runs
}
}
}
#[test]
fn test_executor_metrics_hook_metrics_recorded() {
fn test_record_block_execution_metrics() {
let snapshotter = setup_test_recorder();
let metrics = EngineApiMetrics::default();
@@ -562,42 +420,17 @@ mod tests {
metrics.executor.gas_per_second.set(0.0);
metrics.executor.gas_used_histogram.record(0.0);
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, _rx) = mpsc::channel();
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
// Create a state with some data
let state = {
let mut state = EvmState::default();
let storage =
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
state.insert(
Default::default(),
Account {
info: AccountInfo {
balance: U256::from(100),
nonce: 10,
code_hash: B256::random(),
code: Default::default(),
},
storage,
status: AccountStatus::default(),
transaction_id: 0,
},
);
state
let output = BlockExecutionOutput::<Receipt> {
state: BundleState::default(),
result: BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 21000,
blob_gas_used: 0,
},
};
let executor = MockExecutor::new(state);
// Execute (will fail but should still update some metrics)
let _result = metrics.execute_metered::<_, EmptyDB>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
);
metrics.record_block_execution(&output, Duration::from_millis(100));
let snapshot = snapshotter.snapshot().into_vec();

View File

@@ -30,11 +30,14 @@ use reth_payload_primitives::{
};
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
use reth_provider::{
BlockReader, DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StateProviderBox,
StateProviderFactory, StateReader, TransactionVariant, TrieReader,
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
use reth_trie_db::ChangesetCache;
use revm::state::EvmState;
use state::TreeState;
use std::{fmt::Debug, ops, sync::Arc, time::Instant};
@@ -82,6 +85,12 @@ pub mod state;
/// backfill this gap.
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
/// The minimum number of blocks to retain in the changeset cache after eviction.
///
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
/// even when the finalized block is not set (e.g., on L2s like Optimism).
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
/// A builder for creating state providers that can be used across threads.
#[derive(Clone, Debug)]
pub struct StateProviderBuilder<N: NodePrimitives, P> {
@@ -271,6 +280,8 @@ where
engine_kind: EngineApiKind,
/// The EVM configuration.
evm_config: C,
/// Changeset cache for in-memory trie changesets
changeset_cache: ChangesetCache,
}
impl<N, P: Debug, T: PayloadTypes + Debug, V: Debug, C> std::fmt::Debug
@@ -295,6 +306,7 @@ where
.field("metrics", &self.metrics)
.field("engine_kind", &self.engine_kind)
.field("evm_config", &self.evm_config)
.field("changeset_cache", &self.changeset_cache)
.finish()
}
}
@@ -307,11 +319,13 @@ where
+ StateProviderFactory
+ StateReader<Receipt = N::Receipt>
+ HashedPostStateProvider
+ TrieReader
+ Clone
+ 'static,
<P as DatabaseProviderFactory>::Provider:
BlockReader<Block = N::Block, Header = N::BlockHeader>,
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
V: EngineValidator<T>,
@@ -331,6 +345,7 @@ where
config: TreeConfig,
engine_kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
) -> Self {
let (incoming_tx, incoming) = crossbeam_channel::unbounded();
@@ -351,6 +366,7 @@ where
incoming_tx,
engine_kind,
evm_config,
changeset_cache,
}
}
@@ -370,6 +386,7 @@ where
config: TreeConfig,
kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
) -> (Sender<FromEngine<EngineApiRequest<T, N>, N::Block>>, UnboundedReceiver<EngineApiEvent<N>>)
{
let best_block_number = provider.best_block_number().unwrap_or(0);
@@ -401,6 +418,7 @@ where
config,
kind,
evm_config,
changeset_cache,
);
let incoming = task.incoming_tx.clone();
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
@@ -1365,6 +1383,29 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the eviction threshold.
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
// the finalized block if set.
let min_threshold =
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
let eviction_threshold =
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
// Use the minimum of finalized block and retention threshold to be conservative
finalized.number.min(min_threshold)
} else {
// When finalized is not set (e.g., on L2s), use the retention threshold
min_threshold
};
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
self.on_new_persisted_block()?;
Ok(())
}
@@ -1451,9 +1492,13 @@ where
self.on_maybe_tree_event(res.event.take())?;
}
if let Err(ref err) = output {
error!(target: "engine::tree", %err, ?state, "Error processing forkchoice update");
}
self.metrics.engine.forkchoice_updated.update_response_metrics(
start,
&mut self.metrics.engine.new_payload.latest_at,
&mut self.metrics.engine.new_payload.latest_finish_at,
has_attrs,
&output,
);
@@ -1473,10 +1518,12 @@ where
let gas_used = payload.gas_used();
let num_hash = payload.num_hash();
let mut output = self.on_new_payload(payload);
self.metrics
.engine
.new_payload
.update_response_metrics(start, &output, gas_used);
self.metrics.engine.new_payload.update_response_metrics(
start,
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
&output,
gas_used,
);
let maybe_event =
output.as_mut().ok().and_then(|out| out.event.take());
@@ -1651,6 +1698,18 @@ where
)));
return Ok(());
}
} else {
// We don't have the head block or any of its ancestors buffered. Request
// a download for the head block which will then trigger further sync.
debug!(
target: "engine::tree",
head_hash = %sync_target_state.head_block_hash,
"Backfill complete but head block not buffered, requesting download"
);
self.emit_event(EngineApiEvent::Download(DownloadRequest::single_block(
sync_target_state.head_block_hash,
)));
return Ok(());
}
// try to close the gap by executing buffered blocks that are child blocks of the new head
@@ -1818,6 +1877,7 @@ where
/// or the database. If the required historical data (such as trie change sets) has been
/// pruned for a given block, this operation will return an error. On archive nodes, it
/// can retrieve any block.
#[instrument(level = "debug", target = "engine::tree", skip(self))]
fn canonical_block_by_hash(&self, hash: B256) -> ProviderResult<Option<ExecutedBlock<N>>> {
trace!(target: "engine::tree", ?hash, "Fetching executed block by hash");
// check memory first
@@ -1830,12 +1890,23 @@ where
.sealed_block_with_senders(hash.into(), TransactionVariant::WithHash)?
.ok_or_else(|| ProviderError::HeaderNotFound(hash.into()))?
.split_sealed();
let execution_output = self
let mut execution_output = self
.provider
.get_state(block.header().number())?
.ok_or_else(|| ProviderError::StateForNumberNotFound(block.header().number()))?;
let hashed_state = self.provider.hashed_post_state(execution_output.state());
let trie_updates = self.provider.get_block_trie_updates(block.number())?;
debug!(
target: "engine::tree",
number = ?block.number(),
"computing block trie updates",
);
let db_provider = self.provider.database_provider_ro()?;
let trie_updates = reth_trie_db::compute_block_trie_updates(
&self.changeset_cache,
&db_provider,
block.number(),
)?;
let sorted_hashed_state = Arc::new(hashed_state.into_sorted());
let sorted_trie_updates = Arc::new(trie_updates);
@@ -1843,9 +1914,19 @@ where
let trie_data =
ComputedTrieData::without_trie_input(sorted_hashed_state, sorted_trie_updates);
let execution_output = Arc::new(BlockExecutionOutput {
state: execution_output.bundle,
result: BlockExecutionResult {
receipts: execution_output.receipts.pop().unwrap_or_default(),
requests: execution_output.requests.pop().unwrap_or_default(),
gas_used: block.gas_used(),
blob_gas_used: block.blob_gas_used().unwrap_or_default(),
},
});
Ok(Some(ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(execution_output),
execution_output,
trie_data,
)))
}

View File

@@ -101,7 +101,7 @@ impl<'a> Iterator for BALSlotIter<'a> {
return None;
}
return Some((address, slot));
return Some((address, StorageKey::from(slot)));
}
// Move to next account
@@ -177,13 +177,11 @@ where
let mut storage_map = HashedStorage::new(false);
for slot_changes in &account_changes.storage_changes {
let hashed_slot = keccak256(slot_changes.slot);
let hashed_slot = keccak256(slot_changes.slot.to_be_bytes::<32>());
// Get the last change for this slot
if let Some(last_change) = slot_changes.changes.last() {
storage_map
.storage
.insert(hashed_slot, U256::from_be_bytes(last_change.new_value.0));
storage_map.storage.insert(hashed_slot, last_change.new_value);
}
}
@@ -237,8 +235,8 @@ mod tests {
let provider = StateProviderTest::default();
let address = Address::random();
let slot = StorageKey::random();
let value = B256::random();
let slot = U256::random();
let value = U256::random();
let slot_changes = SlotChanges { slot, changes: vec![StorageChange::new(0, value)] };
@@ -258,10 +256,10 @@ mod tests {
assert!(result.storages.contains_key(&hashed_address));
let storage = result.storages.get(&hashed_address).unwrap();
let hashed_slot = keccak256(slot);
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
let stored_value = storage.storage.get(&hashed_slot).unwrap();
assert_eq!(*stored_value, U256::from_be_bytes(value.0));
assert_eq!(*stored_value, value);
}
#[test]
@@ -392,15 +390,15 @@ mod tests {
let provider = StateProviderTest::default();
let address = Address::random();
let slot = StorageKey::random();
let slot = U256::random();
// Multiple changes to the same slot - should take the last one
let slot_changes = SlotChanges {
slot,
changes: vec![
StorageChange::new(0, B256::from(U256::from(100).to_be_bytes::<32>())),
StorageChange::new(1, B256::from(U256::from(200).to_be_bytes::<32>())),
StorageChange::new(2, B256::from(U256::from(300).to_be_bytes::<32>())),
StorageChange::new(0, U256::from(100)),
StorageChange::new(1, U256::from(200)),
StorageChange::new(2, U256::from(300)),
],
};
@@ -418,7 +416,7 @@ mod tests {
let hashed_address = keccak256(address);
let storage = result.storages.get(&hashed_address).unwrap();
let hashed_slot = keccak256(slot);
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
let stored_value = storage.storage.get(&hashed_slot).unwrap();
@@ -438,15 +436,15 @@ mod tests {
address: addr1,
storage_changes: vec![
SlotChanges {
slot: StorageKey::from(U256::from(100)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(100),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
SlotChanges {
slot: StorageKey::from(U256::from(101)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(101),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
],
storage_reads: vec![StorageKey::from(U256::from(102))],
storage_reads: vec![U256::from(102)],
balance_changes: vec![],
nonce_changes: vec![],
code_changes: vec![],
@@ -456,10 +454,10 @@ mod tests {
let account2 = AccountChanges {
address: addr2,
storage_changes: vec![SlotChanges {
slot: StorageKey::from(U256::from(200)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(200),
changes: vec![StorageChange::new(0, U256::ZERO)],
}],
storage_reads: vec![StorageKey::from(U256::from(201))],
storage_reads: vec![U256::from(201)],
balance_changes: vec![],
nonce_changes: vec![],
code_changes: vec![],
@@ -470,15 +468,15 @@ mod tests {
address: addr3,
storage_changes: vec![
SlotChanges {
slot: StorageKey::from(U256::from(300)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(300),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
SlotChanges {
slot: StorageKey::from(U256::from(301)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(301),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
],
storage_reads: vec![StorageKey::from(U256::from(302))],
storage_reads: vec![U256::from(302)],
balance_changes: vec![],
nonce_changes: vec![],
code_changes: vec![],

View File

@@ -1,188 +0,0 @@
//! Configured sparse trie enum for switching between serial and parallel implementations.
use alloy_primitives::B256;
use reth_trie::{BranchNodeMasks, Nibbles, ProofTrieNode, TrieNode};
use reth_trie_sparse::{
errors::SparseTrieResult, provider::TrieNodeProvider, LeafLookup, LeafLookupError,
SerialSparseTrie, SparseTrieInterface, SparseTrieUpdates,
};
use reth_trie_sparse_parallel::ParallelSparseTrie;
use std::borrow::Cow;
/// Enum for switching between serial and parallel sparse trie implementations.
///
/// This type allows runtime selection between different sparse trie implementations,
/// providing flexibility in choosing the appropriate implementation based on workload
/// characteristics.
#[derive(Debug, Clone)]
pub(crate) enum ConfiguredSparseTrie {
/// Serial implementation of the sparse trie.
Serial(Box<SerialSparseTrie>),
/// Parallel implementation of the sparse trie.
Parallel(Box<ParallelSparseTrie>),
}
impl From<SerialSparseTrie> for ConfiguredSparseTrie {
fn from(trie: SerialSparseTrie) -> Self {
Self::Serial(Box::new(trie))
}
}
impl From<ParallelSparseTrie> for ConfiguredSparseTrie {
fn from(trie: ParallelSparseTrie) -> Self {
Self::Parallel(Box::new(trie))
}
}
impl Default for ConfiguredSparseTrie {
fn default() -> Self {
Self::Serial(Default::default())
}
}
impl SparseTrieInterface for ConfiguredSparseTrie {
fn with_root(
self,
root: TrieNode,
masks: Option<BranchNodeMasks>,
retain_updates: bool,
) -> SparseTrieResult<Self> {
match self {
Self::Serial(trie) => {
trie.with_root(root, masks, retain_updates).map(|t| Self::Serial(Box::new(t)))
}
Self::Parallel(trie) => {
trie.with_root(root, masks, retain_updates).map(|t| Self::Parallel(Box::new(t)))
}
}
}
fn with_updates(self, retain_updates: bool) -> Self {
match self {
Self::Serial(trie) => Self::Serial(Box::new(trie.with_updates(retain_updates))),
Self::Parallel(trie) => Self::Parallel(Box::new(trie.with_updates(retain_updates))),
}
}
fn reserve_nodes(&mut self, additional: usize) {
match self {
Self::Serial(trie) => trie.reserve_nodes(additional),
Self::Parallel(trie) => trie.reserve_nodes(additional),
}
}
fn reveal_node(
&mut self,
path: Nibbles,
node: TrieNode,
masks: Option<BranchNodeMasks>,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_node(path, node, masks),
Self::Parallel(trie) => trie.reveal_node(path, node, masks),
}
}
fn reveal_nodes(&mut self, nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_nodes(nodes),
Self::Parallel(trie) => trie.reveal_nodes(nodes),
}
}
fn update_leaf<P: TrieNodeProvider>(
&mut self,
full_path: Nibbles,
value: Vec<u8>,
provider: P,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.update_leaf(full_path, value, provider),
Self::Parallel(trie) => trie.update_leaf(full_path, value, provider),
}
}
fn remove_leaf<P: TrieNodeProvider>(
&mut self,
full_path: &Nibbles,
provider: P,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.remove_leaf(full_path, provider),
Self::Parallel(trie) => trie.remove_leaf(full_path, provider),
}
}
fn root(&mut self) -> B256 {
match self {
Self::Serial(trie) => trie.root(),
Self::Parallel(trie) => trie.root(),
}
}
fn update_subtrie_hashes(&mut self) {
match self {
Self::Serial(trie) => trie.update_subtrie_hashes(),
Self::Parallel(trie) => trie.update_subtrie_hashes(),
}
}
fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec<u8>> {
match self {
Self::Serial(trie) => trie.get_leaf_value(full_path),
Self::Parallel(trie) => trie.get_leaf_value(full_path),
}
}
fn find_leaf(
&self,
full_path: &Nibbles,
expected_value: Option<&Vec<u8>>,
) -> Result<LeafLookup, LeafLookupError> {
match self {
Self::Serial(trie) => trie.find_leaf(full_path, expected_value),
Self::Parallel(trie) => trie.find_leaf(full_path, expected_value),
}
}
fn take_updates(&mut self) -> SparseTrieUpdates {
match self {
Self::Serial(trie) => trie.take_updates(),
Self::Parallel(trie) => trie.take_updates(),
}
}
fn wipe(&mut self) {
match self {
Self::Serial(trie) => trie.wipe(),
Self::Parallel(trie) => trie.wipe(),
}
}
fn clear(&mut self) {
match self {
Self::Serial(trie) => trie.clear(),
Self::Parallel(trie) => trie.clear(),
}
}
fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> {
match self {
Self::Serial(trie) => trie.updates_ref(),
Self::Parallel(trie) => trie.updates_ref(),
}
}
fn shrink_nodes_to(&mut self, size: usize) {
match self {
Self::Serial(trie) => trie.shrink_nodes_to(size),
Self::Parallel(trie) => trie.shrink_nodes_to(size),
}
}
fn shrink_values_to(&mut self, size: usize) {
match self {
Self::Serial(trie) => trie.shrink_values_to(size),
Self::Parallel(trie) => trie.shrink_values_to(size),
}
}
}

View File

@@ -2,10 +2,7 @@
use super::precompile_cache::PrecompileCacheMap;
use crate::tree::{
cached_state::{
CachedStateMetrics, CachedStateProvider, ExecutionCache as StateExecutionCache,
ExecutionCacheBuilder, SavedCache,
},
cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache},
payload_processor::{
prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
sparse_trie::StateRootComputeOutcome,
@@ -15,23 +12,26 @@ use crate::tree::{
};
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip1898::BlockWithParent;
use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_evm::block::StateChangeSource;
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
use rayon::prelude::*;
use reth_evm::{
block::ExecutableTxParts,
execute::{ExecutableTxFor, WithTxEnv},
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
};
use reth_execution_types::ExecutionOutcome;
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockReader, DatabaseProviderROFactory, StateProvider, StateProviderFactory, StateReader,
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
StateProviderFactory, StateReader,
};
use reth_revm::{db::BundleState, state::EvmState};
use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory};
@@ -41,7 +41,7 @@ use reth_trie_parallel::{
};
use reth_trie_sparse::{
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, SparseStateTrie, SparseTrie,
ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie,
};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::{
@@ -57,14 +57,12 @@ use std::{
use tracing::{debug, debug_span, instrument, warn, Span};
pub mod bal;
mod configured_sparse_trie;
pub mod executor;
pub mod multiproof;
pub mod prewarm;
pub mod receipt_root_task;
pub mod sparse_trie;
use configured_sparse_trie::ConfiguredSparseTrie;
/// Default parallelism thresholds to use with the [`ParallelSparseTrie`].
///
/// These values were determined by performing benchmarks using gradually increasing values to judge
@@ -98,7 +96,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
<I as ExecutableTxTuple>::Error,
<N as NodePrimitives>::Receipt,
>;
@@ -112,11 +110,11 @@ where
/// The executor used by to spawn tasks.
executor: WorkloadExecutor,
/// The most recent cache used for execution.
execution_cache: ExecutionCache,
execution_cache: PayloadExecutionCache,
/// Metrics for trie operations
trie_metrics: MultiProofTaskMetrics,
/// Cross-block cache size in bytes.
cross_block_cache_size: u64,
cross_block_cache_size: usize,
/// Whether transactions should not be executed on prewarming task.
disable_transaction_prewarming: bool,
/// Whether state cache should be disable
@@ -130,14 +128,12 @@ where
/// A cleared `SparseStateTrie`, kept around to be reused for the state root computation so
/// that allocations can be minimized.
sparse_state_trie: Arc<
parking_lot::Mutex<
Option<ClearedSparseStateTrie<ConfiguredSparseTrie, ConfiguredSparseTrie>>,
>,
parking_lot::Mutex<Option<ClearedSparseStateTrie<ParallelSparseTrie, ParallelSparseTrie>>>,
>,
/// Whether to disable the parallel sparse trie.
disable_parallel_sparse_trie: bool,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
impl<N, Evm> PayloadProcessor<Evm>
@@ -168,8 +164,8 @@ where
precompile_cache_disabled: config.precompile_cache_disabled(),
precompile_cache_map,
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
}
@@ -241,6 +237,9 @@ where
let (to_sparse_trie, sparse_trie_rx) = channel();
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
// Extract V2 proofs flag early so we can pass it to prewarm
let v2_proofs_enabled = config.enable_proof_v2();
// Handle BAL-based optimization if available
let prewarm_handle = if let Some(bal) = bal {
// When BAL is present, use BAL prewarming and send BAL to multiproof
@@ -257,6 +256,7 @@ where
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
v2_proofs_enabled,
)
} else {
// Normal path: spawn with transaction prewarming
@@ -267,6 +267,7 @@ where
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
v2_proofs_enabled,
)
};
@@ -274,7 +275,6 @@ where
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let v2_proofs_enabled = config.enable_proof_v2();
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
task_ctx,
@@ -286,10 +286,13 @@ where
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
config
.multiproof_chunking_enabled()
.then_some(config.effective_multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof,
);
)
.with_v2_proofs_enabled(v2_proofs_enabled);
// spawn multi-proof task
let parent_span = span.clone();
@@ -299,7 +302,7 @@ where
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics) = saved_cache.split();
let (cache, metrics, _disable_metrics) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
@@ -338,8 +341,9 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -355,8 +359,8 @@ where
&self,
transactions: I,
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Tx>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>>,
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
usize,
) {
let (transactions, convert) = transactions.into();
@@ -371,7 +375,10 @@ where
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
WithTxEnv { tx_env, tx: Arc::new(tx) }
});
// Only send Ok(_) variants to prewarming task.
if let Ok(tx) = &tx {
let _ = prewarm_tx.send(tx.clone());
@@ -406,6 +413,7 @@ where
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
#[expect(clippy::too_many_arguments)]
fn spawn_caching_with<P>(
&self,
env: ExecutionEnv<Evm>,
@@ -414,6 +422,7 @@ where
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
v2_proofs_enabled: bool,
) -> CacheTaskHandle<N::Receipt>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -436,6 +445,7 @@ where
terminate_execution: Arc::new(AtomicBool::new(false)),
precompile_cache_disabled: self.precompile_cache_disabled,
precompile_cache_map: self.precompile_cache_map.clone(),
v2_proofs_enabled,
};
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
@@ -474,8 +484,12 @@ where
cache
} else {
debug!("creating new execution cache on cache miss");
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
let start = Instant::now();
let cache = ExecutionCache::new(self.cross_block_cache_size);
let metrics = CachedStateMetrics::zeroed();
metrics.record_cache_creation(start.elapsed());
SavedCache::new(parent_hash, cache, metrics)
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
@@ -492,7 +506,6 @@ where
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
{
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
let disable_parallel_sparse_trie = self.disable_parallel_sparse_trie;
let trie_metrics = self.trie_metrics.clone();
let span = Span::current();
@@ -502,14 +515,10 @@ where
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration
// if there's none to reuse.
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
let default_trie = SparseTrie::blind_from(if disable_parallel_sparse_trie {
ConfiguredSparseTrie::Serial(Default::default())
} else {
ConfiguredSparseTrie::Parallel(Box::new(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
))
});
let default_trie = RevealableSparseTrie::blind_from(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
);
ClearedSparseStateTrie::from_state_trie(
SparseStateTrie::new()
.with_accounts_trie(default_trie.clone())
@@ -518,12 +527,13 @@ where
)
});
let task = SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
);
let task =
SparseTrieTask::<_, ParallelSparseTrie, ParallelSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
);
let (result, trie) = task.run();
// Send state root computation result
@@ -557,6 +567,7 @@ where
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
) {
let disable_cache_metrics = self.disable_cache_metrics;
self.execution_cache.update_with_guard(|cached| {
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
debug!(
@@ -564,26 +575,27 @@ where
parent_hash = %block_with_parent.parent,
"Cannot find cache for parent hash, skip updating cache with new state for inserted executed block",
);
return;
return
}
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
existing.split()
}
let (caches, cache_metrics, _) = match cached.take() {
Some(existing) => existing.split(),
None => (
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
ExecutionCache::new(self.cross_block_cache_size),
CachedStateMetrics::zeroed(),
false,
),
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
let new_cache =
SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
return;
return
}
new_cache.update_metrics();
@@ -647,7 +659,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
}
/// Returns a clone of the caches used by prewarming
pub(super) fn caches(&self) -> Option<StateExecutionCache> {
pub(super) fn caches(&self) -> Option<ExecutionCache> {
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.cache().clone())
}
@@ -665,13 +677,15 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
/// Terminates the entire caching task.
///
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` allows sharing with the main execution
/// path without cloning the expensive `BundleState`.
///
/// Returns a sender for the channel that should be notified on block validation success.
pub(super) fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
) {
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
self.prewarm_handle.terminate_caching(execution_outcome)
}
@@ -707,15 +721,21 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
/// Terminates the entire pre-warming task.
///
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
#[must_use = "sender must be used and notified on block validation success"]
pub(super) fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
) {
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
if let Some(tx) = self.to_prewarm_task.take() {
let event = PrewarmTaskEvent::Terminate { execution_outcome };
let (valid_block_tx, valid_block_rx) = mpsc::channel();
let event = PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx };
let _ = tx.send(event);
Some(valid_block_tx)
} else {
None
}
}
}
@@ -724,7 +744,10 @@ impl<R> Drop for CacheTaskHandle<R> {
fn drop(&mut self) {
// Ensure we always terminate on drop - send None without needing Send + Sync bounds
if let Some(tx) = self.to_prewarm_task.take() {
let _ = tx.send(PrewarmTaskEvent::Terminate { execution_outcome: None });
let _ = tx.send(PrewarmTaskEvent::Terminate {
execution_outcome: None,
valid_block_rx: mpsc::channel().1,
});
}
}
}
@@ -740,27 +763,29 @@ impl<R> Drop for CacheTaskHandle<R> {
/// ## Cache Safety
///
/// **CRITICAL**: Cache update operations require exclusive access. All concurrent cache users
/// (such as prewarming tasks) must be terminated before calling `update_with_guard`, otherwise
/// the cache may be corrupted or cleared.
/// (such as prewarming tasks) must be terminated before calling
/// [`PayloadExecutionCache::update_with_guard`], otherwise the cache may be corrupted or cleared.
///
/// ## Cache vs Prewarming Distinction
///
/// **`ExecutionCache`**:
/// **[`PayloadExecutionCache`]**:
/// - Stores parent block's execution state after completion
/// - Used to fetch parent data for next block's execution
/// - Must be exclusively accessed during save operations
///
/// **`PrewarmCacheTask`**:
/// **[`PrewarmCacheTask`]**:
/// - Speculatively loads accounts/storage that might be used in transaction execution
/// - Prepares data for state root proof computation
/// - Runs concurrently but must not interfere with cache saves
#[derive(Clone, Debug, Default)]
struct ExecutionCache {
struct PayloadExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl ExecutionCache {
impl PayloadExecutionCache {
/// Returns the cache for `parent_hash` if it's available for use.
///
/// A cache is considered available when:
@@ -796,8 +821,16 @@ impl ExecutionCache {
"Existing cache found"
);
if hash_matches && available {
return Some(c.clone());
if available {
// If the has is available (no other threads are using it), but has a mismatching
// parent hash, we can just clear it and keep using without re-creating from
// scratch.
if !hash_matches {
c.clear();
}
return Some(c.clone())
} else if hash_matches {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
debug!(target: "engine::caching", %parent_hash, "No cache found");
@@ -834,6 +867,15 @@ impl ExecutionCache {
}
}
/// Metrics for execution cache operations.
#[derive(Metrics, Clone)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ExecutionCacheMetrics {
/// Counter for when the execution cache was unavailable because other threads
/// (e.g., prewarming) are still using it.
pub(crate) execution_cache_in_use: Counter,
}
/// EVM context required to execute a block.
#[derive(Debug, Clone)]
pub struct ExecutionEnv<Evm: ConfigureEvm> {
@@ -860,9 +902,9 @@ where
#[cfg(test)]
mod tests {
use super::ExecutionCache;
use super::PayloadExecutionCache;
use crate::tree::{
cached_state::{CachedStateMetrics, ExecutionCacheBuilder, SavedCache},
cached_state::{CachedStateMetrics, ExecutionCache, SavedCache},
payload_processor::{
evm_state_to_hashed_post_state, executor::WorkloadExecutor, PayloadProcessor,
},
@@ -886,18 +928,19 @@ mod tests {
use reth_revm::db::BundleState;
use reth_testing_utils::generators;
use reth_trie::{test_utils::state_root, HashedPostState};
use reth_trie_db::ChangesetCache;
use revm_primitives::{Address, HashMap, B256, KECCAK_EMPTY, U256};
use revm_state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot};
use std::sync::Arc;
fn make_saved_cache(hash: B256) -> SavedCache {
let execution_cache = ExecutionCacheBuilder::default().build_caches(1_000);
let execution_cache = ExecutionCache::new(1_000);
SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed())
}
#[test]
fn execution_cache_allows_single_checkout() {
let execution_cache = ExecutionCache::default();
let execution_cache = PayloadExecutionCache::default();
let hash = B256::from([1u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
@@ -916,7 +959,7 @@ mod tests {
#[test]
fn execution_cache_checkout_releases_on_drop() {
let execution_cache = ExecutionCache::default();
let execution_cache = PayloadExecutionCache::default();
let hash = B256::from([2u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
@@ -932,19 +975,21 @@ mod tests {
}
#[test]
fn execution_cache_mismatch_parent_returns_none() {
let execution_cache = ExecutionCache::default();
fn execution_cache_mismatch_parent_clears_and_returns() {
let execution_cache = PayloadExecutionCache::default();
let hash = B256::from([3u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
let miss = execution_cache.get_cache_for(B256::from([4u8; 32]));
assert!(miss.is_none(), "checkout should fail for different parent hash");
// When the parent hash doesn't match, the cache is cleared and returned for reuse
let different_hash = B256::from([4u8; 32]);
let cache = execution_cache.get_cache_for(different_hash);
assert!(cache.is_some(), "cache should be returned for reuse after clearing")
}
#[test]
fn execution_cache_update_after_release_succeeds() {
let execution_cache = ExecutionCache::default();
let execution_cache = PayloadExecutionCache::default();
let initial = B256::from([5u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(initial)));
@@ -1058,7 +1103,9 @@ mod tests {
nonce: rng.random::<u64>(),
code_hash: KECCAK_EMPTY,
code: Some(Default::default()),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::Touched,
transaction_id: 0,
@@ -1141,7 +1188,7 @@ mod tests {
std::convert::identity,
),
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(provider_factory),
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
&TreeConfig::default(),
None, // No BAL for test
);

View File

@@ -11,15 +11,20 @@ use reth_metrics::Metrics;
use reth_provider::AccountReader;
use reth_revm::state::EvmState;
use reth_trie::{
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
added_removed_keys::MultiAddedRemovedKeys, proof_v2, HashedPostState, HashedStorage,
MultiProofTargets,
};
#[cfg(test)]
use reth_trie_parallel::stats::ParallelTrieTracker;
use reth_trie_parallel::{
proof::ParallelProof,
proof_task::{
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
},
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
};
use revm_primitives::map::{hash_map, B256Map};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tracing::{debug, error, instrument, trace};
@@ -62,12 +67,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
#[derive(Default, Debug)]
#[derive(Debug)]
pub struct SparseTrieUpdate {
/// The state update that was used to calculate the proof
pub(crate) state: HashedPostState,
/// The calculated multiproof
pub(crate) multiproof: DecodedMultiProof,
pub(crate) multiproof: ProofResult,
}
impl SparseTrieUpdate {
@@ -79,7 +84,11 @@ impl SparseTrieUpdate {
/// Construct update from multiproof.
#[cfg(test)]
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
let stats = ParallelTrieTracker::default().finish();
Ok(Self {
state: HashedPostState::default(),
multiproof: ProofResult::Legacy(multiproof.try_into()?, stats),
})
}
/// Extend update with contents of the other.
@@ -93,7 +102,7 @@ impl SparseTrieUpdate {
#[derive(Debug)]
pub(super) enum MultiProofMessage {
/// Prefetch proof targets
PrefetchProofs(MultiProofTargets),
PrefetchProofs(VersionedMultiProofTargets),
/// New state update from transaction execution with its source
StateUpdate(Source, EvmState),
/// State update that can be applied to the sparse trie without any new proofs.
@@ -140,22 +149,27 @@ impl ProofSequencer {
/// Adds a proof with the corresponding state update and returns all sequential proofs and state
/// updates if we have a continuous sequence
fn add_proof(&mut self, sequence: u64, update: SparseTrieUpdate) -> Vec<SparseTrieUpdate> {
if sequence >= self.next_to_deliver {
// Optimization: fast path for in-order delivery to avoid BTreeMap overhead.
// If this is the expected sequence, return it immediately without buffering.
if sequence == self.next_to_deliver {
let mut consecutive_proofs = Vec::with_capacity(1);
consecutive_proofs.push(update);
self.next_to_deliver += 1;
// Check if we have subsequent proofs in the pending buffer
while let Some(pending) = self.pending_proofs.remove(&self.next_to_deliver) {
consecutive_proofs.push(pending);
self.next_to_deliver += 1;
}
return consecutive_proofs;
}
if sequence > self.next_to_deliver {
self.pending_proofs.insert(sequence, update);
}
let mut consecutive_proofs = Vec::with_capacity(self.pending_proofs.len());
let mut current_sequence = self.next_to_deliver;
// keep collecting proofs and state updates as long as we have consecutive sequence numbers
while let Some(pending) = self.pending_proofs.remove(&current_sequence) {
consecutive_proofs.push(pending);
current_sequence += 1;
}
self.next_to_deliver += consecutive_proofs.len() as u64;
consecutive_proofs
Vec::new()
}
/// Returns true if we still have pending proofs
@@ -217,12 +231,155 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
hashed_state
}
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
/// regardless of which variant the latter is.
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
match src {
VersionedMultiProofTargets::Legacy(targets) => {
dest.extend_ref(targets);
}
VersionedMultiProofTargets::V2(targets) => {
// Add all account targets
for target in &targets.account_targets {
dest.entry(target.key()).or_default();
}
// Add all storage targets
for (hashed_address, slots) in &targets.storage_targets {
let slot_set = dest.entry(*hashed_address).or_default();
for slot in slots {
slot_set.insert(slot.key());
}
}
}
}
}
/// A set of multiproof targets which can be either in the legacy or V2 representations.
#[derive(Debug)]
pub(super) enum VersionedMultiProofTargets {
/// Legacy targets
Legacy(MultiProofTargets),
/// V2 targets
V2(MultiProofTargetsV2),
}
impl VersionedMultiProofTargets {
/// Returns true if there are no account or storage targets.
fn is_empty(&self) -> bool {
match self {
Self::Legacy(targets) => targets.is_empty(),
Self::V2(targets) => targets.is_empty(),
}
}
/// Returns the number of account targets in the multiproof target
fn account_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the number of storage targets in the multiproof target
fn storage_targets_len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
Self::V2(targets) => {
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Returns the number of accounts in the multiproof targets.
fn len(&self) -> usize {
match self {
Self::Legacy(targets) => targets.len(),
Self::V2(targets) => targets.account_targets.len(),
}
}
/// Returns the total storage slot count across all accounts.
fn storage_count(&self) -> usize {
match self {
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
}
}
/// Returns the number of items that will be considered during chunking.
fn chunking_length(&self) -> usize {
match self {
Self::Legacy(targets) => targets.chunking_length(),
Self::V2(targets) => {
// For V2, count accounts + storage slots
targets.account_targets.len() +
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
}
}
/// Retains the targets representing the difference with another `MultiProofTargets`.
/// Removes all targets that are already present in `other`.
fn retain_difference(&mut self, other: &MultiProofTargets) {
match self {
Self::Legacy(targets) => {
targets.retain_difference(other);
}
Self::V2(targets) => {
// Remove account targets that exist in other
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
// For each account in storage_targets, remove slots that exist in other
targets.storage_targets.retain(|hashed_address, slots| {
if let Some(other_slots) = other.get(hashed_address) {
slots.retain(|slot| !other_slots.contains(&slot.key()));
!slots.is_empty()
} else {
true
}
});
}
}
}
/// Extends this `VersionedMultiProofTargets` with the contents of another.
///
/// Panics if the variants do not match.
fn extend(&mut self, other: Self) {
match (self, other) {
(Self::Legacy(dest), Self::Legacy(src)) => {
dest.extend(src);
}
(Self::V2(dest), Self::V2(src)) => {
dest.account_targets.extend(src.account_targets);
for (addr, slots) in src.storage_targets {
dest.storage_targets.entry(addr).or_default().extend(slots);
}
}
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
}
}
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
match self {
Self::Legacy(targets) => {
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
}
Self::V2(targets) => {
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
}
}
}
}
/// Input parameters for dispatching a multiproof calculation.
#[derive(Debug)]
struct MultiproofInput {
source: Option<Source>,
hashed_state_update: HashedPostState,
proof_targets: MultiProofTargets,
proof_targets: VersionedMultiProofTargets,
proof_sequence_number: u64,
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
@@ -257,8 +414,6 @@ pub struct MultiproofManager {
proof_result_tx: CrossbeamSender<ProofResultMessage>,
/// Metrics
metrics: MultiProofTaskMetrics,
/// Whether to use V2 storage proofs
v2_proofs_enabled: bool,
}
impl MultiproofManager {
@@ -272,9 +427,7 @@ impl MultiproofManager {
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
Self { metrics, proof_worker_handle, proof_result_tx }
}
/// Dispatches a new multiproof calculation to worker pools.
@@ -319,41 +472,48 @@ impl MultiproofManager {
multi_added_removed_keys,
} = multiproof_input;
let account_targets = proof_targets.len();
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
trace!(
target: "engine::tree::payload_processor::multiproof",
proof_sequence_number,
?proof_targets,
account_targets,
storage_targets,
account_targets = proof_targets.account_targets_len(),
storage_targets = proof_targets.storage_targets_len(),
?source,
"Dispatching multiproof to workers"
);
let start = Instant::now();
// Extend prefix sets with targets
let frozen_prefix_sets =
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
// Workers will send ProofResultMessage directly to proof_result_rx
let proof_result_sender = ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
);
// Dispatch account multiproof to worker pool with result sender
let input = AccountMultiproofInput {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
// Workers will send ProofResultMessage directly to proof_result_rx
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
proof_sequence_number,
hashed_state_update,
start,
),
v2_proofs_enabled: self.v2_proofs_enabled,
let input = match proof_targets {
VersionedMultiProofTargets::Legacy(proof_targets) => {
// Extend prefix sets with targets
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
&Default::default(),
&proof_targets,
);
AccountMultiproofInput::Legacy {
targets: proof_targets,
prefix_sets: frozen_prefix_sets,
collect_branch_node_masks: true,
multi_added_removed_keys,
proof_result_sender,
}
}
VersionedMultiProofTargets::V2(proof_targets) => {
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
}
};
// Dispatch account multiproof to worker pool with result sender
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
return;
@@ -555,6 +715,9 @@ pub(super) struct MultiProofTask {
/// there are any active workers and force chunking across workers. This is to prevent tasks
/// which are very long from hitting a single worker.
max_targets_for_chunking: usize,
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
/// will be produced by state updates.
v2_proofs_enabled: bool,
}
impl MultiProofTask {
@@ -586,9 +749,16 @@ impl MultiProofTask {
),
metrics,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
v2_proofs_enabled: false,
}
}
/// Enables V2 proof target generation on state updates.
pub(super) const fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
self.v2_proofs_enabled = v2_proofs_enabled;
self
}
/// Handles request for proof prefetch.
///
/// Returns how many multiproof tasks were dispatched for the prefetch request.
@@ -596,25 +766,40 @@ impl MultiProofTask {
level = "debug",
target = "engine::tree::payload_processor::multiproof",
skip_all,
fields(accounts = targets.len(), chunks = 0)
fields(accounts = targets.account_targets_len(), chunks = 0)
)]
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
// Remove already fetched proof targets to avoid redundant work.
targets.retain_difference(&self.fetched_proof_targets);
self.fetched_proof_targets.extend_ref(&targets);
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
// Make sure all target accounts have an `AddedRemovedKeySet` in the
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
// we still want to optimistically fetch extension children for the leaf addition case.
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
let multi_added_removed_keys = Arc::new(self.multi_added_removed_keys.clone());
// V2 multiproofs don't need this.
//
// Only clone the AddedRemovedKeys for accounts in the targets, not the entire accumulated
// set, to avoid O(n) cloning with many buffered blocks.
let multi_added_removed_keys =
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
Some(Arc::new(MultiAddedRemovedKeys {
account: self.multi_added_removed_keys.account.clone(),
storages: legacy_targets
.keys()
.filter_map(|k| {
self.multi_added_removed_keys.storages.get(k).map(|v| (*k, v.clone()))
})
.collect(),
}))
} else {
None
};
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
self.metrics
.prefetch_proof_targets_storages_histogram
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
.record(targets.storage_count() as f64);
let chunking_len = targets.chunking_length();
let available_account_workers =
@@ -628,7 +813,7 @@ impl MultiProofTask {
self.max_targets_for_chunking,
available_account_workers,
available_storage_workers,
MultiProofTargets::chunks,
VersionedMultiProofTargets::chunks,
|proof_targets| {
self.multiproof_manager.dispatch(MultiproofInput {
source: None,
@@ -636,7 +821,7 @@ impl MultiProofTask {
proof_targets,
proof_sequence_number: self.proof_sequencer.next_sequence(),
state_root_message_sender: self.tx.clone(),
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
multi_added_removed_keys: multi_added_removed_keys.clone(),
});
},
);
@@ -705,7 +890,33 @@ impl MultiProofTask {
}
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
let multi_added_removed_keys = Arc::new(self.multi_added_removed_keys.clone());
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
account: self.multi_added_removed_keys.account.clone(),
storages: {
let mut storages = B256Map::with_capacity_and_hasher(
not_fetched_state_update.storages.len(),
Default::default(),
);
for account in not_fetched_state_update
.storages
.keys()
.chain(not_fetched_state_update.accounts.keys())
{
if let hash_map::Entry::Vacant(entry) = storages.entry(*account) {
entry.insert(
self.multi_added_removed_keys
.storages
.get(account)
.cloned()
.unwrap_or_default(),
);
}
}
storages
},
});
let chunking_len = not_fetched_state_update.chunking_length();
let mut spawned_proof_targets = MultiProofTargets::default();
@@ -713,6 +924,7 @@ impl MultiProofTask {
self.multiproof_manager.proof_worker_handle.available_account_workers();
let available_storage_workers =
self.multiproof_manager.proof_worker_handle.available_storage_workers();
let num_chunks = dispatch_with_chunking(
not_fetched_state_update,
chunking_len,
@@ -726,8 +938,9 @@ impl MultiProofTask {
&hashed_state_update,
&self.fetched_proof_targets,
&multi_added_removed_keys,
self.v2_proofs_enabled,
);
spawned_proof_targets.extend_ref(&proof_targets);
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
self.multiproof_manager.dispatch(MultiproofInput {
source: Some(source),
@@ -827,7 +1040,10 @@ impl MultiProofTask {
batch_metrics.proofs_processed += 1;
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate { state, multiproof: Default::default() },
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -854,8 +1070,7 @@ impl MultiProofTask {
}
let account_targets = merged_targets.len();
let storage_targets =
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
let storage_targets = merged_targets.storage_count();
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
trace!(
target: "engine::tree::payload_processor::multiproof",
@@ -959,7 +1174,10 @@ impl MultiProofTask {
if let Some(combined_update) = self.on_proof(
sequence_number,
SparseTrieUpdate { state, multiproof: Default::default() },
SparseTrieUpdate {
state,
multiproof: ProofResult::empty(self.v2_proofs_enabled),
},
) {
let _ = self.to_sparse_trie.send(combined_update);
}
@@ -1062,7 +1280,7 @@ impl MultiProofTask {
let update = SparseTrieUpdate {
state: proof_result.state,
multiproof: proof_result_data.proof,
multiproof: proof_result_data,
};
if let Some(combined_update) =
@@ -1152,7 +1370,7 @@ struct MultiproofBatchCtx {
/// received.
updates_finished_time: Option<Instant>,
/// Reusable buffer for accumulating prefetch targets during batching.
accumulated_prefetch_targets: Vec<MultiProofTargets>,
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
}
impl MultiproofBatchCtx {
@@ -1198,40 +1416,77 @@ fn get_proof_targets(
state_update: &HashedPostState,
fetched_proof_targets: &MultiProofTargets,
multi_added_removed_keys: &MultiAddedRemovedKeys,
) -> MultiProofTargets {
let mut targets = MultiProofTargets::default();
v2_enabled: bool,
) -> VersionedMultiProofTargets {
if v2_enabled {
let mut targets = MultiProofTargetsV2::default();
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
// first collect all new accounts (not previously fetched)
for &hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(&hashed_address) {
targets.account_targets.push(hashed_address.into());
}
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
continue
}
let changed_slots = storage
.storage
.keys()
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
.map(|slot| Into::<proof_v2::Target>::into(*slot))
.collect::<Vec<_>>();
if !changed_slots.is_empty() {
targets.account_targets.push((*hashed_address).into());
targets.storage_targets.insert(*hashed_address, changed_slots);
}
}
VersionedMultiProofTargets::V2(targets)
} else {
let mut targets = MultiProofTargets::default();
// first collect all new accounts (not previously fetched)
for hashed_address in state_update.accounts.keys() {
if !fetched_proof_targets.contains_key(hashed_address) {
targets.insert(*hashed_address, HashSet::default());
}
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
VersionedMultiProofTargets::Legacy(targets)
}
// then process storage slots for all accounts in the state update
for (hashed_address, storage) in &state_update.storages {
let fetched = fetched_proof_targets.get(hashed_address);
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
let mut changed_slots = storage
.storage
.keys()
.filter(|slot| {
!fetched.is_some_and(|f| f.contains(*slot)) ||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
})
.peekable();
// If the storage is wiped, we still need to fetch the account proof.
if storage.wiped && fetched.is_none() {
targets.entry(*hashed_address).or_default();
}
if changed_slots.peek().is_some() {
targets.entry(*hashed_address).or_default().extend(changed_slots);
}
}
targets
}
/// Dispatches work items as a single unit or in chunks based on target size and worker
@@ -1272,16 +1527,18 @@ where
#[cfg(test)]
mod tests {
use crate::tree::cached_state::CachedStateProvider;
use super::*;
use crate::tree::cached_state::{CachedStateProvider, ExecutionCacheBuilder};
use alloy_eip7928::{AccountChanges, BalanceChange};
use alloy_primitives::Address;
use reth_provider::{
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, TrieReader,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, StorageChangeSetReader,
};
use reth_trie::MultiProof;
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle};
use revm_primitives::{B256, U256};
use std::sync::{Arc, OnceLock};
@@ -1302,17 +1559,18 @@ mod tests {
where
F: DatabaseProviderFactory<
Provider: BlockReader
+ TrieReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + Clone
+ Send
+ 'static,
{
let rt_handle = get_test_runtime_handle();
let overlay_factory = OverlayStateProviderFactory::new(factory);
let changeset_cache = ChangesetCache::new();
let overlay_factory = OverlayStateProviderFactory::new(factory, changeset_cache);
let task_ctx = ProofTaskCtx::new(overlay_factory);
let proof_handle = ProofWorkerHandle::new(rt_handle, task_ctx, 1, 1, false);
let (to_sparse_trie, _receiver) = std::sync::mpsc::channel();
@@ -1324,14 +1582,14 @@ mod tests {
fn create_cached_provider<F>(factory: F) -> CachedStateProvider<StateProviderBox>
where
F: DatabaseProviderFactory<
Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader,
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader,
> + Clone
+ Send
+ 'static,
{
let db_provider = factory.database_provider_ro().unwrap();
let state_provider: StateProviderBox = Box::new(LatestStateProvider::new(db_provider));
let cache = ExecutionCacheBuilder::default().build_caches(1000);
let cache = crate::tree::cached_state::ExecutionCache::new(1000);
CachedStateProvider::new(state_provider, cache, Default::default())
}
@@ -1436,12 +1694,24 @@ mod tests {
state
}
fn unwrap_legacy_targets(targets: VersionedMultiProofTargets) -> MultiProofTargets {
match targets {
VersionedMultiProofTargets::Legacy(targets) => targets,
VersionedMultiProofTargets::V2(_) => panic!("Expected Legacy targets"),
}
}
#[test]
fn test_get_proof_targets_new_account_targets() {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// should return all accounts as targets since nothing was fetched before
assert_eq!(targets.len(), state.accounts.len());
@@ -1455,7 +1725,12 @@ mod tests {
let state = create_get_proof_targets_state();
let fetched = MultiProofTargets::default();
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// verify storage slots are included for accounts with storage
for (addr, storage) in &state.storages {
@@ -1483,7 +1758,12 @@ mod tests {
// mark the account as already fetched
fetched.insert(*fetched_addr, HashSet::default());
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// should not include the already fetched account since it has no storage updates
assert!(!targets.contains_key(fetched_addr));
@@ -1503,7 +1783,12 @@ mod tests {
fetched_slots.insert(fetched_slot);
fetched.insert(*addr, fetched_slots);
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// should not include the already fetched storage slot
let target_slots = &targets[addr];
@@ -1516,7 +1801,12 @@ mod tests {
let state = HashedPostState::default();
let fetched = MultiProofTargets::default();
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
assert!(targets.is_empty());
}
@@ -1543,7 +1833,12 @@ mod tests {
fetched_slots.insert(slot1);
fetched.insert(addr1, fetched_slots);
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
assert!(targets.contains_key(&addr2));
assert!(!targets[&addr1].contains(&slot1));
@@ -1569,7 +1864,12 @@ mod tests {
assert!(!state.accounts.contains_key(&addr));
assert!(!fetched.contains_key(&addr));
let targets = get_proof_targets(&state, &fetched, &MultiAddedRemovedKeys::new());
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&MultiAddedRemovedKeys::new(),
false,
));
// verify that we still get the storage slots for the unmodified account
assert!(targets.contains_key(&addr));
@@ -1611,7 +1911,12 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
// slot1 should be included despite being fetched, because it's marked as removed
assert!(targets.contains_key(&addr));
@@ -1638,7 +1943,12 @@ mod tests {
storage.storage.insert(slot1, U256::from(100));
state.storages.insert(addr, storage);
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
// account should be included because storage is wiped and account wasn't fetched
assert!(targets.contains_key(&addr));
@@ -1681,7 +1991,12 @@ mod tests {
removed_state.storages.insert(addr, removed_storage);
multi_added_removed_keys.update_with_state(&removed_state);
let targets = get_proof_targets(&state, &fetched, &multi_added_removed_keys);
let targets = unwrap_legacy_targets(get_proof_targets(
&state,
&fetched,
&multi_added_removed_keys,
false,
));
// only slots in the state update can be included, so slot3 should not appear
assert!(!targets.contains_key(&addr));
@@ -1708,9 +2023,12 @@ mod tests {
targets3.insert(addr3, HashSet::default());
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets3)))
.unwrap();
let proofs_requested =
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
@@ -1724,11 +2042,12 @@ mod tests {
assert_eq!(num_batched, 3);
assert_eq!(merged_targets.len(), 3);
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(merged_targets.contains_key(&addr3));
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(legacy_targets.contains_key(&addr3));
task.on_prefetch_proof(merged_targets)
task.on_prefetch_proof(VersionedMultiProofTargets::Legacy(legacy_targets))
} else {
panic!("Expected PrefetchProofs message");
};
@@ -1771,7 +2090,9 @@ mod tests {
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: None,
},
original_info: Box::new(revm_state::AccountInfo::default()),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
@@ -1788,7 +2109,9 @@ mod tests {
nonce: 2,
code_hash: Default::default(),
code: Default::default(),
account_id: None,
},
original_info: Box::new(revm_state::AccountInfo::default()),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
@@ -1799,11 +2122,16 @@ mod tests {
// Queue: [PrefetchProofs1, PrefetchProofs2, StateUpdate1, StateUpdate2, PrefetchProofs3]
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(targets1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets1)))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(targets2)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update1)).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update2)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(targets3.clone())).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
targets3.clone(),
)))
.unwrap();
// Step 1: Receive and batch PrefetchProofs (should get targets1 + targets2)
let mut pending_msg: Option<MultiProofMessage> = None;
@@ -1829,9 +2157,10 @@ mod tests {
// Should have batched exactly 2 PrefetchProofs (not 3!)
assert_eq!(num_batched, 2, "Should batch only until different message type");
assert_eq!(merged_targets.len(), 2);
assert!(merged_targets.contains_key(&addr1));
assert!(merged_targets.contains_key(&addr2));
assert!(!merged_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
let legacy_targets = unwrap_legacy_targets(merged_targets);
assert!(legacy_targets.contains_key(&addr1));
assert!(legacy_targets.contains_key(&addr2));
assert!(!legacy_targets.contains_key(&addr3), "addr3 should NOT be in first batch");
} else {
panic!("Expected PrefetchProofs message");
}
@@ -1856,7 +2185,8 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
assert!(targets.contains_key(&addr3));
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&addr3));
}
_ => panic!("PrefetchProofs3 was lost!"),
}
@@ -1890,7 +2220,9 @@ mod tests {
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: None,
},
original_info: Box::new(revm_state::AccountInfo::default()),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
@@ -1900,9 +2232,13 @@ mod tests {
let source = StateChangeSource::Transaction(99);
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(prefetch1)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(prefetch1)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), state_update)).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
tx.send(MultiProofMessage::PrefetchProofs(VersionedMultiProofTargets::Legacy(
prefetch2.clone(),
)))
.unwrap();
let mut ctx = MultiproofBatchCtx::new(Instant::now());
let mut batch_metrics = MultiproofBatchMetrics::default();
@@ -1935,7 +2271,8 @@ mod tests {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
assert!(targets.contains_key(&prefetch_addr2));
let legacy_targets = unwrap_legacy_targets(targets);
assert!(legacy_targets.contains_key(&prefetch_addr2));
}
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
}

View File

@@ -16,24 +16,25 @@ use crate::tree::{
payload_processor::{
bal::{total_slots, BALSlotIter},
executor::WorkloadExecutor,
multiproof::MultiProofMessage,
ExecutionCache as PayloadExecutionCache,
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
PayloadExecutionCache,
},
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
ExecutionEnv, StateProviderBuilder,
};
use alloy_consensus::transaction::TxHashRef;
use alloy_eip7928::BlockAccessList;
use alloy_eips::Typed2718;
use alloy_evm::Database;
use alloy_primitives::{keccak256, map::B256Set, B256};
use crossbeam_channel::Sender as CrossbeamSender;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use metrics::{Counter, Gauge, Histogram};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
use reth_execution_types::ExecutionOutcome;
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{AccountReader, BlockReader, StateProvider, StateProviderFactory, StateReader};
use reth_provider::{
AccountReader, BlockExecutionOutput, BlockReader, StateProvider, StateProviderFactory,
StateReader,
};
use reth_revm::{database::StateProviderDatabase, state::EvmState};
use reth_trie::MultiProofTargets;
use std::{
@@ -64,19 +65,6 @@ struct IndexedTransaction<Tx> {
tx: Tx,
}
/// Maximum standard Ethereum transaction type value.
///
/// Standard transaction types are:
/// - Type 0: Legacy transactions (original Ethereum)
/// - Type 1: EIP-2930 (access list transactions)
/// - Type 2: EIP-1559 (dynamic fee transactions)
/// - Type 3: EIP-4844 (blob transactions)
/// - Type 4: EIP-7702 (set code authorization transactions)
///
/// Any transaction with a type > 4 is considered a non-standard/system transaction,
/// typically used by L2s for special purposes (e.g., Optimism deposit transactions use type 126).
const MAX_STANDARD_TX_TYPE: u8 = 4;
/// A task that is responsible for caching and prewarming the cache by executing transactions
/// individually in parallel.
///
@@ -175,8 +163,8 @@ where
transaction_count_hint.min(max_concurrency)
};
// Initialize worker handles container
let handles = ctx.clone().spawn_workers(workers_needed, &executor, actions_tx.clone(), done_tx.clone());
// Spawn workers
let tx_sender = ctx.clone().spawn_workers(workers_needed, &executor, actions_tx.clone(), done_tx.clone());
// Distribute transactions to workers
let mut tx_index = 0usize;
@@ -191,37 +179,18 @@ where
}
let indexed_tx = IndexedTransaction { index: tx_index, tx };
let is_system_tx = indexed_tx.tx.tx().ty() > MAX_STANDARD_TX_TYPE;
// System transactions (type > 4) in the first position set critical metadata
// that affects all subsequent transactions (e.g., L1 block info on L2s).
// Broadcast the first system transaction to all workers to ensure they have
// the critical state. This is particularly important for L2s like Optimism
// where the first deposit transaction (type 126) contains essential block metadata.
if tx_index == 0 && is_system_tx {
for handle in &handles {
// Ignore send errors: workers listen to terminate_execution and may
// exit early when signaled. Sending to a disconnected worker is
// possible and harmless and should happen at most once due to
// the terminate_execution check above.
let _ = handle.send(indexed_tx.clone());
}
} else {
// Round-robin distribution for all other transactions
let worker_idx = tx_index % workers_needed;
// Ignore send errors: workers listen to terminate_execution and may
// exit early when signaled. Sending to a disconnected worker is
// possible and harmless and should happen at most once due to
// the terminate_execution check above.
let _ = handles[worker_idx].send(indexed_tx);
}
// Send transaction to the workers
// Ignore send errors: workers listen to terminate_execution and may
// exit early when signaled.
let _ = tx_sender.send(indexed_tx);
tx_index += 1;
}
// drop handle and wait for all tasks to finish and drop theirs
// drop sender and wait for all tasks to finish
drop(done_tx);
drop(handles);
drop(tx_sender);
while done_rx.recv().is_ok() {}
let _ = actions_tx
@@ -235,7 +204,7 @@ where
}
/// If configured and the tx returned proof targets, emit the targets the transaction produced
fn send_multi_proof_targets(&self, targets: Option<MultiProofTargets>) {
fn send_multi_proof_targets(&self, targets: Option<VersionedMultiProofTargets>) {
if self.is_execution_terminated() {
// if execution is already terminated then we dont need to send more proof fetch
// messages
@@ -259,7 +228,11 @@ where
///
/// This method is called from `run()` only after all execution tasks are complete.
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn save_cache(self, execution_outcome: Arc<ExecutionOutcome<N::Receipt>>) {
fn save_cache(
self,
execution_outcome: Arc<BlockExecutionOutput<N::Receipt>>,
valid_block_rx: mpsc::Receiver<()>,
) {
let start = Instant::now();
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
@@ -272,12 +245,13 @@ where
execution_cache.update_with_guard(|cached| {
// consumes the `SavedCache` held by the prewarming task, which releases its usage
// guard
let (caches, cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics);
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
// Insert state into cache while holding the lock
// Access the BundleState through the shared ExecutionOutcome
if new_cache.cache().insert_state(execution_outcome.state()).is_err() {
if new_cache.cache().insert_state(&execution_outcome.state).is_err() {
// Clear the cache on error to prevent having a polluted cache
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
@@ -286,9 +260,16 @@ where
new_cache.update_metrics();
// Replace the shared cache with the new one; the previous cache (if any) is
// dropped.
*cached = Some(new_cache);
if valid_block_rx.recv().is_ok() {
// Replace the shared cache with the new one; the previous cache (if any) is
// dropped.
*cached = Some(new_cache);
} else {
// Block was invalid; caches were already mutated by insert_state above,
// so we must clear to prevent using polluted state
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on invalid block");
}
});
let elapsed = start.elapsed();
@@ -419,9 +400,10 @@ where
// completed executing a set of transactions
self.send_multi_proof_targets(proof_targets);
}
PrewarmTaskEvent::Terminate { execution_outcome } => {
PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx } => {
trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal");
final_execution_outcome = Some(execution_outcome);
final_execution_outcome =
Some(execution_outcome.map(|outcome| (outcome, valid_block_rx)));
if finished_execution {
// all tasks are done, we can exit, which will save caches and exit
@@ -446,8 +428,8 @@ where
debug!(target: "engine::tree::payload_processor::prewarm", "Completed prewarm execution");
// save caches and finish using the shared ExecutionOutcome
if let Some(Some(execution_outcome)) = final_execution_outcome {
self.save_cache(execution_outcome);
if let Some(Some((execution_outcome, valid_block_rx))) = final_execution_outcome {
self.save_cache(execution_outcome, valid_block_rx);
}
}
}
@@ -469,6 +451,8 @@ where
pub(super) terminate_execution: Arc<AtomicBool>,
pub(super) precompile_cache_disabled: bool,
pub(super) precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
/// Whether V2 proof calculation is enabled.
pub(super) v2_proofs_enabled: bool,
}
impl<N, P, Evm> PrewarmContext<N, P, Evm>
@@ -477,10 +461,12 @@ where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
Evm: ConfigureEvm<Primitives = N> + 'static,
{
/// Splits this context into an evm, an evm config, metrics, and the atomic bool for terminating
/// execution.
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
/// execution, and whether V2 proofs are enabled.
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
fn evm_for_ctx(
self,
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
let Self {
env,
evm_config,
@@ -490,6 +476,7 @@ where
terminate_execution,
precompile_cache_disabled,
precompile_cache_map,
v2_proofs_enabled,
} = self;
let mut state_provider = match provider.build() {
@@ -539,10 +526,10 @@ where
});
}
Some((evm, metrics, terminate_execution))
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
}
/// Accepts an [`mpsc::Receiver`] of transactions and a handle to prewarm task. Executes
/// Accepts a [`CrossbeamReceiver`] of transactions and a handle to prewarm task. Executes
/// transactions and streams [`PrewarmTaskEvent::Outcome`] messages for each transaction.
///
/// This function processes transactions sequentially from the receiver and emits outcome events
@@ -554,22 +541,30 @@ where
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn transact_batch<Tx>(
self,
txs: mpsc::Receiver<IndexedTransaction<Tx>>,
txs: CrossbeamReceiver<IndexedTransaction<Tx>>,
sender: Sender<PrewarmTaskEvent<N::Receipt>>,
done_tx: Sender<()>,
) where
Tx: ExecutableTxFor<Evm>,
{
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
else {
return
};
while let Ok(IndexedTransaction { index, tx }) = {
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", "recv tx")
.entered();
txs.recv()
} {
let enter =
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm tx", index, tx_hash=%tx.tx().tx_hash())
.entered();
let enter = debug_span!(
target: "engine::tree::payload_processor::prewarm",
"prewarm tx",
index,
tx_hash = %tx.tx().tx_hash(),
is_success = tracing::field::Empty,
)
.entered();
// create the tx env
let start = Instant::now();
@@ -581,7 +576,8 @@ where
break
}
let res = match evm.transact(&tx) {
let (tx_env, tx) = tx.into_parts();
let res = match evm.transact(tx_env) {
Ok(res) => res,
Err(err) => {
trace!(
@@ -618,7 +614,8 @@ where
let _enter =
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm outcome", index, tx_hash=%tx.tx().tx_hash())
.entered();
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
let (targets, storage_targets) =
multiproof_targets_from_state(res.state, v2_proofs_enabled);
metrics.prefetch_storage_targets.record(storage_targets as f64);
let _ = sender.send(PrewarmTaskEvent::Outcome { proof_targets: Some(targets) });
drop(_enter);
@@ -631,35 +628,31 @@ where
let _ = done_tx.send(());
}
/// Spawns a worker task for transaction execution and returns its sender channel.
/// Spawns worker tasks that pull transactions from a shared channel.
///
/// Returns the sender for distributing transactions to workers.
fn spawn_workers<Tx>(
self,
workers_needed: usize,
task_executor: &WorkloadExecutor,
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
done_tx: Sender<()>,
) -> Vec<mpsc::Sender<IndexedTransaction<Tx>>>
) -> CrossbeamSender<IndexedTransaction<Tx>>
where
Tx: ExecutableTxFor<Evm> + Send + 'static,
{
let mut handles = Vec::with_capacity(workers_needed);
let mut receivers = Vec::with_capacity(workers_needed);
let (tx_sender, tx_receiver) = crossbeam_channel::unbounded();
for _ in 0..workers_needed {
let (tx, rx) = mpsc::channel();
handles.push(tx);
receivers.push(rx);
}
// Spawn a separate task spawning workers in parallel.
// Spawn workers that all pull from the shared receiver
let executor = task_executor.clone();
let span = Span::current();
task_executor.spawn_blocking(move || {
let _enter = span.entered();
for (idx, rx) in receivers.into_iter().enumerate() {
for idx in 0..workers_needed {
let ctx = self.clone();
let actions_tx = actions_tx.clone();
let done_tx = done_tx.clone();
let rx = tx_receiver.clone();
let span = debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm worker", idx);
executor.spawn_blocking(move || {
let _enter = span.entered();
@@ -668,7 +661,7 @@ where
}
});
handles
tx_sender
}
/// Spawns a worker task for BAL slot prefetching.
@@ -763,9 +756,22 @@ where
}
}
/// Returns a set of [`MultiProofTargets`] and the total amount of storage targets, based on the
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
/// on the given state.
fn multiproof_targets_from_state(
state: EvmState,
v2_enabled: bool,
) -> (VersionedMultiProofTargets, usize) {
if v2_enabled {
multiproof_targets_v2_from_state(state)
} else {
multiproof_targets_legacy_from_state(state)
}
}
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
/// given state.
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize) {
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
let mut targets = MultiProofTargets::with_capacity(state.len());
let mut storage_targets = 0;
for (addr, account) in state {
@@ -795,7 +801,50 @@ fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize)
targets.insert(keccak256(addr), storage_set);
}
(targets, storage_targets)
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
}
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
/// storage targets, based on the given state.
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
use reth_trie::proof_v2;
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
let mut targets = MultiProofTargetsV2::default();
let mut storage_target_count = 0;
for (addr, account) in state {
// if the account was not touched, or if the account was selfdestructed, do not
// fetch proofs for it
//
// Since selfdestruct can only happen in the same transaction, we can skip
// prefetching proofs for selfdestructed accounts
//
// See: https://eips.ethereum.org/EIPS/eip-6780
if !account.is_touched() || account.is_selfdestructed() {
continue
}
let hashed_address = keccak256(addr);
targets.account_targets.push(hashed_address.into());
let mut storage_slots = Vec::with_capacity(account.storage.len());
for (key, slot) in account.storage {
// do nothing if unchanged
if !slot.is_changed() {
continue
}
let hashed_slot = keccak256(B256::new(key.to_be_bytes()));
storage_slots.push(proof_v2::Target::from(hashed_slot));
}
storage_target_count += storage_slots.len();
if !storage_slots.is_empty() {
targets.storage_targets.insert(hashed_address, storage_slots);
}
}
(VersionedMultiProofTargets::V2(targets), storage_target_count)
}
/// The events the pre-warm task can handle.
@@ -810,12 +859,17 @@ pub(super) enum PrewarmTaskEvent<R> {
Terminate {
/// The final execution outcome. Using `Arc` allows sharing with the main execution
/// path without cloning the expensive `BundleState`.
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
/// Receiver for the block validation result.
///
/// Cache saving is racing the state root validation. We optimistically construct the
/// updated cache but only save it once we know the block is valid.
valid_block_rx: mpsc::Receiver<()>,
},
/// The outcome of a pre-warm task
Outcome {
/// The prepared proof targets based on the evm state outcome
proof_targets: Option<MultiProofTargets>,
proof_targets: Option<VersionedMultiProofTargets>,
},
/// Finished executing all transactions
FinishedTxExecution {

View File

@@ -0,0 +1,259 @@
//! Receipt root computation in a background task.
//!
//! This module provides a streaming receipt root builder that computes the receipt trie root
//! in a background thread. Receipts are sent via a channel with their index, and for each
//! receipt received, the builder incrementally flushes leaves to the underlying
//! [`OrderedTrieRootEncodedBuilder`] when possible. When the channel closes, the task returns the
//! computed root.
use alloy_eips::Encodable2718;
use alloy_primitives::{Bloom, B256};
use crossbeam_channel::Receiver;
use reth_primitives_traits::Receipt;
use reth_trie_common::ordered_root::OrderedTrieRootEncodedBuilder;
use tokio::sync::oneshot;
/// Receipt with index, ready to be sent to the background task for encoding and trie building.
#[derive(Debug, Clone)]
pub struct IndexedReceipt<R> {
/// The transaction index within the block.
pub index: usize,
/// The receipt.
pub receipt: R,
}
impl<R> IndexedReceipt<R> {
/// Creates a new indexed receipt.
#[inline]
pub const fn new(index: usize, receipt: R) -> Self {
Self { index, receipt }
}
}
/// Handle for running the receipt root computation in a background task.
///
/// This struct holds the channels needed to receive receipts and send the result.
/// Use [`Self::run`] to execute the computation (typically in a spawned blocking task).
#[derive(Debug)]
pub struct ReceiptRootTaskHandle<R> {
/// Receiver for indexed receipts.
receipt_rx: Receiver<IndexedReceipt<R>>,
/// Sender for the computed result.
result_tx: oneshot::Sender<(B256, Bloom)>,
}
impl<R: Receipt> ReceiptRootTaskHandle<R> {
/// Creates a new handle from the receipt receiver and result sender channels.
pub const fn new(
receipt_rx: Receiver<IndexedReceipt<R>>,
result_tx: oneshot::Sender<(B256, Bloom)>,
) -> Self {
Self { receipt_rx, result_tx }
}
/// Runs the receipt root computation, consuming the handle.
///
/// This method receives indexed receipts from the channel, encodes them,
/// and builds the trie incrementally. When all receipts have been received
/// (channel closed), it sends the result through the oneshot channel.
///
/// This is designed to be called inside a blocking task (e.g., via
/// `executor.spawn_blocking(move || handle.run(receipts_len))`).
///
/// # Arguments
///
/// * `receipts_len` - The total number of receipts expected. This is needed to correctly order
/// the trie keys according to RLP encoding rules.
pub fn run(self, receipts_len: usize) {
let mut builder = OrderedTrieRootEncodedBuilder::new(receipts_len);
let mut aggregated_bloom = Bloom::ZERO;
let mut encode_buf = Vec::new();
let mut received_count = 0usize;
for indexed_receipt in self.receipt_rx {
let receipt_with_bloom = indexed_receipt.receipt.with_bloom_ref();
encode_buf.clear();
receipt_with_bloom.encode_2718(&mut encode_buf);
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
builder.push_unchecked(indexed_receipt.index, &encode_buf);
received_count += 1;
}
let Ok(root) = builder.finalize() else {
// Finalize fails if we didn't receive exactly `receipts_len` receipts. This can
// happen if execution was aborted early (e.g., invalid transaction encountered).
// We return without sending a result, allowing the caller to handle the abort.
tracing::error!(
target: "engine::tree::payload_processor",
expected = receipts_len,
received = received_count,
"Receipt root task received incomplete receipts, execution likely aborted"
);
return;
};
let _ = self.result_tx.send((root, aggregated_bloom));
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_consensus::{proofs::calculate_receipt_root, TxReceipt};
use alloy_primitives::{b256, hex, Address, Bytes, Log};
use crossbeam_channel::bounded;
use reth_ethereum_primitives::{Receipt, TxType};
#[tokio::test]
async fn test_receipt_root_task_empty() {
let (_tx, rx) = bounded::<IndexedReceipt<Receipt>>(1);
let (result_tx, result_rx) = oneshot::channel();
drop(_tx);
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
tokio::task::spawn_blocking(move || handle.run(0)).await.unwrap();
let (root, bloom) = result_rx.await.unwrap();
// Empty trie root
assert_eq!(root, reth_trie_common::EMPTY_ROOT_HASH);
assert_eq!(bloom, Bloom::ZERO);
}
#[tokio::test]
async fn test_receipt_root_task_single_receipt() {
let receipts: Vec<Receipt> = vec![Receipt::default()];
let (tx, rx) = bounded(1);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
for (i, receipt) in receipts.clone().into_iter().enumerate() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (root, _bloom) = result_rx.await.unwrap();
// Verify against the standard calculation
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
let expected_root = calculate_receipt_root(&receipts_with_bloom);
assert_eq!(root, expected_root);
}
#[tokio::test]
async fn test_receipt_root_task_multiple_receipts() {
let receipts: Vec<Receipt> = vec![Receipt::default(); 5];
let (tx, rx) = bounded(4);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
for (i, receipt) in receipts.into_iter().enumerate() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (root, bloom) = result_rx.await.unwrap();
// Verify against expected values from existing test
assert_eq!(
root,
b256!("0x61353b4fb714dc1fccacbf7eafc4273e62f3d1eed716fe41b2a0cd2e12c63ebc")
);
assert_eq!(
bloom,
Bloom::from(hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
);
}
#[tokio::test]
async fn test_receipt_root_matches_standard_calculation() {
// Create some receipts with actual data
let receipts = vec![
Receipt {
tx_type: TxType::Legacy,
cumulative_gas_used: 21000,
success: true,
logs: vec![],
},
Receipt {
tx_type: TxType::Eip1559,
cumulative_gas_used: 42000,
success: true,
logs: vec![Log {
address: Address::ZERO,
data: alloy_primitives::LogData::new_unchecked(vec![B256::ZERO], Bytes::new()),
}],
},
Receipt {
tx_type: TxType::Eip2930,
cumulative_gas_used: 63000,
success: false,
logs: vec![],
},
];
// Calculate expected values first (before we move receipts)
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
let expected_root = calculate_receipt_root(&receipts_with_bloom);
let expected_bloom =
receipts_with_bloom.iter().fold(Bloom::ZERO, |bloom, r| bloom | r.bloom_ref());
// Calculate using the task
let (tx, rx) = bounded(4);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
for (i, receipt) in receipts.into_iter().enumerate() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (task_root, task_bloom) = result_rx.await.unwrap();
assert_eq!(task_root, expected_root);
assert_eq!(task_bloom, expected_bloom);
}
#[tokio::test]
async fn test_receipt_root_task_out_of_order() {
let receipts: Vec<Receipt> = vec![Receipt::default(); 5];
// Calculate expected values first (before we move receipts)
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
let expected_root = calculate_receipt_root(&receipts_with_bloom);
let (tx, rx) = bounded(4);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
// Send in reverse order to test out-of-order handling
for (i, receipt) in receipts.into_iter().enumerate().rev() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (root, _bloom) = result_rx.await.unwrap();
assert_eq!(root, expected_root);
}
}

View File

@@ -4,11 +4,11 @@ use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTr
use alloy_primitives::B256;
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_trie::{updates::TrieUpdates, Nibbles};
use reth_trie_parallel::root::ParallelStateRootError;
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, SerialSparseTrie, SparseStateTrie, SparseTrieInterface,
ClearedSparseStateTrie, SerialSparseTrie, SparseStateTrie, SparseTrie,
};
use smallvec::SmallVec;
use std::{
@@ -38,8 +38,8 @@ where
BPF: TrieNodeProviderFactory + Send + Sync + Clone,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
A: SparseTrieInterface + Send + Sync + Default,
S: SparseTrieInterface + Send + Sync + Default + Clone,
A: SparseTrie + Send + Sync + Default,
S: SparseTrie + Send + Sync + Default + Clone,
{
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
pub(super) fn new_with_cleared_trie(
@@ -97,8 +97,8 @@ where
debug!(
target: "engine::root",
num_updates,
account_proofs = update.multiproof.account_subtree.len(),
storage_proofs = update.multiproof.storages.len(),
account_proofs = update.multiproof.account_proofs_len(),
storage_proofs = update.multiproof.storage_proofs_len(),
"Updating sparse trie"
);
@@ -150,14 +150,21 @@ where
BPF: TrieNodeProviderFactory + Send + Sync,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
A: SparseTrieInterface + Send + Sync + Default,
S: SparseTrieInterface + Send + Sync + Default + Clone,
A: SparseTrie + Send + Sync + Default,
S: SparseTrie + Send + Sync + Default + Clone,
{
trace!(target: "engine::root::sparse", "Updating sparse trie");
let started_at = Instant::now();
// Reveal new accounts and storage slots.
trie.reveal_decoded_multiproof(multiproof)?;
match multiproof {
ProofResult::Legacy(decoded, _) => {
trie.reveal_decoded_multiproof(decoded)?;
}
ProofResult::V2(decoded_v2) => {
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
}
}
let reveal_multiproof_elapsed = started_at.elapsed();
trace!(
target: "engine::root::sparse",

View File

@@ -7,17 +7,19 @@ use crate::tree::{
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
sparse_trie::StateRootComputeOutcome,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
StateProviderDatabase, TreeConfig,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
StateProviderBuilder, StateProviderDatabase, TreeConfig,
};
use alloy_consensus::transaction::Either;
use alloy_consensus::transaction::{Either, TxHashRef};
use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip1898::BlockWithParent, NumHash};
use alloy_evm::Evm;
use alloy_primitives::B256;
use crate::tree::payload_processor::receipt_root_task::{IndexedReceipt, ReceiptRootTaskHandle};
use rayon::prelude::*;
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock};
use reth_consensus::{ConsensusError, FullConsensus};
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock, LazyOverlay};
use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom};
use reth_engine_primitives::{
ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator,
};
@@ -35,12 +37,13 @@ use reth_primitives_traits::{
};
use reth_provider::{
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, ExecutionOutcome,
HashedPostStateProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader,
StateProvider, StateProviderFactory, StateReader, TrieReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader, StorageChangeSetReader,
};
use reth_revm::db::State;
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot, TrieInputSorted};
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
use revm_primitives::Address;
use std::{
@@ -129,6 +132,8 @@ where
metrics: EngineApiMetrics,
/// Validator for the payload.
validator: V,
/// Changeset cache for in-memory trie changesets
changeset_cache: ChangesetCache,
}
impl<N, P, Evm, V> BasicEngineValidator<P, Evm, V>
@@ -136,10 +141,10 @@ where
N: NodePrimitives,
P: DatabaseProviderFactory<
Provider: BlockReader
+ TrieReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ ChangeSetReader
@@ -160,6 +165,7 @@ where
validator: V,
config: TreeConfig,
invalid_block_hook: Box<dyn InvalidBlockHook<N>>,
changeset_cache: ChangesetCache,
) -> Self {
let precompile_cache_map = PrecompileCacheMap::default();
let payload_processor = PayloadProcessor::new(
@@ -179,6 +185,7 @@ where
invalid_block_hook,
metrics: EngineApiMetrics::default(),
validator,
changeset_cache,
}
}
@@ -363,7 +370,6 @@ where
}
let parent_hash = input.parent_hash();
let block_num_hash = input.num_hash();
trace!(target: "engine::tree::payload_validator", "Fetching block state provider");
let _enter =
@@ -418,13 +424,23 @@ where
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
.map(Arc::new);
// Create lazy overlay from ancestors - this doesn't block, allowing execution to start
// before the trie data is ready. The overlay will be computed on first access.
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state());
// Create overlay factory for payload processor (StateRootTask path needs it for
// multiproofs)
let overlay_factory =
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
.with_block_hash(Some(anchor_hash))
.with_lazy_overlay(lazy_overlay);
// Spawn the appropriate processor based on strategy
let mut handle = ensure_ok!(self.spawn_payload_processor(
env.clone(),
txs,
provider_builder,
parent_hash,
ctx.state(),
overlay_factory.clone(),
strategy,
block_access_list,
));
@@ -440,24 +456,54 @@ where
state_provider = Box::new(InstrumentedStateProvider::new(state_provider, "engine"));
}
// Execute the block and handle any execution errors
let (output, senders) = match self.execute_block(state_provider, env, &input, &mut handle) {
Ok(output) => output,
Err(err) => return self.handle_execution_error(input, err, &parent_block),
};
// Execute the block and handle any execution errors.
// The receipt root task is spawned before execution and receives receipts incrementally
// as transactions complete, allowing parallel computation during execution.
let (output, senders, receipt_root_rx) =
match self.execute_block(state_provider, env, &input, &mut handle) {
Ok(output) => output,
Err(err) => return self.handle_execution_error(input, err, &parent_block),
};
// After executing the block we can stop prewarming transactions
handle.stop_prewarming_execution();
// Create ExecutionOutcome early so we can terminate caching before validation and state
// root computation. Using Arc allows sharing with both the caching task and the deferred
// trie task without cloning the expensive BundleState.
let output = Arc::new(output);
// Terminate caching task early since execution is complete and caching is no longer
// needed. This frees up resources while state root computation continues.
let valid_block_tx = handle.terminate_caching(Some(output.clone()));
let block = self.convert_to_block(input)?.with_senders(senders);
// Wait for the receipt root computation to complete.
let receipt_root_bloom = receipt_root_rx
.blocking_recv()
.inspect_err(|_| {
tracing::error!(
target: "engine::tree::payload_validator",
"Receipt root task dropped sender without result, receipt root calculation likely aborted"
);
})
.ok();
let hashed_state = ensure_ok_post_block!(
self.validate_post_execution(&block, &parent_block, &output, &mut ctx),
self.validate_post_execution(
&block,
&parent_block,
&output,
&mut ctx,
receipt_root_bloom
),
block
);
let root_time = Instant::now();
let mut maybe_state_root = None;
let mut state_root_task_failed = false;
match strategy {
StateRootStrategy::StateRootTask => {
@@ -476,20 +522,18 @@ where
block_state_root = ?block.header().state_root(),
"State root task returned incorrect state root"
);
state_root_task_failed = true;
}
}
Err(error) => {
debug!(target: "engine::tree::payload_validator", %error, "State root task failed");
state_root_task_failed = true;
}
}
}
StateRootStrategy::Parallel => {
debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm");
match self.compute_state_root_parallel(
block.parent_hash(),
&hashed_state,
ctx.state(),
) {
match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) {
Ok(result) => {
let elapsed = root_time.elapsed();
info!(
@@ -525,9 +569,14 @@ where
}
let (root, updates) = ensure_ok_post_block!(
self.compute_state_root_serial(block.parent_hash(), &hashed_state, ctx.state()),
self.compute_state_root_serial(overlay_factory.clone(), &hashed_state),
block
);
if state_root_task_failed {
self.metrics.block_validation.state_root_task_fallback_success_total.increment(1);
}
(root, updates, root_time.elapsed())
};
@@ -555,14 +604,18 @@ where
.into())
}
// Create ExecutionOutcome and wrap in Arc for sharing with both the caching task
// and the deferred trie task. This avoids cloning the expensive BundleState.
let execution_outcome = Arc::new(ExecutionOutcome::from((output, block_num_hash.number)));
if let Some(valid_block_tx) = valid_block_tx {
let _ = valid_block_tx.send(());
}
// Terminate prewarming task with the shared execution outcome
handle.terminate_caching(Some(Arc::clone(&execution_outcome)));
Ok(self.spawn_deferred_trie_task(block, execution_outcome, &ctx, hashed_state, trie_output))
Ok(self.spawn_deferred_trie_task(
block,
output,
&ctx,
hashed_state,
trie_output,
overlay_factory,
))
}
/// Return sealed block header from database or in-memory state by hash.
@@ -598,15 +651,29 @@ where
Ok(())
}
/// Executes a block with the given state provider
/// Executes a block with the given state provider.
///
/// This method orchestrates block execution:
/// 1. Sets up the EVM with state database and precompile caching
/// 2. Spawns a background task for incremental receipt root computation
/// 3. Executes transactions with metrics collection via state hooks
/// 4. Merges state transitions and records execution metrics
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
#[expect(clippy::type_complexity)]
fn execute_block<S, Err, T>(
&mut self,
state_provider: S,
env: ExecutionEnv<Evm>,
input: &BlockOrPayload<T>,
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err, N::Receipt>,
) -> Result<(BlockExecutionOutput<N::Receipt>, Vec<Address>), InsertBlockErrorKind>
) -> Result<
(
BlockExecutionOutput<N::Receipt>,
Vec<Address>,
tokio::sync::oneshot::Receiver<(B256, alloy_primitives::Bloom)>,
),
InsertBlockErrorKind,
>
where
S: StateProvider + Send,
Err: core::error::Error + Send + Sync + 'static,
@@ -645,22 +712,131 @@ where
});
}
// Spawn background task to compute receipt root and logs bloom incrementally.
// Unbounded channel is used since tx count bounds capacity anyway (max ~30k txs per block).
let receipts_len = input.transaction_count();
let (receipt_tx, receipt_rx) = crossbeam_channel::unbounded();
let (result_tx, result_rx) = tokio::sync::oneshot::channel();
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
// Wrap the state hook with metrics collection
let inner_hook = Box::new(handle.state_hook());
let state_hook =
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
let transaction_count = input.transaction_count();
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
let execution_start = Instant::now();
let state_hook = Box::new(handle.state_hook());
let (output, senders) = self.metrics.execute_metered(
// Execute all transactions and finalize
let (executor, senders) = self.execute_transactions(
executor,
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
input.transaction_count(),
state_hook,
transaction_count,
handle.iter_transactions(),
&receipt_tx,
)?;
let execution_finish = Instant::now();
let execution_time = execution_finish.duration_since(execution_start);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
Ok((output, senders))
drop(receipt_tx);
// Finish execution and get the result
let post_exec_start = Instant::now();
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result))?;
self.metrics.record_post_execution(post_exec_start.elapsed());
// Merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.take_bundle() };
let execution_duration = execution_start.elapsed();
self.metrics.record_block_execution(&output, execution_duration);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
Ok((output, senders, result_rx))
}
/// Executes transactions and collects senders, streaming receipts to a background task.
///
/// This method handles:
/// - Applying pre-execution changes (e.g., beacon root updates)
/// - Executing each transaction with timing metrics
/// - Streaming receipts to the receipt root computation task
/// - Collecting transaction senders for later use
///
/// Returns the executor (for finalization) and the collected senders.
fn execute_transactions<E, Tx, InnerTx, Err>(
&self,
mut executor: E,
transaction_count: usize,
transactions: impl Iterator<Item = Result<Tx, Err>>,
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
) -> Result<(E, Vec<Address>), BlockExecutionError>
where
E: BlockExecutor<Receipt = N::Receipt>,
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
InnerTx: TxHashRef,
Err: core::error::Error + Send + Sync + 'static,
{
let mut senders = Vec::with_capacity(transaction_count);
// Apply pre-execution changes (e.g., beacon root update)
let pre_exec_start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.in_scope(|| executor.apply_pre_execution_changes())?;
self.metrics.record_pre_execution(pre_exec_start.elapsed());
// Execute transactions
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
let mut transactions = transactions.into_iter();
loop {
// Measure time spent waiting for next transaction from iterator
// (e.g., parallel signature recovery)
let wait_start = Instant::now();
let Some(tx_result) = transactions.next() else { break };
self.metrics.record_transaction_wait(wait_start.elapsed());
let tx = tx_result.map_err(BlockExecutionError::other)?;
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
senders.push(tx_signer);
let span = debug_span!(
target: "engine::tree",
"execute tx",
?tx_hash,
gas_used = tracing::field::Empty,
);
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let tx_start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.metrics.record_transaction_execution(tx_start.elapsed());
// Send the latest receipt to the background task for incremental root computation
if let Some(receipt) = executor.receipts().last() {
let tx_index = executor.receipts().len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
enter.record("gas_used", gas_used);
}
drop(exec_span);
Ok((executor, senders))
}
/// Compute state root for the given hashed post state in parallel.
///
/// Uses an overlay factory which provides the state of the parent block, along with the
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
/// trie updates for this block.
///
/// # Returns
///
/// Returns `Ok(_)` if computed successfully.
@@ -668,58 +844,39 @@ where
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
fn compute_state_root_parallel(
&self,
parent_hash: B256,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &HashedPostState,
state: &EngineApiTreeState<N>,
) -> Result<(B256, TrieUpdates), ParallelStateRootError> {
let (mut input, block_hash) = self.compute_trie_input(parent_hash, state)?;
// Extend state overlay with current block's sorted state.
input.prefix_sets.extend(hashed_state.construct_prefix_sets());
let sorted_hashed_state = hashed_state.clone_into_sorted();
Arc::make_mut(&mut input.state).extend_ref(&sorted_hashed_state);
let TrieInputSorted { nodes, state, prefix_sets: prefix_sets_mut } = input;
let factory = OverlayStateProviderFactory::new(self.provider.clone())
.with_block_hash(Some(block_hash))
.with_trie_overlay(Some(nodes))
.with_hashed_state_overlay(Some(state));
// The `hashed_state` argument is already taken into account as part of the overlay, but we
// The `hashed_state` argument will be taken into account as part of the overlay, but we
// need to use the prefix sets which were generated from it to indicate to the
// ParallelStateRoot which parts of the trie need to be recomputed.
let prefix_sets = prefix_sets_mut.freeze();
ParallelStateRoot::new(factory, prefix_sets).incremental_root_with_updates()
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
let overlay_factory =
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
ParallelStateRoot::new(overlay_factory, prefix_sets).incremental_root_with_updates()
}
/// Compute state root for the given hashed post state in serial.
///
/// Uses an overlay factory which provides the state of the parent block, along with the
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
/// trie updates for this block.
fn compute_state_root_serial(
&self,
parent_hash: B256,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &HashedPostState,
state: &EngineApiTreeState<N>,
) -> ProviderResult<(B256, TrieUpdates)> {
let (mut input, block_hash) = self.compute_trie_input(parent_hash, state)?;
// The `hashed_state` argument will be taken into account as part of the overlay, but we
// need to use the prefix sets which were generated from it to indicate to the
// StateRoot which parts of the trie need to be recomputed.
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
let overlay_factory =
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
// Extend state overlay with current block's sorted state.
input.prefix_sets.extend(hashed_state.construct_prefix_sets());
let sorted_hashed_state = hashed_state.clone_into_sorted();
Arc::make_mut(&mut input.state).extend_ref(&sorted_hashed_state);
let TrieInputSorted { nodes, state, .. } = input;
let prefix_sets = hashed_state.construct_prefix_sets();
let factory = OverlayStateProviderFactory::new(self.provider.clone())
.with_block_hash(Some(block_hash))
.with_trie_overlay(Some(nodes))
.with_hashed_state_overlay(Some(state));
let provider = factory.database_provider_ro()?;
let provider = overlay_factory.database_provider_ro()?;
Ok(StateRoot::new(&provider, &provider)
.with_prefix_sets(prefix_sets.freeze())
.with_prefix_sets(prefix_sets)
.root_with_updates()?)
}
@@ -729,6 +886,9 @@ where
/// - parent header validation
/// - post-execution consensus validation
/// - state-root based post-execution validation
///
/// If `receipt_root_bloom` is provided, it will be used instead of computing the receipt root
/// and logs bloom from the receipts.
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
fn validate_post_execution<T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>>(
&self,
@@ -736,6 +896,7 @@ where
parent_block: &SealedHeader<N::BlockHeader>,
output: &BlockExecutionOutput<N::Receipt>,
ctx: &mut TreeCtx<'_, N>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<HashedPostState, InsertBlockErrorKind>
where
V: PayloadValidator<T, Block = N::Block>,
@@ -762,7 +923,9 @@ where
let _enter =
debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution")
.entered();
if let Err(err) = self.consensus.validate_block_post_execution(block, output) {
if let Err(err) =
self.consensus.validate_block_post_execution(block, output, receipt_root_bloom)
{
// call post-block hook
self.on_invalid_block(parent_block, block, output, None, ctx.state_mut());
return Err(err.into())
@@ -802,6 +965,11 @@ where
///
/// The method handles strategy fallbacks if the preferred approach fails, ensuring
/// block execution always completes with a valid state root.
///
/// # Arguments
///
/// * `overlay_factory` - Pre-computed overlay factory for multiproof generation
/// (`StateRootTask`)
#[allow(clippy::too_many_arguments)]
#[instrument(
level = "debug",
@@ -814,8 +982,7 @@ where
env: ExecutionEnv<Evm>,
txs: T,
provider_builder: StateProviderBuilder<N, P>,
parent_hash: B256,
state: &EngineApiTreeState<N>,
overlay_factory: OverlayStateProviderFactory<P>,
strategy: StateRootStrategy,
block_access_list: Option<Arc<BlockAccessList>>,
) -> Result<
@@ -828,32 +995,14 @@ where
> {
match strategy {
StateRootStrategy::StateRootTask => {
// Compute trie input
let trie_input_start = Instant::now();
let (trie_input, block_hash) = self.compute_trie_input(parent_hash, state)?;
// Create OverlayStateProviderFactory with sorted trie data for multiproofs
let TrieInputSorted { nodes, state, .. } = trie_input;
let multiproof_provider_factory =
OverlayStateProviderFactory::new(self.provider.clone())
.with_block_hash(Some(block_hash))
.with_trie_overlay(Some(nodes))
.with_hashed_state_overlay(Some(state));
// Record trie input duration including OverlayStateProviderFactory setup
self.metrics
.block_validation
.trie_input_duration
.record(trie_input_start.elapsed().as_secs_f64());
let spawn_start = Instant::now();
// Use the pre-computed overlay factory for multiproofs
let handle = self.payload_processor.spawn(
env,
txs,
provider_builder,
multiproof_provider_factory,
overlay_factory,
&self.config,
block_access_list,
);
@@ -947,99 +1096,36 @@ where
self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates);
}
/// Computes [`TrieInputSorted`] for the provided parent hash by combining database state
/// with in-memory overlays.
/// Creates a [`LazyOverlay`] for the parent block without blocking.
///
/// The goal of this function is to take in-memory blocks and generate a [`TrieInputSorted`]
/// that extends from the highest persisted ancestor up through the parent. This enables state
/// root computation and proof generation without requiring all blocks to be persisted
/// first.
/// Returns a lazy overlay that will compute the trie input on first access, and the anchor
/// block hash (the highest persisted ancestor). This allows execution to start immediately
/// while the trie input computation is deferred until the overlay is actually needed.
///
/// It works as follows:
/// 1. Collect in-memory overlay blocks using [`crate::tree::TreeState::blocks_by_hash`]. This
/// returns the highest persisted ancestor hash (`block_hash`) and the list of in-memory
/// blocks building on top of it.
/// 2. Fast path: If the tip in-memory block's trie input is already anchored to `block_hash`
/// (its `anchor_hash` matches `block_hash`), reuse it directly.
/// 3. Slow path: Build a new [`TrieInputSorted`] by aggregating the overlay blocks (from oldest
/// to newest) on top of the database state at `block_hash`.
#[instrument(
level = "debug",
target = "engine::tree::payload_validator",
skip_all,
fields(parent_hash)
)]
fn compute_trie_input(
&self,
/// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay.
fn get_parent_lazy_overlay(
parent_hash: B256,
state: &EngineApiTreeState<N>,
) -> ProviderResult<(TrieInputSorted, B256)> {
let wait_start = Instant::now();
let (block_hash, blocks) =
) -> (Option<LazyOverlay>, B256) {
let (anchor_hash, blocks) =
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
// Fast path: if the tip block's anchor matches the persisted ancestor hash, reuse its
// TrieInput. This means the TrieInputSorted already aggregates all in-memory overlays
// from that ancestor, so we can avoid re-aggregation.
if let Some(tip_block) = blocks.first() {
let data = tip_block.trie_data();
if let (Some(anchor_hash), Some(trie_input)) =
(data.anchor_hash(), data.trie_input().cloned()) &&
anchor_hash == block_hash
{
trace!(target: "engine::tree::payload_validator", %block_hash,"Reusing trie input with matching anchor hash");
self.metrics
.block_validation
.deferred_trie_wait_duration
.record(wait_start.elapsed().as_secs_f64());
return Ok(((*trie_input).clone(), block_hash));
}
}
if blocks.is_empty() {
debug!(target: "engine::tree::payload_validator", "Parent found on disk");
} else {
debug!(target: "engine::tree::payload_validator", historical = ?block_hash, blocks = blocks.len(), "Parent found in memory");
debug!(target: "engine::tree::payload_validator", "Parent found on disk, no lazy overlay needed");
return (None, anchor_hash);
}
// Extend with contents of parent in-memory blocks directly in sorted form.
let input = Self::merge_overlay_trie_input(&blocks);
debug!(
target: "engine::tree::payload_validator",
%anchor_hash,
num_blocks = blocks.len(),
"Creating lazy overlay for in-memory blocks"
);
self.metrics
.block_validation
.deferred_trie_wait_duration
.record(wait_start.elapsed().as_secs_f64());
Ok((input, block_hash))
}
// Extract deferred trie data handles (non-blocking)
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
/// Aggregates multiple in-memory blocks into a single [`TrieInputSorted`] by combining their
/// state changes.
///
/// The input `blocks` vector is ordered newest -> oldest (see `TreeState::blocks_by_hash`).
/// We iterate it in reverse so we start with the oldest block's trie data and extend forward
/// toward the newest, ensuring newer state takes precedence.
fn merge_overlay_trie_input(blocks: &[ExecutedBlock<N>]) -> TrieInputSorted {
let mut input = TrieInputSorted::default();
let mut blocks_iter = blocks.iter().rev().peekable();
if let Some(first) = blocks_iter.next() {
let data = first.trie_data();
input.state = data.hashed_state;
input.nodes = data.trie_updates;
// Only clone and mutate if there are more in-memory blocks.
if blocks_iter.peek().is_some() {
let state_mut = Arc::make_mut(&mut input.state);
let nodes_mut = Arc::make_mut(&mut input.nodes);
for block in blocks_iter {
let data = block.trie_data();
state_mut.extend_ref(data.hashed_state.as_ref());
nodes_mut.extend_ref(data.trie_updates.as_ref());
}
}
}
input
(Some(LazyOverlay::new(anchor_hash, handles)), anchor_hash)
}
/// Spawns a background task to compute and sort trie data for the executed block.
@@ -1061,10 +1147,11 @@ where
fn spawn_deferred_trie_task(
&self,
block: RecoveredBlock<N::Block>,
execution_outcome: Arc<ExecutionOutcome<N::Receipt>>,
execution_outcome: Arc<BlockExecutionOutput<N::Receipt>>,
ctx: &TreeCtx<'_, N>,
hashed_state: HashedPostState,
trie_output: TrieUpdates,
overlay_factory: OverlayStateProviderFactory<P>,
) -> ExecutedBlock<N> {
// Capture parent hash and ancestor overlays for deferred trie input construction.
let (anchor_hash, overlay_blocks) = ctx
@@ -1088,9 +1175,21 @@ where
let deferred_handle_task = deferred_trie_data.clone();
let block_validation_metrics = self.metrics.block_validation.clone();
// Capture block info and cache handle for changeset computation
let block_hash = block.hash();
let block_number = block.number();
let changeset_cache = self.changeset_cache.clone();
// Spawn background task to compute trie data. Calling `wait_cloned` will compute from
// the stored inputs and cache the result, so subsequent calls return immediately.
let compute_trie_input_task = move || {
let _span = debug_span!(
target: "engine::tree::payload_validator",
"compute_trie_input_task",
block_number
)
.entered();
let result = panic::catch_unwind(AssertUnwindSafe(|| {
let compute_start = Instant::now();
let computed = deferred_handle_task.wait_cloned();
@@ -1113,6 +1212,40 @@ where
.anchored_overlay_hashed_state_size
.record(anchored.trie_input.state.total_len() as f64);
}
// Compute and cache changesets using the computed trie_updates
let changeset_start = Instant::now();
// Get a provider from the overlay factory for trie cursor access
let changeset_result =
overlay_factory.database_provider_ro().and_then(|provider| {
reth_trie::changesets::compute_trie_changesets(
&provider,
&computed.trie_updates,
)
.map_err(ProviderError::Database)
});
match changeset_result {
Ok(changesets) => {
debug!(
target: "engine::tree::changeset",
?block_number,
elapsed = ?changeset_start.elapsed(),
"Computed and caching changesets"
);
changeset_cache.insert(block_hash, block_number, Arc::new(changesets));
}
Err(e) => {
warn!(
target: "engine::tree::changeset",
?block_number,
?e,
"Failed to compute changesets in deferred trie task"
);
}
}
}));
if result.is_err() {
@@ -1209,10 +1342,10 @@ impl<N, Types, P, Evm, V> EngineValidator<Types> for BasicEngineValidator<P, Evm
where
P: DatabaseProviderFactory<
Provider: BlockReader
+ TrieReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ BlockNumReader,
> + BlockReader<Header = N::BlockHeader>
+ StateProviderFactory
@@ -1262,7 +1395,7 @@ where
fn on_inserted_executed_block(&self, block: ExecutedBlock<N>) {
self.payload_processor.on_inserted_executed_block(
block.recovered_block.block_with_parent(),
block.execution_output.state(),
&block.execution_output.state,
);
}
}

View File

@@ -7,6 +7,7 @@ use crate::{
PersistTarget, TreeConfig,
},
};
use reth_trie_db::ChangesetCache;
use alloy_eips::eip1898::BlockWithParent;
use alloy_primitives::{
@@ -26,7 +27,7 @@ use reth_ethereum_engine_primitives::EthEngineTypes;
use reth_ethereum_primitives::{Block, EthPrimitives};
use reth_evm_ethereum::MockEvmConfig;
use reth_primitives_traits::Block as _;
use reth_provider::{test_utils::MockEthProvider, ExecutionOutcome};
use reth_provider::test_utils::MockEthProvider;
use std::{
collections::BTreeMap,
str::FromStr,
@@ -192,6 +193,7 @@ impl TestHarness {
let payload_builder = PayloadBuilderHandle::new(to_payload_service);
let evm_config = MockEvmConfig::default();
let changeset_cache = ChangesetCache::new();
let engine_validator = BasicEngineValidator::new(
provider.clone(),
consensus.clone(),
@@ -199,6 +201,7 @@ impl TestHarness {
payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache.clone(),
);
let tree = EngineApiTreeHandler::new(
@@ -215,6 +218,7 @@ impl TestHarness {
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
EngineApiKind::Ethereum,
evm_config,
changeset_cache,
);
let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone());
@@ -388,6 +392,7 @@ impl ValidatorTestHarness {
let provider = harness.provider.clone();
let payload_validator = MockEngineValidator;
let evm_config = MockEvmConfig::default();
let changeset_cache = ChangesetCache::new();
let validator = BasicEngineValidator::new(
provider,
@@ -396,6 +401,7 @@ impl ValidatorTestHarness {
payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache,
);
Self { harness, validator, metrics: TestMetrics::default() }
@@ -832,7 +838,7 @@ fn test_tree_state_on_new_head_deep_fork() {
for block in &chain_a {
test_harness.tree.state.tree_state.insert_executed(ExecutedBlock::new(
Arc::new(block.clone()),
Arc::new(ExecutionOutcome::default()),
Arc::new(BlockExecutionOutput::default()),
empty_trie_data(),
));
}
@@ -841,7 +847,7 @@ fn test_tree_state_on_new_head_deep_fork() {
for block in &chain_b {
test_harness.tree.state.tree_state.insert_executed(ExecutedBlock::new(
Arc::new(block.clone()),
Arc::new(ExecutionOutcome::default()),
Arc::new(BlockExecutionOutput::default()),
empty_trie_data(),
));
}
@@ -1002,6 +1008,15 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() {
_ => panic!("Unexpected event: {event:#?}"),
}
// After backfill completes with head not buffered, we also request head download
let event = test_harness.from_tree_rx.recv().await.unwrap();
match event {
EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => {
assert_eq!(hash_set, HashSet::from_iter([main_chain_last_hash]));
}
_ => panic!("Unexpected event: {event:#?}"),
}
let _ = test_harness
.tree
.on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain

View File

@@ -52,7 +52,7 @@ pub fn read_dir(
checksums.next().transpose()?.ok_or_eyre("Got less checksums than ERA files")?;
}
entries.sort_by(|(left, _), (right, _)| left.cmp(right));
entries.sort_by_key(|(left, _)| *left);
Ok(stream::iter(entries.into_iter().skip_while(move |(n, _)| *n < start_index).map(
move |(_, path)| {

Some files were not shown because too many files have changed in this diff Show More