Compare commits

..

209 Commits

Author SHA1 Message Date
joshieDo
ef46b62e5e fix(persistence): use unwind commit order for remove_blocks_above
The persistence service was using `database_provider_rw()` for unwind
operations, which uses normal commit order (SF → RocksDB → MDBX).

This could cause crash-recovery issues:
1. Blocks are persisted to RocksDB history shards
2. Unwind is triggered (RemoveBlocksAbove)
3. RocksDB shards are truncated and committed
4. Crash occurs before MDBX commit
5. On restart, MDBX checkpoints show old state
6. Same blocks are re-persisted
7. append_storage_history_shard fails with UnsortedInput because
   the shard already contains those block numbers

Fix: Use `unwind_provider_rw()` which ensures MDBX is committed first,
so on crash recovery MDBX checkpoints correctly reflect the unwind state.
2026-02-02 16:32:42 +00:00
Matthias Seitz
102a6944ba perf(trie): avoid clearing already-cached sparse trie (#21702)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 13:03:07 +00:00
Alexey Shekhirin
1592e51d34 feat(engine): add CLI args for sparse trie pruning configuration (#21703) 2026-02-02 12:52:31 +00:00
Arsenii Kulikov
4280ccf470 fix: short-circuit in reveal_account_v2_proof_nodes on empty nodes (#21701) 2026-02-02 12:18:45 +00:00
Alexey Shekhirin
05ab98107c fix(reth-bench): gracefully stop when transaction source exhausted (#21700) 2026-02-02 11:10:58 +00:00
Brian Picciano
49128ed28f fix(trie): Return full_key from update_leaves unless it is not a child of the missing path (#21699)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-02-02 11:07:56 +00:00
Huber
f74e594292 perf(trie): dispatch V2 storage proofs in lexicographical order (#21684) 2026-02-02 09:31:47 +00:00
Georgios Konstantopoulos
e7d4a05e36 perf(trie): fix allocation hot paths with capacity hints and buffer reuse (#21466)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-02 06:58:45 +00:00
Georgios Konstantopoulos
9382a4c713 fix(prune): use batched pruning loop with edge feature to prevent OOM (#21649)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 02:38:00 +00:00
DaniPopes
28409558f9 perf: add ParallelBridgeBuffered trait to replace par_bridge (#21674) 2026-02-02 00:58:43 +00:00
DaniPopes
5ef32726db refactor: add with_* compressor utility methods (#21680) 2026-02-01 20:43:25 +00:00
Snezhkko
60c3bef1e8 fix(zstd): use transaction dictionary for tx compressor (#21382) 2026-02-01 20:12:51 +00:00
iPLAY888
af96eeae56 refactor(provider): deduplicate segment-to-stage mapping in static file manager (#21670) 2026-02-01 20:09:32 +00:00
Georgios Konstantopoulos
5528aae8f6 fix(engine): wait for persistence service thread before RocksDB drop (#21640)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 19:55:45 +00:00
Georgios Konstantopoulos
83364aa2d6 fix(prune): migrate invalid receipts prune config to Distance(64) (#21677)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 19:44:14 +00:00
DaniPopes
749a742bcf chore(deps): update metrics-derive 0.1.1 (#21673) 2026-02-01 19:38:38 +00:00
ethfanWilliam
2970624413 chore: avoid eager evaluation in base_fee_params_at_timestamp (#21536) 2026-02-01 19:04:42 +00:00
Matthias Seitz
7e18aa4be8 fix(rpc): change debug_set_head number parameter to U64 (#21678)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 18:59:22 +00:00
YK
9f8c22e2c3 feat(prune): prune rocksdb account and storage history indices (#21331)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 18:42:17 +00:00
Georgios Konstantopoulos
3d699ac9c6 perf(trie): reuse account RLP buffer in SparseTrieCacheTask (#21644)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 15:20:11 +00:00
かりんとう
9be31d504d fix(trie): silence unused param warnings in sparse-parallel no_std build (#21657) 2026-02-01 13:05:39 +00:00
github-actions[bot]
34cc65cfe6 chore(deps): weekly cargo update (#21660)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-02-01 13:03:13 +00:00
Matthias Seitz
6e161f0fc9 perf: batch finalized/safe block commits with SaveBlocks (#21663)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 13:02:59 +00:00
iPLAY888
63a3e18404 fix: remove unnecessary alloc (#21665) 2026-02-01 13:01:11 +00:00
Matthias Seitz
7d10e791b2 refactor(engine): improve payload processor tx iterator (#21658)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:44:10 +00:00
Georgios Konstantopoulos
a9b2c1d454 feat(rpc): make blob sidecar upcasting opt-in (#21624)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:25:46 +00:00
CPerezz
9127563914 fix: cleanup entire temp directory when using testing_node (#18399)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 16:46:11 +00:00
Georgios Konstantopoulos
a500fb22ba fix(metrics): rename save_blocks_block_count to save_blocks_batch_size (#21654)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 12:59:09 +00:00
Matthias Seitz
e869cd4670 perf(engine): skip DB lookup for new blocks in insert_block_or_payload (#21650)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 03:35:20 +00:00
DaniPopes
de69654b73 chore(deps): breaking bumps (#21584)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 00:44:09 +00:00
DaniPopes
8d28c4c8f2 chore(trie): add set_* methods alongside with_* builders (#21639)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:42:57 +00:00
Georgios Konstantopoulos
bfe778ab51 perf(trie): use Entry API to avoid empty Vec allocation in extend (#21645)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:29:21 +00:00
DaniPopes
e523a76fb8 chore(trie): clear RevealableSparseTrie in place (#21638)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:27:43 +00:00
DaniPopes
cd12ae58f2 docs(CLAUDE.md): tweaks (#21646) 2026-01-30 22:26:34 +00:00
Georgios Konstantopoulos
370a548f34 refactor(db): derive Clone for DatabaseEnv (#21641)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 21:54:50 +00:00
pepes
781128eece chore(db-api): simplify DatabaseMetrics impl for Arc (#21635) 2026-01-30 18:43:19 +00:00
Julian Meyer
435d915422 chore: make engine tree crate items public (#21487) 2026-01-30 18:40:30 +00:00
Georgios Konstantopoulos
3ec065295e refactor(trie): replace SmallVec with Vec in sparse trie buffers (#21637)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-01-30 18:34:15 +00:00
Matthias Seitz
e1bc6d0f08 feat(engine): preserve sparse trie across payload validations (#21534)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-30 18:34:13 +00:00
Georgios Konstantopoulos
29072639d6 perf(trie): remove shrink_to_fit calls from SparseSubtrieBuffers::clear (#21630)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-01-30 18:02:43 +00:00
Brian Picciano
f90b5c8a7f fix(trie): cleanup modified branch masks in update_leaf on reveal failure (#21629) 2026-01-30 16:06:28 +00:00
Chase Wright
d4fa6806b7 fix(ethstats): WSS Handling (#21595)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 15:15:41 +00:00
Matthias Seitz
63742ab4ae fix(debug-client): fix off-by-one in block hash buffer lookup (#21628)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 15:15:26 +00:00
Matthias Seitz
08122bc1ea perf: use biased select and prioritize engine events (#21556)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 14:31:27 +00:00
Georgios Konstantopoulos
83afaf1aa7 feat(grafana): add gauge panels for save_blocks _last metrics (#21604)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-30 14:08:32 +00:00
Alexey Shekhirin
d72300c685 fix(net): include disconnect reason in P2PStreamError display (#21626) 2026-01-30 14:04:58 +00:00
Matthias Seitz
faf64c712e feat(cli): add reth db state command for historical contract storage (#21570)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 14:03:19 +00:00
theo
b3d532ce9d chore(op-reth): move op-dependent examples into crates/optimism/examples/ (#21495)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-30 14:02:12 +00:00
Georgios Konstantopoulos
9d064be77e feat(rpc): add EIP-7934 block size validation to testing_buildBlockV1 (#21623)
Co-authored-by: Alexey <alexey@tempo.xyz>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 13:57:51 +00:00
Matus Kysel
e3c256340e feat(txpool): add EIP-7594 blob sidecar toggle (#21622) 2026-01-30 12:27:06 +00:00
ligt
d0df549ddb chore(engine-tree): simplify impl trait bound (#21621) 2026-01-30 11:55:23 +00:00
Arsenii Kulikov
7ccb43ea13 perf: cache fetched proof targets in SparseTrieCacheTask (#21612)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 11:44:36 +00:00
Arsenii Kulikov
20f48b1e50 fix(proof_v2): make sure that all storage proofs are delivered (#21611)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-30 11:21:17 +00:00
Dan Cline
0470c65e6c feat(cli): add --metrics param to reth prune (#21613)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 03:24:25 +00:00
Georgios Konstantopoulos
9de1f0905e feat(prune): add static file pruning support for sender recovery (#21598) 2026-01-30 01:09:38 +00:00
joshieDo
327a1a6681 test(stages): add pipeline forward sync and unwind test (#21602)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 00:49:28 +00:00
Dan Cline
b8f27b73ad chore: fix unused parallel trie const without std (#21610) 2026-01-29 23:05:32 +00:00
かりんとう
7ec5ff6483 refactor(reth-bench): dedupe derive_ws_rpc_url helper (#21585) 2026-01-29 22:50:22 +00:00
Georgios Konstantopoulos
f98af4ad9f feat(rpc): default --testing.skip-invalid-transactions to true (#21603)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 22:03:19 +00:00
joshieDo
d8e912f66b fix(provider): prune account changesets from static files in remove_state_above (#21605) 2026-01-29 21:57:28 +00:00
Georgios Konstantopoulos
0572c4e0ca feat(metrics): add _last gauge metrics for save_blocks timings (#21597) 2026-01-29 21:34:27 +00:00
joshieDo
67a7a1c2d1 chore: revert "test(stages): add pipeline forward sync and unwind test" (#21601) 2026-01-29 22:36:47 +01:00
joshieDo
2b1833576b test(stages): add pipeline forward sync and unwind test (#21553)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 21:13:07 +00:00
Dan Cline
5592c362d4 feat(grafana): add reth-persistence dashboard (#21594) 2026-01-29 21:05:07 +00:00
Georgios Konstantopoulos
6beec25f43 fix(grafana): order MerkleChangeSets checkpoint after MerkleExecute (#21581) 2026-01-29 20:40:26 +00:00
Arsenii Kulikov
19bf580f93 feat: sparse trie as cache (#21583)
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-29 19:11:48 +00:00
joshieDo
796ba6d5dc chore(trie): remove unused direct MDBX changeset readers (#21580) 2026-01-29 17:50:19 +00:00
Georgios Konstantopoulos
5307dfc22b chore: update RPC URLs from ithaca.xyz to reth.rs (#21574)
Co-authored-by: Tim Beiko <tim@ethereum.org>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 17:06:13 +00:00
Brian Picciano
f380ed1581 fix(engine): Try to always compute storage root in V2 proofs when account proof is present, fallback if not (#21579) 2026-01-29 16:58:59 +00:00
DaniPopes
f7313c755c chore(deps): bump codspeed (#21578) 2026-01-29 16:50:09 +00:00
Georgios Konstantopoulos
3bc2191590 chore: remove cargo-chef from Dockerfile.depot (#21577) 2026-01-29 16:28:44 +00:00
Brian Picciano
320f2a6015 fix(trie): PST: Fix update_leaf atomicity, remove update_leaves revealed tracking, fix callback calling (#21573) 2026-01-29 16:18:42 +00:00
Georgios Konstantopoulos
70bfdafd26 fix(provider): check executed block before returning historical state (#21571)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-29 13:54:50 +00:00
YK
e9fe0283a9 fix(provider): use storage-aware methods in unwind_trie_state_from (#21561) 2026-01-29 11:54:12 +00:00
Alexey Shekhirin
92b8857625 fix(reth-bench): stop fetcher when reaching chain tip (#21568) 2026-01-29 11:34:15 +00:00
YK
2d71243cf6 feat(trie): add update_leaves method to SparseTrieExt (#21525)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-29 11:25:08 +00:00
かりんとう
732bf712aa refactor(reth-bench): dedupe read_input and load_jwt_secret helpers (#21555) 2026-01-29 10:17:51 +00:00
Dan Cline
0901c2ca8b fix(reth-bench): retry testing_buildBlockV1 when payload gas < target (#21547)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-29 10:08:54 +00:00
Matthias Seitz
2352158b3d fix(reth-bench): return error instead of panic on invalid payload (#21557)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 00:35:19 +00:00
Georgios Konstantopoulos
1a98605ce6 chore(net): downgrade fork id mismatch log to trace (#21554)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 22:41:42 +00:00
DaniPopes
8d37f76d23 chore: move scripts from .github/assets to .github/scripts (#21539)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 22:14:37 +00:00
Dan Cline
2d9cf4c989 chore: fix unused warns in sparse trie (#21546) 2026-01-28 21:48:59 +00:00
DaniPopes
f5ca71d2fb chore(deps): cargo update (#21538)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 19:49:15 +00:00
Alexey Shekhirin
8d58c98034 feat(reth-bench): add reporting and wait options to replay-payloads (#21537) 2026-01-28 19:13:19 +00:00
Matthias Seitz
50e0591540 perf(tree): optimistically prepare canonical overlay (#21475)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 18:16:04 +00:00
joshieDo
013dfdf8c8 fix(prune): add minimum 64 block retention for receipts and bodies (#21520) 2026-01-28 18:10:07 +00:00
joshieDo
effa0ab4c7 fix(provider): read changesets from static files during unwind (#21528)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 15:52:08 +00:00
SS
543a85e9f3 fix: simplify UTF-8 decoding in StreamCodec by using Result::ok (#21524) 2026-01-28 15:12:55 +00:00
theo
88eb0beeb2 chore(op-reth): remove op-reth dependencies from core reth library crates (#21492) 2026-01-28 14:53:17 +00:00
YK
747c0169a7 feat(trie): add prune method to SparseTrieInterface (#21427)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-28 13:55:21 +00:00
Georgios Konstantopoulos
497985ca86 fix(prune): improve pruner log readability (#21522)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-28 13:41:55 +00:00
bobtajson
48a999a81b refactor: using iterator over references (#21506) 2026-01-28 12:46:19 +00:00
ligt
d53858b3e2 chore(engine): simplify EngineApiTreeHandler type inference (#21503) 2026-01-28 12:43:30 +00:00
David Klank
6aa91b0020 perf(trie-db): preallocate vectors in changeset computation (#21465) 2026-01-28 12:39:08 +00:00
katikatidimon
e0a0a0d5fb refactor: remove redundant clone() in CursorSubNode::new (#21493) 2026-01-28 12:33:10 +00:00
joshieDo
231292b58e fix(provider): cap static file changeset iteration to highest available block (#21510) 2026-01-28 11:03:49 +00:00
Brian Picciano
42765890b5 feat(trie): Enable proofs v2 by default (#21434) 2026-01-28 10:54:50 +00:00
Matus Kysel
8417ddc0e8 fix(engine): guard receipt streaming against duplicate indices (#21512) 2026-01-28 10:48:11 +00:00
かりんとう
1ca62d0696 fix(rpc): populate block fields in mev_simBundle logs (#21491) 2026-01-27 22:59:58 +00:00
katikatidimon
928bf37297 perf: avoid cloning prefix sets in TrieWitness::compute (#21352) 2026-01-27 22:26:31 +00:00
Matthias Seitz
aa5b12af44 refactor(db): make Tx::inner field private with accessor (#21490)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 22:06:41 +00:00
katikatidimon
f12acf17e6 chore(txpool): remove redundant locals clone in config (#21477) 2026-01-27 21:37:44 +00:00
joshieDo
2e05cec84b fix: ensure edge enables history in rocksdb (#21478) 2026-01-27 18:43:25 +00:00
Matthias Seitz
9eaa5a6303 chore: remove Sync bound from cursor associated types (#21486)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 18:31:40 +00:00
Georgios Konstantopoulos
ba8c8354e5 fix(reth-bench): retry up to 5 times on failed transaction fetches in big blocks generate (#21483) 2026-01-27 16:10:53 +00:00
Arsenii Kulikov
af3601c65d feat: more metrics (#21481) 2026-01-27 15:17:49 +00:00
Brian Picciano
bff11ab663 refactor(trie): reuse shared StorageProofCalculator for V2 sync storage roots and add deferred encoder metrics (#21424)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 14:54:56 +00:00
joshieDo
08cd1cbda6 fix(static-files): apply minimal blocks per file to all segments (#21479)
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-01-27 14:01:32 +00:00
Georgios Konstantopoulos
e4e05e9ef9 refactor: align RocksDbArgs defaults with StorageSettings::base() (#21472)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-27 13:13:25 +00:00
joshieDo
c8245594bc fix(db): write genesis history to correct storage backend (#21471)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 11:59:06 +00:00
Dan Cline
ed40ce8c4c chore: simplify account_changesets_range (#21457) 2026-01-27 11:44:11 +00:00
YK
1e734936d8 fix(provider): skip storage changeset writes when routed to static files (#21468) 2026-01-27 10:34:44 +00:00
YK
11d9f38077 test(e2e): comprehensive RocksDB storage E2E tests (#21423) 2026-01-27 07:08:57 +00:00
Matthias Seitz
226ce14ca1 perf(trie): use is_zero() check to avoid copy in is_storage_empty (#21459)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 00:42:26 +00:00
Dan Cline
a6e1dea2d7 chore: add logging for internal fcu errors (#21456) 2026-01-26 23:24:48 +00:00
Georgios Konstantopoulos
71ed68e944 perf(db): flatten HashedPostState before persisting (#21422) 2026-01-26 22:49:01 +00:00
DaniPopes
adecbd7814 chore: log docker sccache stats (#21455) 2026-01-26 22:30:20 +00:00
Matthias Seitz
26a37f3c00 chore: use Default::default() for TransactionInfo for forward compatibility (#21454)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 22:15:41 +00:00
DaniPopes
0bfa7fa5fa ci: typorino (#21453) 2026-01-26 21:39:35 +00:00
Georgios Konstantopoulos
18bec10a0b perf(docker): use shared cache mounts for parallel builds (#21451)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:00:45 +00:00
DaniPopes
1e33821e19 ci: use depot cache in Dockerfile.depot (#21450) 2026-01-26 20:37:33 +00:00
ethfanWilliam
da92733be8 fix: use unwrap_or_else for lazy evaluation of BlobParams::cancun (#21442) 2026-01-26 20:19:28 +00:00
DaniPopes
c41c8e6cae chore: reduce number of nightly builds (#21446) 2026-01-26 20:06:09 +00:00
DaniPopes
1ccc174e7b chore: remove unused docker from makefile (#21445) 2026-01-26 19:53:55 +00:00
ethfanWilliam
f1459fcf91 fix(stages): retain RocksDB TempDir in TestStageDB to prevent premature deletion (#21444) 2026-01-26 19:43:11 +00:00
Dan Cline
94235d64a8 fix(pruner): prune account and storage changeset static files (#21346) 2026-01-26 19:28:18 +00:00
Dan Cline
7fe60017cf chore(metrics): add a gas_last metric similar to new_payload_last (#21437) 2026-01-26 17:54:20 +00:00
Brian Picciano
f9ec2fafa0 refactor(trie): always use ParallelSparseTrie, deprecate config flags (#21435) 2026-01-26 17:02:06 +00:00
Arsenii Kulikov
768a687189 perf: use shared channel for prewarm workers (#21429) 2026-01-26 15:49:44 +00:00
Rez
b87cde5479 feat: configurable EVM execution limits (#21088)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-26 15:27:09 +00:00
figtracer
ab685579f0 feat(rpc): add transaction hash caching to EthStateCache (#21180)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:37:53 +00:00
Matthias Seitz
c7faafd183 fix(rpc): add block timestamp validation in eth_simulateV1 (#21397)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:12:28 +00:00
Matthias Seitz
935a2cc056 fix(rpc): use correct error codes for eth_simulateV1 reverts and halts (#21412)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:06:38 +00:00
Matthias Seitz
507cf58db0 fix(rpc): add block number validation in eth_simulateV1 (#21396)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 13:47:20 +00:00
Matthias Seitz
6cfd369d17 fix(rpc): populate block_hash in eth_simulateV1 logs (#21413)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:41:19 +00:00
Matthias Seitz
934f462d01 feat(cli): make stopping on invalid block the default for reth import (#21403)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:41:06 +00:00
Matthias Seitz
d4f28b02ff feat(rpc): implement movePrecompileToAddress for eth_simulateV1 (#21414)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:40:12 +00:00
Matthias Seitz
963bfeeeed fix(rpc): set prevrandao to zero for eth_simulateV1 simulated blocks (#21399)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:39:37 +00:00
Matthias Seitz
adbe6d9da0 fix(rpc): cap simulate_v1 default gas limit to RPC gas cap (#21402)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:39:15 +00:00
Matthias Seitz
6d19c0ed8e fix(engine): only warn for critical capability mismatches (#21398)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:36:49 +00:00
Andrey Kolishchak
4baf2baec4 fix(net): FetchFullBlockRangeFuture can get stuck forever after partial body fetch + error (#21411)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:34:07 +00:00
emmmm
0b5f79e8c9 docs(rpc): add reth_subscribePersistedBlock method (#21420) 2026-01-26 10:48:35 +00:00
Georgios Konstantopoulos
afe164baca test: add E2E test for RocksDB provider functionality (#21419)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-26 10:24:10 +00:00
Hwangjae Lee
31fdbe914c docs(tracing): fix incorrect example description in lib.rs (#21417)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-26 10:19:36 +00:00
Ahsen Kamal
6870747246 feat(payload): add fn for system transaction check (#21407)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-25 14:47:22 +00:00
Fallengirl
0ad8c772e1 fix(era-utils): export correct era1 CompressedBody payload (#21409) 2026-01-25 14:36:24 +00:00
github-actions[bot]
5440d0d89a chore(deps): weekly cargo update (#21406)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-25 10:39:48 +00:00
Matthias Seitz
0eea4d76e9 chore: remove unused imports in storage-api (#21400)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-24 15:49:21 +00:00
YK
8a1702cd74 fix(rocksdb): filter history writes to only changed accounts/storage (#21339)
Co-authored-by: Tempo AI <ai@tempo.xyz>
2026-01-24 13:07:16 +00:00
cui
7feb56d5f6 feat: prealloc vec (#21391) 2026-01-24 11:30:34 +00:00
cui
0aa922c4e8 feat: change from stable sort to unstable sort (#21387) 2026-01-24 11:21:47 +00:00
Matthias Seitz
ccff9a08f0 chore: fix clippy unnecessary_sort_by lint (#21385) 2026-01-24 03:13:49 +00:00
Matthias Seitz
eb788cc7cf fix(docker): pass vergen git vars as build args (#21384)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-24 03:21:43 +01:00
Dan Cline
fb05a0654f fix(engine): use LazyTrieData::deferred for chain notification (#21383)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-23 22:32:21 +00:00
ethfanWilliam
d5a36dcc00 perf(trie): parallelize merge_ancestors_into_overlay extend ops (#21379) 2026-01-23 22:26:07 +00:00
iPLAY888
ffbef9e3cd chore: removed needless collect (#21381) 2026-01-23 21:59:19 +00:00
Dan Cline
820c112e8e feat(engine): add metric for forkchoiceUpdated response -> newPayload (#21380) 2026-01-23 21:57:15 +00:00
Alexey Shekhirin
9285f7eafc ci: use depot for docker (#20380)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-23 21:14:55 +00:00
joshieDo
9a4c6d8a11 feat(rocksdb): static file based healing for TransactionHashNumbers (#21343) 2026-01-23 20:11:47 +00:00
Dan Cline
963c26550a fix(trie): only clone required keys in on_prefetch_proofs (#21378) 2026-01-23 21:13:01 +01:00
joshieDo
3648483512 feat(rocksdb): add WAL size tracking metric and Grafana dashboard (#21295)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 19:59:10 +00:00
joshieDo
ab418642b4 fix(stages): commit RocksDB batches before flush and configure immediate WAL cleanup (#21374)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-23 19:28:52 +00:00
joshieDo
decb56fae1 feat(rocksdb): changeset-based crash recovery healing for history indices (#21341) 2026-01-23 19:28:10 +00:00
Matthias Seitz
ee1ec8f9f0 perf(trie): parallelize COW extend operations with rayon (#21375) 2026-01-23 19:31:04 +01:00
Georgios Konstantopoulos
d7bf87da52 feat(engine): add metric for state root task fallback success (#21371) 2026-01-23 18:21:44 +00:00
Georgios Konstantopoulos
dd0c6d279f revert: perf(trie): parallelize merge_ancestors_into_overlay (#21202) (#21370) 2026-01-23 19:09:19 +01:00
Alexey Shekhirin
c137ed836f perf(engine): fixed-cache for execution cache (#21128)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Tempo AI <ai@tempo.xyz>
2026-01-23 17:57:42 +00:00
Dan Cline
a543752f7d chore(reth-bench): make from-block a required flag (#21372) 2026-01-23 17:52:33 +00:00
joshieDo
b814893221 feat(stages): flush RocksDB at end of history and tx_lookup stages (#21367) 2026-01-23 17:02:53 +00:00
Georgios Konstantopoulos
fcef82261d fix(libmdbx): handle errors gracefully in TransactionInner::drop (#21368) 2026-01-23 16:37:15 +00:00
iPLAY888
d3846d98a9 refactor: refactor get_idle_peer_for to use Iterator::find (#21321) 2026-01-23 15:56:09 +00:00
Alexey Shekhirin
1f536cce65 test(e2e): selfdestruct pre- and post-Dencun (#21363) 2026-01-23 15:41:08 +00:00
Matthias Seitz
0ddaf1b26c feat(engine): add BAL metrics type for EIP-7928 (#21356) 2026-01-23 15:17:33 +00:00
Gigi
830cd5e355 chore: update snmalloc upstream repository link (#21360) 2026-01-23 14:57:46 +00:00
Georgios Konstantopoulos
f77d7d5983 feat(reth-bench): support human-readable gas format in generate-big-block (#21361) 2026-01-23 14:24:34 +00:00
Georgios Konstantopoulos
a2237c534e feat(p2p): add reth p2p enode command (#21357)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 13:23:44 +00:00
Arsenii Kulikov
1bd8fab887 feat(txpool): add Block associated type to TransactionValidator trait (#21359) 2026-01-23 13:16:05 +00:00
Matthias Seitz
22a68756c7 fix(tree): evict changeset cache even when finalized block is unset (#21354) 2026-01-23 11:26:57 +00:00
Hwangjae Lee
d99c0ffd62 chore(etc): update ethereum-metrics-exporter GitHub URL (#21348)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-23 10:59:53 +00:00
Georgios Konstantopoulos
ad476e2b5c chore: add yongkangc as codeowner for crates/storage/provider (#21349) 2026-01-23 07:18:18 +00:00
Matthias Seitz
6df249c1f1 feat(engine): stub Amsterdam engine API endpoints (newPayloadV5, getPayloadV6, BALs) (#21344)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
Co-authored-by: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com>
2026-01-22 20:48:11 +00:00
Arsenii Kulikov
5a076df09a feat: allow setting custom debug block provider (#21345)
Co-authored-by: Karl <yh975593284@gmail.com>
2026-01-22 20:40:26 +00:00
YK
f07629eac0 perf: avoid creating RocksDB transactions for legacy MDBX-only nodes (#21325) 2026-01-22 20:30:52 +00:00
Dan Cline
f643e93c35 feat(reth-bench): send-invalid-payload command (#21335) 2026-01-22 19:42:19 +00:00
Matthias Seitz
653362a436 ci: align check-alloy workflow with main clippy job (#21329) 2026-01-22 20:48:53 +01:00
Seola Oh
a02508600c chore(txpool): explicitly deref RwLockReadGuard in PartialEq impl (#21336) 2026-01-22 19:35:00 +00:00
Georgios Konstantopoulos
937a7f226d fix(rpc): use Default for SimulateError to prepare for alloy breaking change (#21319)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 19:14:58 +00:00
joshieDo
a0df561117 fix(rocksdb): periodic batch commits in stages to prevent OOM (#21334)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 19:04:56 +00:00
Arsenii Kulikov
be5a4ac7a6 feat: bump alloy and alloy-evm (#21337) 2026-01-22 18:43:24 +00:00
Georgios Konstantopoulos
0c854b6f14 fix(net): limit pending pool imports for broadcast transactions (#21254)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-22 18:32:07 +00:00
Georgios Konstantopoulos
28a31cd579 fix: use unwrap_or_else for lazy evaluation of StorageSettings::legacy (#21332) 2026-01-22 17:02:15 +00:00
Brian Picciano
da12451c9c chore(trie): Cleanup unused trie changesets code (#21323) 2026-01-22 16:57:46 +00:00
Georgios Konstantopoulos
247ce3c4e9 feat(storage): warn storage settings diff at startup (#21320)
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-22 16:40:10 +00:00
iPLAY888
bf43ebaa29 fix(cli): handle invalid hex in db list --search (#21315) 2026-01-22 16:18:36 +00:00
Matthias Seitz
a01ecce73f test: add e2e tests for invalid payload handling via Engine API (#21288) 2026-01-22 15:55:36 +00:00
Arsenii Kulikov
3e55c6ca6e fix: always check upper subtrie for keys (#21276)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-22 15:47:50 +00:00
Brian Picciano
2ac7d719f3 feat(trie): add V2 account proof computation and refactor proof types (reapply) (#21316)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:46:01 +00:00
andrewshab
965705ff88 fix: remove collect (#21318)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:24:51 +00:00
Dan Cline
ebe2ca1366 feat: add StaticFileSegment::StorageChangeSets (#20896) 2026-01-22 15:03:47 +00:00
Matthias Seitz
cc242f83fd feat(rpc): respect history expiry in eth_getLogs per EIP-4444 (#21304) 2026-01-22 14:55:50 +00:00
joshieDo
12cf3d685b fix(provider): add CommitOrder for RocksDB/MDBX unwind atomicity (#21311) 2026-01-22 14:54:47 +00:00
Matthias Seitz
ad5b533ad1 chore: rm patches (#21317) 2026-01-22 15:48:53 +01:00
joshieDo
118f15f345 feat(rocksdb): disable bloom filter for default column family (#21312) 2026-01-22 13:47:34 +00:00
joshieDo
97481f69e5 perf(rocksdb): disable compression and bloom filters for TransactionHashNumbers CF (#21310) 2026-01-22 13:31:16 +00:00
Georgios Konstantopoulos
f692ac7d1e perf(prune): use bulk table clear for PruneMode::Full (#21302) 2026-01-22 13:01:17 +00:00
andrewshab
4b1c341ced fix: remove redundant clone (#21300) 2026-01-22 12:43:19 +00:00
Georgios Konstantopoulos
865f8f8951 perf(prune): sort tx hashes for efficient TransactionLookup pruning (#21297) 2026-01-22 12:10:07 +00:00
joshieDo
492fc20fd1 fix(cli): clear rocksdb tables in drop-stage command (#21299)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 12:09:36 +00:00
Sergei Shulepov
ad9886abb8 fix(mdbx): mark reserve as unsafe (#21263) 2026-01-22 12:03:12 +00:00
453 changed files with 22640 additions and 7970 deletions

2
.github/CODEOWNERS vendored
View File

@@ -37,7 +37,7 @@ crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @mattsse

View File

@@ -38,6 +38,6 @@ for pid in "${saving_pids[@]}"; do
done
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/assets/hive/no_sim_build.diff
git apply ../.github/scripts/hive/no_sim_build.diff
go build .
mv ./hive ../hive_assets/

View File

@@ -60,7 +60,6 @@ jobs:
tail -50 Cargo.toml
- name: Check workspace
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings

View File

@@ -1,54 +0,0 @@
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
# images from this workflow will be tagged with the git sha of the branch used
# and will NOT tag it as `latest`.
name: docker-git
on:
workflow_dispatch: {}
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
GIT_SHA: ${{ github.sha }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the git-sha-tagged reth image'
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
- name: 'Build and push the git-sha-tagged op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,65 +0,0 @@
# Publishes the nightly Docker image.
name: docker-nightly
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the nightly reth image'
command: 'make PROFILE=maxperf docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
- name: 'Build and push the nightly op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
steps:
- uses: actions/checkout@v6
- name: Remove bloatware
uses: laverdet/remove-bloatware@v1.0.0
with:
docker: true
lang: rust
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,4 +1,9 @@
# Publishes the Docker image.
# Publishes Docker images.
#
# Triggers:
# - Push tag v*: builds release (RC or latest)
# - Schedule: builds nightly + profiling
# - Manual: builds git-sha or nightly
name: docker
@@ -6,84 +11,94 @@ on:
push:
tags:
- v*
env:
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
build_type:
description: "Build type"
required: true
type: choice
options:
- git-sha
- nightly
default: git-sha
dry_run:
description: "Skip pushing images (dry run)"
required: false
type: boolean
default: false
jobs:
build-rc:
if: contains(github.ref, '-rc')
name: build and push as release candidate
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
build:
if: ${{ !contains(github.ref, '-rc') }}
name: build and push as latest
name: Build Docker images
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
id-token: write
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Set up Depot CLI
uses: depot/setup-action@v1
- name: Log in to GHCR
uses: docker/login-action@v3
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get git info for vergen
id: git
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
echo "dirty=false" >> "$GITHUB_OUTPUT"
- name: Determine build parameters
id: params
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
REGISTRY="ghcr.io/${{ github.repository_owner }}"
if [[ "${{ github.event_name }}" == "push" ]]; then
VERSION="${GITHUB_REF#refs/tags/}"
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION},${REGISTRY}/op-reth:latest" >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
uses: depot/bake-action@v1
env:
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
with:
project: ${{ vars.DEPOT_PROJECT_ID }}
files: docker-bake.hcl
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
optimism.tags=${{ steps.params.outputs.optimism_tags }}

View File

@@ -44,3 +44,24 @@ jobs:
--exclude 'op-reth' \
--exclude 'reth' \
-E 'binary(e2e_testsuite)'
rocksdb:
name: e2e-rocksdb
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run RocksDB e2e tests
run: |
cargo nextest run \
--locked --features "edge" \
-p reth-e2e-test-utils \
-E 'binary(rocksdb)'

View File

@@ -58,11 +58,11 @@ jobs:
uses: actions/cache@v5
with:
path: ./hive_assets
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
- name: Build hive assets
if: steps.cache-hive.outputs.cache-hit != 'true'
run: .github/assets/hive/build_simulators.sh
run: .github/scripts/hive/build_simulators.sh
- name: Load cached Docker images
if: steps.cache-hive.outputs.cache-hit == 'true'
@@ -213,7 +213,7 @@ jobs:
path: /tmp
- name: Load Docker images
run: .github/assets/hive/load_images.sh
run: .github/scripts/hive/load_images.sh
- name: Move hive binary
run: |
@@ -241,11 +241,11 @@ jobs:
FILTER="/"
fi
echo "filter: $FILTER"
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
- name: Print simulator output
if: ${{ failure() }}

View File

@@ -22,7 +22,7 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.network }}
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
if: github.event_name != 'schedule'
runs-on: depot-ubuntu-latest-4
env:
@@ -30,13 +30,17 @@ jobs:
strategy:
matrix:
network: ["ethereum", "optimism"]
storage: ["stable", "edge"]
exclude:
- network: optimism
storage: edge
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- name: Install Geth
run: .github/assets/install_geth.sh
run: .github/scripts/install_geth.sh
- uses: taiki-e/install-action@nextest
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
@@ -46,7 +50,7 @@ jobs:
name: Run tests
run: |
cargo nextest run \
--locked --features "asm-keccak ${{ matrix.network }}" \
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
--workspace --exclude ef-tests \
-E "kind(test) and not binary(e2e_testsuite)"
- if: matrix.network == 'optimism'

View File

@@ -19,5 +19,5 @@ jobs:
uses: actions/github-script@v8
with:
script: |
const label_pr = require('./.github/assets/label_pr.js')
const label_pr = require('./.github/scripts/label_pr.js')
await label_pr({github, context})

View File

@@ -76,7 +76,7 @@ jobs:
- name: Run Wasm checks
run: |
sudo apt update && sudo apt install gcc-multilib
.github/assets/check_wasm.sh
.github/scripts/check_wasm.sh
riscv:
runs-on: depot-ubuntu-latest
@@ -94,7 +94,7 @@ jobs:
cache-on-failure: true
- uses: dcarbone/install-jq-action@v3
- name: Run RISC-V checks
run: .github/assets/check_rv32imac.sh
run: .github/scripts/check_rv32imac.sh
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})

View File

@@ -43,7 +43,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
file: .github/assets/hive/Dockerfile
file: .github/scripts/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |

View File

@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
2. **Linting**: Run clippy with all features
```bash
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
```
3. **Testing**: Use nextest for faster test execution
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
Before submitting changes, ensure:
1. **Format Check**: `cargo +nightly fmt --all --check`
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
2. **Clippy**: No warnings
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### Opening PRs against <https://github.com/paradigmxyz/reth>
Label PRs appropriately, first check the available labels and then apply the relevant ones:
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
}
```
5. **Run checks**:
5. **Run checks** (IMPORTANT!):
```bash
cargo +nightly fmt --all
cargo clippy --all-features
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
cargo test -p reth-discv4
```
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
cargo +nightly fmt --all
# Run lints
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
cargo +nightly clippy --workspace --all-features
# Run tests
cargo nextest run --workspace

1209
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -77,6 +77,10 @@ members = [
"crates/optimism/cli",
"crates/optimism/consensus",
"crates/optimism/evm/",
"crates/optimism/examples/custom-node",
"crates/optimism/examples/engine-api-access",
"crates/optimism/examples/exex-hello-world",
"crates/optimism/examples/op-db-access",
"crates/optimism/flashblocks/",
"crates/optimism/hardforks/",
"crates/optimism/node/",
@@ -145,7 +149,6 @@ members = [
"examples/beacon-api-sse/",
"examples/bsc-p2p",
"examples/custom-dev-node/",
"examples/custom-node/",
"examples/custom-engine-types/",
"examples/custom-evm/",
"examples/custom-hardforks/",
@@ -154,10 +157,7 @@ members = [
"examples/custom-payload-builder/",
"examples/custom-rlpx-subprotocol",
"examples/custom-rpc-middleware",
"examples/custom-node",
"examples/db-access",
"examples/engine-api-access",
"examples/exex-hello-world",
"examples/exex-subscription",
"examples/exex-test",
"examples/full-contract-state",
@@ -168,7 +168,6 @@ members = [
"examples/node-builder-api/",
"examples/node-custom-rpc/",
"examples/node-event-hooks/",
"examples/op-db-access/",
"examples/polygon-p2p/",
"examples/rpc-db/",
"examples/precompile-cache/",
@@ -481,52 +480,52 @@ revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.0"
revm-inspectors = "0.34.1"
# eth
alloy-dyn-abi = "1.5.4"
alloy-primitives = { version = "1.5.4", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.4", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.4.3"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
# op
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -561,7 +560,7 @@ humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
lz4 = "1.28.1"
modular-bitfield = "0.11.2"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.2", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
@@ -588,15 +587,15 @@ tracing-appender = "0.2"
url = { version = "2.3", default-features = false }
zstd = "0.13"
byteorder = "1"
mini-moka = "0.10"
fixed-cache = { version = "0.1.7", features = ["stats"] }
moka = "0.12"
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
tar-no-std = { version = "0.4.2", default-features = false }
miniz_oxide = { version = "0.9.0", default-features = false }
chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-derive = "0.1.1"
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.20.0" }
@@ -608,7 +607,7 @@ quote = "1.0"
# tokio
tokio = { version = "1.44.2", default-features = false }
tokio-stream = "0.1.11"
tokio-tungstenite = "0.26.2"
tokio-tungstenite = "0.28.0"
tokio-util = { version = "0.7.4", features = ["codec"] }
# async
@@ -621,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
hyper = "1.3"
hyper-util = "0.1.5"
pin-project = "1.0.12"
reqwest = { version = "0.12", default-features = false }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
tracing-futures = "0.2"
tower = "0.5"
tower-http = "0.6"
@@ -641,7 +640,6 @@ jsonrpsee-types = "0.26.0"
http = "1.0"
http-body = "1.0"
http-body-util = "0.1.2"
jsonwebtoken = "9"
proptest-arbitrary-interop = "0.1.0"
# crypto
@@ -655,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
c-kzg = "2.1.5"
# config
toml = "0.8"
toml = "0.9"
# rocksdb
rocksdb = { version = "0.24" }
@@ -671,19 +669,19 @@ tracing-opentelemetry = "0.32"
# misc-testing
arbitrary = "1.3"
assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.5"
proptest-derive = "0.7"
similar-asserts = { version = "1.5.0", features = ["serde"] }
tempfile = "3.20"
test-fuzz = "7"
rstest = "0.24.0"
rstest = "0.26.1"
test-case = "3"
# ssz encoding
ethereum_ssz = "0.9.0"
ethereum_ssz_derive = "0.9.0"
ethereum_ssz = "0.10.1"
ethereum_ssz_derive = "0.10.1"
# allocators
jemalloc_pprof = { version = "0.8", default-features = false }
@@ -695,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
aes = "0.8.1"
ahash = "0.8"
anyhow = "1.0"
bindgen = { version = "0.71", default-features = false }
block-padding = "0.3.2"
bindgen = { version = "0.72", default-features = false }
block-padding = "0.3"
cc = "1.2.15"
cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossterm = "0.28.0"
crossterm = "0.29.0"
csv = "1.3.0"
ctrlc = "3.4"
ctr = "0.9.2"
@@ -715,7 +713,7 @@ hmac = "0.12.1"
human_bytes = "0.4.1"
indexmap = "2"
interprocess = "2.2.0"
lz4_flex = { version = "0.11", default-features = false }
lz4_flex = { version = "0.12", default-features = false }
memmap2 = "0.9.4"
mev-share-sse = { version = "0.5.0", default-features = false }
num-traits = "0.2.15"
@@ -723,17 +721,17 @@ page_size = "0.6.0"
parity-scale-codec = "3.2.1"
plain_hasher = "0.2"
pretty_assertions = "1.4"
ratatui = { version = "0.29", default-features = false }
ringbuffer = "0.15.0"
ratatui = { version = "0.30", default-features = false }
ringbuffer = "0.16.0"
rmp-serde = "1.3"
roaring = "0.10.2"
roaring = "0.11.3"
rolling-file = "0.2.0"
sha3 = "0.10.5"
snap = "1.1.1"
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
socket2 = { version = "0.6", default-features = false }
sysinfo = { version = "0.38", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "0.3.3"
tracing-logfmt = "=0.3.5"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
@@ -790,39 +788,10 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

View File

@@ -1,15 +0,0 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth"]

84
Dockerfile.depot Normal file
View File

@@ -0,0 +1,84 @@
# syntax=docker/dockerfile:1
# Unified Dockerfile for reth and op-reth, optimized for Depot builds
# Usage:
# reth: --build-arg BINARY=reth
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
FROM rust:1 AS builder
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
RUN apt-get update && apt-get install -y libclang-dev pkg-config
# Install sccache for compilation caching
RUN cargo install sccache --locked
ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
# Binary to build (reth or op-reth)
ARG BINARY=reth
# Manifest path for the binary
ARG MANIFEST_PATH=bin/reth
# Build profile, release by default
ARG BUILD_PROFILE=release
ENV BUILD_PROFILE=$BUILD_PROFILE
# Extra Cargo flags
ARG RUSTFLAGS=""
ENV RUSTFLAGS="$RUSTFLAGS"
# Extra Cargo features
ARG FEATURES=""
ENV FEATURES=$FEATURES
# Git info for vergen (since .git is excluded from Docker context)
ARG VERGEN_GIT_SHA=""
ARG VERGEN_GIT_DESCRIBE=""
ARG VERGEN_GIT_DIRTY="false"
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Build application
COPY --exclude=.git . .
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml
RUN sccache --show-stats || true
# Copy binary to a known location (ARG not resolved in COPY)
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
cp /app/target/release/$BINARY /app/binary
FROM ubuntu:24.04 AS runtime
WORKDIR /app
# Binary name for entrypoint
ARG BINARY=reth
# Install runtime dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Copy binary from build stage and create canonical symlink for entrypoint
COPY --from=builder /app/binary /usr/local/bin/
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
chmod +x /usr/local/bin/$BINARY
# Copy licenses
COPY LICENSE-* ./
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth-binary"]

View File

@@ -1,15 +0,0 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/op-reth"]

134
Makefile
View File

@@ -35,9 +35,6 @@ EEST_TESTS_TAG := v4.5.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
# The docker image name
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
##@ Help
.PHONY: help
@@ -242,137 +239,6 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
--features "$(FEATURES)" \
--profile "$(PROFILE)"
##@ Docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-git-sha
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-latest
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call docker_build_push,nightly,nightly)
.PHONY: docker-build-push-nightly-edge-profiling
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Create a cross-arch Docker image with the given tags and push it
define docker_build_push
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
docker buildx build --file ./Dockerfile.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Optimism docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-git-sha
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-latest
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call op_docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly,nightly)
.PHONY: op-docker-build-push-nightly-edge-profiling
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly-profiling
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
$(call docker_build_push,nightly-profiling,nightly-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly-profiling
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
# Create a cross-arch Docker image with the given tags and push it
define op_docker_build_push
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
docker buildx build --file ./DockerfileOp.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Other
.PHONY: clean

View File

@@ -56,7 +56,7 @@ ctrlc.workspace = true
shlex.workspace = true
[target.'cfg(unix)'.dependencies]
nix = { version = "0.29", features = ["signal", "process"] }
nix = { version = "0.31", features = ["signal", "process"] }
[features]
default = ["jemalloc"]

View File

@@ -274,10 +274,10 @@ impl Args {
/// Get the default RPC URL for a given chain
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
match chain.id() {
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
8453 => "https://base.reth.rs/rpc", // base
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
}
}

View File

@@ -32,7 +32,7 @@ alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
# reqwest
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
# tower
tower.workspace = true

View File

@@ -3,7 +3,7 @@
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
@@ -22,29 +22,6 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
@@ -237,50 +214,3 @@ const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -3,7 +3,9 @@
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::authenticated_transport::AuthenticatedTransportConnect;
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
@@ -130,13 +132,24 @@ impl<S: TransactionSource> TransactionCollector<S> {
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
/// Returns a `CollectionResult` with transactions, gas info, and next block.
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
self.collect_gas(start_block, self.target_gas).await
}
/// Collect transactions up to a specific gas target.
///
/// This is used both for initial collection and for retry top-ups.
pub async fn collect_gas(
&self,
start_block: u64,
gas_target: u64,
) -> eyre::Result<CollectionResult> {
let mut transactions: Vec<RawTransaction> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < self.target_gas {
while total_gas < gas_target {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(block = current_block, "Block not found, stopping");
@@ -149,12 +162,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
continue;
}
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
if total_gas + tx.gas_used <= gas_target {
total_gas += tx.gas_used;
transactions.push(tx);
}
if total_gas >= self.target_gas {
if total_gas >= gas_target {
break;
}
}
@@ -162,7 +175,7 @@ impl<S: TransactionSource> TransactionCollector<S> {
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = self.target_gas.saturating_sub(total_gas);
let remaining_gas = gas_target.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
@@ -170,12 +183,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
info!(
total_txs = transactions.len(),
total_gas,
gas_sent = total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok((transactions, total_gas, current_block))
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
}
}
@@ -202,13 +215,26 @@ pub struct Command {
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
target_gas: u64,
/// Starting block number to fetch transactions from.
/// If not specified, starts from the engine's latest block.
/// Block number to start fetching transactions from (required).
///
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
/// The command collects transactions from historical blocks starting at this number
/// to pack into large blocks.
///
/// How to determine this value:
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
/// - If gas limit ramping has already been performed: use the block number that was the chain
/// tip BEFORE ramping began (you must track this yourself)
///
/// Using a block after ramping started will cause transaction collection to fail
/// because those blocks contain synthetic transactions that cannot be replayed.
#[arg(long, value_name = "FROM_BLOCK")]
from_block: Option<u64>,
from_block: u64,
/// Execute the payload (call newPayload + forkchoiceUpdated).
/// If false, only builds the payload and prints it.
@@ -237,6 +263,80 @@ struct BuiltPayload {
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
/// The actual gas used in the built block.
gas_used: u64,
}
/// Result of collecting transactions from blocks.
#[derive(Debug)]
pub struct CollectionResult {
/// Collected transactions with their gas info.
pub transactions: Vec<RawTransaction>,
/// Total gas sent (sum of historical `gas_used` for all collected txs).
pub gas_sent: u64,
/// Next block number to continue collecting from.
pub next_block: u64,
}
/// Constants for retry logic.
const MAX_BUILD_RETRIES: u32 = 5;
/// Maximum retries for fetching a transaction batch.
const MAX_FETCH_RETRIES: u32 = 5;
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
const MIN_TARGET_SLACK: u64 = 1_000_000;
/// Maximum gas to request in retries (10x target as safety cap).
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
/// Fetches a batch of transactions with retry logic.
///
/// Returns `None` if all retries are exhausted.
async fn fetch_batch_with_retry<S: TransactionSource>(
collector: &TransactionCollector<S>,
block: u64,
) -> Option<CollectionResult> {
for attempt in 1..=MAX_FETCH_RETRIES {
match collector.collect(block).await {
Ok(result) => return Some(result),
Err(e) => {
if attempt == MAX_FETCH_RETRIES {
warn!(attempt, error = %e, "Failed to fetch transactions after max retries");
return None;
}
warn!(attempt, error = %e, "Failed to fetch transactions, retrying...");
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
}
None
}
/// Outcome of a build attempt check.
enum RetryOutcome {
/// Payload is close enough to target gas.
Success,
/// Max retries reached, accept what we have.
MaxRetries,
/// Need more transactions with the specified gas amount.
NeedMore(u64),
}
/// Buffer for receiving transaction batches from the fetcher.
///
/// This abstracts over the channel to allow the main loop to request
/// batches on demand, including for retries.
struct TxBuffer {
receiver: mpsc::Receiver<CollectionResult>,
}
impl TxBuffer {
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
Self { receiver }
}
/// Take the next available batch from the fetcher.
async fn take_batch(&mut self) -> Option<CollectionResult> {
self.receiver.recv().await
}
}
impl Command {
@@ -284,7 +384,7 @@ impl Command {
format!("Failed to create output directory: {:?}", self.output_dir)
})?;
let start_block = self.from_block.unwrap_or(parent_number);
let start_block = self.from_block;
// Use pipelined execution when generating multiple payloads
if self.count > 1 {
@@ -297,19 +397,20 @@ impl Command {
)
.await?;
} else {
// Single payload - collect transactions and build
// Single payload - collect transactions and build with retry
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
let result = collector.collect(start_block).await?;
if transactions.is_empty() {
if result.transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential(
self.execute_sequential_with_retry(
&auth_provider,
&testing_provider,
transactions,
&collector,
result,
parent_hash,
parent_timestamp,
)
@@ -320,32 +421,34 @@ impl Command {
Ok(())
}
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
/// Sequential execution path with retry logic for underfilled payloads.
async fn execute_sequential_with_retry<S: TransactionSource>(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
transactions: Vec<Bytes>,
collector: &TransactionCollector<S>,
initial_result: CollectionResult,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let mut current_result = initial_result;
for i in 0..self.count {
let built = self
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.build_with_retry(
testing_provider,
collector,
&mut current_result,
i,
parent_hash,
parent_timestamp,
)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
info!(payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
}
@@ -356,7 +459,62 @@ impl Command {
Ok(())
}
/// Pipelined execution - fetches transactions and builds payloads in background.
/// Build a payload with retry logic when `gas_used` is below target.
///
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
/// are needed to hit the target gas.
async fn build_with_retry<S: TransactionSource>(
&self,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let additional =
collector.collect_gas(result.next_block, additional_gas).await?;
result.transactions.extend(additional.transactions);
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
result.next_block = additional.next_block;
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
}
/// Pipelined execution - fetches transactions in background, builds with retry.
///
/// The fetcher continuously produces transaction batches. The main loop consumes them,
/// builds payloads with retry logic (requesting more transactions if underfilled),
/// and executes each payload before moving to the next.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
@@ -365,167 +523,229 @@ impl Command {
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Create channel for transaction batches - fetcher sends CollectionResult
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(error = %e, "Failed to create transaction source");
return;
return None;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
});
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
let is_last = i == self.count - 1;
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
if batch.transactions.is_empty() {
info!(block = current_block, "Reached chain tip, stopping fetcher");
break;
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
tx_count = batch.transactions.len(),
gas_sent = batch.gas_sent,
blocks = format!("{}..{}", current_block, batch.next_block),
"Fetched transaction batch"
);
self.build_payload(
current_block = batch.next_block;
if tx_sender.send(batch).await.is_err() {
break;
}
}
Some(current_block)
});
// Transaction buffer: holds transactions from batches + any extras from retries
let mut tx_buffer = TxBuffer::new(tx_receiver);
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
for i in 0..self.count {
// Get initial batch of transactions for this payload
let Some(mut result) = tx_buffer.take_batch().await else {
info!(
payloads_built = i,
payloads_requested = self.count,
"Transaction source exhausted, stopping"
);
break;
};
if result.transactions.is_empty() {
info!(
payloads_built = i,
payloads_requested = self.count,
"No more transactions available, stopping"
);
break;
}
// Build with retry - may need to request more transactions
let built = self
.build_with_retry_buffered(
testing_provider,
&transactions,
&mut tx_buffer,
&mut result,
i,
parent_hash,
parent_timestamp,
)
.await?
};
.await?;
self.save_payload(&current_payload)?;
self.save_payload(&built)?;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
let current_block_hash = built.block_hash;
let current_timestamp = built.timestamp;
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
// Execute payload
info!(payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_receiver);
drop(tx_buffer);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
/// Build a payload with retry logic, using the buffered transaction source.
async fn build_with_retry_buffered(
&self,
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
tx_buffer: &mut TxBuffer,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let mut collected_gas = 0u64;
while collected_gas < additional_gas {
if let Some(batch) = tx_buffer.take_batch().await {
collected_gas += batch.gas_sent;
result.transactions.extend(batch.transactions);
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
result.next_block = batch.next_block;
} else {
warn!("Transaction fetcher exhausted, proceeding with available transactions");
break;
}
}
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
}
/// Static version for use in spawned tasks.
/// Determines the outcome of a build attempt.
fn check_retry_outcome(
&self,
built: &BuiltPayload,
index: u64,
attempt: u32,
gas_sent: u64,
) -> RetryOutcome {
let gas_used = built.gas_used;
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
attempts = attempt,
"Payload built successfully"
);
return RetryOutcome::Success;
}
if attempt == MAX_BUILD_RETRIES {
warn!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
gas_sent,
"Underfilled after max retries, accepting payload"
);
return RetryOutcome::MaxRetries;
}
if gas_used == 0 {
warn!(
payload = index + 1,
"Zero gas used in payload, requesting fixed chunk of additional transactions"
);
return RetryOutcome::NeedMore(self.target_gas);
}
let gas_sent_needed_total =
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
if additional == 0 {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
"No additional transactions needed based on ratio"
);
return RetryOutcome::Success;
}
let ratio = gas_used as f64 / gas_sent as f64;
info!(
payload = index + 1,
gas_used,
gas_sent,
ratio = format!("{:.4}", ratio),
additional_gas = additional,
"Underfilled, collecting more transactions for retry"
);
RetryOutcome::NeedMore(additional)
}
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
@@ -563,8 +783,9 @@ impl Command {
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
let gas_used = inner.gas_used;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
}
/// Save a payload to disk.

View File

@@ -1,6 +1,56 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use eyre::Result;
use std::io::{BufReader, Read};
/// Read input from either a file path or stdin.
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
Ok(match path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly.
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
match jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.to_string())),
}
}
None => Ok(None),
}
}
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
@@ -194,3 +244,50 @@ pub(crate) async fn get_payload_with_sidecar(
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -15,7 +15,9 @@ pub use generate_big_block::{
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod persistence_waiter;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -74,6 +76,18 @@ pub enum Subcommands {
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -89,6 +103,7 @@ impl BenchmarkCommand {
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}

View File

@@ -15,28 +15,23 @@ use crate::{
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
};
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_provider::Provider;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use url::Url;
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -105,7 +100,11 @@ impl Command {
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let sub = self.setup_persistence_subscription().await?;
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
@@ -245,293 +244,22 @@ impl Command {
results.into_iter().unzip();
if let Some(ref path) = self.benchmark.output {
write_benchmark_results(path, &gas_output_results, combined_results)?;
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
let gas_output = TotalGasOutput::new(gas_output_results)?;
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
blocks_processed=?gas_output.blocks_processed,
"Total Ggas/s: {:.4}",
gas_output.total_gigagas_per_second()
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
);
Ok(())
}
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `--ws-rpc-url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %self.benchmark.engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
let ws_url = self.derive_ws_rpc_url()?;
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -6,7 +6,7 @@ use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{path::Path, time::Duration};
use std::{fs, path::Path, time::Duration};
use tracing::info;
/// This is the suffix for gas output csv files.
@@ -158,29 +158,58 @@ pub(crate) struct TotalGasRow {
pub(crate) struct TotalGasOutput {
/// The total gas used in the benchmark.
pub(crate) total_gas_used: u64,
/// The total duration of the benchmark.
/// The total wall-clock duration of the benchmark (includes wait times).
pub(crate) total_duration: Duration,
/// The total gas used per second.
pub(crate) total_gas_per_second: f64,
/// The total execution-only duration (excludes wait times).
pub(crate) execution_duration: Duration,
/// The number of blocks processed.
pub(crate) blocks_processed: u64,
}
impl TotalGasOutput {
/// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`].
/// Create a new [`TotalGasOutput`] from gas rows only.
///
/// Use this when execution-only timing is not available (e.g., `new_payload_only`).
/// `execution_duration` will equal `total_duration`.
pub(crate) fn new(rows: Vec<TotalGasRow>) -> eyre::Result<Self> {
// the duration is obtained from the last row
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64();
Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed })
Ok(Self {
total_gas_used,
total_duration,
execution_duration: total_duration,
blocks_processed,
})
}
/// Return the total gigagas per second.
/// Create a new [`TotalGasOutput`] from gas rows and combined results.
///
/// - `rows`: Used for total gas and wall-clock duration
/// - `combined_results`: Used for execution-only duration (sum of `total_latency`)
pub(crate) fn with_combined_results(
rows: Vec<TotalGasRow>,
combined_results: &[CombinedResult],
) -> eyre::Result<Self> {
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
// Sum execution-only time from combined results
let execution_duration: Duration = combined_results.iter().map(|r| r.total_latency).sum();
Ok(Self { total_gas_used, total_duration, execution_duration, blocks_processed })
}
/// Return the total gigagas per second based on wall-clock time.
pub(crate) fn total_gigagas_per_second(&self) -> f64 {
self.total_gas_per_second / GIGAGAS as f64
self.total_gas_used as f64 / self.total_duration.as_secs_f64() / GIGAGAS as f64
}
/// Return the execution-only gigagas per second (excludes wait times).
pub(crate) fn execution_gigagas_per_second(&self) -> f64 {
self.total_gas_used as f64 / self.execution_duration.as_secs_f64() / GIGAGAS as f64
}
}
@@ -192,8 +221,10 @@ impl TotalGasOutput {
pub(crate) fn write_benchmark_results(
output_dir: &Path,
gas_results: &[TotalGasRow],
combined_results: Vec<CombinedResult>,
combined_results: &[CombinedResult],
) -> eyre::Result<()> {
fs::create_dir_all(output_dir)?;
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;

View File

@@ -0,0 +1,304 @@
//! Persistence waiting utilities for benchmarks.
//!
//! Provides waiting behavior to control benchmark pacing:
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
//! - **Persistence-based waits**: Wait for blocks to be persisted using
//! `reth_subscribePersistedBlock` subscription
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_transport_ws::WsConnect;
use eyre::Context;
use futures::StreamExt;
use std::time::Duration;
use tracing::{debug, info};
/// Default `WebSocket` RPC port for reth.
const DEFAULT_WS_RPC_PORT: u16 = 8546;
use url::Url;
/// Default timeout for waiting on persistence.
pub(crate) const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `ws_rpc_url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since we may only have the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
pub(crate) fn derive_ws_rpc_url(
ws_rpc_url: Option<&str>,
engine_rpc_url: &str,
) -> eyre::Result<Url> {
if let Some(ws_url) = ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url
.set_port(Some(DEFAULT_WS_RPC_PORT))
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
pub(crate) struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
pub(crate) async fn setup_persistence_subscription(
ws_url: Url,
) -> eyre::Result<PersistenceSubscription> {
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
pub(crate) struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
pub(crate) const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Instant;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -2,10 +2,27 @@
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::output::GasRampPayloadFile,
bench::{
output::{
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
valid_payload::{call_forkchoice_updated, call_new_payload},
};
use alloy_primitives::B256;
@@ -14,11 +31,16 @@ use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use clap::Parser;
use eyre::Context;
use reqwest::Url;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_api::EngineApiMessageVersion;
use std::path::PathBuf;
use std::{
path::PathBuf,
time::{Duration, Instant},
};
use tracing::{debug, info};
use url::Url;
/// `reth bench replay-payloads` command
///
@@ -51,6 +73,42 @@ pub struct Command {
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
/// Optional output directory for benchmark results (CSV files).
#[arg(long, value_name = "OUTPUT")]
output: Option<PathBuf>,
/// How long to wait after a forkchoice update before sending the next payload.
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(
long = "persistence-threshold",
value_name = "PERSISTENCE_THRESHOLD",
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
verbatim_doc_comment
)]
persistence_threshold: u64,
/// Optional `WebSocket` RPC URL for persistence subscription.
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
ws_rpc_url: Option<String>,
}
/// A loaded payload ready for execution.
@@ -78,6 +136,33 @@ impl Command {
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
// Log mode configuration
if let Some(duration) = self.wait_time {
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options (duration takes precedence)
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
let sub = setup_persistence_subscription(ws_url).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
PERSISTENCE_CHECKPOINT_TIMEOUT,
))
}
(None, false) => None,
};
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
@@ -144,6 +229,11 @@ impl Command {
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
if let Some(w) = &mut waiter {
w.on_block(payload.block_number).await?;
}
parent_hash = payload.file.block_hash;
}
@@ -151,22 +241,112 @@ impl Command {
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
let mut results = Vec::new();
let total_benchmark_duration = Instant::now();
for (i, payload) in payloads.iter().enumerate() {
info!(
let envelope = &payload.envelope;
let block_hash = payload.block_hash;
let execution_payload = &envelope.envelope_inner.execution_payload;
let inner_payload = &execution_payload.payload_inner.payload_inner;
let gas_used = inner_payload.gas_used;
let gas_limit = inner_payload.gas_limit;
let block_number = inner_payload.block_number;
let transaction_count =
execution_payload.payload_inner.payload_inner.transactions.len() as u64;
debug!(
payload = i + 1,
total = payloads.len(),
index = payload.index,
block_hash = %payload.block_hash,
block_hash = %block_hash,
"Executing payload (newPayload + FCU)"
);
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
let start = Instant::now();
info!(payload = i + 1, "Payload executed successfully");
parent_hash = payload.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = auth_provider
.new_payload_v4(
execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let combined_result = CombinedResult {
block_number,
gas_limit,
transaction_count,
new_payload_result,
fcu_latency,
total_latency,
};
let current_duration = total_benchmark_duration.elapsed();
info!(%combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
let gas_row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
debug!(?status, ?fcu_result, "Payload executed successfully");
parent_hash = block_hash;
}
info!(count = payloads.len(), "All payloads replayed successfully");
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
if let Some(ref path) = self.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
);
Ok(())
}
@@ -284,49 +464,4 @@ impl Command {
Ok(payloads)
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: &ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
info!(?status, "newPayloadV4 response");
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
info!(?fcu_result, "forkchoiceUpdatedV3 response");
Ok(())
}
}

View File

@@ -0,0 +1,219 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
pub(super) struct InvalidationConfig {
// Explicit value overrides (Option<T>)
pub(super) parent_hash: Option<B256>,
pub(super) fee_recipient: Option<Address>,
pub(super) state_root: Option<B256>,
pub(super) receipts_root: Option<B256>,
pub(super) logs_bloom: Option<Bloom>,
pub(super) prev_randao: Option<B256>,
pub(super) block_number: Option<u64>,
pub(super) gas_limit: Option<u64>,
pub(super) gas_used: Option<u64>,
pub(super) timestamp: Option<u64>,
pub(super) extra_data: Option<Bytes>,
pub(super) base_fee_per_gas: Option<u64>,
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
pub(super) invalidate_state_root: bool,
pub(super) invalidate_receipts_root: bool,
pub(super) invalidate_gas_used: bool,
pub(super) invalidate_block_number: bool,
pub(super) invalidate_timestamp: bool,
pub(super) invalidate_base_fee: bool,
pub(super) invalidate_transactions: bool,
pub(super) invalidate_block_hash: bool,
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
}
impl InvalidationConfig {
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
self.block_hash.is_some() || self.invalidate_block_hash
}
/// Applies invalidations to a V1 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
let mut changes = Vec::new();
// Explicit value overrides
if let Some(parent_hash) = self.parent_hash {
payload.parent_hash = parent_hash;
changes.push(format!("parent_hash = {parent_hash}"));
}
if let Some(fee_recipient) = self.fee_recipient {
payload.fee_recipient = fee_recipient;
changes.push(format!("fee_recipient = {fee_recipient}"));
}
if let Some(state_root) = self.state_root {
payload.state_root = state_root;
changes.push(format!("state_root = {state_root}"));
}
if let Some(receipts_root) = self.receipts_root {
payload.receipts_root = receipts_root;
changes.push(format!("receipts_root = {receipts_root}"));
}
if let Some(logs_bloom) = self.logs_bloom {
payload.logs_bloom = logs_bloom;
changes.push("logs_bloom = <custom>".to_string());
}
if let Some(prev_randao) = self.prev_randao {
payload.prev_randao = prev_randao;
changes.push(format!("prev_randao = {prev_randao}"));
}
if let Some(block_number) = self.block_number {
payload.block_number = block_number;
changes.push(format!("block_number = {block_number}"));
}
if let Some(gas_limit) = self.gas_limit {
payload.gas_limit = gas_limit;
changes.push(format!("gas_limit = {gas_limit}"));
}
if let Some(gas_used) = self.gas_used {
payload.gas_used = gas_used;
changes.push(format!("gas_used = {gas_used}"));
}
if let Some(timestamp) = self.timestamp {
payload.timestamp = timestamp;
changes.push(format!("timestamp = {timestamp}"));
}
if let Some(ref extra_data) = self.extra_data {
payload.extra_data = extra_data.clone();
changes.push(format!("extra_data = {} bytes", extra_data.len()));
}
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
}
if let Some(block_hash) = self.block_hash {
payload.block_hash = block_hash;
changes.push(format!("block_hash = {block_hash}"));
}
// Auto-invalidation flags
if self.invalidate_parent_hash {
let random_hash = B256::random();
payload.parent_hash = random_hash;
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
}
if self.invalidate_state_root {
payload.state_root = B256::ZERO;
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
}
if self.invalidate_receipts_root {
payload.receipts_root = B256::ZERO;
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
}
if self.invalidate_gas_used {
let invalid_gas = payload.gas_limit + 1;
payload.gas_used = invalid_gas;
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
}
if self.invalidate_block_number {
let invalid_number = payload.block_number + 999;
payload.block_number = invalid_number;
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
}
if self.invalidate_timestamp {
payload.timestamp = 0;
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
}
if self.invalidate_base_fee {
payload.base_fee_per_gas = U256::ZERO;
changes
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
}
if self.invalidate_transactions {
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
payload.transactions.insert(0, invalid_tx);
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
}
if self.invalidate_block_hash {
let random_hash = B256::random();
payload.block_hash = random_hash;
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
}
changes
}
/// Applies invalidations to a V2 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
// Handle withdrawals invalidation (V2+)
if self.invalidate_withdrawals {
let fake_withdrawal = Withdrawal {
index: u64::MAX,
validator_index: u64::MAX,
address: Address::ZERO,
amount: u64::MAX,
};
payload.withdrawals.push(fake_withdrawal);
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
}
changes
}
/// Applies invalidations to a V3 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
// Explicit overrides for V3 fields
if let Some(blob_gas_used) = self.blob_gas_used {
payload.blob_gas_used = blob_gas_used;
changes.push(format!("blob_gas_used = {blob_gas_used}"));
}
if let Some(excess_blob_gas) = self.excess_blob_gas {
payload.excess_blob_gas = excess_blob_gas;
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
}
// Auto-invalidation for V3 fields
if self.invalidate_blob_gas_used {
payload.blob_gas_used = u64::MAX;
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
}
if self.invalidate_excess_blob_gas {
payload.excess_blob_gas = u64::MAX;
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -0,0 +1,347 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use invalidation::InvalidationConfig;
use super::helpers::{load_jwt_secret, read_input};
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::Write;
/// Command for generating and sending an invalid `engine_newPayload` request.
///
/// Takes a valid block and modifies fields to make it invalid for testing
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
#[derive(Debug, Parser)]
pub struct Command {
// ==================== Input Options ====================
/// Path to the JSON file containing the block. If not specified, stdin will be used.
#[arg(short, long, help_heading = "Input Options")]
path: Option<String>,
/// The engine RPC URL to use.
#[arg(
short,
long,
help_heading = "Input Options",
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
required_unless_present("mode")
)]
rpc_url: Option<String>,
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
/// itself.
#[arg(short, long, help_heading = "Input Options")]
jwt_secret: Option<String>,
/// The newPayload version to use (3 or 4).
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
new_payload_version: u8,
/// The output mode to use.
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
mode: Mode,
// ==================== Explicit Value Overrides ====================
/// Override the parent hash with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_hash: Option<B256>,
/// Override the fee recipient (coinbase) with a specific address.
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
fee_recipient: Option<Address>,
/// Override the state root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
state_root: Option<B256>,
/// Override the receipts root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
receipts_root: Option<B256>,
/// Override the block number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
block_number: Option<u64>,
/// Override the gas limit with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_limit: Option<u64>,
/// Override the gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_used: Option<u64>,
/// Override the timestamp with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
timestamp: Option<u64>,
/// Override the base fee per gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
base_fee_per_gas: Option<u64>,
/// Override the block hash with a specific value (skips hash recalculation).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
block_hash: Option<B256>,
/// Override the blob gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
blob_gas_used: Option<u64>,
/// Override the excess blob gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
excess_blob_gas: Option<u64>,
/// Override the parent beacon block root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_beacon_block_root: Option<B256>,
/// Override the requests hash with a specific value (EIP-7685).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_parent_hash: bool,
/// Invalidate the state root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_state_root: bool,
/// Invalidate the receipts root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_receipts_root: bool,
/// Invalidate the gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_gas_used: bool,
/// Invalidate the block number by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_number: bool,
/// Invalidate the timestamp by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_timestamp: bool,
/// Invalidate the base fee by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_base_fee: bool,
/// Invalidate the transactions by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_transactions: bool,
/// Invalidate the block hash by not recalculating it after modifications.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_hash: bool,
/// Invalidate the withdrawals by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_withdrawals: bool,
/// Invalidate the blob gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_blob_gas_used: bool,
/// Invalidate the excess blob gas by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_excess_blob_gas: bool,
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
skip_hash_recalc: bool,
/// Print what would be done without actually sending the payload.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
dry_run: bool,
}
#[derive(Debug, Clone, clap::ValueEnum)]
enum Mode {
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
/// payload into the `cast` command.
Execute,
/// Print the `cast` command. Caution: this may not work with large blocks because of the
/// command length limit.
Cast,
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
Json,
}
impl Command {
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
parent_hash: self.parent_hash,
fee_recipient: self.fee_recipient,
state_root: self.state_root,
receipts_root: self.receipts_root,
logs_bloom: None,
prev_randao: None,
block_number: self.block_number,
gas_limit: self.gas_limit,
gas_used: self.gas_used,
timestamp: self.timestamp,
extra_data: None,
base_fee_per_gas: self.base_fee_per_gas,
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
invalidate_gas_used: self.invalidate_gas_used,
invalidate_block_number: self.invalidate_block_number,
invalidate_timestamp: self.invalidate_timestamp,
invalidate_base_fee: self.invalidate_base_fee,
invalidate_transactions: self.invalidate_transactions,
invalidate_block_hash: self.invalidate_block_hash,
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
}
}
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = read_input(self.path.as_deref())?;
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
.into_consensus();
let config = self.build_invalidation_config();
let parent_beacon_block_root =
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
if !skip_recalc {
let new_hash = match execution_payload.clone().into_block_raw() {
Ok(block) => block.header.hash_slow(),
Err(e) => {
eprintln!(
"Warning: Could not recalculate block hash: {e}. Using original hash."
);
match &execution_payload {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
}
}
};
match &mut execution_payload {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
}
}
if self.dry_run {
println!("=== Dry Run ===");
println!("Changes that would be applied:");
for change in &changes {
println!(" - {}", change);
}
if changes.is_empty() {
println!(" (no changes)");
}
if skip_recalc {
println!(" - Block hash recalculation: SKIPPED");
} else {
println!(" - Block hash recalculation: PERFORMED");
}
println!("\nResulting payload JSON:");
let json = serde_json::to_string_pretty(&execution_payload)?;
println!("{}", json);
return Ok(());
}
let json_request = if use_v4 {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
requests_hash.unwrap_or_default(),
))?
} else {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
))?
};
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
}
if let Some(secret) = &jwt_secret {
command.arg("--jwt-secret").arg(secret);
}
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
process
.stdin
.take()
.ok_or_eyre("stdin not available")?
.write_all(json_request.as_bytes())?;
process.wait()?;
}
Mode::Cast => {
let mut cmd = format!(
"cast rpc engine_newPayloadV{} --raw '{}'",
self.new_payload_version, json_request
);
if let Some(rpc_url) = self.rpc_url {
cmd += &format!(" --rpc-url {rpc_url}");
}
if let Some(secret) = &jwt_secret {
cmd += &format!(" --jwt-secret {secret}");
}
println!("{cmd}");
}
Mode::Json => {
println!("{json_request}");
}
}
Ok(())
}
}

View File

@@ -1,10 +1,11 @@
use super::helpers::{load_jwt_secret, read_input};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::{BufReader, Read, Write};
use std::io::Write;
/// Command for generating and sending an `engine_newPayload` request constructed from an RPC
/// block.
@@ -51,38 +52,13 @@ enum Mode {
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.clone())),
}
}
None => Ok(None),
}
}
/// Execute the generate payload command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
// Load block
let block_json = self.read_input()?;
let block_json = read_input(self.path.as_deref())?;
// Load JWT secret
let jwt_secret = self.load_jwt_secret()?;
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
// Parse the block
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?

View File

@@ -260,7 +260,9 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
while !status.is_valid() {
if status.is_invalid() {
error!(?status, ?params, "Invalid {method}",);
panic!("Invalid {method}: {status:?}");
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
format!("Invalid {method}: {status:?}"),
))))
}
if status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(

View File

@@ -25,8 +25,8 @@
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!

View File

@@ -206,11 +206,33 @@ impl DeferredTrieData {
Default::default(), // prefix_sets are per-block, not cumulative
);
// Only trigger COW clone if there's actually data to add.
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
#[cfg(feature = "rayon")]
{
rayon::join(
|| {
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
},
|| {
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
},
);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
#[cfg(not(feature = "rayon"))]
{
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
}
overlay
}
@@ -243,53 +265,8 @@ impl DeferredTrieData {
/// In normal operation, the parent always has a cached overlay and this
/// function is never called.
///
/// When the `rayon` feature is enabled, uses parallel collection and merge:
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
/// 2. Merges hashed state and trie updates in parallel with each other
/// 3. Uses tree reduction within each merge for O(log n) depth
#[cfg(feature = "rayon")]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
// Early exit: no ancestors means just wrap current block's data
if ancestors.is_empty() {
return TrieInputSorted::new(
Arc::new(sorted_trie_updates.clone()),
Arc::new(sorted_hashed_state.clone()),
Default::default(),
);
}
// Collect ancestor data, unzipping states and updates into Arc slices
let (states, updates): (Vec<_>, Vec<_>) = ancestors
.iter()
.map(|a| {
let data = a.wait_cloned();
(data.hashed_state, data.trie_updates)
})
.unzip();
// Merge state and nodes in parallel with each other using tree reduction
let (state, nodes) = rayon::join(
|| {
let mut merged = HashedPostStateSorted::merge_parallel(&states);
merged.extend_ref_and_sort(sorted_hashed_state);
merged
},
|| {
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
merged.extend_ref_and_sort(sorted_trie_updates);
merged
},
);
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
}
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
#[cfg(not(feature = "rayon"))]
/// Iterates ancestors oldest -> newest, then extends with current block's data,
/// so later state takes precedence.
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
@@ -307,8 +284,17 @@ impl DeferredTrieData {
}
// Extend with current block's sorted data last (takes precedence)
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
#[cfg(feature = "rayon")]
rayon::join(
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
);
#[cfg(not(feature = "rayon"))]
{
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
}
overlay
}

View File

@@ -17,7 +17,10 @@ use reth_primitives_traits::{
SignedTransaction,
};
use reth_storage_api::StateProviderBox;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, TrieInputSorted};
use reth_trie::{
updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, SortedTrieData,
TrieInputSorted,
};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tokio::sync::{broadcast, watch};
@@ -948,22 +951,36 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
match blocks {
[] => Chain::default(),
[first, rest @ ..] => {
let trie_data_handle = first.trie_data_handle();
let mut chain = Chain::from_block(
first.recovered_block().clone(),
ExecutionOutcome::from((
first.execution_outcome().clone(),
first.block_number(),
)),
LazyTrieData::ready(first.hashed_state(), first.trie_updates()),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
);
for exec in rest {
let trie_data_handle = exec.trie_data_handle();
chain.append_block(
exec.recovered_block().clone(),
ExecutionOutcome::from((
exec.execution_outcome().clone(),
exec.block_number(),
)),
LazyTrieData::ready(exec.hashed_state(), exec.trie_updates()),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
);
}
chain

View File

@@ -25,6 +25,7 @@ pub use alloy_chains::{Chain, ChainKind, NamedChain};
/// Re-export for convenience
pub use reth_ethereum_forks::*;
pub use alloy_evm::EvmLimitParams;
pub use api::EthChainSpec;
pub use info::ChainInfo;
#[cfg(any(test, feature = "test-utils"))]

View File

@@ -541,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
}
}
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
}
}
}

View File

@@ -50,6 +50,7 @@ reth-stages-types = { workspace = true, optional = true }
reth-static-file-types = { workspace = true, features = ["clap"] }
reth-static-file.workspace = true
reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true
@@ -77,6 +78,7 @@ lz4.workspace = true
zstd.workspace = true
serde.workspace = true
serde_json.workspace = true
parking_lot.workspace = true
tar.workspace = true
tracing.workspace = true
backon.workspace = true

View File

@@ -121,13 +121,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
let (db, sfp) = match access {
AccessRights::RW => (
Arc::new(init_db(db_path, self.db.database_args())?),
init_db(db_path, self.db.database_args())?,
StaticFileProviderBuilder::read_write(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?,
),
AccessRights::RO | AccessRights::RoInconsistent => {
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
(open_db_read_only(&db_path, self.db.database_args())?, {
let provider = StaticFileProviderBuilder::read_only(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?;
@@ -160,16 +160,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
fn create_provider_factory<N: CliNodeTypes>(
&self,
config: &Config,
db: Arc<DatabaseEnv>,
db: DatabaseEnv,
static_file_provider: StaticFileProvider<N::Primitives>,
rocksdb_provider: RocksDBProvider,
access: AccessRights,
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let prune_modes = config.prune.segments.clone();
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
db,
self.chain.clone(),
static_file_provider,
@@ -200,7 +200,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
// Builds and executes an unwind-only pipeline
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
.add_stages(DefaultStages::new(
factory.clone(),
tip_rx,
@@ -229,7 +229,7 @@ pub struct Environment<N: NodeTypes> {
/// Configuration for reth node
pub config: Config,
/// Provider factory.
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
/// Datadir path.
pub data_dir: ChainPath<DataDirPath>,
}
@@ -261,8 +261,8 @@ impl AccessRights {
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
type FullTypesAdapter<T> = FullNodeTypesAdapter<
T,
Arc<DatabaseEnv>,
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
DatabaseEnv,
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
>;
/// Helper trait with a common set of requirements for the

View File

@@ -17,7 +17,6 @@ use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProvider
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
@@ -90,7 +89,7 @@ impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
@@ -117,7 +116,7 @@ fn checksum_hasher() -> impl Hasher {
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,

View File

@@ -9,7 +9,7 @@ use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use std::{hash::Hasher, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
@@ -36,7 +36,7 @@ impl RocksDbTable {
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {

View File

@@ -16,7 +16,6 @@ use std::{
hash::Hash,
io::Write,
path::{Path, PathBuf},
sync::Arc,
};
use tracing::{info, warn};
@@ -56,7 +55,7 @@ impl Command {
/// then written to a file in the output directory.
pub fn execute<T: NodeTypes>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("Make sure the node is not running when running `reth db diff`!");
// open second db

View File

@@ -21,6 +21,7 @@ use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
/// The arguments for the `reth db get` command
@@ -82,6 +83,33 @@ impl Command {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
let key = table_key::<tables::StorageChangeSets>(&key)?;
let provider = tool.provider_factory.static_file_provider();
if let Some(storage_key) = storage_key {
let entry = provider.get_storage_before_block(
key.block_number(),
key.address(),
storage_key,
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(());
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
@@ -112,6 +140,9 @@ impl Command {
AccountChangesetMask::MASK,
)
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changesets handled above");
}
};
// handle account changesets differently if a subkey is provided.
@@ -190,6 +221,9 @@ impl Command {
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changeset static files are special cased before this match")
}
}
}
}

View File

@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
use reth_db_common::{DbTool, ListFilter};
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
use std::{cell::RefCell, sync::Arc};
use std::cell::RefCell;
use tracing::error;
#[derive(Parser, Debug)]
@@ -55,25 +55,27 @@ impl Command {
/// Execute `db list` command
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
self.table.view(&ListTableViewer { tool, args: &self })
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
let search = match self.search.as_deref() {
Some(search) => {
if let Some(search) = search.strip_prefix("0x") {
return hex::decode(search).unwrap()
hex::decode(search).wrap_err(
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
)?
} else {
search.as_bytes().to_vec()
}
search.as_bytes().to_vec()
})
.unwrap_or_default();
}
None => Vec::new(),
};
ListFilter {
Ok(ListFilter {
skip: self.skip,
len: self.len,
search,
@@ -82,12 +84,12 @@ impl Command {
min_value_size: self.min_value_size,
reverse: self.reverse,
only_count: self.count,
}
})
}
}
struct ListTableViewer<'a, N: NodeTypes> {
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
args: &'a Command,
}
@@ -99,8 +101,8 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
// We may be using the tui for a long time
tx.disable_long_read_transaction_safety();
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let table_db = tx.inner().open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner().db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let total_entries = stats.entries();
let final_entry_idx = total_entries.saturating_sub(1);
if self.args.skip > final_entry_idx {
@@ -115,7 +117,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
}
let list_filter = self.args.list_filter();
let list_filter = self.args.list_filter()?;
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;

View File

@@ -17,6 +17,7 @@ mod get;
mod list;
mod repair_trie;
mod settings;
mod state;
mod static_file_header;
mod stats;
/// DB List TUI
@@ -65,6 +66,8 @@ pub enum Subcommands {
Settings(settings::Command),
/// Gets storage size information for an account
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
State(state::Command),
}
/// Initializes a provider factory with specified access rights, and then execute with the provided
@@ -162,7 +165,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
command.execute(&tool, ctx.task_executor, &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {
@@ -198,6 +201,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::State(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(&tool)?;
});
}
}
Ok(())

View File

@@ -69,6 +69,11 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -115,6 +120,7 @@ impl Command {
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
@@ -167,6 +173,14 @@ impl Command {
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
}
// Write updated settings

View File

@@ -0,0 +1,412 @@
use alloy_primitives::{Address, BlockNumber, B256, U256};
use clap::Parser;
use parking_lot::Mutex;
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRO},
database::Database,
tables,
transaction::DbTx,
};
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_provider::providers::ProviderNodeTypes;
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
use std::{
collections::BTreeSet,
thread,
time::{Duration, Instant},
};
use tracing::{error, info};
/// Log progress every 5 seconds
const LOG_INTERVAL: Duration = Duration::from_secs(30);
/// The arguments for the `reth db state` command
#[derive(Parser, Debug)]
pub struct Command {
/// The account address to get state for
address: Address,
/// Block number to query state at (uses current state if not provided)
#[arg(long, short)]
block: Option<BlockNumber>,
/// Maximum number of storage slots to display
#[arg(long, short, default_value = "100")]
limit: usize,
/// Output format (table, json, csv)
#[arg(long, short, default_value = "table")]
format: OutputFormat,
}
impl Command {
/// Execute `db state` command
pub fn execute<N: NodeTypesWithDB + ProviderNodeTypes>(
self,
tool: &DbTool<N>,
) -> eyre::Result<()> {
let address = self.address;
let limit = self.limit;
if let Some(block) = self.block {
self.execute_historical(tool, address, block, limit)
} else {
self.execute_current(tool, address, limit)
}
}
fn execute_current<N: NodeTypesWithDB + ProviderNodeTypes>(
&self,
tool: &DbTool<N>,
address: Address,
limit: usize,
) -> eyre::Result<()> {
let entries = tool.provider_factory.db_ref().view(|tx| {
// Get account info
let account = tx.get::<tables::PlainAccountState>(address)?;
// Get storage entries
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
let walker = cursor.walk_dup(Some(address), None)?;
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
Ok::<_, eyre::Report>((account, entries))
})??;
let (account, storage_entries) = entries;
self.print_results(address, None, account, &storage_entries);
Ok(())
}
fn execute_historical<N: NodeTypesWithDB + ProviderNodeTypes>(
&self,
tool: &DbTool<N>,
address: Address,
block: BlockNumber,
limit: usize,
) -> eyre::Result<()> {
let provider = tool.provider_factory.history_by_block_number(block)?;
// Get account info at that block
let account = provider.basic_account(&address)?;
// Check storage settings to determine where history is stored
let storage_settings = tool.provider_factory.cached_storage_settings();
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
// For historical queries, enumerate keys from history indices only
// (not PlainStorageState, which reflects current state)
let mut storage_keys = BTreeSet::new();
if history_in_rocksdb {
error!(
target: "reth::cli",
"Historical storage queries with RocksDB backend are not yet supported. \
Use MDBX for storage history or query current state without --block."
);
return Ok(());
}
// Collect keys from MDBX StorageChangeSets using parallel scanning
self.collect_mdbx_storage_keys_parallel(tool, address, &mut storage_keys)?;
info!(
target: "reth::cli",
address = %address,
block = block,
total_keys = storage_keys.len(),
"Found storage keys to query"
);
// Now query each key at the historical block using the StateProvider
// This handles both MDBX and RocksDB backends transparently
let mut entries = Vec::new();
let mut last_log = Instant::now();
for (idx, key) in storage_keys.iter().enumerate() {
match provider.storage(address, *key) {
Ok(Some(value)) if value != U256::ZERO => {
entries.push((*key, value));
}
_ => {}
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
block = block,
keys_total = storage_keys.len(),
slots_scanned = idx,
slots_found = entries.len(),
"Scanning historical storage slots"
);
last_log = Instant::now();
}
}
self.print_results(address, Some(block), account, &entries);
Ok(())
}
/// Collects storage keys from MDBX StorageChangeSets using parallel block range scanning.
fn collect_mdbx_storage_keys_parallel<N: NodeTypesWithDB + ProviderNodeTypes>(
&self,
tool: &DbTool<N>,
address: Address,
keys: &mut BTreeSet<B256>,
) -> eyre::Result<()> {
const CHUNK_SIZE: u64 = 500_000; // 500k blocks per thread
let num_threads = std::thread::available_parallelism()
.map(|p| p.get().saturating_sub(1).max(1))
.unwrap_or(4);
// Get the current tip block
let tip = tool.provider_factory.provider()?.best_block_number()?;
if tip == 0 {
return Ok(());
}
info!(
target: "reth::cli",
address = %address,
tip,
chunk_size = CHUNK_SIZE,
num_threads,
"Starting parallel MDBX changeset scan"
);
// Shared state for collecting keys
let collected_keys: Mutex<BTreeSet<B256>> = Mutex::new(BTreeSet::new());
let total_entries_scanned = Mutex::new(0usize);
// Create chunk ranges
let mut chunks: Vec<(u64, u64)> = Vec::new();
let mut start = 0u64;
while start <= tip {
let end = (start + CHUNK_SIZE - 1).min(tip);
chunks.push((start, end));
start = end + 1;
}
let chunks_ref = &chunks;
let next_chunk = Mutex::new(0usize);
let next_chunk_ref = &next_chunk;
let collected_keys_ref = &collected_keys;
let total_entries_ref = &total_entries_scanned;
thread::scope(|s| {
let handles: Vec<_> = (0..num_threads)
.map(|thread_id| {
s.spawn(move || {
loop {
// Get next chunk to process
let chunk_idx = {
let mut idx = next_chunk_ref.lock();
if *idx >= chunks_ref.len() {
return Ok::<_, eyre::Report>(());
}
let current = *idx;
*idx += 1;
current
};
let (chunk_start, chunk_end) = chunks_ref[chunk_idx];
// Open a new read transaction for this chunk
tool.provider_factory.db_ref().view(|tx| {
tx.disable_long_read_transaction_safety();
let mut changeset_cursor =
tx.cursor_read::<tables::StorageChangeSets>()?;
let start_key =
reth_db_api::models::BlockNumberAddress((chunk_start, address));
let end_key =
reth_db_api::models::BlockNumberAddress((chunk_end, address));
let mut local_keys = BTreeSet::new();
let mut entries_in_chunk = 0usize;
if let Ok(walker) = changeset_cursor.walk_range(start_key..=end_key)
{
for (block_addr, storage_entry) in walker.flatten() {
if block_addr.address() == address {
local_keys.insert(storage_entry.key);
}
entries_in_chunk += 1;
}
}
// Merge into global state
collected_keys_ref.lock().extend(local_keys);
*total_entries_ref.lock() += entries_in_chunk;
info!(
target: "reth::cli",
thread_id,
chunk_start,
chunk_end,
entries_in_chunk,
"Thread completed chunk"
);
Ok::<_, eyre::Report>(())
})??;
}
})
})
.collect();
for handle in handles {
handle.join().map_err(|_| eyre::eyre!("Thread panicked"))??;
}
Ok::<_, eyre::Report>(())
})?;
let final_keys = collected_keys.into_inner();
let total = *total_entries_scanned.lock();
info!(
target: "reth::cli",
address = %address,
total_entries = total,
unique_keys = final_keys.len(),
"Finished parallel MDBX changeset scan"
);
keys.extend(final_keys);
Ok(())
}
fn print_results(
&self,
address: Address,
block: Option<BlockNumber>,
account: Option<reth_primitives_traits::Account>,
storage: &[(alloy_primitives::B256, U256)],
) {
match self.format {
OutputFormat::Table => {
println!("Account: {address}");
if let Some(b) = block {
println!("Block: {b}");
} else {
println!("Block: latest");
}
println!();
if let Some(acc) = account {
println!("Nonce: {}", acc.nonce);
println!("Balance: {} wei", acc.balance);
if let Some(code_hash) = acc.bytecode_hash {
println!("Code hash: {code_hash}");
}
} else {
println!("Account not found");
}
println!();
println!("Storage ({} slots):", storage.len());
println!("{:-<130}", "");
println!("{:<66} | {:<64}", "Slot", "Value");
println!("{:-<130}", "");
for (key, value) in storage {
println!("{key} | {value:#066x}");
}
}
OutputFormat::Json => {
let output = serde_json::json!({
"address": address.to_string(),
"block": block,
"account": account.map(|a| serde_json::json!({
"nonce": a.nonce,
"balance": a.balance.to_string(),
"code_hash": a.bytecode_hash.map(|h| h.to_string()),
})),
"storage": storage.iter().map(|(k, v)| {
serde_json::json!({
"key": k.to_string(),
"value": format!("{v:#066x}"),
})
}).collect::<Vec<_>>(),
});
println!("{}", serde_json::to_string_pretty(&output).unwrap());
}
OutputFormat::Csv => {
println!("slot,value");
for (key, value) in storage {
println!("{key},{value:#066x}");
}
}
}
}
}
#[derive(Debug, Clone, Default, clap::ValueEnum)]
pub enum OutputFormat {
#[default]
Table,
Json,
Csv,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_state_args() {
let cmd = Command::try_parse_from([
"state",
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045",
"--block",
"1000000",
])
.unwrap();
assert_eq!(
cmd.address,
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045".parse::<Address>().unwrap()
);
assert_eq!(cmd.block, Some(1000000));
}
#[test]
fn parse_state_args_no_block() {
let cmd = Command::try_parse_from(["state", "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045"])
.unwrap();
assert_eq!(cmd.block, None);
}
}

View File

@@ -16,7 +16,7 @@ use reth_provider::{
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
use std::time::Duration;
#[derive(Parser, Debug)]
/// The arguments for the `reth db stats` command
@@ -48,7 +48,7 @@ impl Command {
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
data_dir: ChainPath<DataDirPath>,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
if self.checksum {
let checksum_report = self.checksum_report(tool)?;
@@ -72,7 +72,7 @@ impl Command {
Ok(())
}
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
&self,
tool: &DbTool<N>,
) -> eyre::Result<ComfyTable> {
@@ -92,10 +92,10 @@ impl Command {
db_tables.sort();
let mut total_size = 0;
for db_table in db_tables {
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
let table_db = tx.inner().open_db(Some(db_table)).wrap_err("Could not open db.")?;
let stats = tx
.inner
.inner()
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {db_table}"))?;
@@ -136,9 +136,9 @@ impl Command {
.add_cell(Cell::new(human_bytes(total_size as f64)));
table.add_row(row);
let freelist = tx.inner.env().freelist()?;
let freelist = tx.inner().env().freelist()?;
let pagesize =
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
tx.inner().db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
let freelist_size = freelist * pagesize;
let mut row = Row::new();
@@ -205,6 +205,16 @@ impl Command {
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
let wal_size = tool.provider_factory.rocksdb_provider().wal_size_bytes();
let mut row = Row::new();
row.add_cell(Cell::new("WAL"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(wal_size as f64)))
.add_cell(Cell::new(""));
table.add_row(row);
}
table

View File

@@ -227,8 +227,9 @@ where
// Handle errors
if let Err(err) = res {
error!("{:?}", err)
error!("{err}");
}
Ok(())
}
}
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
) -> io::Result<()>
where
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
io::Error: From<B::Error>,
{
let mut last_tick = Instant::now();
let mut running = true;

View File

@@ -26,6 +26,14 @@ pub struct ImportCommand<C: ChainSpecParser> {
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
/// Fail immediately when an invalid block is encountered.
///
/// By default, the import will stop at the last valid block if an invalid block is
/// encountered during execution or validation, leaving the database at the last valid
/// block state. When this flag is set, the import will instead fail with an error.
#[arg(long, verbatim_doc_comment)]
fail_on_invalid_block: bool,
/// The path(s) to block file(s) for import.
///
/// The online stages (headers and bodies) are replaced by a file import, after which the
@@ -52,7 +60,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len());
let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len };
let import_config = ImportConfig {
no_state: self.no_state,
chunk_len: self.chunk_len,
fail_on_invalid_block: self.fail_on_invalid_block,
};
let executor = components.evm_config().clone();
let consensus = Arc::new(components.consensus().clone());
@@ -81,7 +93,20 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
total_decoded_blocks += result.total_decoded_blocks;
total_decoded_txns += result.total_decoded_txns;
if !result.is_complete() {
// Check if we stopped due to an invalid block
if result.stopped_on_invalid_block {
info!(target: "reth::cli",
"Stopped at last valid block {} due to invalid block {} in file: {}. Imported {} blocks, {} transactions",
result.last_valid_block.unwrap_or(0),
result.bad_block.unwrap_or(0),
path.display(),
result.total_imported_blocks,
result.total_imported_txns);
// Stop importing further files and exit successfully
break;
}
if !result.is_successful() {
return Err(eyre::eyre!(
"Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions",
path.display(),
@@ -98,7 +123,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
}
info!(target: "reth::cli",
"All files imported successfully. Total: {}/{} blocks, {}/{} transactions",
"Import complete. Total: {}/{} blocks, {}/{} transactions",
total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns);
Ok(())
@@ -139,4 +164,20 @@ mod tests {
assert_eq!(args.paths[1], PathBuf::from("file2.rlp"));
assert_eq!(args.paths[2], PathBuf::from("file3.rlp"));
}
#[test]
fn parse_import_command_with_fail_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "--fail-on-invalid-block", "chain.rlp"]);
assert!(args.fail_on_invalid_block);
assert_eq!(args.paths.len(), 1);
assert_eq!(args.paths[0], PathBuf::from("chain.rlp"));
}
#[test]
fn parse_import_command_default_stops_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "chain.rlp"]);
assert!(!args.fail_on_invalid_block);
}
}

View File

@@ -22,11 +22,11 @@ use reth_provider::{
StageCheckpointReader,
};
use reth_prune::PruneModes;
use reth_stages::{prelude::*, Pipeline, StageId, StageSet};
use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
use reth_static_file::StaticFileProducer;
use std::{path::Path, sync::Arc};
use tokio::sync::watch;
use tracing::{debug, error, info};
use tracing::{debug, error, info, warn};
/// Configuration for importing blocks from RLP files.
#[derive(Debug, Clone, Default)]
@@ -35,6 +35,9 @@ pub struct ImportConfig {
pub no_state: bool,
/// Chunk byte length to read from file.
pub chunk_len: Option<u64>,
/// If true, fail immediately when an invalid block is encountered.
/// By default (false), the import stops at the last valid block and exits successfully.
pub fail_on_invalid_block: bool,
}
/// Result of an import operation.
@@ -48,6 +51,12 @@ pub struct ImportResult {
pub total_imported_blocks: usize,
/// Total number of transactions imported into the database.
pub total_imported_txns: usize,
/// Whether the import was stopped due to an invalid block.
pub stopped_on_invalid_block: bool,
/// The block number that was invalid, if any.
pub bad_block: Option<u64>,
/// The last valid block number when stopped due to invalid block.
pub last_valid_block: Option<u64>,
}
impl ImportResult {
@@ -56,6 +65,14 @@ impl ImportResult {
self.total_decoded_blocks == self.total_imported_blocks &&
self.total_decoded_txns == self.total_imported_txns
}
/// Returns true if the import was successful, considering stop-on-invalid-block mode.
///
/// In stop-on-invalid-block mode, a partial import is considered successful if we
/// stopped due to an invalid block (leaving the DB at the last valid block).
pub fn is_successful(&self) -> bool {
self.is_complete() || self.stopped_on_invalid_block
}
}
/// Imports blocks from an RLP-encoded file into the database.
@@ -103,6 +120,11 @@ where
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
// Track if we stopped due to an invalid block
let mut stopped_on_invalid_block = false;
let mut bad_block_number: Option<u64> = None;
let mut last_valid_block_number: Option<u64> = None;
while let Some(file_client) =
reader.next_chunk::<BlockTy<N>>(consensus.clone(), Some(sealed_header)).await?
{
@@ -137,12 +159,51 @@ where
// Run pipeline
info!(target: "reth::import", "Starting sync pipeline");
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
if import_config.fail_on_invalid_block {
// Original behavior: fail on unwind
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
}
} else {
// Default behavior: Use run_loop() to handle unwinds gracefully
let result = tokio::select! {
res = pipeline.run_loop() => res,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
};
match result {
Ok(ControlFlow::Unwind { target, bad_block }) => {
// An invalid block was encountered; stop at last valid block
let bad = bad_block.block.number;
warn!(
target: "reth::import",
bad_block = bad,
last_valid_block = target,
"Invalid block encountered during import; stopping at last valid block"
);
stopped_on_invalid_block = true;
bad_block_number = Some(bad);
last_valid_block_number = Some(target);
break;
}
Ok(ControlFlow::Continue { block_number }) => {
debug!(target: "reth::import", block_number, "Pipeline chunk completed");
}
Ok(ControlFlow::NoProgress { block_number }) => {
debug!(target: "reth::import", ?block_number, "Pipeline made no progress");
}
Err(e) => {
// Propagate other pipeline errors
return Err(e.into());
}
}
}
sealed_header = provider_factory
@@ -160,9 +221,20 @@ where
total_decoded_txns,
total_imported_blocks,
total_imported_txns,
stopped_on_invalid_block,
bad_block: bad_block_number,
last_valid_block: last_valid_block_number,
};
if !result.is_complete() {
if result.stopped_on_invalid_block {
info!(target: "reth::import",
total_imported_blocks,
total_imported_txns,
bad_block = ?result.bad_block,
last_valid_block = ?result.last_valid_block,
"Import stopped at last valid block due to invalid block"
);
} else if !result.is_complete() {
error!(target: "reth::import",
total_decoded_blocks,
total_imported_blocks,

View File

@@ -2,7 +2,7 @@ use futures::Future;
use reth_cli::chainspec::ChainSpecParser;
use reth_db::DatabaseEnv;
use reth_node_builder::{NodeBuilder, WithLaunchContext};
use std::{fmt, sync::Arc};
use std::fmt;
/// A trait for launching a reth node with custom configuration strategies.
///
@@ -30,7 +30,7 @@ where
/// * `builder_args` - Extension arguments for configuration
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>>;
}
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
where
C: ChainSpecParser,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
@@ -77,13 +77,13 @@ where
C: ChainSpecParser,
Ext: clap::Args + fmt::Debug,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>> {
(self.func)(builder, builder_args)

View File

@@ -206,7 +206,7 @@ where
let db_path = data_dir.db();
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
if with_unused_ports {
node_config = node_config.with_unused_ports();

View File

@@ -0,0 +1,34 @@
//! Enode identifier command
use clap::Parser;
use reth_cli_util::get_secret_key;
use reth_network_peers::NodeRecord;
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf,
};
/// Print the enode identifier for a given secret key.
#[derive(Parser, Debug)]
pub struct Command {
/// Path to the secret key file for discovery.
pub discovery_secret: PathBuf,
/// Optional IP address to include in the enode URL.
///
/// If not provided, defaults to 0.0.0.0.
#[arg(long)]
pub ip: Option<IpAddr>,
}
impl Command {
/// Execute the enode command.
pub fn execute(self) -> eyre::Result<()> {
let sk = get_secret_key(&self.discovery_secret)?;
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
let addr = SocketAddr::new(ip, 30303);
let enr = NodeRecord::from_secret_key(addr, &sk);
println!("{enr}");
Ok(())
}
}

View File

@@ -18,6 +18,7 @@ use reth_node_core::{
};
pub mod bootnode;
pub mod enode;
pub mod rlpx;
/// `reth p2p` command
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Subcommands::Bootnode(command) => {
command.execute().await?;
}
Subcommands::Enode(command) => {
command.execute()?;
}
}
Ok(())
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
Subcommands::Body { args, .. } => Some(&args.chain),
Subcommands::Rlpx(_) => None,
Subcommands::Bootnode(_) => None,
Subcommands::Enode(_) => None,
}
}
}
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
Rlpx(rlpx::Command),
/// Bootnode command
Bootnode(bootnode::Command),
/// Print enode identifier
Enode(enode::Command),
}
#[derive(Debug, Clone, Parser)]
@@ -225,4 +232,16 @@ mod tests {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
}
#[test]
fn parse_enode_cmd() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret"]);
}
#[test]
fn parse_enode_cmd_with_ip() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
}
}

View File

@@ -1,26 +1,62 @@
//! Command that runs pruning without any limits.
//! Command that runs pruning.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{args::MetricArgs, version::version_metadata};
use reth_node_metrics::{
chain::ChainSpecInfo,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
use tracing::info;
/// Prunes according to the configuration without any limits
/// Prunes according to the configuration
#[derive(Debug, Parser)]
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
env: EnvironmentArgs<C>,
/// Prometheus metrics configuration.
#[command(flatten)]
metrics: MetricArgs,
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
/// Execute the `prune` command
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
self,
ctx: CliContext,
) -> eyre::Result<()> {
let env = self.env.init::<N>(AccessRights::RW)?;
let provider_factory = env.provider_factory;
let config = env.config.prune;
let data_dir = env.data_dir;
if let Some(listen_addr) = self.metrics.prometheus {
let config = MetricServerConfig::new(
listen_addr,
VersionInfo {
version: version_metadata().cargo_pkg_version.as_ref(),
build_timestamp: version_metadata().vergen_build_timestamp.as_ref(),
cargo_features: version_metadata().vergen_cargo_features.as_ref(),
git_sha: version_metadata().vergen_git_sha.as_ref(),
target_triple: version_metadata().vergen_cargo_target_triple.as_ref(),
build_profile: version_metadata().build_profile_name.as_ref(),
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
MetricServer::new(config).serve().await?;
}
// Copy data from database to static files
info!(target: "reth::cli", "Copying data from database to static files...");
@@ -33,13 +69,43 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
// Delete data which has been copied to static files.
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
// Use batched pruning with a limit to bound memory, running in a loop until complete.
const DELETE_LIMIT: usize = 200_000;
let mut pruner = PrunerBuilder::new(config)
.delete_limit(usize::MAX)
.delete_limit(DELETE_LIMIT)
.build_with_provider_factory(provider_factory);
pruner.run(prune_tip)?;
info!(target: "reth::cli", "Pruned data from database");
let mut total_pruned = 0usize;
loop {
let output = pruner.run(prune_tip)?;
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
total_pruned = total_pruned.saturating_add(batch_pruned);
// Check if all segments are finished (not just the overall progress,
// since the pruner sets overall progress from the last segment only)
let all_segments_finished =
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
if all_segments_finished {
break;
}
if batch_pruned == 0 {
return Err(eyre::eyre!(
"pruner made no progress but reported more data remaining; \
aborting to prevent infinite loop"
));
}
info!(
target: "reth::cli",
batch_pruned,
total_pruned,
"Pruning batch complete, continuing..."
);
}
info!(target: "reth::cli", total_pruned, "Pruned data from database");
}
Ok(())

View File

@@ -15,7 +15,8 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
StorageSettingsCache,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -90,11 +91,14 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(highest_block)?;
}
}
}
}
let provider_rw = tool.provider_factory.database_provider_rw()?;
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
let tx = provider_rw.tx_ref();
match self.stage {
@@ -168,8 +172,20 @@ impl<C: ChainSpecParser> Command<C> {
)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
}
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
}
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
@@ -177,7 +193,14 @@ impl<C: ChainSpecParser> Command<C> {
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
}
StageEnum::TxLookup => {
tx.clear::<tables::TransactionHashNumbers>()?;
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;
} else {
tx.clear::<tables::TransactionHashNumbers>()?;
}
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;

View File

@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
consensus: C,
) -> eyre::Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
C: FullConsensus<E::Primitives> + 'static,
{
@@ -39,7 +39,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -10,10 +10,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: BlockNumber,
to: BlockNumber,
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -9,10 +9,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: u64,
to: u64,
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
consensus: impl FullConsensus<N::Primitives> + 'static,
) -> Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
{
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
@@ -59,7 +59,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -158,7 +158,7 @@ enum Subcommands {
impl Subcommands {
/// Returns the block to unwind to. The returned block will stay in database.
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
&self,
factory: ProviderFactory<N>,
) -> eyre::Result<u64> {

View File

@@ -438,6 +438,8 @@ pub struct BlocksPerFileConfig {
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
pub storage_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -451,6 +453,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
@@ -469,6 +472,10 @@ impl StaticFilesConfig {
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
eyre::ensure!(
storage_change_sets != Some(0),
"Storage changesets segment blocks per file must be greater than 0"
);
Ok(())
}
@@ -480,6 +487,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
let mut map = StaticFileMap::default();
@@ -492,6 +500,7 @@ impl StaticFilesConfig {
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
StaticFileSegment::StorageChangeSets => storage_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {

View File

@@ -1,14 +1,11 @@
//! Collection of methods for block validation.
use alloy_consensus::{BlockHeader as _, Transaction, EMPTY_OMMER_ROOT_HASH};
use alloy_consensus::{BlockHeader as _, EMPTY_OMMER_ROOT_HASH};
use alloy_eips::{eip4844::DATA_GAS_PER_BLOB, eip7840::BlobParams};
use reth_chainspec::{EthChainSpec, EthereumHardfork, EthereumHardforks};
use reth_consensus::{ConsensusError, TxGasLimitTooHighErr};
use reth_consensus::ConsensusError;
use reth_primitives_traits::{
constants::{
GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MAX_TX_GAS_LIMIT_OSAKA, MINIMUM_GAS_LIMIT,
},
transaction::TxHashRef,
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
Block, BlockBody, BlockHeader, GotExpected, SealedBlock, SealedHeader,
};
@@ -146,7 +143,7 @@ pub fn validate_block_pre_execution<B, ChainSpec>(
) -> Result<(), ConsensusError>
where
B: Block,
ChainSpec: EthereumHardforks,
ChainSpec: EthChainSpec + EthereumHardforks,
{
post_merge_hardfork_fields(block, chain_spec)?;
@@ -154,19 +151,6 @@ where
if let Err(error) = block.ensure_transaction_root_valid() {
return Err(ConsensusError::BodyTransactionRootDiff(error.into()))
}
// EIP-7825 validation
if chain_spec.is_osaka_active_at_timestamp(block.timestamp()) {
for tx in block.body().transactions() {
if tx.gas_limit() > MAX_TX_GAS_LIMIT_OSAKA {
return Err(TxGasLimitTooHighErr {
tx_hash: *tx.tx_hash(),
gas_limit: tx.gas_limit(),
max_allowed: MAX_TX_GAS_LIMIT_OSAKA,
}
.into());
}
}
}
Ok(())
}

View File

@@ -29,7 +29,7 @@ auto_impl.workspace = true
derive_more.workspace = true
futures.workspace = true
eyre.workspace = true
reqwest = { workspace = true, features = ["rustls-tls"] }
reqwest.workspace = true
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["time"] }
serde_json.workspace = true

View File

@@ -38,12 +38,8 @@ pub trait BlockProvider: Send + Sync + 'static {
offset: usize,
) -> impl Future<Output = eyre::Result<B256>> + Send {
async move {
let stored_hash = previous_block_hashes
.len()
.checked_sub(offset)
.and_then(|index| previous_block_hashes.get(index));
if let Some(hash) = stored_hash {
return Ok(*hash);
if let Some(hash) = get_hash_at_offset(previous_block_hashes, offset) {
return Ok(hash);
}
// Return zero hash if the chain isn't long enough to have the block at the offset.
@@ -83,7 +79,7 @@ where
/// Spawn the client to start sending FCUs and new payloads by periodically fetching recent
/// blocks.
pub async fn run(self) {
let mut previous_block_hashes = AllocRingBuffer::new(64);
let mut previous_block_hashes = AllocRingBuffer::new(65);
let mut block_stream = {
let (tx, rx) = mpsc::channel::<P::Block>(64);
let block_provider = self.block_provider.clone();
@@ -99,7 +95,7 @@ where
let block_hash = payload.block_hash();
let block_number = payload.block_number();
previous_block_hashes.push(block_hash);
previous_block_hashes.enqueue(block_hash);
// Send new events to execution client
let _ = self.engine_handle.new_payload(payload).await;
@@ -142,3 +138,60 @@ where
}
}
}
/// Looks up a block hash from the ring buffer at the given offset from the most recent entry.
///
/// Returns `None` if the buffer doesn't have enough entries to satisfy the offset.
fn get_hash_at_offset(buffer: &AllocRingBuffer<B256>, offset: usize) -> Option<B256> {
buffer.len().checked_sub(offset + 1).and_then(|index| buffer.get(index).copied())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_get_hash_at_offset() {
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
// Empty buffer returns None for any offset
assert_eq!(get_hash_at_offset(&buffer, 0), None);
assert_eq!(get_hash_at_offset(&buffer, 1), None);
// Push hashes 0..65
for i in 0..65u8 {
buffer.enqueue(B256::with_last_byte(i));
}
// offset=0 should return the most recent (64)
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(64)));
// offset=32 (safe block) should return hash 32
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(32)));
// offset=64 (finalized block) should return hash 0 (the oldest)
assert_eq!(get_hash_at_offset(&buffer, 64), Some(B256::with_last_byte(0)));
// offset=65 exceeds buffer, should return None
assert_eq!(get_hash_at_offset(&buffer, 65), None);
}
#[test]
fn test_get_hash_at_offset_insufficient_entries() {
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
// With only 1 entry, only offset=0 works
buffer.enqueue(B256::with_last_byte(1));
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
assert_eq!(get_hash_at_offset(&buffer, 1), None);
assert_eq!(get_hash_at_offset(&buffer, 32), None);
assert_eq!(get_hash_at_offset(&buffer, 64), None);
// With 33 entries, offset=32 works but offset=64 doesn't
for i in 2..=33u8 {
buffer.enqueue(B256::with_last_byte(i));
}
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
assert_eq!(get_hash_at_offset(&buffer, 64), None);
}
}

View File

@@ -72,3 +72,11 @@ derive_more.workspace = true
[[test]]
name = "e2e_testsuite"
path = "tests/e2e-testsuite/main.rs"
[[test]]
name = "rocksdb"
path = "tests/rocksdb/main.rs"
required-features = ["edge"]
[features]
edge = ["reth-node-core/edge", "reth-provider/rocksdb", "reth-cli-commands/edge"]

View File

@@ -103,7 +103,10 @@ where
N: NodeBuilderHelper,
{
E2ETestSetupBuilder::new(num_nodes, chain_spec, attributes_generator)
.with_tree_config_modifier(move |_| tree_config.clone())
.with_tree_config_modifier(move |base| {
// Apply caller's tree_config but preserve the small cache size from base
tree_config.clone().with_cross_block_cache_size(base.cross_block_cache_size())
})
.with_node_config_modifier(move |config| config.set_dev(is_dev))
.with_connect_nodes(connect_nodes)
.build()

View File

@@ -112,11 +112,13 @@ where
..NetworkArgs::default()
};
// Apply tree config modifier if present
// Apply tree config modifier if present, with test-appropriate defaults
let base_tree_config =
reth_node_api::TreeConfig::default().with_cross_block_cache_size(1024 * 1024);
let tree_config = if let Some(modifier) = self.tree_config_modifier {
modifier(reth_node_api::TreeConfig::default())
modifier(base_tree_config)
} else {
reth_node_api::TreeConfig::default()
base_tree_config
};
let mut nodes = (0..self.num_nodes)

View File

@@ -114,19 +114,22 @@ pub async fn setup_engine_with_chain_import(
// Initialize the database using init_db (same as CLI import command)
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args)?;
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, db_args)?;
// Create a provider factory with the initialized database (use regular DB, not
// TempDatabase) We need to specify the node types properly for the adapter
let provider_factory = ProviderFactory::<
NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
>::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
)?;
let provider_factory =
ProviderFactory::<NodeTypesWithDBAdapter<EthereumNode, DatabaseEnv>>::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(
static_files_path.clone(),
)?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)?;
// Initialize genesis if needed
reth_db_common::init::init_genesis(&provider_factory)?;
@@ -317,17 +320,17 @@ mod tests {
// Import the chain
{
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, db_args).unwrap();
let provider_factory: ProviderFactory<
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
> = ProviderFactory::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path.clone())
.with_default_tables()
.build()
.unwrap(),
)
@@ -381,17 +384,17 @@ mod tests {
// Now reopen the database and verify checkpoints are still there
{
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let provider_factory: ProviderFactory<
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
> = ProviderFactory::new(
db,
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_only(static_files_path, false)
.unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)
@@ -490,7 +493,10 @@ mod tests {
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path).unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path).build().unwrap(),
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)
.expect("failed to create provider factory");

View File

@@ -38,6 +38,18 @@ impl TransactionTestContext {
signed.encoded_2718().into()
}
/// Creates a transfer with a specific nonce and signs it, returning bytes.
/// Uses high `max_fee_per_gas` (1000 gwei) to ensure tx acceptance regardless of basefee.
pub async fn transfer_tx_bytes_with_nonce(
chain_id: u64,
wallet: PrivateKeySigner,
nonce: u64,
) -> Bytes {
let tx = tx(chain_id, 21000, None, None, nonce, Some(1000e9 as u128));
let signed = Self::sign_tx(wallet, tx).await;
signed.encoded_2718().into()
}
/// Creates a deployment transaction and signs it, returning an envelope.
pub async fn deploy_tx(
chain_id: u64,

View File

@@ -0,0 +1,591 @@
//! E2E tests for `RocksDB` provider functionality.
#![cfg(all(feature = "edge", unix))]
use alloy_consensus::BlockHeader;
use alloy_primitives::B256;
use alloy_rpc_types_eth::{Transaction, TransactionReceipt};
use eyre::Result;
use jsonrpsee::core::client::ClientT;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
use reth_db::tables;
use reth_e2e_test_utils::{transaction::TransactionTestContext, wallet, E2ETestSetupBuilder};
use reth_node_core::args::RocksDbArgs;
use reth_node_ethereum::EthereumNode;
use reth_payload_builder::EthPayloadBuilderAttributes;
use reth_provider::{RocksDBProviderFactory, StorageSettings};
use std::{sync::Arc, time::Duration};
const ROCKSDB_POLL_TIMEOUT: Duration = Duration::from_secs(60);
const ROCKSDB_POLL_INTERVAL: Duration = Duration::from_millis(50);
/// Polls RPC until the given `tx_hash` is visible as pending (not yet mined).
/// Prevents race conditions where `advance_block` is called before txs are in the pool.
/// Returns the pending transaction.
async fn wait_for_pending_tx<C: ClientT>(client: &C, tx_hash: B256) -> Transaction {
let start = std::time::Instant::now();
loop {
let tx: Option<Transaction> = client
.request("eth_getTransactionByHash", [tx_hash])
.await
.expect("RPC request failed");
if let Some(tx) = tx {
assert!(
tx.block_number.is_none(),
"Expected pending tx but tx_hash={tx_hash:?} is already mined in block {:?}",
tx.block_number
);
return tx;
}
assert!(
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
"Timed out after {:?} waiting for tx_hash={tx_hash:?} to appear in pending pool",
start.elapsed()
);
tokio::time::sleep(ROCKSDB_POLL_INTERVAL).await;
}
}
/// Polls `RocksDB` until the given `tx_hash` appears in `TransactionHashNumbers`.
/// Returns the `tx_number` on success, or panics on timeout.
async fn poll_tx_in_rocksdb<P: RocksDBProviderFactory>(provider: &P, tx_hash: B256) -> u64 {
let start = std::time::Instant::now();
let mut interval = ROCKSDB_POLL_INTERVAL;
loop {
// Re-acquire handle each iteration to avoid stale snapshot reads
let rocksdb = provider.rocksdb_provider();
let tx_number: Option<u64> =
rocksdb.get::<tables::TransactionHashNumbers>(tx_hash).expect("RocksDB get failed");
if let Some(n) = tx_number {
return n;
}
assert!(
start.elapsed() < ROCKSDB_POLL_TIMEOUT,
"Timed out after {:?} waiting for tx_hash={tx_hash:?} in RocksDB",
start.elapsed()
);
tokio::time::sleep(interval).await;
// Simple backoff: 50ms -> 100ms -> 200ms (capped)
interval = std::cmp::min(interval * 2, Duration::from_millis(200));
}
}
/// Returns the test chain spec for `RocksDB` tests.
fn test_chain_spec() -> Arc<ChainSpec> {
Arc::new(
ChainSpecBuilder::default()
.chain(MAINNET.chain)
.genesis(
serde_json::from_str(include_str!("../../src/testsuite/assets/genesis.json"))
.expect("failed to parse genesis.json"),
)
.cancun_activated()
.build(),
)
}
/// Returns test payload attributes for the given timestamp.
fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
let attributes = alloy_rpc_types_engine::PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
};
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}
/// Verifies that `RocksDB` CLI defaults match `StorageSettings::base()`.
#[test]
fn test_rocksdb_defaults_match_storage_settings() {
let args = RocksDbArgs::default();
let settings = StorageSettings::base();
assert_eq!(
args.tx_hash, settings.transaction_hash_numbers_in_rocksdb,
"tx_hash default should match StorageSettings::base()"
);
assert_eq!(
args.storages_history, settings.storages_history_in_rocksdb,
"storages_history default should match StorageSettings::base()"
);
assert_eq!(
args.account_history, settings.account_history_in_rocksdb,
"account_history default should match StorageSettings::base()"
);
}
/// Smoke test: node boots with `RocksDB` routing enabled.
#[tokio::test]
async fn test_rocksdb_node_startup() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let (nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Verify RocksDB provider is functional (can query without error)
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let missing_hash = B256::from([0xab; 32]);
let result: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
assert!(result.is_none(), "Missing hash should return None");
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
Ok(())
}
/// Block mining works with `RocksDB` storage.
#[tokio::test]
async fn test_rocksdb_block_mining() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _wallet) =
E2ETestSetupBuilder::<EthereumNode, _>::new(1, chain_spec, test_attributes_generator)
.build()
.await?;
assert_eq!(nodes.len(), 1);
let genesis_hash = nodes[0].block_hash(0);
assert_ne!(genesis_hash, B256::ZERO);
// Mine 3 blocks with transactions
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client should be available");
for i in 1..=3u64 {
let raw_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), i - 1)
.await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Wait for tx to enter pending pool before mining
wait_for_pending_tx(&client, tx_hash).await;
let payload = nodes[0].advance_block().await?;
let block = payload.block();
assert_eq!(block.number(), i);
assert_ne!(block.hash(), B256::ZERO);
// Verify tx was actually included in the block
let receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
let receipt = receipt.expect("Receipt should exist after mining");
assert_eq!(receipt.block_number, Some(i), "Tx should be in block {i}");
}
// Verify all blocks are stored
for i in 0..=3 {
let block_hash = nodes[0].block_hash(i);
assert_ne!(block_hash, B256::ZERO);
}
Ok(())
}
/// Tx hash lookup exercises `TransactionHashNumbers` table.
#[tokio::test]
async fn test_rocksdb_transaction_queries() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Inject and mine a transaction
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client should be available");
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Wait for tx to enter pending pool before mining
wait_for_pending_tx(&client, tx_hash).await;
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Query each transaction by hash
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
let tx = tx.expect("Transaction should be found");
assert_eq!(tx.block_number, Some(1));
let receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [tx_hash]).await?;
let receipt = receipt.expect("Receipt should be found");
assert_eq!(receipt.block_number, Some(1));
assert!(receipt.status());
// Direct RocksDB assertion - poll with timeout since persistence is async
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
assert_eq!(tx_number, 0, "First tx should have TxNumber 0");
// Verify missing hash returns None
let missing_hash = B256::from([0xde; 32]);
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let missing_tx_number: Option<u64> =
rocksdb.get::<tables::TransactionHashNumbers>(missing_hash)?;
assert!(missing_tx_number.is_none());
let missing_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [missing_hash]).await?;
assert!(missing_tx.is_none(), "expected no transaction for missing hash");
let missing_receipt: Option<TransactionReceipt> =
client.request("eth_getTransactionReceipt", [missing_hash]).await?;
assert!(missing_receipt.is_none(), "expected no receipt for missing hash");
Ok(())
}
/// Multiple transactions in the same block are correctly persisted to `RocksDB`.
#[tokio::test]
async fn test_rocksdb_multi_tx_same_block() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
// Create 3 txs from the same wallet with sequential nonces
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client");
let mut tx_hashes = Vec::new();
for nonce in 0..3 {
let raw_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce)
.await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
tx_hashes.push(tx_hash);
}
// Wait for all txs to appear in pending pool before mining
for tx_hash in &tx_hashes {
wait_for_pending_tx(&client, *tx_hash).await;
}
// Mine one block containing all 3 txs
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Verify block contains all 3 txs
let block: Option<alloy_rpc_types_eth::Block> =
client.request("eth_getBlockByNumber", ("0x1", true)).await?;
let block = block.expect("Block 1 should exist");
assert_eq!(block.transactions.len(), 3, "Block should contain 3 txs");
// Verify each tx via RPC
for tx_hash in &tx_hashes {
let tx: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash]).await?;
let tx = tx.expect("Transaction should be found");
assert_eq!(tx.block_number, Some(1), "All txs should be in block 1");
}
// Poll RocksDB for all tx hashes and collect tx_numbers
let mut tx_numbers = Vec::new();
for tx_hash in &tx_hashes {
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
tx_numbers.push(n);
}
// Verify tx_numbers form the set {0, 1, 2}
tx_numbers.sort();
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be 0, 1, 2");
Ok(())
}
/// Transactions across multiple blocks have globally continuous `tx_numbers`.
#[tokio::test]
async fn test_rocksdb_txs_across_blocks() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
let client = nodes[0].rpc_client().expect("RPC client");
// Block 1: 2 transactions
let tx_hash_0 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await,
)
.await?;
let tx_hash_1 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await,
)
.await?;
// Wait for both txs to appear in pending pool
wait_for_pending_tx(&client, tx_hash_0).await;
wait_for_pending_tx(&client, tx_hash_1).await;
let payload1 = nodes[0].advance_block().await?;
assert_eq!(payload1.block().number(), 1);
// Block 2: 1 transaction
let tx_hash_2 = nodes[0]
.rpc
.inject_tx(
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await,
)
.await?;
wait_for_pending_tx(&client, tx_hash_2).await;
let payload2 = nodes[0].advance_block().await?;
assert_eq!(payload2.block().number(), 2);
// Verify block contents via RPC
let tx0: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_1]).await?;
let tx2: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash_2]).await?;
assert_eq!(tx0.expect("tx0").block_number, Some(1));
assert_eq!(tx1.expect("tx1").block_number, Some(1));
assert_eq!(tx2.expect("tx2").block_number, Some(2));
// Poll RocksDB and verify global tx_number continuity
let all_tx_hashes = [tx_hash_0, tx_hash_1, tx_hash_2];
let mut tx_numbers = Vec::new();
for tx_hash in &all_tx_hashes {
let n = poll_tx_in_rocksdb(&nodes[0].inner.provider, *tx_hash).await;
tx_numbers.push(n);
}
// Verify they form a continuous sequence {0, 1, 2}
tx_numbers.sort();
assert_eq!(tx_numbers, vec![0, 1, 2], "TxNumbers should be globally continuous: 0, 1, 2");
// Re-query block 1 txs after block 2 is mined (regression guard)
let tx0_again: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash_0]).await?;
assert!(tx0_again.is_some(), "Block 1 tx should still be queryable after block 2");
Ok(())
}
/// Pending transactions should NOT appear in `RocksDB` until mined.
#[tokio::test]
async fn test_rocksdb_pending_tx_not_in_storage() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
let signer = wallets[0].clone();
// Inject tx but do NOT mine
let raw_tx = TransactionTestContext::transfer_tx_bytes(chain_id, signer).await;
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
// Verify tx is in pending pool via RPC
let client = nodes[0].rpc_client().expect("RPC client");
wait_for_pending_tx(&client, tx_hash).await;
let pending_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash]).await?;
assert!(pending_tx.is_some(), "Pending tx should be visible via RPC");
assert!(pending_tx.unwrap().block_number.is_none(), "Pending tx should have no block_number");
// Assert tx is NOT in RocksDB before mining (single check - tx is confirmed pending)
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
let tx_number: Option<u64> = rocksdb.get::<tables::TransactionHashNumbers>(tx_hash)?;
assert!(
tx_number.is_none(),
"Pending tx should NOT be in RocksDB before mining, but found tx_number={:?}",
tx_number
);
// Now mine the block
let payload = nodes[0].advance_block().await?;
assert_eq!(payload.block().number(), 1);
// Poll until tx appears in RocksDB
let tx_number = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash).await;
assert_eq!(tx_number, 0, "First tx should have tx_number 0");
// Verify tx is now mined via RPC
let mined_tx: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash]).await?;
assert_eq!(mined_tx.expect("mined tx").block_number, Some(1));
Ok(())
}
/// Reorg with `RocksDB`: verifies that unwind correctly reads changesets from
/// storage-aware locations (static files vs MDBX) rather than directly from MDBX.
///
/// This test exercises `unwind_trie_state_from` which previously failed with
/// `UnsortedInput` errors because it read changesets directly from MDBX tables
/// instead of using storage-aware methods that check `storage_changesets_in_static_files`.
#[tokio::test]
async fn test_rocksdb_reorg_unwind() -> Result<()> {
reth_tracing::init_test_tracing();
let chain_spec = test_chain_spec();
let chain_id = chain_spec.chain().id();
let (mut nodes, _tasks, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
1,
chain_spec.clone(),
test_attributes_generator,
)
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
.build()
.await?;
assert_eq!(nodes.len(), 1);
// Use two separate wallets to avoid nonce conflicts during reorg
let wallets = wallet::Wallet::new(2).with_chain_id(chain_id).wallet_gen();
let signer1 = wallets[0].clone();
let signer2 = wallets[1].clone();
let client = nodes[0].rpc_client().expect("RPC client");
// Mine block 1 with a transaction from signer1
let raw_tx1 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 0).await;
let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?;
wait_for_pending_tx(&client, tx_hash1).await;
let payload1 = nodes[0].advance_block().await?;
let block1_hash = payload1.block().hash();
assert_eq!(payload1.block().number(), 1);
// Poll until tx1 appears in RocksDB (ensures persistence happened)
let tx_number1 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
assert_eq!(tx_number1, 0, "First tx should have tx_number 0");
// Mine block 2 with transaction from signer1 (nonce 1)
let raw_tx2 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 1).await;
let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?;
wait_for_pending_tx(&client, tx_hash2).await;
let payload2 = nodes[0].advance_block().await?;
assert_eq!(payload2.block().number(), 2);
// Poll until tx2 appears in RocksDB
let tx_number2 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
assert_eq!(tx_number2, 1, "Second tx should have tx_number 1");
// Mine block 3 with transaction from signer1 (nonce 2)
let raw_tx3 =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer1.clone(), 2).await;
let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?;
wait_for_pending_tx(&client, tx_hash3).await;
let payload3 = nodes[0].advance_block().await?;
assert_eq!(payload3.block().number(), 3);
// Poll until tx3 appears in RocksDB
let tx_number3 = poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
assert_eq!(tx_number3, 2, "Third tx should have tx_number 2");
// Now create an alternate block 2 using signer2 (different wallet, avoids nonce conflict)
// Inject a tx from signer2 (nonce 0) before building the alternate block
let raw_alt_tx =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 0).await;
let alt_tx_hash = nodes[0].rpc.inject_tx(raw_alt_tx).await?;
wait_for_pending_tx(&client, alt_tx_hash).await;
// Build an alternate payload (this builds on top of the current head, i.e., block 3)
// But we want to reorg back to block 1, so we'll use the payload and then FCU to it
let alt_payload = nodes[0].new_payload().await?;
let alt_block_hash = nodes[0].submit_payload(alt_payload.clone()).await?;
// Trigger reorg: make the alternate chain canonical by sending FCU pointing to block 1's hash
// as finalized, which should trigger an unwind of blocks 2 and 3
// The alt block becomes the new head
nodes[0].update_forkchoice(block1_hash, alt_block_hash).await?;
// Give time for the reorg to complete
tokio::time::sleep(Duration::from_millis(500)).await;
// Verify we can still query transactions and the chain is consistent
// If unwind_trie_state_from failed, this would have errored during reorg
let latest: Option<alloy_rpc_types_eth::Block> =
client.request("eth_getBlockByNumber", ("latest", false)).await?;
let latest = latest.expect("Latest block should exist");
// The alt block is at height 4 (on top of block 3)
assert!(latest.header.number >= 3, "Should be at height >= 3 after operation");
// tx1 from block 1 should still be there
let tx1: Option<Transaction> = client.request("eth_getTransactionByHash", [tx_hash1]).await?;
assert!(tx1.is_some(), "tx1 from block 1 should still be queryable");
assert_eq!(tx1.unwrap().block_number, Some(1));
// Mine another block to verify the chain can continue
let raw_tx_final =
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer2.clone(), 1).await;
let tx_hash_final = nodes[0].rpc.inject_tx(raw_tx_final).await?;
wait_for_pending_tx(&client, tx_hash_final).await;
let final_payload = nodes[0].advance_block().await?;
assert!(final_payload.block().number() > 3, "Should be able to mine block after reorg");
// Verify tx_final is included
let tx_final: Option<Transaction> =
client.request("eth_getTransactionByHash", [tx_hash_final]).await?;
assert!(tx_final.is_some(), "final tx should be in latest block");
Ok(())
}

View File

@@ -32,9 +32,7 @@ futures-util.workspace = true
# misc
eyre.workspace = true
tracing.workspace = true
op-alloy-rpc-types-engine = { workspace = true, optional = true }
reth-optimism-chainspec = { workspace = true, optional = true }
[lints]
workspace = true
@@ -42,7 +40,6 @@ workspace = true
[features]
op = [
"dep:op-alloy-rpc-types-engine",
"dep:reth-optimism-chainspec",
"reth-payload-primitives/op",
"reth-primitives-traits/op",
]

View File

@@ -72,13 +72,18 @@ where
&self,
parent: &SealedHeader<ChainSpec::Header>,
) -> op_alloy_rpc_types_engine::OpPayloadAttributes {
/// Dummy system transaction for dev mode.
/// OP Mainnet transaction at index 0 in block 124665056.
///
/// <https://optimistic.etherscan.io/tx/0x312e290cf36df704a2217b015d6455396830b0ce678b860ebfcc30f41403d7b1>
const TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056: [u8; 251] = alloy_primitives::hex!(
"7ef8f8a0683079df94aa5b9cf86687d739a60a9b4f0835e520ec4d664e2e415dca17a6df94deaddeaddeaddeaddeaddeaddeaddeaddead00019442000000000000000000000000000000000000158080830f424080b8a4440a5e200000146b000f79c500000000000000040000000066d052e700000000013ad8a3000000000000000000000000000000000000000000000000000000003ef1278700000000000000000000000000000000000000000000000000000000000000012fdf87b89884a61e74b322bbcf60386f543bfae7827725efaaf0ab1de2294a590000000000000000000000006887246668a3b87f54deb3b94ba47a6f63f32985"
);
op_alloy_rpc_types_engine::OpPayloadAttributes {
payload_attributes: self.build(parent),
// Add dummy system transaction
transactions: Some(vec![
reth_optimism_chainspec::constants::TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056
.into(),
]),
transactions: Some(vec![TX_SET_L1_BLOCK_OP_MAINNET_BLOCK_124665056.into()]),
no_tx_pool: None,
gas_limit: None,
eip_1559_params: None,

View File

@@ -34,6 +34,11 @@ fn default_account_worker_count() -> usize {
/// The size of proof targets chunk to spawn in one multiproof calculation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
/// computation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
/// Default number of reserved CPU cores for non-reth processes.
///
/// This will be deducted from the thread count of main reth global threadpool.
@@ -42,10 +47,31 @@ pub const DEFAULT_RESERVED_CPU_CORES: usize = 1;
/// Default maximum concurrency for prewarm task.
pub const DEFAULT_PREWARM_MAX_CONCURRENCY: usize = 16;
/// Default depth for sparse trie pruning.
///
/// Nodes at this depth and below are converted to hash stubs to reduce memory.
/// Depth 4 means we keep roughly 16^4 = 65536 potential branch paths at most.
pub const DEFAULT_SPARSE_TRIE_PRUNE_DEPTH: usize = 4;
/// Default maximum number of storage tries to keep after pruning.
///
/// Storage tries beyond this limit are cleared (but allocations preserved).
pub const DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES: usize = 100;
const DEFAULT_BLOCK_BUFFER_LIMIT: u32 = EPOCH_SLOTS as u32 * 2;
const DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH: u32 = 256;
const DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE: usize = 4;
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: u64 = 4 * 1024 * 1024 * 1024;
const DEFAULT_CROSS_BLOCK_CACHE_SIZE: usize = default_cross_block_cache_size();
const fn default_cross_block_cache_size() -> usize {
if cfg!(test) {
1024 * 1024 // 1 MB in tests
} else if cfg!(target_pointer_width = "32") {
usize::MAX // max possible on wasm32 / 32-bit
} else {
4 * 1024 * 1024 * 1024 // 4 GB on 64-bit
}
}
/// Determines if the host has enough parallelism to run the payload processor.
///
@@ -95,12 +121,10 @@ pub struct TreeConfig {
disable_state_cache: bool,
/// Whether to disable parallel prewarming.
disable_prewarming: bool,
/// Whether to disable the parallel sparse trie state root algorithm.
disable_parallel_sparse_trie: bool,
/// Whether to enable state provider metrics.
state_provider_metrics: bool,
/// Cross-block cache size in bytes.
cross_block_cache_size: u64,
cross_block_cache_size: usize,
/// Whether the host has enough parallelism to run state root task.
has_enough_parallelism: bool,
/// Whether multiproof task should chunk proof targets.
@@ -135,10 +159,16 @@ pub struct TreeConfig {
storage_worker_count: usize,
/// Number of account proof worker threads.
account_worker_count: usize,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable V2 storage proofs.
disable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
/// Whether to enable sparse trie as cache.
enable_sparse_trie_as_cache: bool,
/// Depth for sparse trie pruning after state root computation.
sparse_trie_prune_depth: usize,
/// Maximum number of storage tries to retain after pruning.
sparse_trie_max_storage_tries: usize,
}
impl Default for TreeConfig {
@@ -153,7 +183,6 @@ impl Default for TreeConfig {
always_compare_trie_updates: false,
disable_state_cache: false,
disable_prewarming: false,
disable_parallel_sparse_trie: false,
state_provider_metrics: false,
cross_block_cache_size: DEFAULT_CROSS_BLOCK_CACHE_SIZE,
has_enough_parallelism: has_enough_parallelism(),
@@ -167,8 +196,11 @@ impl Default for TreeConfig {
allow_unwind_canonical_header: false,
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
enable_proof_v2: false,
disable_proof_v2: false,
disable_cache_metrics: false,
enable_sparse_trie_as_cache: false,
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
}
}
}
@@ -186,9 +218,8 @@ impl TreeConfig {
always_compare_trie_updates: bool,
disable_state_cache: bool,
disable_prewarming: bool,
disable_parallel_sparse_trie: bool,
state_provider_metrics: bool,
cross_block_cache_size: u64,
cross_block_cache_size: usize,
has_enough_parallelism: bool,
multiproof_chunking_enabled: bool,
multiproof_chunk_size: usize,
@@ -200,8 +231,10 @@ impl TreeConfig {
allow_unwind_canonical_header: bool,
storage_worker_count: usize,
account_worker_count: usize,
enable_proof_v2: bool,
disable_proof_v2: bool,
disable_cache_metrics: bool,
sparse_trie_prune_depth: usize,
sparse_trie_max_storage_tries: usize,
) -> Self {
Self {
persistence_threshold,
@@ -213,7 +246,6 @@ impl TreeConfig {
always_compare_trie_updates,
disable_state_cache,
disable_prewarming,
disable_parallel_sparse_trie,
state_provider_metrics,
cross_block_cache_size,
has_enough_parallelism,
@@ -227,8 +259,11 @@ impl TreeConfig {
allow_unwind_canonical_header,
storage_worker_count,
account_worker_count,
enable_proof_v2,
disable_proof_v2,
disable_cache_metrics,
enable_sparse_trie_as_cache: false,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
}
}
@@ -267,6 +302,18 @@ impl TreeConfig {
self.multiproof_chunk_size
}
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
/// and the chunk size is at the default value.
pub const fn effective_multiproof_chunk_size(&self) -> usize {
if !self.disable_proof_v2 &&
self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
{
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
} else {
self.multiproof_chunk_size
}
}
/// Return the number of reserved CPU cores for non-reth processes
pub const fn reserved_cpu_cores(&self) -> usize {
self.reserved_cpu_cores
@@ -283,11 +330,6 @@ impl TreeConfig {
self.state_provider_metrics
}
/// Returns whether or not the parallel sparse trie is disabled.
pub const fn disable_parallel_sparse_trie(&self) -> bool {
self.disable_parallel_sparse_trie
}
/// Returns whether or not state cache is disabled.
pub const fn disable_state_cache(&self) -> bool {
self.disable_state_cache
@@ -305,7 +347,7 @@ impl TreeConfig {
}
/// Returns the cross-block cache size.
pub const fn cross_block_cache_size(&self) -> u64 {
pub const fn cross_block_cache_size(&self) -> usize {
self.cross_block_cache_size
}
@@ -408,7 +450,7 @@ impl TreeConfig {
}
/// Setter for cross block cache size.
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: u64) -> Self {
pub const fn with_cross_block_cache_size(mut self, cross_block_cache_size: usize) -> Self {
self.cross_block_cache_size = cross_block_cache_size;
self
}
@@ -425,15 +467,6 @@ impl TreeConfig {
self
}
/// Setter for whether to disable the parallel sparse trie
pub const fn with_disable_parallel_sparse_trie(
mut self,
disable_parallel_sparse_trie: bool,
) -> Self {
self.disable_parallel_sparse_trie = disable_parallel_sparse_trie;
self
}
/// Setter for whether multiproof task should chunk proof targets.
pub const fn with_multiproof_chunking_enabled(
mut self,
@@ -495,8 +528,12 @@ impl TreeConfig {
}
/// Setter for the number of storage proof worker threads.
pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self {
self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT);
///
/// No-op if it's [`None`].
pub fn with_storage_worker_count_opt(mut self, storage_worker_count: Option<usize>) -> Self {
if let Some(count) = storage_worker_count {
self.storage_worker_count = count.max(MIN_WORKER_COUNT);
}
self
}
@@ -506,19 +543,23 @@ impl TreeConfig {
}
/// Setter for the number of account proof worker threads.
pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self {
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
///
/// No-op if it's [`None`].
pub fn with_account_worker_count_opt(mut self, account_worker_count: Option<usize>) -> Self {
if let Some(count) = account_worker_count {
self.account_worker_count = count.max(MIN_WORKER_COUNT);
}
self
}
/// Return whether V2 storage proofs are enabled.
pub const fn enable_proof_v2(&self) -> bool {
self.enable_proof_v2
/// Return whether V2 storage proofs are disabled.
pub const fn disable_proof_v2(&self) -> bool {
self.disable_proof_v2
}
/// Setter for whether to enable V2 storage proofs.
pub const fn with_enable_proof_v2(mut self, enable_proof_v2: bool) -> Self {
self.enable_proof_v2 = enable_proof_v2;
/// Setter for whether to disable V2 storage proofs.
pub const fn with_disable_proof_v2(mut self, disable_proof_v2: bool) -> Self {
self.disable_proof_v2 = disable_proof_v2;
self
}
@@ -532,4 +573,37 @@ impl TreeConfig {
self.disable_cache_metrics = disable_cache_metrics;
self
}
/// Returns whether sparse trie as cache is enabled.
pub const fn enable_sparse_trie_as_cache(&self) -> bool {
self.enable_sparse_trie_as_cache
}
/// Setter for whether to enable sparse trie as cache.
pub const fn with_enable_sparse_trie_as_cache(mut self, value: bool) -> Self {
self.enable_sparse_trie_as_cache = value;
self
}
/// Returns the sparse trie prune depth.
pub const fn sparse_trie_prune_depth(&self) -> usize {
self.sparse_trie_prune_depth
}
/// Setter for sparse trie prune depth.
pub const fn with_sparse_trie_prune_depth(mut self, depth: usize) -> Self {
self.sparse_trie_prune_depth = depth;
self
}
/// Returns the maximum number of storage tries to retain after pruning.
pub const fn sparse_trie_max_storage_tries(&self) -> usize {
self.sparse_trie_max_storage_tries
}
/// Setter for maximum storage tries to retain.
pub const fn with_sparse_trie_max_storage_tries(mut self, max_tries: usize) -> Self {
self.sparse_trie_max_storage_tries = max_tries;
self
}
}

View File

@@ -62,7 +62,8 @@ pub trait EngineTypes:
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
> + DeserializeOwned
+ Serialize
{
@@ -106,6 +107,14 @@ pub trait EngineTypes:
+ Send
+ Sync
+ 'static;
/// Execution Payload V6 envelope type.
type ExecutionPayloadEnvelopeV6: DeserializeOwned
+ Serialize
+ Clone
+ Unpin
+ Send
+ Sync
+ 'static;
}
/// Type that validates the payloads processed by the engine API.

View File

@@ -17,7 +17,6 @@ reth-engine-tree.workspace = true
reth-evm.workspace = true
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
reth-stages-api.workspace = true

View File

@@ -14,7 +14,6 @@ pub use reth_engine_tree::{
chain::{ChainEvent, ChainOrchestrator},
engine::EngineApiEvent,
};
use reth_ethereum_primitives::EthPrimitives;
use reth_evm::ConfigureEvm;
use reth_network_p2p::BlockClient;
use reth_node_types::{BlockTy, NodeTypes};
@@ -97,11 +96,11 @@ where
let downloader = BasicBlockDownloader::new(client, consensus.clone());
let persistence_handle =
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx);
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
let (to_tree_tx, from_tree) = EngineApiTreeHandler::<N::Primitives, _, _, _, _>::spawn_new(
let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new(
blockchain_db,
consensus,
payload_validator,

View File

@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-primitives-traits = { workspace = true, features = ["rayon"] }
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
@@ -53,7 +53,7 @@ revm-primitives.workspace = true
futures.workspace = true
thiserror.workspace = true
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] }
mini-moka = { workspace = true, features = ["sync"] }
fixed-cache.workspace = true
moka = { workspace = true, features = ["sync"] }
smallvec.workspace = true

View File

@@ -20,7 +20,7 @@ pub(crate) struct PersistenceMetrics {
/// How long it took for blocks to be saved
pub(crate) save_blocks_duration_seconds: Histogram,
/// How many blocks we persist at once.
pub(crate) save_blocks_block_count: Histogram,
pub(crate) save_blocks_batch_size: Histogram,
/// How long it took for blocks to be pruned
pub(crate) prune_before_duration_seconds: Histogram,
}

View File

@@ -12,7 +12,11 @@ use reth_provider::{
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
use reth_stages_api::{MetricEvent, MetricEventsSender};
use std::{
sync::mpsc::{Receiver, SendError, Sender},
sync::{
mpsc::{Receiver, SendError, Sender},
Arc,
},
thread::JoinHandle,
time::Instant,
};
use thiserror::Error;
@@ -40,6 +44,12 @@ where
metrics: PersistenceMetrics,
/// Sender for sync metrics - we only submit sync metrics for persisted blocks
sync_metrics_tx: MetricEventsSender,
/// Pending finalized block number to be committed with the next block save.
/// This avoids triggering a separate fsync for each finalized block update.
pending_finalized_block: Option<u64>,
/// Pending safe block number to be committed with the next block save.
/// This avoids triggering a separate fsync for each safe block update.
pending_safe_block: Option<u64>,
}
impl<N> PersistenceService<N>
@@ -53,7 +63,15 @@ where
pruner: PrunerWithFactory<ProviderFactory<N>>,
sync_metrics_tx: MetricEventsSender,
) -> Self {
Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx }
Self {
provider,
incoming,
pruner,
metrics: PersistenceMetrics::default(),
sync_metrics_tx,
pending_finalized_block: None,
pending_safe_block: None,
}
}
/// Prunes block data before the given block number according to the configured prune
@@ -106,14 +124,10 @@ where
}
}
PersistenceAction::SaveFinalizedBlock(finalized_block) => {
let provider = self.provider.database_provider_rw()?;
provider.save_finalized_block_number(finalized_block)?;
provider.commit()?;
self.pending_finalized_block = Some(finalized_block);
}
PersistenceAction::SaveSafeBlock(safe_block) => {
let provider = self.provider.database_provider_rw()?;
provider.save_safe_block_number(safe_block)?;
provider.commit()?;
self.pending_safe_block = Some(safe_block);
}
}
}
@@ -126,7 +140,10 @@ where
) -> Result<Option<BlockNumHash>, PersistenceError> {
debug!(target: "engine::persistence", ?new_tip_num, "Removing blocks");
let start_time = Instant::now();
let provider_rw = self.provider.database_provider_rw()?;
// Use unwind provider to ensure correct commit order (MDBX → RocksDB → SF).
// This prevents crash-recovery issues where RocksDB is committed but MDBX is not,
// which would leave stale block numbers in RocksDB history shards.
let provider_rw = self.provider.unwind_provider_rw()?;
let new_tip_hash = provider_rw.block_hash(new_tip_num)?;
provider_rw.remove_block_and_execution_above(new_tip_num)?;
@@ -138,26 +155,39 @@ where
}
fn on_save_blocks(
&self,
&mut self,
blocks: Vec<ExecutedBlock<N::Primitives>>,
) -> Result<Option<BlockNumHash>, PersistenceError> {
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
let block_count = blocks.len();
// Take any pending finalized/safe block updates to commit together
let pending_finalized = self.pending_finalized_block.take();
let pending_safe = self.pending_safe_block.take();
debug!(target: "engine::persistence", ?block_count, first=?first_block, last=?last_block, "Saving range of blocks");
let start_time = Instant::now();
if last_block.is_some() {
let provider_rw = self.provider.database_provider_rw()?;
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
// Commit pending finalized/safe block updates in the same transaction
if let Some(finalized) = pending_finalized {
provider_rw.save_finalized_block_number(finalized)?;
}
if let Some(safe) = pending_safe {
provider_rw.save_safe_block_number(safe)?;
}
provider_rw.commit()?;
}
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
self.metrics.save_blocks_block_count.record(block_count as f64);
self.metrics.save_blocks_batch_size.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
Ok(last_block)
@@ -204,15 +234,25 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
pub struct PersistenceHandle<N: NodePrimitives = EthPrimitives> {
/// The channel used to communicate with the persistence service
sender: Sender<PersistenceAction<N>>,
/// Guard that joins the service thread when all handles are dropped.
/// Uses `Arc` so the handle remains `Clone`.
_service_guard: Arc<ServiceGuard>,
}
impl<T: NodePrimitives> PersistenceHandle<T> {
/// Create a new [`PersistenceHandle`] from a [`Sender<PersistenceAction>`].
pub const fn new(sender: Sender<PersistenceAction<T>>) -> Self {
Self { sender }
///
/// This is intended for testing purposes where you want to mock the persistence service.
/// For production use, prefer [`spawn_service`](Self::spawn_service).
pub fn new(sender: Sender<PersistenceAction<T>>) -> Self {
Self { sender, _service_guard: Arc::new(ServiceGuard(None)) }
}
/// Create a new [`PersistenceHandle`], and spawn the persistence service.
///
/// The returned handle can be cloned and shared. When all clones are dropped, the service
/// thread will be joined, ensuring graceful shutdown before resources (like `RocksDB`) are
/// released.
pub fn spawn_service<N>(
provider_factory: ProviderFactory<N>,
pruner: PrunerWithFactory<ProviderFactory<N>>,
@@ -224,13 +264,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
// create the initial channels
let (db_service_tx, db_service_rx) = std::sync::mpsc::channel();
// construct persistence handle
let persistence_handle = PersistenceHandle::new(db_service_tx);
// spawn the persistence service
let db_service =
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
std::thread::Builder::new()
let join_handle = std::thread::Builder::new()
.name("Persistence Service".to_string())
.spawn(|| {
if let Err(err) = db_service.run() {
@@ -239,7 +276,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
})
.unwrap();
persistence_handle
PersistenceHandle {
sender: db_service_tx,
_service_guard: Arc::new(ServiceGuard(Some(join_handle))),
}
}
/// Sends a specific [`PersistenceAction`] in the contained channel. The caller is responsible
@@ -267,7 +307,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
}
/// Persists the finalized block number on disk.
/// Queues the finalized block number to be persisted on disk.
///
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
/// call to avoid triggering a separate fsync for each update.
pub fn save_finalized_block_number(
&self,
finalized_block: u64,
@@ -275,7 +318,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block))
}
/// Persists the safe block number on disk.
/// Queues the safe block number to be persisted on disk.
///
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
/// call to avoid triggering a separate fsync for each update.
pub fn save_safe_block_number(
&self,
safe_block: u64,
@@ -297,6 +343,27 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
}
}
/// Guard that joins the persistence service thread when dropped.
///
/// This ensures graceful shutdown - the service thread completes before resources like
/// `RocksDB` are released. Stored in an `Arc` inside [`PersistenceHandle`] so the handle
/// can be cloned while sharing the same guard.
struct ServiceGuard(Option<JoinHandle<()>>);
impl std::fmt::Debug for ServiceGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("ServiceGuard").field(&self.0.as_ref().map(|_| "...")).finish()
}
}
impl Drop for ServiceGuard {
fn drop(&mut self) {
if let Some(join_handle) = self.0.take() {
let _ = join_handle.join();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -323,12 +390,12 @@ mod tests {
#[test]
fn test_save_blocks_empty() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let blocks = vec![];
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let hash = rx.recv().unwrap();
assert_eq!(hash, None);
@@ -337,7 +404,7 @@ mod tests {
#[test]
fn test_save_blocks_single_block() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let block_number = 0;
let mut test_block_builder = TestBlockBuilder::eth();
let executed =
@@ -347,7 +414,7 @@ mod tests {
let blocks = vec![executed];
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx
.recv_timeout(std::time::Duration::from_secs(10))
@@ -360,14 +427,14 @@ mod tests {
#[test]
fn test_save_blocks_multiple_blocks() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let mut test_block_builder = TestBlockBuilder::eth();
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);
}
@@ -375,7 +442,7 @@ mod tests {
#[test]
fn test_save_blocks_multiple_calls() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let ranges = [0..1, 1..2, 2..4, 4..5];
let mut test_block_builder = TestBlockBuilder::eth();
@@ -384,7 +451,7 @@ mod tests {
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);

File diff suppressed because it is too large Load Diff

View File

@@ -13,15 +13,18 @@ use std::time::{Duration, Instant};
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
pub(crate) struct EngineApiMetrics {
pub struct EngineApiMetrics {
/// Engine API-specific metrics.
pub(crate) engine: EngineMetrics,
pub engine: EngineMetrics,
/// Block executor metrics.
pub(crate) executor: ExecutorMetrics,
pub executor: ExecutorMetrics,
/// Metrics for block validation
pub(crate) block_validation: BlockValidationMetrics,
pub block_validation: BlockValidationMetrics,
/// Canonical chain and reorg related metrics
pub tree: TreeMetrics,
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
#[allow(dead_code)]
pub(crate) bal: BalMetrics,
}
impl EngineApiMetrics {
@@ -29,7 +32,7 @@ impl EngineApiMetrics {
///
/// This method updates metrics for execution time, gas usage, and the number
/// of accounts, storage slots and bytecodes updated.
pub(crate) fn record_block_execution<R>(
pub fn record_block_execution<R>(
&self,
output: &BlockExecutionOutput<R>,
execution_duration: Duration,
@@ -56,27 +59,27 @@ impl EngineApiMetrics {
}
/// Returns a reference to the executor metrics for use in state hooks.
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
pub const fn executor_metrics(&self) -> &ExecutorMetrics {
&self.executor
}
/// Records the duration of block pre-execution changes (e.g., beacon root update).
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
pub fn record_pre_execution(&self, elapsed: Duration) {
self.executor.pre_execution_histogram.record(elapsed);
}
/// Records the duration of block post-execution changes (e.g., finalization).
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
pub fn record_post_execution(&self, elapsed: Duration) {
self.executor.post_execution_histogram.record(elapsed);
}
/// Records the time spent waiting for the next transaction from the iterator.
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
pub fn record_transaction_wait(&self, elapsed: Duration) {
self.executor.transaction_wait_histogram.record(elapsed);
}
/// Records the duration of a single transaction execution.
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
pub fn record_transaction_execution(&self, elapsed: Duration) {
self.executor.transaction_execution_histogram.record(elapsed);
}
}
@@ -84,7 +87,7 @@ impl EngineApiMetrics {
/// Metrics for the entire blockchain tree
#[derive(Metrics)]
#[metrics(scope = "blockchain_tree")]
pub(crate) struct TreeMetrics {
pub struct TreeMetrics {
/// The highest block number in the canonical chain
pub canonical_chain_height: Gauge,
/// The number of reorgs
@@ -100,7 +103,7 @@ pub(crate) struct TreeMetrics {
/// Metrics for the `EngineApi`.
#[derive(Metrics)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct EngineMetrics {
pub struct EngineMetrics {
/// Engine API forkchoiceUpdated response type metrics
#[metric(skip)]
pub(crate) forkchoice_updated: ForkchoiceUpdatedMetrics,
@@ -239,6 +242,8 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) new_payload_error: Counter,
/// The total gas of valid new payload messages received.
pub(crate) new_payload_total_gas: Histogram,
/// The gas used for the last valid new payload.
pub(crate) new_payload_total_gas_last: Gauge,
/// The gas per second of valid new payload messages received.
pub(crate) new_payload_gas_per_second: Histogram,
/// The gas per second for the last new payload call.
@@ -251,6 +256,8 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) time_between_new_payloads: Histogram,
/// Time from previous payload start to current payload start (total interval).
pub(crate) new_payload_interval: Histogram,
/// Time diff between forkchoice updated call response and the next new payload call request.
pub(crate) forkchoice_updated_new_payload_time_diff: Histogram,
}
impl NewPayloadStatusMetrics {
@@ -258,6 +265,7 @@ impl NewPayloadStatusMetrics {
pub(crate) fn update_response_metrics(
&mut self,
start: Instant,
latest_forkchoice_updated_at: &mut Option<Instant>,
result: &Result<TreeOutcome<PayloadStatus>, InsertBlockFatalError>,
gas_used: u64,
) {
@@ -277,6 +285,7 @@ impl NewPayloadStatusMetrics {
PayloadStatusEnum::Valid => {
self.new_payload_valid.increment(1);
self.new_payload_total_gas.record(gas_used as f64);
self.new_payload_total_gas_last.set(gas_used as f64);
let gas_per_second = gas_used as f64 / elapsed.as_secs_f64();
self.new_payload_gas_per_second.record(gas_per_second);
self.new_payload_gas_per_second_last.set(gas_per_second);
@@ -290,46 +299,79 @@ impl NewPayloadStatusMetrics {
self.new_payload_messages.increment(1);
self.new_payload_latency.record(elapsed);
self.new_payload_last.set(elapsed);
if let Some(latest_forkchoice_updated_at) = latest_forkchoice_updated_at.take() {
self.forkchoice_updated_new_payload_time_diff
.record(start - latest_forkchoice_updated_at);
}
}
}
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
///
/// See also <https://github.com/ethereum/execution-metrics/issues/5>
#[allow(dead_code)]
#[derive(Metrics, Clone)]
#[metrics(scope = "execution.block_access_list")]
pub(crate) struct BalMetrics {
/// Size of the BAL in bytes for the current block.
pub(crate) size_bytes: Gauge,
/// Total number of blocks with valid BALs.
pub(crate) valid_total: Counter,
/// Total number of blocks with invalid BALs.
pub(crate) invalid_total: Counter,
/// Time taken to validate the BAL against actual execution.
pub(crate) validation_time_seconds: Histogram,
/// Number of account changes in the BAL.
pub(crate) account_changes: Gauge,
/// Number of storage changes in the BAL.
pub(crate) storage_changes: Gauge,
/// Number of balance changes in the BAL.
pub(crate) balance_changes: Gauge,
/// Number of nonce changes in the BAL.
pub(crate) nonce_changes: Gauge,
/// Number of code changes in the BAL.
pub(crate) code_changes: Gauge,
}
/// Metrics for non-execution related block validation.
#[derive(Metrics, Clone)]
#[metrics(scope = "sync.block_validation")]
pub(crate) struct BlockValidationMetrics {
pub struct BlockValidationMetrics {
/// Total number of storage tries updated in the state root calculation
pub(crate) state_root_storage_tries_updated_total: Counter,
pub state_root_storage_tries_updated_total: Counter,
/// Total number of times the parallel state root computation fell back to regular.
pub(crate) state_root_parallel_fallback_total: Counter,
pub state_root_parallel_fallback_total: Counter,
/// Total number of times the state root task failed but the fallback succeeded.
pub state_root_task_fallback_success_total: Counter,
/// Latest state root duration, ie the time spent blocked waiting for the state root.
pub(crate) state_root_duration: Gauge,
pub state_root_duration: Gauge,
/// Histogram for state root duration ie the time spent blocked waiting for the state root
pub(crate) state_root_histogram: Histogram,
pub state_root_histogram: Histogram,
/// Histogram of deferred trie computation duration.
pub(crate) deferred_trie_compute_duration: Histogram,
pub deferred_trie_compute_duration: Histogram,
/// Payload conversion and validation latency
pub(crate) payload_validation_duration: Gauge,
pub payload_validation_duration: Gauge,
/// Histogram of payload validation latency
pub(crate) payload_validation_histogram: Histogram,
pub payload_validation_histogram: Histogram,
/// Payload processor spawning duration
pub(crate) spawn_payload_processor: Histogram,
pub spawn_payload_processor: Histogram,
/// Post-execution validation duration
pub(crate) post_execution_validation_duration: Histogram,
pub post_execution_validation_duration: Histogram,
/// Total duration of the new payload call
pub(crate) total_duration: Histogram,
pub total_duration: Histogram,
/// Size of `HashedPostStateSorted` (`total_len`)
pub(crate) hashed_post_state_size: Histogram,
pub hashed_post_state_size: Histogram,
/// Size of `TrieUpdatesSorted` (`total_len`)
pub(crate) trie_updates_sorted_size: Histogram,
pub trie_updates_sorted_size: Histogram,
/// Size of `AnchoredTrieInput` overlay `TrieUpdatesSorted` (`total_len`)
pub(crate) anchored_overlay_trie_updates_size: Histogram,
pub anchored_overlay_trie_updates_size: Histogram,
/// Size of `AnchoredTrieInput` overlay `HashedPostStateSorted` (`total_len`)
pub(crate) anchored_overlay_hashed_state_size: Histogram,
pub anchored_overlay_hashed_state_size: Histogram,
}
impl BlockValidationMetrics {
/// Records a new state root time, updating both the histogram and state root gauge
pub(crate) fn record_state_root(&self, trie_output: &TrieUpdates, elapsed_as_secs: f64) {
pub fn record_state_root(&self, trie_output: &TrieUpdates, elapsed_as_secs: f64) {
self.state_root_storage_tries_updated_total
.increment(trie_output.storage_tries_ref().len() as u64);
self.state_root_duration.set(elapsed_as_secs);
@@ -338,7 +380,7 @@ impl BlockValidationMetrics {
/// Records a new payload validation time, updating both the histogram and the payload
/// validation gauge
pub(crate) fn record_payload_validation(&self, elapsed_as_secs: f64) {
pub fn record_payload_validation(&self, elapsed_as_secs: f64) {
self.payload_validation_duration.set(elapsed_as_secs);
self.payload_validation_histogram.record(elapsed_as_secs);
}

View File

@@ -3,7 +3,7 @@ use crate::{
chain::FromOrchestrator,
engine::{DownloadRequest, EngineApiEvent, EngineApiKind, EngineApiRequest, FromEngine},
persistence::PersistenceHandle,
tree::{error::InsertPayloadError, metrics::EngineApiMetrics, payload_validator::TreeCtx},
tree::{error::InsertPayloadError, payload_validator::TreeCtx},
};
use alloy_consensus::BlockHeader;
use alloy_eips::{eip1898::BlockWithParent, merge::EPOCH_SLOTS, BlockNumHash, NumHash};
@@ -30,9 +30,10 @@ use reth_payload_primitives::{
};
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
use reth_provider::{
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
StateProviderBox, StateProviderFactory, StateReader, StorageChangeSetReader,
TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
@@ -54,7 +55,7 @@ pub mod error;
pub mod instrumented_state;
mod invalid_headers;
mod metrics;
mod payload_processor;
pub mod payload_processor;
pub mod payload_validator;
mod persistence_state;
pub mod precompile_cache;
@@ -65,7 +66,9 @@ mod trie_updates;
use crate::tree::error::AdvancePersistenceError;
pub use block_buffer::BlockBuffer;
pub use cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache};
pub use invalid_headers::InvalidHeaderCache;
pub use metrics::EngineApiMetrics;
pub use payload_processor::*;
pub use payload_validator::{BasicEngineValidator, EngineValidator};
pub use persistence_state::PersistenceState;
@@ -84,6 +87,12 @@ pub mod state;
/// backfill this gap.
pub(crate) const MIN_BLOCKS_FOR_PIPELINE_RUN: u64 = EPOCH_SLOTS;
/// The minimum number of blocks to retain in the changeset cache after eviction.
///
/// This ensures that recent trie changesets are kept in memory for potential reorgs,
/// even when the finalized block is not set (e.g., on L2s like Optimism).
const CHANGESET_CACHE_RETENTION_BLOCKS: u64 = 64;
/// A builder for creating state providers that can be used across threads.
#[derive(Clone, Debug)]
pub struct StateProviderBuilder<N: NodePrimitives, P> {
@@ -151,6 +160,16 @@ impl<N: NodePrimitives> EngineApiTreeState<N> {
forkchoice_state_tracker: ForkchoiceStateTracker::default(),
}
}
/// Returns a reference to the tree state.
pub const fn tree_state(&self) -> &TreeState<N> {
&self.tree_state
}
/// Returns true if the block has been marked as invalid.
pub fn has_invalid_header(&mut self, hash: &B256) -> bool {
self.invalid_headers.get(hash).is_some()
}
}
/// The outcome of a tree operation.
@@ -314,10 +333,10 @@ where
+ HashedPostStateProvider
+ Clone
+ 'static,
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
P::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ BlockNumReader,
+ StorageChangeSetReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
V: EngineValidator<T>,
@@ -1376,21 +1395,42 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
let min_threshold = last_persisted_block_number.saturating_sub(64);
let eviction_threshold = finalized.number.min(min_threshold);
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = finalized.number,
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
}
// Evict trie changesets for blocks below the eviction threshold.
// Keep at least CHANGESET_CACHE_RETENTION_BLOCKS from the persisted tip, and also respect
// the finalized block if set.
let min_threshold =
last_persisted_block_number.saturating_sub(CHANGESET_CACHE_RETENTION_BLOCKS);
let eviction_threshold =
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
// Use the minimum of finalized block and retention threshold to be conservative
finalized.number.min(min_threshold)
} else {
// When finalized is not set (e.g., on L2s), use the retention threshold
min_threshold
};
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = ?self.canonical_in_memory_state.get_finalized_num_hash().map(|f| f.number),
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
// Invalidate cached overlay since the anchor has changed
self.state.tree_state.invalidate_cached_overlay();
self.on_new_persisted_block()?;
// Re-prepare overlay for the current canonical head with the new anchor.
// Spawn a background task to trigger computation so it's ready when the next payload
// arrives.
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
rayon::spawn(move || {
let _ = overlay.get();
});
}
Ok(())
}
@@ -1476,6 +1516,10 @@ where
self.on_maybe_tree_event(res.event.take())?;
}
if let Err(ref err) = output {
error!(target: "engine::tree", %err, ?state, "Error processing forkchoice update");
}
self.metrics.engine.forkchoice_updated.update_response_metrics(
start,
&mut self.metrics.engine.new_payload.latest_finish_at,
@@ -1498,10 +1542,12 @@ where
let gas_used = payload.gas_used();
let num_hash = payload.num_hash();
let mut output = self.on_new_payload(payload);
self.metrics
.engine
.new_payload
.update_response_metrics(start, &output, gas_used);
self.metrics.engine.new_payload.update_response_metrics(
start,
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
&output,
gas_used,
);
let maybe_event =
output.as_mut().ok().and_then(|out| out.event.take());
@@ -2567,19 +2613,27 @@ where
let block_num_hash = block_id.block;
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
match self.sealed_header_by_hash(block_num_hash.hash) {
Err(err) => {
let block = convert_to_block(self, input)?;
return Err(InsertBlockError::new(block, err.into()).into());
// Check if block already exists - first in memory, then DB only if it could be persisted
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
}
// Only query DB if block could be persisted (number <= last persisted block).
// New blocks from CL always have number > last persisted, so skip DB lookup for them.
if block_num_hash.number <= self.persistence_state.last_persisted_block.number {
match self.provider.sealed_header_by_hash(block_num_hash.hash) {
Err(err) => {
let block = convert_to_block(self, input)?;
return Err(InsertBlockError::new(block, err.into()).into());
}
Ok(Some(_)) => {
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
}
Ok(None) => {}
}
Ok(Some(_)) => {
// We now assume that we already have this block in the tree. However, we need to
// run the conversion to ensure that the block hash is valid.
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid))
}
_ => {}
};
}
// Ensure that the parent state is available.
match self.state_provider_builder(block_id.parent) {

View File

@@ -21,7 +21,7 @@ pub fn total_slots(bal: &BlockAccessList) -> usize {
/// first, followed by read-only slots. The iterator intelligently skips accounts and slots
/// outside the specified range for efficient traversal.
#[derive(Debug)]
pub(crate) struct BALSlotIter<'a> {
pub struct BALSlotIter<'a> {
bal: &'a BlockAccessList,
range: Range<usize>,
current_index: usize,
@@ -34,7 +34,7 @@ pub(crate) struct BALSlotIter<'a> {
impl<'a> BALSlotIter<'a> {
/// Creates a new iterator over storage slots within the specified range.
pub(crate) fn new(bal: &'a BlockAccessList, range: Range<usize>) -> Self {
pub fn new(bal: &'a BlockAccessList, range: Range<usize>) -> Self {
let mut iter = Self { bal, range, current_index: 0, account_idx: 0, slot_idx: 0 };
iter.skip_to_range_start();
iter

View File

@@ -1,188 +0,0 @@
//! Configured sparse trie enum for switching between serial and parallel implementations.
use alloy_primitives::B256;
use reth_trie::{BranchNodeMasks, Nibbles, ProofTrieNode, TrieNode};
use reth_trie_sparse::{
errors::SparseTrieResult, provider::TrieNodeProvider, LeafLookup, LeafLookupError,
SerialSparseTrie, SparseTrieInterface, SparseTrieUpdates,
};
use reth_trie_sparse_parallel::ParallelSparseTrie;
use std::borrow::Cow;
/// Enum for switching between serial and parallel sparse trie implementations.
///
/// This type allows runtime selection between different sparse trie implementations,
/// providing flexibility in choosing the appropriate implementation based on workload
/// characteristics.
#[derive(Debug, Clone)]
pub(crate) enum ConfiguredSparseTrie {
/// Serial implementation of the sparse trie.
Serial(Box<SerialSparseTrie>),
/// Parallel implementation of the sparse trie.
Parallel(Box<ParallelSparseTrie>),
}
impl From<SerialSparseTrie> for ConfiguredSparseTrie {
fn from(trie: SerialSparseTrie) -> Self {
Self::Serial(Box::new(trie))
}
}
impl From<ParallelSparseTrie> for ConfiguredSparseTrie {
fn from(trie: ParallelSparseTrie) -> Self {
Self::Parallel(Box::new(trie))
}
}
impl Default for ConfiguredSparseTrie {
fn default() -> Self {
Self::Serial(Default::default())
}
}
impl SparseTrieInterface for ConfiguredSparseTrie {
fn with_root(
self,
root: TrieNode,
masks: Option<BranchNodeMasks>,
retain_updates: bool,
) -> SparseTrieResult<Self> {
match self {
Self::Serial(trie) => {
trie.with_root(root, masks, retain_updates).map(|t| Self::Serial(Box::new(t)))
}
Self::Parallel(trie) => {
trie.with_root(root, masks, retain_updates).map(|t| Self::Parallel(Box::new(t)))
}
}
}
fn with_updates(self, retain_updates: bool) -> Self {
match self {
Self::Serial(trie) => Self::Serial(Box::new(trie.with_updates(retain_updates))),
Self::Parallel(trie) => Self::Parallel(Box::new(trie.with_updates(retain_updates))),
}
}
fn reserve_nodes(&mut self, additional: usize) {
match self {
Self::Serial(trie) => trie.reserve_nodes(additional),
Self::Parallel(trie) => trie.reserve_nodes(additional),
}
}
fn reveal_node(
&mut self,
path: Nibbles,
node: TrieNode,
masks: Option<BranchNodeMasks>,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_node(path, node, masks),
Self::Parallel(trie) => trie.reveal_node(path, node, masks),
}
}
fn reveal_nodes(&mut self, nodes: Vec<ProofTrieNode>) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_nodes(nodes),
Self::Parallel(trie) => trie.reveal_nodes(nodes),
}
}
fn update_leaf<P: TrieNodeProvider>(
&mut self,
full_path: Nibbles,
value: Vec<u8>,
provider: P,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.update_leaf(full_path, value, provider),
Self::Parallel(trie) => trie.update_leaf(full_path, value, provider),
}
}
fn remove_leaf<P: TrieNodeProvider>(
&mut self,
full_path: &Nibbles,
provider: P,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.remove_leaf(full_path, provider),
Self::Parallel(trie) => trie.remove_leaf(full_path, provider),
}
}
fn root(&mut self) -> B256 {
match self {
Self::Serial(trie) => trie.root(),
Self::Parallel(trie) => trie.root(),
}
}
fn update_subtrie_hashes(&mut self) {
match self {
Self::Serial(trie) => trie.update_subtrie_hashes(),
Self::Parallel(trie) => trie.update_subtrie_hashes(),
}
}
fn get_leaf_value(&self, full_path: &Nibbles) -> Option<&Vec<u8>> {
match self {
Self::Serial(trie) => trie.get_leaf_value(full_path),
Self::Parallel(trie) => trie.get_leaf_value(full_path),
}
}
fn find_leaf(
&self,
full_path: &Nibbles,
expected_value: Option<&Vec<u8>>,
) -> Result<LeafLookup, LeafLookupError> {
match self {
Self::Serial(trie) => trie.find_leaf(full_path, expected_value),
Self::Parallel(trie) => trie.find_leaf(full_path, expected_value),
}
}
fn take_updates(&mut self) -> SparseTrieUpdates {
match self {
Self::Serial(trie) => trie.take_updates(),
Self::Parallel(trie) => trie.take_updates(),
}
}
fn wipe(&mut self) {
match self {
Self::Serial(trie) => trie.wipe(),
Self::Parallel(trie) => trie.wipe(),
}
}
fn clear(&mut self) {
match self {
Self::Serial(trie) => trie.clear(),
Self::Parallel(trie) => trie.clear(),
}
}
fn updates_ref(&self) -> Cow<'_, SparseTrieUpdates> {
match self {
Self::Serial(trie) => trie.updates_ref(),
Self::Parallel(trie) => trie.updates_ref(),
}
}
fn shrink_nodes_to(&mut self, size: usize) {
match self {
Self::Serial(trie) => trie.shrink_nodes_to(size),
Self::Parallel(trie) => trie.shrink_nodes_to(size),
}
}
fn shrink_values_to(&mut self, size: usize) {
match self {
Self::Serial(trie) => trie.shrink_values_to(size),
Self::Parallel(trie) => trie.shrink_values_to(size),
}
}
}

View File

@@ -2,22 +2,19 @@
use super::precompile_cache::PrecompileCacheMap;
use crate::tree::{
cached_state::{
CachedStateMetrics, CachedStateProvider, ExecutionCache as StateExecutionCache,
ExecutionCacheBuilder, SavedCache,
},
cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache},
payload_processor::{
prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
sparse_trie::StateRootComputeOutcome,
},
sparse_trie::SparseTrieTask,
sparse_trie::{SparseTrieCacheTask, SparseTrieTask, SpawnedSparseTrieTask},
StateProviderBuilder, TreeConfig,
};
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip1898::BlockWithParent;
use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_evm::block::StateChangeSource;
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
@@ -25,6 +22,7 @@ use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
use rayon::prelude::*;
use reth_evm::{
block::ExecutableTxParts,
execute::{ExecutableTxFor, WithTxEnv},
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
@@ -41,10 +39,7 @@ use reth_trie_parallel::{
proof_task::{ProofTaskCtx, ProofWorkerHandle},
root::ParallelStateRootError,
};
use reth_trie_sparse::{
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, SparseStateTrie, SparseTrie,
};
use reth_trie_sparse::{RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::{
collections::BTreeMap,
@@ -59,14 +54,14 @@ use std::{
use tracing::{debug, debug_span, instrument, warn, Span};
pub mod bal;
mod configured_sparse_trie;
pub mod executor;
pub mod multiproof;
mod preserved_sparse_trie;
pub mod prewarm;
pub mod receipt_root_task;
pub mod sparse_trie;
use configured_sparse_trie::ConfiguredSparseTrie;
use preserved_sparse_trie::{PreservedSparseTrie, SharedPreservedSparseTrie};
/// Default parallelism thresholds to use with the [`ParallelSparseTrie`].
///
@@ -101,7 +96,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxTuple>::Tx>,
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
<I as ExecutableTxTuple>::Error,
<N as NodePrimitives>::Receipt,
>;
@@ -115,11 +110,11 @@ where
/// The executor used by to spawn tasks.
executor: WorkloadExecutor,
/// The most recent cache used for execution.
execution_cache: ExecutionCache,
execution_cache: PayloadExecutionCache,
/// Metrics for trie operations
trie_metrics: MultiProofTaskMetrics,
/// Cross-block cache size in bytes.
cross_block_cache_size: u64,
cross_block_cache_size: usize,
/// Whether transactions should not be executed on prewarming task.
disable_transaction_prewarming: bool,
/// Whether state cache should be disable
@@ -130,17 +125,16 @@ where
precompile_cache_disabled: bool,
/// Precompile cache map.
precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
/// A cleared `SparseStateTrie`, kept around to be reused for the state root computation so
/// that allocations can be minimized.
sparse_state_trie: Arc<
parking_lot::Mutex<
Option<ClearedSparseStateTrie<ConfiguredSparseTrie, ConfiguredSparseTrie>>,
>,
>,
/// Whether to disable the parallel sparse trie.
disable_parallel_sparse_trie: bool,
/// A pruned `SparseStateTrie`, kept around as a cache of already revealed trie nodes and to
/// re-use allocated memory. Stored with the block hash it was computed for to enable trie
/// preservation across sequential payload validations.
sparse_state_trie: SharedPreservedSparseTrie,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Sparse trie prune depth.
sparse_trie_prune_depth: usize,
/// Maximum storage tries to retain after pruning.
sparse_trie_max_storage_tries: usize,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
@@ -151,7 +145,7 @@ where
Evm: ConfigureEvm<Primitives = N>,
{
/// Returns a reference to the workload executor driving payload tasks.
pub(super) const fn executor(&self) -> &WorkloadExecutor {
pub const fn executor(&self) -> &WorkloadExecutor {
&self.executor
}
@@ -172,9 +166,10 @@ where
disable_state_cache: config.disable_state_cache(),
precompile_cache_disabled: config.precompile_cache_disabled(),
precompile_cache_map,
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
sparse_state_trie: SharedPreservedSparseTrie::default(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
sparse_trie_prune_depth: config.sparse_trie_prune_depth(),
sparse_trie_max_storage_tries: config.sparse_trie_max_storage_tries(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
@@ -240,13 +235,18 @@ where
+ 'static,
{
// start preparing transactions immediately
let (prewarm_rx, execution_rx, transaction_count_hint) =
self.spawn_tx_iterator(transactions);
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
let span = Span::current();
let (to_sparse_trie, sparse_trie_rx) = channel();
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
// Extract V2 proofs flag early so we can pass it to prewarm
let v2_proofs_enabled = !config.disable_proof_v2();
// Capture parent_state_root before env is moved into spawn_caching_with
let parent_state_root = env.parent_state_root;
// Handle BAL-based optimization if available
let prewarm_handle = if let Some(bal) = bal {
// When BAL is present, use BAL prewarming and send BAL to multiproof
@@ -259,20 +259,20 @@ where
self.spawn_caching_with(
env,
prewarm_rx,
transaction_count_hint,
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
v2_proofs_enabled,
)
} else {
// Normal path: spawn with transaction prewarming
self.spawn_caching_with(
env,
prewarm_rx,
transaction_count_hint,
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
v2_proofs_enabled,
)
};
@@ -280,7 +280,6 @@ where
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let v2_proofs_enabled = config.enable_proof_v2();
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
task_ctx,
@@ -289,36 +288,46 @@ where
v2_proofs_enabled,
);
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof,
);
if !config.enable_sparse_trie_as_cache() {
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof.clone(),
from_multi_proof.clone(),
)
.with_v2_proofs_enabled(v2_proofs_enabled);
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
self.executor.spawn_blocking(move || {
let _enter = parent_span.entered();
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
Box::new(provider)
};
multi_proof_task.run(provider);
});
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
self.executor.spawn_blocking(move || {
let _enter = parent_span.entered();
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics, _disable_metrics) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
Box::new(provider)
};
multi_proof_task.run(provider);
});
}
// wire the sparse trie to the state root response receiver
let (state_root_tx, state_root_rx) = channel();
// Spawn the sparse trie task using any stored trie and parallel trie configuration.
self.spawn_sparse_trie_task(sparse_trie_rx, proof_handle, state_root_tx);
self.spawn_sparse_trie_task(
sparse_trie_rx,
proof_handle,
state_root_tx,
from_multi_proof,
config,
parent_state_root,
);
PayloadHandle {
to_multi_proof: Some(to_multi_proof),
@@ -333,7 +342,7 @@ where
///
/// Returns a [`PayloadHandle`] to communicate with the task.
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
pub(super) fn spawn_cache_exclusive<P, I: ExecutableTxIterator<Evm>>(
pub fn spawn_cache_exclusive<P, I: ExecutableTxIterator<Evm>>(
&self,
env: ExecutionEnv<Evm>,
transactions: I,
@@ -343,9 +352,10 @@ where
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal);
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -361,23 +371,22 @@ where
&self,
transactions: I,
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Tx>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Tx>, I::Error>>,
usize,
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
) {
let (transactions, convert) = transactions.into();
let transactions = transactions.into_par_iter();
let transaction_count_hint = transactions.len();
let (ooo_tx, ooo_rx) = mpsc::channel();
let (prewarm_tx, prewarm_rx) = mpsc::channel();
let (execute_tx, execute_rx) = mpsc::channel();
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
rayon::spawn(move || {
let (transactions, convert) = transactions.into();
transactions.into_par_iter().enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| WithTxEnv { tx_env: tx.to_tx_env(), tx: Arc::new(tx) });
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
WithTxEnv { tx_env, tx: Arc::new(tx) }
});
// Only send Ok(_) variants to prewarming task.
if let Ok(tx) = &tx {
let _ = prewarm_tx.send(tx.clone());
@@ -408,7 +417,7 @@ where
}
});
(prewarm_rx, execute_rx, transaction_count_hint)
(prewarm_rx, execute_rx)
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
@@ -416,10 +425,10 @@ where
&self,
env: ExecutionEnv<Evm>,
mut transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
transaction_count_hint: usize,
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
v2_proofs_enabled: bool,
) -> CacheTaskHandle<N::Receipt>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -442,6 +451,7 @@ where
terminate_execution: Arc::new(AtomicBool::new(false)),
precompile_cache_disabled: self.precompile_cache_disabled,
precompile_cache_map: self.precompile_cache_map.clone(),
v2_proofs_enabled,
};
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
@@ -449,7 +459,6 @@ where
self.execution_cache.clone(),
prewarm_ctx,
to_multi_proof,
transaction_count_hint,
self.prewarm_max_concurrency,
);
@@ -480,75 +489,131 @@ where
cache
} else {
debug!("creating new execution cache on cache miss");
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
let start = Instant::now();
let cache = ExecutionCache::new(self.cross_block_cache_size);
let metrics = CachedStateMetrics::zeroed();
metrics.record_cache_creation(start.elapsed());
SavedCache::new(parent_hash, cache, metrics)
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
/// Spawns the [`SparseTrieTask`] for this payload processor.
///
/// The trie is preserved when the new payload is a child of the previous one.
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
fn spawn_sparse_trie_task<BPF>(
fn spawn_sparse_trie_task(
&self,
sparse_trie_rx: mpsc::Receiver<SparseTrieUpdate>,
proof_worker_handle: BPF,
proof_worker_handle: ProofWorkerHandle,
state_root_tx: mpsc::Sender<Result<StateRootComputeOutcome, ParallelStateRootError>>,
) where
BPF: TrieNodeProviderFactory + Clone + Send + Sync + 'static,
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
{
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
let disable_parallel_sparse_trie = self.disable_parallel_sparse_trie;
from_multi_proof: CrossbeamReceiver<MultiProofMessage>,
config: &TreeConfig,
parent_state_root: B256,
) {
let preserved_sparse_trie = self.sparse_state_trie.clone();
let trie_metrics = self.trie_metrics.clone();
let span = Span::current();
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
let prune_depth = self.sparse_trie_prune_depth;
let max_storage_tries = self.sparse_trie_max_storage_tries;
self.executor.spawn_blocking(move || {
let _enter = span.entered();
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration
// if there's none to reuse.
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
let default_trie = SparseTrie::blind_from(if disable_parallel_sparse_trie {
ConfiguredSparseTrie::Serial(Default::default())
} else {
ConfiguredSparseTrie::Parallel(Box::new(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
))
});
ClearedSparseStateTrie::from_state_trie(
// Reuse a stored SparseStateTrie if available, applying continuation logic.
// If this payload's parent state root matches the preserved trie's anchor,
// we can reuse the pruned trie structure. Otherwise, we clear the trie but
// keep allocations.
let sparse_state_trie = preserved_sparse_trie
.take()
.map(|preserved| preserved.into_trie_for(parent_state_root))
.unwrap_or_else(|| {
debug!(
target: "engine::tree::payload_processor",
"Creating new sparse trie - no preserved trie available"
);
let default_trie = RevealableSparseTrie::blind_from(
ParallelSparseTrie::default().with_parallelism_thresholds(
PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS,
),
);
SparseStateTrie::new()
.with_accounts_trie(default_trie.clone())
.with_default_storage_trie(default_trie)
.with_updates(true),
)
});
.with_updates(true)
});
let task = SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
);
let mut task = if disable_sparse_trie_as_cache {
SpawnedSparseTrieTask::Cleared(SparseTrieTask::new(
sparse_trie_rx,
proof_worker_handle,
trie_metrics.clone(),
sparse_state_trie,
))
} else {
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
from_multi_proof,
proof_worker_handle,
trie_metrics.clone(),
sparse_state_trie,
))
};
let (result, trie) = task.run();
// Send state root computation result
let _ = state_root_tx.send(result);
let result = task.run();
// Capture the computed state_root before sending the result
let computed_state_root = result.as_ref().ok().map(|outcome| outcome.state_root);
// Clear the SparseStateTrie, shrink, and replace it back into the mutex _after_ sending
// results to the next step, so that time spent clearing doesn't block the step after
// this one.
let _enter = debug_span!(target: "engine::tree::payload_processor", "clear").entered();
let mut cleared_trie = ClearedSparseStateTrie::from_state_trie(trie);
// Acquire the guard before sending the result to prevent a race condition:
// Without this, the next block could start after send() but before store(),
// causing take() to return None and forcing it to create a new empty trie
// instead of reusing the preserved one. Holding the guard ensures the next
// block's take() blocks until we've stored the trie for reuse.
let mut guard = preserved_sparse_trie.lock();
// Shrink the sparse trie so that we don't have ever increasing memory.
cleared_trie.shrink_to(
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
);
// Send state root computation result - next block may start but will block on take()
if state_root_tx.send(result).is_err() {
// Receiver dropped - payload was likely invalid or cancelled.
// Clear the trie instead of preserving potentially invalid state.
debug!(
target: "engine::tree::payload_processor",
"State root receiver dropped, clearing trie"
);
let trie = task.into_cleared_trie(
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
);
guard.store(PreservedSparseTrie::cleared(trie));
return;
}
cleared_sparse_trie.lock().replace(cleared_trie);
// Only preserve the trie as anchored if computation succeeded.
// A failed computation may have left the trie in a partially updated state.
let _enter =
debug_span!(target: "engine::tree::payload_processor", "preserve").entered();
if let Some(state_root) = computed_state_root {
let start = std::time::Instant::now();
let trie = task.into_trie_for_reuse(
prune_depth,
max_storage_tries,
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
);
trie_metrics
.into_trie_for_reuse_duration_histogram
.record(start.elapsed().as_secs_f64());
guard.store(PreservedSparseTrie::anchored(trie, state_root));
} else {
debug!(
target: "engine::tree::payload_processor",
"State root computation failed, clearing trie"
);
let trie = task.into_cleared_trie(
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
);
guard.store(PreservedSparseTrie::cleared(trie));
}
});
}
@@ -559,7 +624,7 @@ where
///
/// The cache enables subsequent blocks to reuse account, storage, and bytecode data without
/// hitting the database, maintaining performance consistency.
pub(crate) fn on_inserted_executed_block(
pub fn on_inserted_executed_block(
&self,
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
@@ -572,28 +637,27 @@ where
parent_hash = %block_with_parent.parent,
"Cannot find cache for parent hash, skip updating cache with new state for inserted executed block",
);
return;
return
}
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
let (c, m, _) = existing.split();
(c, m)
}
let (caches, cache_metrics, _) = match cached.take() {
Some(existing) => existing.split(),
None => (
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
ExecutionCache::new(self.cross_block_cache_size),
CachedStateMetrics::zeroed(),
false,
),
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
let new_cache =
SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
return;
return
}
new_cache.update_metrics();
@@ -657,19 +721,19 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
}
/// Returns a clone of the caches used by prewarming
pub(super) fn caches(&self) -> Option<StateExecutionCache> {
pub fn caches(&self) -> Option<ExecutionCache> {
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.cache().clone())
}
/// Returns a clone of the cache metrics used by prewarming
pub(super) fn cache_metrics(&self) -> Option<CachedStateMetrics> {
pub fn cache_metrics(&self) -> Option<CachedStateMetrics> {
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.metrics().clone())
}
/// Terminates the pre-warming transaction processing.
///
/// Note: This does not terminate the task yet.
pub(super) fn stop_prewarming_execution(&self) {
pub fn stop_prewarming_execution(&self) {
self.prewarm_handle.stop_prewarming_execution()
}
@@ -680,7 +744,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
/// path without cloning the expensive `BundleState`.
///
/// Returns a sender for the channel that should be notified on block validation success.
pub(super) fn terminate_caching(
pub fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
@@ -700,7 +764,7 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the
/// prewarm task without cloning the expensive `BundleState`.
#[derive(Debug)]
pub(crate) struct CacheTaskHandle<R> {
pub struct CacheTaskHandle<R> {
/// The shared cache the task operates with.
saved_cache: Option<SavedCache>,
/// Channel to the spawned prewarm task if any
@@ -711,7 +775,7 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
/// Terminates the pre-warming transaction processing.
///
/// Note: This does not terminate the task yet.
pub(super) fn stop_prewarming_execution(&self) {
pub fn stop_prewarming_execution(&self) {
self.to_prewarm_task
.as_ref()
.map(|tx| tx.send(PrewarmTaskEvent::TerminateTransactionExecution).ok());
@@ -722,7 +786,7 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
#[must_use = "sender must be used and notified on block validation success"]
pub(super) fn terminate_caching(
pub fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
@@ -761,29 +825,29 @@ impl<R> Drop for CacheTaskHandle<R> {
/// ## Cache Safety
///
/// **CRITICAL**: Cache update operations require exclusive access. All concurrent cache users
/// (such as prewarming tasks) must be terminated before calling `update_with_guard`, otherwise
/// the cache may be corrupted or cleared.
/// (such as prewarming tasks) must be terminated before calling
/// [`PayloadExecutionCache::update_with_guard`], otherwise the cache may be corrupted or cleared.
///
/// ## Cache vs Prewarming Distinction
///
/// **`ExecutionCache`**:
/// **[`PayloadExecutionCache`]**:
/// - Stores parent block's execution state after completion
/// - Used to fetch parent data for next block's execution
/// - Must be exclusively accessed during save operations
///
/// **`PrewarmCacheTask`**:
/// **[`PrewarmCacheTask`]**:
/// - Speculatively loads accounts/storage that might be used in transaction execution
/// - Prepares data for state root proof computation
/// - Runs concurrently but must not interfere with cache saves
#[derive(Clone, Debug, Default)]
struct ExecutionCache {
pub struct PayloadExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl ExecutionCache {
impl PayloadExecutionCache {
/// Returns the cache for `parent_hash` if it's available for use.
///
/// A cache is considered available when:
@@ -819,11 +883,15 @@ impl ExecutionCache {
"Existing cache found"
);
if hash_matches && available {
return Some(c.clone());
}
if hash_matches && !available {
if available {
// If the has is available (no other threads are using it), but has a mismatching
// parent hash, we can just clear it and keep using without re-creating from
// scratch.
if !hash_matches {
c.clear();
}
return Some(c.clone())
} else if hash_matches {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
@@ -852,7 +920,7 @@ impl ExecutionCache {
///
/// Violating this requirement can result in cache corruption, incorrect state data,
/// and potential consensus failures.
pub(crate) fn update_with_guard<F>(&self, update_fn: F)
pub fn update_with_guard<F>(&self, update_fn: F)
where
F: FnOnce(&mut Option<SavedCache>),
{
@@ -879,6 +947,14 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
pub hash: B256,
/// Hash of the parent block.
pub parent_hash: B256,
/// State root of the parent block.
/// Used for sparse trie continuation: if the preserved trie's anchor matches this,
/// the trie can be reused directly.
pub parent_state_root: B256,
/// Number of transactions in the block.
/// Used to determine parallel worker count for prewarming.
/// A value of 0 indicates the count is unknown.
pub transaction_count: usize,
}
impl<Evm: ConfigureEvm> Default for ExecutionEnv<Evm>
@@ -890,15 +966,17 @@ where
evm_env: Default::default(),
hash: Default::default(),
parent_hash: Default::default(),
parent_state_root: Default::default(),
transaction_count: 0,
}
}
}
#[cfg(test)]
mod tests {
use super::ExecutionCache;
use super::PayloadExecutionCache;
use crate::tree::{
cached_state::{CachedStateMetrics, ExecutionCacheBuilder, SavedCache},
cached_state::{CachedStateMetrics, ExecutionCache, SavedCache},
payload_processor::{
evm_state_to_hashed_post_state, executor::WorkloadExecutor, PayloadProcessor,
},
@@ -928,13 +1006,13 @@ mod tests {
use std::sync::Arc;
fn make_saved_cache(hash: B256) -> SavedCache {
let execution_cache = ExecutionCacheBuilder::default().build_caches(1_000);
let execution_cache = ExecutionCache::new(1_000);
SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed())
}
#[test]
fn execution_cache_allows_single_checkout() {
let execution_cache = ExecutionCache::default();
let execution_cache = PayloadExecutionCache::default();
let hash = B256::from([1u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
@@ -953,7 +1031,7 @@ mod tests {
#[test]
fn execution_cache_checkout_releases_on_drop() {
let execution_cache = ExecutionCache::default();
let execution_cache = PayloadExecutionCache::default();
let hash = B256::from([2u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
@@ -969,19 +1047,21 @@ mod tests {
}
#[test]
fn execution_cache_mismatch_parent_returns_none() {
let execution_cache = ExecutionCache::default();
fn execution_cache_mismatch_parent_clears_and_returns() {
let execution_cache = PayloadExecutionCache::default();
let hash = B256::from([3u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
let miss = execution_cache.get_cache_for(B256::from([4u8; 32]));
assert!(miss.is_none(), "checkout should fail for different parent hash");
// When the parent hash doesn't match, the cache is cleared and returned for reuse
let different_hash = B256::from([4u8; 32]);
let cache = execution_cache.get_cache_for(different_hash);
assert!(cache.is_some(), "cache should be returned for reuse after clearing")
}
#[test]
fn execution_cache_update_after_release_succeeds() {
let execution_cache = ExecutionCache::default();
let execution_cache = PayloadExecutionCache::default();
let initial = B256::from([5u8; 32]);
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(initial)));

Some files were not shown because too many files have changed in this diff Show More