Compare commits

..

341 Commits

Author SHA1 Message Date
Georgios Konstantopoulos
efafd4666d feat(libmdbx): add parallel write preparation module
Introduces a new `parallel` module for preparing MDBX writes across
multiple tables in parallel, then applying them atomically through a
single transaction.

This addresses the I/O bottleneck in MDBX by parallelizing the CPU-bound
data preparation phase while keeping the I/O-bound write phase sequential.

Key components:
- `WriteSet`: Collects write operations for a single table
- `ParallelWriteBatch`: Aggregates multiple WriteSets for atomic apply
- `ParallelWriteBuilder`: Convenience builder for parallel workflows

Usage pattern:
1. Spawn threads to prepare WriteSets for each table in parallel
2. Collect all WriteSets into a ParallelWriteBatch
3. Call batch.apply(env) for single-transaction atomic commit

Includes comprehensive tests demonstrating parallel preparation with
30k ops across 3 tables completing in ~28ms.

Amp-Thread-ID: https://ampcode.com/threads/T-019c2449-0935-761c-8ff6-600d8a37c754
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 16:58:57 +00:00
DaniPopes
3722071a7c chore(deps): bump bytes 1.11.1 (#21755) 2026-02-03 16:31:22 +00:00
DaniPopes
6273530501 perf: use alloy_primitives hasher for dashmaps (#21726)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 15:05:44 +00:00
Alexey Shekhirin
ce29101277 chore(static-files): proper segment writer scoped thread names (#21747) 2026-02-03 14:44:03 +00:00
John Chase
b1b95f9825 fix(discv5): add missing rand feature for test compilation (#21749) 2026-02-03 14:37:39 +00:00
YK
7f970e136a refactor(stages): use with_rocksdb_batch_auto_commit in tx_lookup (#21722) 2026-02-03 14:35:07 +00:00
YK
6b7cc00289 refactor(rocksdb): deduplicate first()/last() implementations (#21738)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:33:44 +00:00
YK
786140a99d perf(static-file): simplify stage checkpoint lookup to avoid allocs (#21730) 2026-02-03 14:32:43 +00:00
YK
ffcb486388 refactor(rocksdb): deduplicate iterator next() implementations (#21737) 2026-02-03 14:31:05 +00:00
YK
59d68f92c4 perf(static-file): hoist cursor creation outside block loop (#21731) 2026-02-03 14:29:07 +00:00
Matthias Seitz
0e0271a612 chore(deps): bump alloy 1.5.2 -> 1.6.1 (#21746)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 14:16:50 +00:00
Minhyuk Kim
df12fee965 feat(txpool): add is_transaction_ready to TransactionPool trait (#21742)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:13:52 +00:00
DaniPopes
11a4f65624 chore: misc tree cleanups (#21691)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 13:34:19 +00:00
Matthias Seitz
a782e1a18a chore: disable changelog workflow on PRs (#21748)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:12:43 +01:00
DaniPopes
2dc76f9abe chore: match statement order in ExecutionCache::new (#21712)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 12:47:15 +00:00
Nicolas SSS
65100971e5 fix(evm): remove unused reth-ethereum-forks (#21695)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 12:33:44 +00:00
Georgios Konstantopoulos
8e21afa9cc feat(trie): add memory_size heuristic for ParallelSparseTrie (#21745)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-03 12:29:57 +00:00
DaniPopes
46a9b9ad3d perf: replace RwLock<HashMap/HashSet> with DashMap/DashSet (#21692)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-03 13:31:05 +01:00
Georgios Konstantopoulos
3f77af4f98 feat: add AI-assisted changelog generation (#21743)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmail.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 12:19:49 +00:00
Arsenii Kulikov
79cabbf89c perf: optimize SparseTrieCacheTask (#21704) 2026-02-03 11:39:10 +00:00
drhgencer
e04afe6e0e fix(rpc): validate toBlock in trace_filter (#21718) 2026-02-03 11:02:57 +00:00
Arsenii Kulikov
ee224fe20f fix: update sparse trie masks (#21716) 2026-02-03 12:01:58 +01:00
DaniPopes
972f23745e chore: remove clone from in memory cursor (#21719) 2026-02-03 04:04:33 +00:00
Dan Cline
49f60822f7 chore: move TransactionLookup as first option (#21721) 2026-02-03 02:30:13 +00:00
Georgios Konstantopoulos
47ebc79c85 feat(rpc): add EIP-7928 eth_getBalanceWithProof and eth_getAccountWithProof (#21720)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 01:12:04 +00:00
Arsenii Kulikov
53f922927a feat: reintroduce --engine.state-root-task-compare-updates (#21717) 2026-02-02 23:48:54 +00:00
Dan Cline
f1f3980d29 fix(cli): actually enable reth-prune rocksdb feature in cli (#21715) 2026-02-02 23:39:04 +00:00
Dan Cline
6946f26d77 fix(cli): delete all static files when PruneModes::Full is configured (#21647) 2026-02-02 17:30:21 +00:00
Arsenii Kulikov
f663d1d110 fix: properly drain pending account updates (#21709) 2026-02-02 17:29:43 +00:00
Huber
f4943abf73 chore(ci): add consts to typos allowlist (#21708) 2026-02-02 17:02:16 +00:00
Matthias Seitz
102a6944ba perf(trie): avoid clearing already-cached sparse trie (#21702)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 13:03:07 +00:00
Alexey Shekhirin
1592e51d34 feat(engine): add CLI args for sparse trie pruning configuration (#21703) 2026-02-02 12:52:31 +00:00
Arsenii Kulikov
4280ccf470 fix: short-circuit in reveal_account_v2_proof_nodes on empty nodes (#21701) 2026-02-02 12:18:45 +00:00
Alexey Shekhirin
05ab98107c fix(reth-bench): gracefully stop when transaction source exhausted (#21700) 2026-02-02 11:10:58 +00:00
Brian Picciano
49128ed28f fix(trie): Return full_key from update_leaves unless it is not a child of the missing path (#21699)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-02-02 11:07:56 +00:00
Huber
f74e594292 perf(trie): dispatch V2 storage proofs in lexicographical order (#21684) 2026-02-02 09:31:47 +00:00
Georgios Konstantopoulos
e7d4a05e36 perf(trie): fix allocation hot paths with capacity hints and buffer reuse (#21466)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-02 06:58:45 +00:00
Georgios Konstantopoulos
9382a4c713 fix(prune): use batched pruning loop with edge feature to prevent OOM (#21649)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 02:38:00 +00:00
DaniPopes
28409558f9 perf: add ParallelBridgeBuffered trait to replace par_bridge (#21674) 2026-02-02 00:58:43 +00:00
DaniPopes
5ef32726db refactor: add with_* compressor utility methods (#21680) 2026-02-01 20:43:25 +00:00
Snezhkko
60c3bef1e8 fix(zstd): use transaction dictionary for tx compressor (#21382) 2026-02-01 20:12:51 +00:00
iPLAY888
af96eeae56 refactor(provider): deduplicate segment-to-stage mapping in static file manager (#21670) 2026-02-01 20:09:32 +00:00
Georgios Konstantopoulos
5528aae8f6 fix(engine): wait for persistence service thread before RocksDB drop (#21640)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 19:55:45 +00:00
Georgios Konstantopoulos
83364aa2d6 fix(prune): migrate invalid receipts prune config to Distance(64) (#21677)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 19:44:14 +00:00
DaniPopes
749a742bcf chore(deps): update metrics-derive 0.1.1 (#21673) 2026-02-01 19:38:38 +00:00
ethfanWilliam
2970624413 chore: avoid eager evaluation in base_fee_params_at_timestamp (#21536) 2026-02-01 19:04:42 +00:00
Matthias Seitz
7e18aa4be8 fix(rpc): change debug_set_head number parameter to U64 (#21678)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 18:59:22 +00:00
YK
9f8c22e2c3 feat(prune): prune rocksdb account and storage history indices (#21331)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 18:42:17 +00:00
Georgios Konstantopoulos
3d699ac9c6 perf(trie): reuse account RLP buffer in SparseTrieCacheTask (#21644)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 15:20:11 +00:00
かりんとう
9be31d504d fix(trie): silence unused param warnings in sparse-parallel no_std build (#21657) 2026-02-01 13:05:39 +00:00
github-actions[bot]
34cc65cfe6 chore(deps): weekly cargo update (#21660)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-02-01 13:03:13 +00:00
Matthias Seitz
6e161f0fc9 perf: batch finalized/safe block commits with SaveBlocks (#21663)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 13:02:59 +00:00
iPLAY888
63a3e18404 fix: remove unnecessary alloc (#21665) 2026-02-01 13:01:11 +00:00
Matthias Seitz
7d10e791b2 refactor(engine): improve payload processor tx iterator (#21658)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:44:10 +00:00
Georgios Konstantopoulos
a9b2c1d454 feat(rpc): make blob sidecar upcasting opt-in (#21624)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:25:46 +00:00
CPerezz
9127563914 fix: cleanup entire temp directory when using testing_node (#18399)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 16:46:11 +00:00
Georgios Konstantopoulos
a500fb22ba fix(metrics): rename save_blocks_block_count to save_blocks_batch_size (#21654)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 12:59:09 +00:00
Matthias Seitz
e869cd4670 perf(engine): skip DB lookup for new blocks in insert_block_or_payload (#21650)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 03:35:20 +00:00
DaniPopes
de69654b73 chore(deps): breaking bumps (#21584)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 00:44:09 +00:00
DaniPopes
8d28c4c8f2 chore(trie): add set_* methods alongside with_* builders (#21639)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:42:57 +00:00
Georgios Konstantopoulos
bfe778ab51 perf(trie): use Entry API to avoid empty Vec allocation in extend (#21645)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:29:21 +00:00
DaniPopes
e523a76fb8 chore(trie): clear RevealableSparseTrie in place (#21638)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:27:43 +00:00
DaniPopes
cd12ae58f2 docs(CLAUDE.md): tweaks (#21646) 2026-01-30 22:26:34 +00:00
Georgios Konstantopoulos
370a548f34 refactor(db): derive Clone for DatabaseEnv (#21641)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 21:54:50 +00:00
pepes
781128eece chore(db-api): simplify DatabaseMetrics impl for Arc (#21635) 2026-01-30 18:43:19 +00:00
Julian Meyer
435d915422 chore: make engine tree crate items public (#21487) 2026-01-30 18:40:30 +00:00
Georgios Konstantopoulos
3ec065295e refactor(trie): replace SmallVec with Vec in sparse trie buffers (#21637)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-01-30 18:34:15 +00:00
Matthias Seitz
e1bc6d0f08 feat(engine): preserve sparse trie across payload validations (#21534)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-30 18:34:13 +00:00
Georgios Konstantopoulos
29072639d6 perf(trie): remove shrink_to_fit calls from SparseSubtrieBuffers::clear (#21630)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-01-30 18:02:43 +00:00
Brian Picciano
f90b5c8a7f fix(trie): cleanup modified branch masks in update_leaf on reveal failure (#21629) 2026-01-30 16:06:28 +00:00
Chase Wright
d4fa6806b7 fix(ethstats): WSS Handling (#21595)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 15:15:41 +00:00
Matthias Seitz
63742ab4ae fix(debug-client): fix off-by-one in block hash buffer lookup (#21628)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 15:15:26 +00:00
Matthias Seitz
08122bc1ea perf: use biased select and prioritize engine events (#21556)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 14:31:27 +00:00
Georgios Konstantopoulos
83afaf1aa7 feat(grafana): add gauge panels for save_blocks _last metrics (#21604)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-30 14:08:32 +00:00
Alexey Shekhirin
d72300c685 fix(net): include disconnect reason in P2PStreamError display (#21626) 2026-01-30 14:04:58 +00:00
Matthias Seitz
faf64c712e feat(cli): add reth db state command for historical contract storage (#21570)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 14:03:19 +00:00
theo
b3d532ce9d chore(op-reth): move op-dependent examples into crates/optimism/examples/ (#21495)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-30 14:02:12 +00:00
Georgios Konstantopoulos
9d064be77e feat(rpc): add EIP-7934 block size validation to testing_buildBlockV1 (#21623)
Co-authored-by: Alexey <alexey@tempo.xyz>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 13:57:51 +00:00
Matus Kysel
e3c256340e feat(txpool): add EIP-7594 blob sidecar toggle (#21622) 2026-01-30 12:27:06 +00:00
ligt
d0df549ddb chore(engine-tree): simplify impl trait bound (#21621) 2026-01-30 11:55:23 +00:00
Arsenii Kulikov
7ccb43ea13 perf: cache fetched proof targets in SparseTrieCacheTask (#21612)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 11:44:36 +00:00
Arsenii Kulikov
20f48b1e50 fix(proof_v2): make sure that all storage proofs are delivered (#21611)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-30 11:21:17 +00:00
Dan Cline
0470c65e6c feat(cli): add --metrics param to reth prune (#21613)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 03:24:25 +00:00
Georgios Konstantopoulos
9de1f0905e feat(prune): add static file pruning support for sender recovery (#21598) 2026-01-30 01:09:38 +00:00
joshieDo
327a1a6681 test(stages): add pipeline forward sync and unwind test (#21602)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 00:49:28 +00:00
Dan Cline
b8f27b73ad chore: fix unused parallel trie const without std (#21610) 2026-01-29 23:05:32 +00:00
かりんとう
7ec5ff6483 refactor(reth-bench): dedupe derive_ws_rpc_url helper (#21585) 2026-01-29 22:50:22 +00:00
Georgios Konstantopoulos
f98af4ad9f feat(rpc): default --testing.skip-invalid-transactions to true (#21603)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 22:03:19 +00:00
joshieDo
d8e912f66b fix(provider): prune account changesets from static files in remove_state_above (#21605) 2026-01-29 21:57:28 +00:00
Georgios Konstantopoulos
0572c4e0ca feat(metrics): add _last gauge metrics for save_blocks timings (#21597) 2026-01-29 21:34:27 +00:00
joshieDo
67a7a1c2d1 chore: revert "test(stages): add pipeline forward sync and unwind test" (#21601) 2026-01-29 22:36:47 +01:00
joshieDo
2b1833576b test(stages): add pipeline forward sync and unwind test (#21553)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 21:13:07 +00:00
Dan Cline
5592c362d4 feat(grafana): add reth-persistence dashboard (#21594) 2026-01-29 21:05:07 +00:00
Georgios Konstantopoulos
6beec25f43 fix(grafana): order MerkleChangeSets checkpoint after MerkleExecute (#21581) 2026-01-29 20:40:26 +00:00
Arsenii Kulikov
19bf580f93 feat: sparse trie as cache (#21583)
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-29 19:11:48 +00:00
joshieDo
796ba6d5dc chore(trie): remove unused direct MDBX changeset readers (#21580) 2026-01-29 17:50:19 +00:00
Georgios Konstantopoulos
5307dfc22b chore: update RPC URLs from ithaca.xyz to reth.rs (#21574)
Co-authored-by: Tim Beiko <tim@ethereum.org>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 17:06:13 +00:00
Brian Picciano
f380ed1581 fix(engine): Try to always compute storage root in V2 proofs when account proof is present, fallback if not (#21579) 2026-01-29 16:58:59 +00:00
DaniPopes
f7313c755c chore(deps): bump codspeed (#21578) 2026-01-29 16:50:09 +00:00
Georgios Konstantopoulos
3bc2191590 chore: remove cargo-chef from Dockerfile.depot (#21577) 2026-01-29 16:28:44 +00:00
Brian Picciano
320f2a6015 fix(trie): PST: Fix update_leaf atomicity, remove update_leaves revealed tracking, fix callback calling (#21573) 2026-01-29 16:18:42 +00:00
Georgios Konstantopoulos
70bfdafd26 fix(provider): check executed block before returning historical state (#21571)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-29 13:54:50 +00:00
YK
e9fe0283a9 fix(provider): use storage-aware methods in unwind_trie_state_from (#21561) 2026-01-29 11:54:12 +00:00
Alexey Shekhirin
92b8857625 fix(reth-bench): stop fetcher when reaching chain tip (#21568) 2026-01-29 11:34:15 +00:00
YK
2d71243cf6 feat(trie): add update_leaves method to SparseTrieExt (#21525)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-29 11:25:08 +00:00
かりんとう
732bf712aa refactor(reth-bench): dedupe read_input and load_jwt_secret helpers (#21555) 2026-01-29 10:17:51 +00:00
Dan Cline
0901c2ca8b fix(reth-bench): retry testing_buildBlockV1 when payload gas < target (#21547)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-29 10:08:54 +00:00
Matthias Seitz
2352158b3d fix(reth-bench): return error instead of panic on invalid payload (#21557)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-29 00:35:19 +00:00
Georgios Konstantopoulos
1a98605ce6 chore(net): downgrade fork id mismatch log to trace (#21554)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 22:41:42 +00:00
DaniPopes
8d37f76d23 chore: move scripts from .github/assets to .github/scripts (#21539)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 22:14:37 +00:00
Dan Cline
2d9cf4c989 chore: fix unused warns in sparse trie (#21546) 2026-01-28 21:48:59 +00:00
DaniPopes
f5ca71d2fb chore(deps): cargo update (#21538)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 19:49:15 +00:00
Alexey Shekhirin
8d58c98034 feat(reth-bench): add reporting and wait options to replay-payloads (#21537) 2026-01-28 19:13:19 +00:00
Matthias Seitz
50e0591540 perf(tree): optimistically prepare canonical overlay (#21475)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 18:16:04 +00:00
joshieDo
013dfdf8c8 fix(prune): add minimum 64 block retention for receipts and bodies (#21520) 2026-01-28 18:10:07 +00:00
joshieDo
effa0ab4c7 fix(provider): read changesets from static files during unwind (#21528)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 15:52:08 +00:00
SS
543a85e9f3 fix: simplify UTF-8 decoding in StreamCodec by using Result::ok (#21524) 2026-01-28 15:12:55 +00:00
theo
88eb0beeb2 chore(op-reth): remove op-reth dependencies from core reth library crates (#21492) 2026-01-28 14:53:17 +00:00
YK
747c0169a7 feat(trie): add prune method to SparseTrieInterface (#21427)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-28 13:55:21 +00:00
Georgios Konstantopoulos
497985ca86 fix(prune): improve pruner log readability (#21522)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-28 13:41:55 +00:00
bobtajson
48a999a81b refactor: using iterator over references (#21506) 2026-01-28 12:46:19 +00:00
ligt
d53858b3e2 chore(engine): simplify EngineApiTreeHandler type inference (#21503) 2026-01-28 12:43:30 +00:00
David Klank
6aa91b0020 perf(trie-db): preallocate vectors in changeset computation (#21465) 2026-01-28 12:39:08 +00:00
katikatidimon
e0a0a0d5fb refactor: remove redundant clone() in CursorSubNode::new (#21493) 2026-01-28 12:33:10 +00:00
joshieDo
231292b58e fix(provider): cap static file changeset iteration to highest available block (#21510) 2026-01-28 11:03:49 +00:00
Brian Picciano
42765890b5 feat(trie): Enable proofs v2 by default (#21434) 2026-01-28 10:54:50 +00:00
Matus Kysel
8417ddc0e8 fix(engine): guard receipt streaming against duplicate indices (#21512) 2026-01-28 10:48:11 +00:00
かりんとう
1ca62d0696 fix(rpc): populate block fields in mev_simBundle logs (#21491) 2026-01-27 22:59:58 +00:00
katikatidimon
928bf37297 perf: avoid cloning prefix sets in TrieWitness::compute (#21352) 2026-01-27 22:26:31 +00:00
Matthias Seitz
aa5b12af44 refactor(db): make Tx::inner field private with accessor (#21490)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 22:06:41 +00:00
katikatidimon
f12acf17e6 chore(txpool): remove redundant locals clone in config (#21477) 2026-01-27 21:37:44 +00:00
joshieDo
2e05cec84b fix: ensure edge enables history in rocksdb (#21478) 2026-01-27 18:43:25 +00:00
Matthias Seitz
9eaa5a6303 chore: remove Sync bound from cursor associated types (#21486)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 18:31:40 +00:00
Georgios Konstantopoulos
ba8c8354e5 fix(reth-bench): retry up to 5 times on failed transaction fetches in big blocks generate (#21483) 2026-01-27 16:10:53 +00:00
Arsenii Kulikov
af3601c65d feat: more metrics (#21481) 2026-01-27 15:17:49 +00:00
Brian Picciano
bff11ab663 refactor(trie): reuse shared StorageProofCalculator for V2 sync storage roots and add deferred encoder metrics (#21424)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 14:54:56 +00:00
joshieDo
08cd1cbda6 fix(static-files): apply minimal blocks per file to all segments (#21479)
Co-authored-by: Claude Sonnet 4.5 <noreply@anthropic.com>
2026-01-27 14:01:32 +00:00
Georgios Konstantopoulos
e4e05e9ef9 refactor: align RocksDbArgs defaults with StorageSettings::base() (#21472)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-27 13:13:25 +00:00
joshieDo
c8245594bc fix(db): write genesis history to correct storage backend (#21471)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 11:59:06 +00:00
Dan Cline
ed40ce8c4c chore: simplify account_changesets_range (#21457) 2026-01-27 11:44:11 +00:00
YK
1e734936d8 fix(provider): skip storage changeset writes when routed to static files (#21468) 2026-01-27 10:34:44 +00:00
YK
11d9f38077 test(e2e): comprehensive RocksDB storage E2E tests (#21423) 2026-01-27 07:08:57 +00:00
Matthias Seitz
226ce14ca1 perf(trie): use is_zero() check to avoid copy in is_storage_empty (#21459)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 00:42:26 +00:00
Dan Cline
a6e1dea2d7 chore: add logging for internal fcu errors (#21456) 2026-01-26 23:24:48 +00:00
Georgios Konstantopoulos
71ed68e944 perf(db): flatten HashedPostState before persisting (#21422) 2026-01-26 22:49:01 +00:00
DaniPopes
adecbd7814 chore: log docker sccache stats (#21455) 2026-01-26 22:30:20 +00:00
Matthias Seitz
26a37f3c00 chore: use Default::default() for TransactionInfo for forward compatibility (#21454)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 22:15:41 +00:00
DaniPopes
0bfa7fa5fa ci: typorino (#21453) 2026-01-26 21:39:35 +00:00
Georgios Konstantopoulos
18bec10a0b perf(docker): use shared cache mounts for parallel builds (#21451)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:00:45 +00:00
DaniPopes
1e33821e19 ci: use depot cache in Dockerfile.depot (#21450) 2026-01-26 20:37:33 +00:00
ethfanWilliam
da92733be8 fix: use unwrap_or_else for lazy evaluation of BlobParams::cancun (#21442) 2026-01-26 20:19:28 +00:00
DaniPopes
c41c8e6cae chore: reduce number of nightly builds (#21446) 2026-01-26 20:06:09 +00:00
DaniPopes
1ccc174e7b chore: remove unused docker from makefile (#21445) 2026-01-26 19:53:55 +00:00
ethfanWilliam
f1459fcf91 fix(stages): retain RocksDB TempDir in TestStageDB to prevent premature deletion (#21444) 2026-01-26 19:43:11 +00:00
Dan Cline
94235d64a8 fix(pruner): prune account and storage changeset static files (#21346) 2026-01-26 19:28:18 +00:00
Dan Cline
7fe60017cf chore(metrics): add a gas_last metric similar to new_payload_last (#21437) 2026-01-26 17:54:20 +00:00
Brian Picciano
f9ec2fafa0 refactor(trie): always use ParallelSparseTrie, deprecate config flags (#21435) 2026-01-26 17:02:06 +00:00
Arsenii Kulikov
768a687189 perf: use shared channel for prewarm workers (#21429) 2026-01-26 15:49:44 +00:00
Rez
b87cde5479 feat: configurable EVM execution limits (#21088)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-26 15:27:09 +00:00
figtracer
ab685579f0 feat(rpc): add transaction hash caching to EthStateCache (#21180)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:37:53 +00:00
Matthias Seitz
c7faafd183 fix(rpc): add block timestamp validation in eth_simulateV1 (#21397)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:12:28 +00:00
Matthias Seitz
935a2cc056 fix(rpc): use correct error codes for eth_simulateV1 reverts and halts (#21412)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 14:06:38 +00:00
Matthias Seitz
507cf58db0 fix(rpc): add block number validation in eth_simulateV1 (#21396)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 13:47:20 +00:00
Matthias Seitz
6cfd369d17 fix(rpc): populate block_hash in eth_simulateV1 logs (#21413)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:41:19 +00:00
Matthias Seitz
934f462d01 feat(cli): make stopping on invalid block the default for reth import (#21403)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:41:06 +00:00
Matthias Seitz
d4f28b02ff feat(rpc): implement movePrecompileToAddress for eth_simulateV1 (#21414)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:40:12 +00:00
Matthias Seitz
963bfeeeed fix(rpc): set prevrandao to zero for eth_simulateV1 simulated blocks (#21399)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:39:37 +00:00
Matthias Seitz
adbe6d9da0 fix(rpc): cap simulate_v1 default gas limit to RPC gas cap (#21402)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:39:15 +00:00
Matthias Seitz
6d19c0ed8e fix(engine): only warn for critical capability mismatches (#21398)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:36:49 +00:00
Andrey Kolishchak
4baf2baec4 fix(net): FetchFullBlockRangeFuture can get stuck forever after partial body fetch + error (#21411)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 12:34:07 +00:00
emmmm
0b5f79e8c9 docs(rpc): add reth_subscribePersistedBlock method (#21420) 2026-01-26 10:48:35 +00:00
Georgios Konstantopoulos
afe164baca test: add E2E test for RocksDB provider functionality (#21419)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-26 10:24:10 +00:00
Hwangjae Lee
31fdbe914c docs(tracing): fix incorrect example description in lib.rs (#21417)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-26 10:19:36 +00:00
Ahsen Kamal
6870747246 feat(payload): add fn for system transaction check (#21407)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-25 14:47:22 +00:00
Fallengirl
0ad8c772e1 fix(era-utils): export correct era1 CompressedBody payload (#21409) 2026-01-25 14:36:24 +00:00
github-actions[bot]
5440d0d89a chore(deps): weekly cargo update (#21406)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-25 10:39:48 +00:00
Matthias Seitz
0eea4d76e9 chore: remove unused imports in storage-api (#21400)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-24 15:49:21 +00:00
YK
8a1702cd74 fix(rocksdb): filter history writes to only changed accounts/storage (#21339)
Co-authored-by: Tempo AI <ai@tempo.xyz>
2026-01-24 13:07:16 +00:00
cui
7feb56d5f6 feat: prealloc vec (#21391) 2026-01-24 11:30:34 +00:00
cui
0aa922c4e8 feat: change from stable sort to unstable sort (#21387) 2026-01-24 11:21:47 +00:00
Matthias Seitz
ccff9a08f0 chore: fix clippy unnecessary_sort_by lint (#21385) 2026-01-24 03:13:49 +00:00
Matthias Seitz
eb788cc7cf fix(docker): pass vergen git vars as build args (#21384)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-24 03:21:43 +01:00
Dan Cline
fb05a0654f fix(engine): use LazyTrieData::deferred for chain notification (#21383)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-23 22:32:21 +00:00
ethfanWilliam
d5a36dcc00 perf(trie): parallelize merge_ancestors_into_overlay extend ops (#21379) 2026-01-23 22:26:07 +00:00
iPLAY888
ffbef9e3cd chore: removed needless collect (#21381) 2026-01-23 21:59:19 +00:00
Dan Cline
820c112e8e feat(engine): add metric for forkchoiceUpdated response -> newPayload (#21380) 2026-01-23 21:57:15 +00:00
Alexey Shekhirin
9285f7eafc ci: use depot for docker (#20380)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-23 21:14:55 +00:00
joshieDo
9a4c6d8a11 feat(rocksdb): static file based healing for TransactionHashNumbers (#21343) 2026-01-23 20:11:47 +00:00
Dan Cline
963c26550a fix(trie): only clone required keys in on_prefetch_proofs (#21378) 2026-01-23 21:13:01 +01:00
joshieDo
3648483512 feat(rocksdb): add WAL size tracking metric and Grafana dashboard (#21295)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 19:59:10 +00:00
joshieDo
ab418642b4 fix(stages): commit RocksDB batches before flush and configure immediate WAL cleanup (#21374)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-01-23 19:28:52 +00:00
joshieDo
decb56fae1 feat(rocksdb): changeset-based crash recovery healing for history indices (#21341) 2026-01-23 19:28:10 +00:00
Matthias Seitz
ee1ec8f9f0 perf(trie): parallelize COW extend operations with rayon (#21375) 2026-01-23 19:31:04 +01:00
Georgios Konstantopoulos
d7bf87da52 feat(engine): add metric for state root task fallback success (#21371) 2026-01-23 18:21:44 +00:00
Georgios Konstantopoulos
dd0c6d279f revert: perf(trie): parallelize merge_ancestors_into_overlay (#21202) (#21370) 2026-01-23 19:09:19 +01:00
Alexey Shekhirin
c137ed836f perf(engine): fixed-cache for execution cache (#21128)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Tempo AI <ai@tempo.xyz>
2026-01-23 17:57:42 +00:00
Dan Cline
a543752f7d chore(reth-bench): make from-block a required flag (#21372) 2026-01-23 17:52:33 +00:00
joshieDo
b814893221 feat(stages): flush RocksDB at end of history and tx_lookup stages (#21367) 2026-01-23 17:02:53 +00:00
Georgios Konstantopoulos
fcef82261d fix(libmdbx): handle errors gracefully in TransactionInner::drop (#21368) 2026-01-23 16:37:15 +00:00
iPLAY888
d3846d98a9 refactor: refactor get_idle_peer_for to use Iterator::find (#21321) 2026-01-23 15:56:09 +00:00
Alexey Shekhirin
1f536cce65 test(e2e): selfdestruct pre- and post-Dencun (#21363) 2026-01-23 15:41:08 +00:00
Matthias Seitz
0ddaf1b26c feat(engine): add BAL metrics type for EIP-7928 (#21356) 2026-01-23 15:17:33 +00:00
Gigi
830cd5e355 chore: update snmalloc upstream repository link (#21360) 2026-01-23 14:57:46 +00:00
Georgios Konstantopoulos
f77d7d5983 feat(reth-bench): support human-readable gas format in generate-big-block (#21361) 2026-01-23 14:24:34 +00:00
Georgios Konstantopoulos
a2237c534e feat(p2p): add reth p2p enode command (#21357)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-23 13:23:44 +00:00
Arsenii Kulikov
1bd8fab887 feat(txpool): add Block associated type to TransactionValidator trait (#21359) 2026-01-23 13:16:05 +00:00
Matthias Seitz
22a68756c7 fix(tree): evict changeset cache even when finalized block is unset (#21354) 2026-01-23 11:26:57 +00:00
Hwangjae Lee
d99c0ffd62 chore(etc): update ethereum-metrics-exporter GitHub URL (#21348)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-23 10:59:53 +00:00
Georgios Konstantopoulos
ad476e2b5c chore: add yongkangc as codeowner for crates/storage/provider (#21349) 2026-01-23 07:18:18 +00:00
Matthias Seitz
6df249c1f1 feat(engine): stub Amsterdam engine API endpoints (newPayloadV5, getPayloadV6, BALs) (#21344)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
Co-authored-by: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com>
2026-01-22 20:48:11 +00:00
Arsenii Kulikov
5a076df09a feat: allow setting custom debug block provider (#21345)
Co-authored-by: Karl <yh975593284@gmail.com>
2026-01-22 20:40:26 +00:00
YK
f07629eac0 perf: avoid creating RocksDB transactions for legacy MDBX-only nodes (#21325) 2026-01-22 20:30:52 +00:00
Dan Cline
f643e93c35 feat(reth-bench): send-invalid-payload command (#21335) 2026-01-22 19:42:19 +00:00
Matthias Seitz
653362a436 ci: align check-alloy workflow with main clippy job (#21329) 2026-01-22 20:48:53 +01:00
Seola Oh
a02508600c chore(txpool): explicitly deref RwLockReadGuard in PartialEq impl (#21336) 2026-01-22 19:35:00 +00:00
Georgios Konstantopoulos
937a7f226d fix(rpc): use Default for SimulateError to prepare for alloy breaking change (#21319)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 19:14:58 +00:00
joshieDo
a0df561117 fix(rocksdb): periodic batch commits in stages to prevent OOM (#21334)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 19:04:56 +00:00
Arsenii Kulikov
be5a4ac7a6 feat: bump alloy and alloy-evm (#21337) 2026-01-22 18:43:24 +00:00
Georgios Konstantopoulos
0c854b6f14 fix(net): limit pending pool imports for broadcast transactions (#21254)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-01-22 18:32:07 +00:00
Georgios Konstantopoulos
28a31cd579 fix: use unwrap_or_else for lazy evaluation of StorageSettings::legacy (#21332) 2026-01-22 17:02:15 +00:00
Brian Picciano
da12451c9c chore(trie): Cleanup unused trie changesets code (#21323) 2026-01-22 16:57:46 +00:00
Georgios Konstantopoulos
247ce3c4e9 feat(storage): warn storage settings diff at startup (#21320)
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-22 16:40:10 +00:00
iPLAY888
bf43ebaa29 fix(cli): handle invalid hex in db list --search (#21315) 2026-01-22 16:18:36 +00:00
Matthias Seitz
a01ecce73f test: add e2e tests for invalid payload handling via Engine API (#21288) 2026-01-22 15:55:36 +00:00
Arsenii Kulikov
3e55c6ca6e fix: always check upper subtrie for keys (#21276)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-22 15:47:50 +00:00
Brian Picciano
2ac7d719f3 feat(trie): add V2 account proof computation and refactor proof types (reapply) (#21316)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:46:01 +00:00
andrewshab
965705ff88 fix: remove collect (#21318)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-22 15:24:51 +00:00
Dan Cline
ebe2ca1366 feat: add StaticFileSegment::StorageChangeSets (#20896) 2026-01-22 15:03:47 +00:00
Matthias Seitz
cc242f83fd feat(rpc): respect history expiry in eth_getLogs per EIP-4444 (#21304) 2026-01-22 14:55:50 +00:00
joshieDo
12cf3d685b fix(provider): add CommitOrder for RocksDB/MDBX unwind atomicity (#21311) 2026-01-22 14:54:47 +00:00
Matthias Seitz
ad5b533ad1 chore: rm patches (#21317) 2026-01-22 15:48:53 +01:00
joshieDo
118f15f345 feat(rocksdb): disable bloom filter for default column family (#21312) 2026-01-22 13:47:34 +00:00
joshieDo
97481f69e5 perf(rocksdb): disable compression and bloom filters for TransactionHashNumbers CF (#21310) 2026-01-22 13:31:16 +00:00
Georgios Konstantopoulos
f692ac7d1e perf(prune): use bulk table clear for PruneMode::Full (#21302) 2026-01-22 13:01:17 +00:00
andrewshab
4b1c341ced fix: remove redundant clone (#21300) 2026-01-22 12:43:19 +00:00
Georgios Konstantopoulos
865f8f8951 perf(prune): sort tx hashes for efficient TransactionLookup pruning (#21297) 2026-01-22 12:10:07 +00:00
joshieDo
492fc20fd1 fix(cli): clear rocksdb tables in drop-stage command (#21299)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-22 12:09:36 +00:00
Sergei Shulepov
ad9886abb8 fix(mdbx): mark reserve as unsafe (#21263) 2026-01-22 12:03:12 +00:00
Matthias Seitz
5c3e45cd6b fix: handle incomplete receipts gracefully in receipt root task (#21285) 2026-01-22 10:52:56 +00:00
Emma Jamieson-Hoare
68fdba32d2 chore(release): prep v1.10.2 release (#21287)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-22 10:50:10 +00:00
Matthias Seitz
8f6a0a2992 ci: add on-demand workflow to check alloy breaking changes (#21267) 2026-01-22 10:47:38 +00:00
Matthias Seitz
ec9c7f8d3e perf(db): use ArrayVec for StoredNibbles key encoding (#21279) 2026-01-22 02:05:50 +00:00
Matthias Seitz
dbdaf068f0 fix(engine): clear execution cache when block validation fails (#21282) 2026-01-22 01:01:22 +00:00
Matthias Seitz
055bf63ee9 refactor: use Default::default() for Header in tests (#21277) 2026-01-21 22:50:10 +00:00
Georgios Konstantopoulos
2305c3ebeb feat(rpc): respect history expiry in block() and map to PrunedHistoryUnavailable (#21270) 2026-01-21 22:22:05 +00:00
joshieDo
eb55c3c3da feat(grafana): add RocksDB metrics dashboard (#21243)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 22:09:42 +00:00
Alexey Shekhirin
72e1467ba3 fix(prune): avoid panic in tx lookup (#21275) 2026-01-21 21:21:53 +00:00
Alexey Shekhirin
74edce0089 revert: feat(trie): add V2 account proof computation and refactor proof types (#21214) (#21274) 2026-01-21 21:07:13 +00:00
Georgios Konstantopoulos
8c645d5762 feat(reth-bench): accept short notation for --target-gas-limit (#21273) 2026-01-21 21:04:10 +00:00
Georgios Konstantopoulos
b7d2ee2566 feat(engine): add metric for execution cache unavailability due to concurrent use (#21265)
Co-authored-by: Tempo AI <ai@tempo.xyz>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-21 20:17:45 +00:00
Matthias Seitz
7609deddda perf(trie): parallelize merge_ancestors_into_overlay (#21202) 2026-01-21 20:08:03 +00:00
Matthias Seitz
ec50fd40b3 chore(chainspec): use ..Default::default() in create_chain_config (#21266) 2026-01-21 19:19:24 +00:00
YK
624ddc5779 feat(stages): add RocksDB support for IndexStorageHistoryStage (#21175) 2026-01-21 17:05:19 +00:00
Georgios Konstantopoulos
dd72cfe23e refactor: remove static_files.to_settings() and add edge feature to RocksDB flags (#21225) 2026-01-21 16:52:24 +00:00
joshieDo
ff8ac97e33 fix(stages): clear ETL collectors on HeaderStage error paths (#21258) 2026-01-21 16:27:30 +00:00
Alexey Shekhirin
0974485863 feat(reth-bench): add --target-gas-limit option to gas-limit-ramp (#21262) 2026-01-21 16:19:22 +00:00
かりんとう
274394e777 fix: fix payload file filter prefix in replay-payloads (#21255) 2026-01-21 16:11:03 +00:00
Emma Jamieson-Hoare
1954c91a60 chore: update CODEOWNERS (#21223)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-21 14:40:54 +00:00
Sergei Shulepov
9cf82c8403 fix: supply a real ptr to mdbx_dbi_flags_ex (#21230) 2026-01-21 14:23:26 +00:00
Brian Picciano
f85fcba872 feat(trie): add V2 account proof computation and refactor proof types (#21214)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 14:18:44 +00:00
joshieDo
ebaa4bda3a feat(rocksdb): add missing observability (#21253) 2026-01-21 14:14:34 +00:00
joshieDo
04d4c9a02f fix(rocksdb): flush all column families on drop and show SST/memtable sizes (#21251)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 12:44:08 +00:00
Arsenii Kulikov
3065a328f9 fix: clear overlay_cache in with_extended_hashed_state_overlay (#21233) 2026-01-21 12:08:24 +00:00
Sergei Shulepov
43a84f1231 refactor(engine): move execution logic from metrics to payload_validator (#21226) 2026-01-21 11:17:30 +00:00
Matthias Seitz
5a5c21cc1b feat(txpool): add IntoIterator for AllPoolTransactions (#21241) 2026-01-21 10:01:32 +00:00
Matthias Seitz
8a8a9126d6 feat(execution-types): add receipts_iter and logs_iter helpers to Chain (#21240) 2026-01-21 09:59:15 +00:00
Emilia Hane
6f73c2447d feat(trie): Add serde-bincode-compat feature to reth-trie (#21235) 2026-01-21 09:42:52 +00:00
Sergei Shulepov
2cae438642 fix: sigsegv handler (#21231)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-21 09:42:36 +00:00
Georgios Konstantopoulos
37b5db0d47 feat(cli): add RocksDB table stats to reth db stats command (#21221)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-21 08:45:17 +00:00
joshieDo
238433e146 fix(rocksdb): flush memtables before dropping (#21234) 2026-01-21 02:19:36 +00:00
Georgios Konstantopoulos
660964a0f5 feat(node): log storage settings after genesis init (#21229) 2026-01-21 00:58:23 +00:00
Matthias Seitz
22b465dd64 chore(trie): remove unnecessary clone in into_sorted_ref (#21232) 2026-01-20 22:57:08 +00:00
Georgios Konstantopoulos
3ff575b877 feat(engine): add --engine.disable-cache-metrics flag (#21228)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-20 22:03:12 +00:00
かりんとう
d12752dc8a feat(engine): add time_between_forkchoice_updated metric (#21227) 2026-01-20 21:06:11 +00:00
Georgios Konstantopoulos
869b5d0851 feat(edge): enable transaction_hash_numbers_in_rocksdb for edge builds (#21224)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 20:02:02 +00:00
Georgios Konstantopoulos
78de3d8f61 perf(db): use Cow::Borrowed in walk_dup to avoid allocation (#21220) 2026-01-20 19:31:50 +00:00
YK
bc79cc44c9 feat(cli): add --rocksdb.* flags for RocksDB table routing (#21191) 2026-01-20 19:29:05 +00:00
Georgios Konstantopoulos
ff8f434dcd feat(cli): add reth db checksum rocksdb command (#21217)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 19:10:34 +00:00
Arsenii Kulikov
9662dc5271 fix: properly save history indices in pipeline (#21222) 2026-01-20 18:20:28 +00:00
Alexey Shekhirin
3ba37082dc fix(reth-bench): replay-payloads prefix (#21219) 2026-01-20 18:36:35 +01:00
Ahsen Kamal
7934294988 perf(trie): dispatch storage proofs in lexicographical order (#21213)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
2026-01-20 17:09:20 +00:00
Georgios Konstantopoulos
7371bd3f29 chore(db-api): remove sharded_key_encode benchmark (#21215)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 17:01:12 +00:00
Georgios Konstantopoulos
80980b8e4d feat(pruning): add DefaultPruningValues for overridable pruning defaults (#21207)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-20 16:58:29 +00:00
Matthias Seitz
2e2cd67663 perf(chain-state): parallelize into_sorted with rayon (#21193) 2026-01-20 16:42:16 +00:00
Georgios Konstantopoulos
4f009728e2 feat(cli): add reth db checksum mdbx/static-file command (#21211)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:11:51 +00:00
Georgios Konstantopoulos
39d5ae73e8 feat(storage): add read-only mode for RocksDB provider (#21210)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:09:51 +00:00
Georgios Konstantopoulos
5ef200eaad perf(db): stack-allocate ShardedKey and StorageShardedKey encoding (#21200)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 15:58:43 +00:00
ethfanWilliam
d002dacc13 chore: remove deprecated and unused ExecuteOutput struct (#20887)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 15:06:26 +00:00
Alexey Shekhirin
bb39cba504 ci: partition bench codspeed job (#20332)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 14:29:48 +00:00
YK
bd144a4c42 feat(stages): add RocksDB support for IndexAccountHistoryStage (#21165)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 14:23:29 +00:00
tonis
a0845bab18 feat: Check CL/Reth capability compatibility (#20348)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 14:19:31 +00:00
Brian Picciano
346cc0da71 feat(trie): add AsyncAccountValueEncoder for V2 proof computation (#21197)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:50:29 +00:00
Matthias Seitz
ea3d4663ae perf(trie): use HashMap reserve heuristic in MultiProof::extend (#21199) 2026-01-20 13:34:41 +00:00
Hwangjae Lee
3667d3b5aa perf(trie): defer child RLP conversion in proof_v2 for async encoder support (#20873)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-20 13:33:08 +00:00
Brian Picciano
7cfb19c98e feat(trie): Add V2 reveal method and target types (#21196)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:25:54 +00:00
joshieDo
5a38871489 fix: set StaticFileArgs defaults for edge (#21208) 2026-01-20 12:39:36 +00:00
Brian Picciano
c825c8c187 chore(trie): Move hybrid check for trie input merges into common code (#21198) 2026-01-20 12:38:46 +00:00
Matthias Seitz
8f37cd08fc feat(engine-api): add EIP-7928 BAL stub methods (#21204) 2026-01-20 11:33:27 +00:00
Matthias Seitz
c9dad4765d chore: bump version to 1.10.1 (#21188) 2026-01-19 14:04:08 +00:00
Dan Cline
1d55abeef3 chore: rename extend_ref methods on sorted data structures (#21043) 2026-01-19 13:04:57 +00:00
Niven
f7460e219c fix(flashblocks): Add flashblock ws connection retry period (#20510) 2026-01-19 12:01:33 +00:00
Georgios Konstantopoulos
0c66315f20 chore(bench): add --disable-tx-gossip to benchmark node args (#21171) 2026-01-19 11:45:56 +00:00
MozirDmitriy
6a2010e595 refactor(stages): reuse history index cache buffers in collect_history_indices (#21017) 2026-01-19 11:39:52 +00:00
Georgios Konstantopoulos
c2435ff6f8 feat(download): resumable snapshot downloads with auto-retry (#21161) 2026-01-19 10:26:24 +00:00
DaniPopes
52ec8e9491 ci: update to tempoxyz (#21176) 2026-01-19 10:21:37 +00:00
Georgios Konstantopoulos
a901d80ee6 chore: apply spelling and typo fixes (#21182) 2026-01-19 10:21:25 +00:00
MoNyAvA
915164078f docs: document minimal storage mode in pruning FAQ (#21025)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-19 10:27:45 +01:00
github-actions[bot]
be3234d848 chore(deps): weekly cargo update (#21167)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-18 14:57:20 +00:00
Matthias Seitz
f624372334 feat(execution-types): add receipts_iter helper (#21162)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-17 19:20:28 +01:00
Matthias Seitz
40bc9d3860 revert: undo Chain crate, add LazyTrieData to trie-common (#21155) 2026-01-17 15:57:09 +00:00
Georgios Konstantopoulos
1ea574417f feat(engine): add new_payload_interval metric (start-to-start) (#21159) 2026-01-17 12:15:45 +00:00
Georgios Konstantopoulos
27e055f790 feat(engine): add time_between_new_payloads metric (#21158) 2026-01-17 10:20:22 +00:00
Georgios Konstantopoulos
d5dc0b27eb fix(storage-api): gate reth-chain dependency behind std feature
The reth-chain crate is inherently std-only (uses BTreeMap, Arc, etc.)
and was breaking the riscv32imac no_std builds by pulling in serde_core
which doesn't support no_std properly.

This makes reth-chain optional and only enables it when std feature is
active, gating the block_writer module that uses Chain behind std.
2026-01-17 08:32:10 +00:00
Georgios Konstantopoulos
c11c13000f perf(storage): batch trie updates across blocks in save_blocks (#21142)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-17 07:15:40 +00:00
Matthias Seitz
6bf43ab24a refactor: use ExecutionOutcome::single instead of tuple From (#21152) 2026-01-17 01:51:26 +00:00
Matthias Seitz
574bde0d6f chore(chain-state): reorganize deferred_trie.rs impl blocks (#21151) 2026-01-17 01:39:29 +00:00
Matthias Seitz
79b8ffb828 feat(primitives-traits): add try_recover_signers for parallel batch recovery (#21103) 2026-01-17 01:24:53 +00:00
Dan Cline
c617d25c36 perf: make Chain use DeferredTrieData (#21137)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-17 01:05:35 +00:00
Georgios Konstantopoulos
b96a30821f fix(engine): request head block download when not buffered after backfill (#21150) 2026-01-17 00:33:27 +00:00
Mablr
012fbf5110 fix(docs/cli): update help.rs to use nightly toolchain (#21149) 2026-01-16 23:35:26 +00:00
Arsenii Kulikov
d7a5d1f872 fix: properly record span fields (#21148) 2026-01-16 23:25:54 +00:00
Matthias Seitz
3a39251f79 fix: release mutex before dropping ancestors in wait_cloned (#21146) 2026-01-16 22:32:23 +00:00
Julian Meyer
f6dbf2d82d feat(db): implement extra dup methods (#20964)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 21:31:52 +00:00
Brian Picciano
13707faf1a feat(consensus): incremental receipt root computation in background task (#21131) 2026-01-16 19:53:59 +00:00
Arsenii Kulikov
6e6415690c perf: start saving cache sooner (#21130) 2026-01-16 18:55:18 +00:00
Matthias Seitz
b81e373d78 chore(deps): bump vergen and vergen-git2 to 9.1.0 (#21141) 2026-01-16 20:00:43 +01:00
Arun Dhyani
a164654145 fix(exex): prevent ExExManager deadlock when buffer clears after being full (#21135) 2026-01-16 18:42:23 +00:00
Matthias Seitz
905bb95f8b perf(engine): defer trie overlay computation with LazyOverlay (#21133) 2026-01-16 18:25:04 +00:00
YK
13c32625bc feat(storage): add EitherReader for routing history queries to MDBX or RocksDB (#21063)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-16 17:44:43 +00:00
YK
1be9fab5bf perf: Optimize multiproof sequencer add_proof (#21129) 2026-01-16 17:33:48 +00:00
Arsenii Kulikov
80eb0d0fb6 refactor: use BlockExecutionOutcome in ExecutedBlock (#21123) 2026-01-16 17:07:19 +00:00
Matthias Seitz
5e178f6ac6 chore(deps): update alloy-evm and alloy-op-evm to 0.26.3 (#21126) 2026-01-16 17:24:45 +01:00
Matthias Seitz
b4b64096c8 perf(cli): use available_parallelism as default for re-execute (#21010) 2026-01-16 16:08:30 +00:00
figtracer
e313de818b chore(provider): pre alloc tx hashes (#21114) 2026-01-16 15:40:47 +00:00
rakita
86c414081a feat: stagging revm v34.0.0 (#20627)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 14:56:27 +00:00
Brian Picciano
a74cb9cbc3 feat(trie): in-memory trie changesets (#20997) 2026-01-16 01:06:31 +00:00
YK
e25411c32b perf(trie): fix extend_sorted_vec O(n log n) → O(n+m) merge (#21098) 2026-01-16 00:17:22 +00:00
Matthias Seitz
ec3323bba0 refactor(chain-state): extract blocks_to_chain helper (#21110) 2026-01-15 23:27:11 +00:00
Dan Cline
26cd132631 fix(reth-bench): use requests hash (#21111) 2026-01-15 19:19:16 +00:00
DaniPopes
079f59c2be perf: reserve in extend_sorted_vec (#21109) 2026-01-15 19:10:20 +00:00
joshieDo
e9b079ad62 feat: add rocksdb to save_blocks (#21003)
Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com>
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-15 18:33:19 +00:00
625 changed files with 36598 additions and 13019 deletions

20
.changelog/config.toml Normal file
View File

@@ -0,0 +1,20 @@
# Changelogs configuration for reth
# https://github.com/wevm/changelogs
# How to bump packages that depend on changed packages
dependent_bump = "patch"
[changelog]
# Generate per-crate changelogs (vs single root changelog)
format = "per-crate"
# Fixed groups: all always share the same version
# reth binaries share version
[[fixed]]
members = ["reth", "op-reth"]
# Packages to ignore (internal/test-only crates)
ignore = [
"reth-testing-utils",
"reth-bench",
]

View File

@@ -0,0 +1,5 @@
---
reth-engine-tree: patch
---
Reordered cache size calculations in `ExecutionCache::new` to group related operations together.

View File

@@ -0,0 +1,6 @@
---
reth: patch
op-reth: patch
---
Added automated changelog generation infrastructure using wevm/changelogs-rs with Claude Code integration. Configured per-crate changelog format with fixed version groups for reth binaries and exclusions for internal test utilities.

View File

@@ -0,0 +1,5 @@
---
reth: patch
---
Updated Alloy dependencies from 1.5.2 to 1.6.1.

43
.github/CODEOWNERS vendored
View File

@@ -1,45 +1,52 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
crates/era/ @mattsse
crates/era-downloader/ @mattsse
crates/era-utils/ @mattsse
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
crates/ethereum/ @mattsse @Rjected
crates/etl/ @joshieDo @shekhirin
crates/evm/ @rakita @mattsse @Rjected
crates/evm/ @mattsse @Rjected @klkvr
crates/exex/ @shekhirin
crates/fs-util/ @mattsse
crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected @fgimenez
crates/optimism/ @mattsse @Rjected
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/ress/ @shekhirin @Rjected
crates/revm/ @mattsse
crates/rpc/ @mattsse @Rjected
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/stateless/ @mattsse
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
crates/storage/db-api/ @joshieDo
crates/storage/db-common/ @Rjected
crates/storage/db/ @joshieDo @rakita
crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @fgimenez
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
bin/reth/ @mattsse @shekhirin @Rjected
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

View File

@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
cmd=(cargo codspeed build --profile profiling)
crates=(
-p reth-primitives
-p reth-trie
-p reth-trie-common
-p reth-trie-sparse
)
"${cmd[@]}" --features test-utils "${crates[@]}"

View File

@@ -11,17 +11,14 @@ go build .
# Run each hive command in the background for each simulator and wait
echo "Building images"
./hive -client reth --sim "ethereum/eels" \
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v2.0.0/fixtures_bal.tar.gz \
--sim.buildarg branch=eips/amsterdam/eip-7928 \
--sim.timelimit 1s || true &
# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &
./hive -client reth --sim "smoke/genesis" -sim.timelimit 1s || true &
./hive -client reth --sim "smoke/network" -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/sync" -sim.timelimit 1s || true &
wait
# Run docker save in parallel, wait and exit on error
@@ -40,7 +37,7 @@ for pid in "${saving_pids[@]}"; do
wait "$pid" || exit
done
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/assets/hive/no_sim_build.diff
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/scripts/hive/no_sim_build.diff
go build .
mv ./hive ../hive_assets/

View File

@@ -24,4 +24,4 @@ done
wait
docker image ls -a
docker image ls -a

View File

@@ -3,9 +3,9 @@
on:
pull_request:
# TODO: Disabled temporarily for https://github.com/CodSpeedHQ/runner/issues/55
# merge_group :
# merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always
@@ -17,6 +17,16 @@ name: bench
jobs:
codspeed:
runs-on: depot-ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
include:
- partition: 1
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
- partition: 2
crates: "-p reth-trie"
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
steps:
- uses: actions/checkout@v6
with:
@@ -32,10 +42,10 @@ jobs:
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: ./.github/scripts/codspeed-build.sh
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run --workspace
run: cargo codspeed run ${{ matrix.crates }}
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}

View File

@@ -74,4 +74,4 @@ jobs:
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
uses: actions/deploy-pages@v4

21
.github/workflows/changelog.yml vendored Normal file
View File

@@ -0,0 +1,21 @@
name: Changelog
on:
workflow_dispatch:
jobs:
changelog:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- run: npm install -g @anthropic-ai/claude-code
- uses: wevm/changelogs-rs/gen@master
with:
ai: 'claude -p'
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

65
.github/workflows/check-alloy.yml vendored Normal file
View File

@@ -0,0 +1,65 @@
# Checks reth compilation against alloy branches to detect breaking changes.
# Run on-demand via workflow_dispatch.
name: Check Alloy Breaking Changes
on:
workflow_dispatch:
inputs:
alloy_branch:
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
required: false
type: string
alloy_evm_branch:
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
required: false
type: string
op_alloy_branch:
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
required: false
type: string
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Apply alloy patches
run: |
ARGS=""
if [ -n "${{ inputs.alloy_branch }}" ]; then
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
fi
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
fi
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
fi
if [ -z "$ARGS" ]; then
echo "No branches specified, nothing to patch"
exit 1
fi
./scripts/patch-alloy.sh $ARGS
echo "=== Final patch section ==="
tail -50 Cargo.toml
- name: Check workspace
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings

View File

@@ -9,7 +9,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -15,6 +15,6 @@ permissions:
jobs:
update:
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
secrets:
token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -1,54 +0,0 @@
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
# images from this workflow will be tagged with the git sha of the branch used
# and will NOT tag it as `latest`.
name: docker-git
on:
workflow_dispatch: {}
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
GIT_SHA: ${{ github.sha }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the git-sha-tagged reth image'
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
- name: 'Build and push the git-sha-tagged op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,65 +0,0 @@
# Publishes the nightly Docker image.
name: docker-nightly
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the nightly reth image'
command: 'make PROFILE=maxperf docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
- name: 'Build and push the nightly op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
steps:
- uses: actions/checkout@v6
- name: Remove bloatware
uses: laverdet/remove-bloatware@v1.0.0
with:
docker: true
lang: rust
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -1,4 +1,9 @@
# Publishes the Docker image.
# Publishes Docker images.
#
# Triggers:
# - Push tag v*: builds release (RC or latest)
# - Schedule: builds nightly + profiling
# - Manual: builds git-sha or nightly
name: docker
@@ -6,84 +11,94 @@ on:
push:
tags:
- v*
env:
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
build_type:
description: "Build type"
required: true
type: choice
options:
- git-sha
- nightly
default: git-sha
dry_run:
description: "Skip pushing images (dry run)"
required: false
type: boolean
default: false
jobs:
build-rc:
if: contains(github.ref, '-rc')
name: build and push as release candidate
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
build:
if: ${{ !contains(github.ref, '-rc') }}
name: build and push as latest
name: Build Docker images
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
id-token: write
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
- name: Set up Depot CLI
uses: depot/setup-action@v1
- name: Log in to GHCR
uses: docker/login-action@v3
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get git info for vergen
id: git
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
echo "dirty=false" >> "$GITHUB_OUTPUT"
- name: Determine build parameters
id: params
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
REGISTRY="ghcr.io/${{ github.repository_owner }}"
if [[ "${{ github.event_name }}" == "push" ]]; then
VERSION="${GITHUB_REF#refs/tags/}"
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION},${REGISTRY}/op-reth:latest" >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum optimism" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "optimism_tags=${REGISTRY}/op-reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
uses: depot/bake-action@v1
env:
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
with:
project: ${{ vars.DEPOT_PROJECT_ID }}
files: docker-bake.hcl
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
optimism.tags=${{ steps.params.outputs.optimism_tags }}

View File

@@ -6,7 +6,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always
@@ -44,3 +44,24 @@ jobs:
--exclude 'op-reth' \
--exclude 'reth' \
-E 'binary(e2e_testsuite)'
rocksdb:
name: e2e-rocksdb
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run RocksDB e2e tests
run: |
cargo nextest run \
--locked --features "edge" \
-p reth-e2e-test-utils \
-E 'binary(rocksdb)'

View File

@@ -6,9 +6,7 @@ on:
workflow_dispatch:
schedule:
- cron: "0 */6 * * *"
pull_request:
branches:
- main
env:
CARGO_TERM_COLOR: always
@@ -44,7 +42,6 @@ jobs:
uses: actions/checkout@v6
with:
repository: ethereum/hive
ref: master
path: hivetests
- name: Get hive commit hash
@@ -61,11 +58,11 @@ jobs:
uses: actions/cache@v5
with:
path: ./hive_assets
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
- name: Build hive assets
if: steps.cache-hive.outputs.cache-hit != 'true'
run: .github/assets/hive/build_simulators.sh
run: .github/scripts/hive/build_simulators.sh
- name: Load cached Docker images
if: steps.cache-hive.outputs.cache-hit == 'true'
@@ -130,29 +127,27 @@ jobs:
# eth_ rpc methods
- sim: ethereum/rpc-compat
include:
# - eth_blockNumber
- eth_blockNumber
- eth_call
# - eth_chainId
# - eth_createAccessList
# - eth_estimateGas
# - eth_feeHistory
# - eth_getBalance
# - eth_getBlockBy
# - eth_getBlockTransactionCountBy
# - eth_getCode
# - eth_getProof
# - eth_getStorage
# - eth_getTransactionBy
# - eth_getTransactionCount
# - eth_getTransactionReceipt
# - eth_sendRawTransaction
# - eth_syncing
# # debug_ rpc methods
# - debug_
- eth_chainId
- eth_createAccessList
- eth_estimateGas
- eth_feeHistory
- eth_getBalance
- eth_getBlockBy
- eth_getBlockTransactionCountBy
- eth_getCode
- eth_getProof
- eth_getStorage
- eth_getTransactionBy
- eth_getTransactionCount
- eth_getTransactionReceipt
- eth_sendRawTransaction
- eth_syncing
# debug_ rpc methods
- debug_
# consume-engine
- sim: ethereum/eels/consume-engine
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-engine
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-engine
@@ -169,10 +164,10 @@ jobs:
limit: .*tests/homestead.*
- sim: ethereum/eels/consume-engine
limit: .*tests/frontier.*
- sim: ethereum/eels/consume-engine
limit: .*tests/paris.*
# consume-rlp
- sim: ethereum/eels/consume-rlp
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-rlp
@@ -189,6 +184,8 @@ jobs:
limit: .*tests/homestead.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/frontier.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/paris.*
needs:
- prepare-reth-stable
- prepare-reth-edge
@@ -216,7 +213,7 @@ jobs:
path: /tmp
- name: Load Docker images
run: .github/assets/hive/load_images.sh
run: .github/scripts/hive/load_images.sh
- name: Move hive binary
run: |
@@ -244,11 +241,11 @@ jobs:
FILTER="/"
fi
echo "filter: $FILTER"
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
- name: Print simulator output
if: ${{ failure() }}

View File

@@ -6,7 +6,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
schedule:
# Run once a day at 3:00 UTC
- cron: "0 3 * * *"
@@ -22,7 +22,7 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.network }}
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
if: github.event_name != 'schedule'
runs-on: depot-ubuntu-latest-4
env:
@@ -30,13 +30,17 @@ jobs:
strategy:
matrix:
network: ["ethereum", "optimism"]
storage: ["stable", "edge"]
exclude:
- network: optimism
storage: edge
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- name: Install Geth
run: .github/assets/install_geth.sh
run: .github/scripts/install_geth.sh
- uses: taiki-e/install-action@nextest
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
@@ -46,7 +50,7 @@ jobs:
name: Run tests
run: |
cargo nextest run \
--locked --features "asm-keccak ${{ matrix.network }}" \
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
--workspace --exclude ef-tests \
-E "kind(test) and not binary(e2e_testsuite)"
- if: matrix.network == 'optimism'

View File

@@ -19,5 +19,5 @@ jobs:
uses: actions/github-script@v8
with:
script: |
const label_pr = require('./.github/assets/label_pr.js')
const label_pr = require('./.github/scripts/label_pr.js')
await label_pr({github, context})

View File

@@ -4,7 +4,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always
@@ -76,7 +76,7 @@ jobs:
- name: Run Wasm checks
run: |
sudo apt update && sudo apt install gcc-multilib
.github/assets/check_wasm.sh
.github/scripts/check_wasm.sh
riscv:
runs-on: depot-ubuntu-latest
@@ -94,7 +94,7 @@ jobs:
cache-on-failure: true
- uses: dcarbone/install-jq-action@v3
- name: Run RISC-V checks
run: .github/assets/check_rv32imac.sh
run: .github/scripts/check_rv32imac.sh
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
@@ -285,7 +285,7 @@ jobs:
- run: zepter run check
deny:
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
uses: tempoxyz/ci/.github/workflows/deny.yml@main
lint-success:
name: lint success

View File

@@ -43,7 +43,7 @@ jobs:
uses: docker/build-push-action@v6
with:
context: .
file: .github/assets/hive/Dockerfile
file: .github/scripts/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |

View File

@@ -6,7 +6,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -4,10 +4,9 @@ name: unit
on:
pull_request:
branches: ["**"]
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -4,9 +4,9 @@ name: windows
on:
push:
branches: ["**"]
branches: [main]
pull_request:
branches: ["**"]
branches: [main]
merge_group:
env:

View File

@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
2. **Linting**: Run clippy with all features
```bash
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
```
3. **Testing**: Use nextest for faster test execution
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
Before submitting changes, ensure:
1. **Format Check**: `cargo +nightly fmt --all --check`
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
2. **Clippy**: No warnings
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### Opening PRs against <https://github.com/paradigmxyz/reth>
Label PRs appropriately, first check the available labels and then apply the relevant ones:
@@ -249,7 +248,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
unsafe impl GlobalAlloc for LimitedAllocator { ... }
// Binary search requires sorted input. Panics on unsorted slices.
fn find_index(items: &[Item], target: &Item) -> Option
fn find_index(items: &[Item], target: &Item) -> Option<usize>
// Timeout set to 5s to match EVM block processing limits
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
}
```
5. **Run checks**:
5. **Run checks** (IMPORTANT!):
```bash
cargo +nightly fmt --all
cargo clippy --all-features
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
cargo test -p reth-discv4
```
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
cargo +nightly fmt --all
# Run lints
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
cargo +nightly clippy --workspace --all-features
# Run tests
cargo nextest run --workspace

1724
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.10.0"
version = "1.10.2"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
@@ -77,6 +77,10 @@ members = [
"crates/optimism/cli",
"crates/optimism/consensus",
"crates/optimism/evm/",
"crates/optimism/examples/custom-node",
"crates/optimism/examples/engine-api-access",
"crates/optimism/examples/exex-hello-world",
"crates/optimism/examples/op-db-access",
"crates/optimism/flashblocks/",
"crates/optimism/hardforks/",
"crates/optimism/node/",
@@ -145,7 +149,6 @@ members = [
"examples/beacon-api-sse/",
"examples/bsc-p2p",
"examples/custom-dev-node/",
"examples/custom-node/",
"examples/custom-engine-types/",
"examples/custom-evm/",
"examples/custom-hardforks/",
@@ -154,10 +157,7 @@ members = [
"examples/custom-payload-builder/",
"examples/custom-rlpx-subprotocol",
"examples/custom-rpc-middleware",
"examples/custom-node",
"examples/db-access",
"examples/engine-api-access",
"examples/exex-hello-world",
"examples/exex-subscription",
"examples/exex-test",
"examples/full-contract-state",
@@ -168,7 +168,6 @@ members = [
"examples/node-builder-api/",
"examples/node-custom-rpc/",
"examples/node-event-hooks/",
"examples/op-db-access/",
"examples/polygon-p2p/",
"examples/rpc-db/",
"examples/precompile-cache/",
@@ -376,11 +375,11 @@ reth-era-utils = { path = "crates/era-utils" }
reth-errors = { path = "crates/errors" }
reth-eth-wire = { path = "crates/net/eth-wire" }
reth-eth-wire-types = { path = "crates/net/eth-wire-types" }
reth-ethereum-payload-builder = { path = "crates/ethereum/payload" }
reth-ethereum-cli = { path = "crates/ethereum/cli", default-features = false }
reth-ethereum-consensus = { path = "crates/ethereum/consensus", default-features = false }
reth-ethereum-engine-primitives = { path = "crates/ethereum/engine-primitives", default-features = false }
reth-ethereum-forks = { path = "crates/ethereum/hardforks", default-features = false }
reth-ethereum-payload-builder = { path = "crates/ethereum/payload" }
reth-ethereum-primitives = { path = "crates/ethereum/primitives", default-features = false }
reth-ethereum = { path = "crates/ethereum/reth" }
reth-etl = { path = "crates/etl" }
@@ -472,7 +471,7 @@ reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-feat
reth-ress-protocol = { path = "crates/ress/protocol" }
reth-ress-provider = { path = "crates/ress/provider" }
# revm v103 (revm v34.0.0 with BAL EIP-7928 support)
# revm
revm = { version = "34.0.0", default-features = false }
revm-bytecode = { version = "8.0.0", default-features = false }
revm-database = { version = "10.0.0", default-features = false }
@@ -480,58 +479,53 @@ revm-state = { version = "9.0.0", default-features = false }
revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
revm-context = { version = "13.0.0", default-features = false }
revm-context-interface = { version = "14.0.0", default-features = false }
revm-inspector = { version = "15.0.0", default-features = false }
revm-handler = { version = "15.0.0", default-features = false }
revm-precompile = { version = "32.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.33.2"
revm-inspectors = "0.34.1"
# eth
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-dyn-abi = "1.5.4"
alloy-primitives = { version = "1.5.4", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.4", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-evm = { version = "0.25.2", default-features = false }
alloy-dyn-abi = "1.4.1"
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
alloy-consensus = { version = "1.6.1", default-features = false }
alloy-contract = { version = "1.6.1", default-features = false }
alloy-eips = { version = "1.6.1", default-features = false }
alloy-genesis = { version = "1.6.1", default-features = false }
alloy-json-rpc = { version = "1.6.1", default-features = false }
alloy-network = { version = "1.6.1", default-features = false }
alloy-network-primitives = { version = "1.6.1", default-features = false }
alloy-provider = { version = "1.6.1", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.6.1", default-features = false }
alloy-rpc-client = { version = "1.6.1", default-features = false }
alloy-rpc-types = { version = "1.6.1", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.6.1", default-features = false }
alloy-rpc-types-anvil = { version = "1.6.1", default-features = false }
alloy-rpc-types-beacon = { version = "1.6.1", default-features = false }
alloy-rpc-types-debug = { version = "1.6.1", default-features = false }
alloy-rpc-types-engine = { version = "1.6.1", default-features = false }
alloy-rpc-types-eth = { version = "1.6.1", default-features = false }
alloy-rpc-types-mev = { version = "1.6.1", default-features = false }
alloy-rpc-types-trace = { version = "1.6.1", default-features = false }
alloy-rpc-types-txpool = { version = "1.6.1", default-features = false }
alloy-serde = { version = "1.6.1", default-features = false }
alloy-signer = { version = "1.6.1", default-features = false }
alloy-signer-local = { version = "1.6.1", default-features = false }
alloy-transport = { version = "1.6.1" }
alloy-transport-http = { version = "1.6.1", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.6.1", default-features = false }
alloy-transport-ws = { version = "1.6.1", default-features = false }
# op
alloy-op-evm = { version = "0.25.2", default-features = false }
alloy-op-evm = { version = "0.27.2", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -549,7 +543,7 @@ backon = { version = "1.2", default-features = false, features = ["std-blocking-
bincode = "1.3"
bitflags = "2.4"
boyer-moore-magiclen = "0.2.16"
bytes = { version = "1.5", default-features = false }
bytes = { version = "1.11.1", default-features = false }
brotli = "8"
cfg-if = "1.0"
clap = "4"
@@ -566,7 +560,7 @@ humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
lz4 = "1.28.1"
modular-bitfield = "0.11.2"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.2", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
@@ -593,15 +587,15 @@ tracing-appender = "0.2"
url = { version = "2.3", default-features = false }
zstd = "0.13"
byteorder = "1"
mini-moka = "0.10"
fixed-cache = { version = "0.1.7", features = ["stats"] }
moka = "0.12"
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
tar-no-std = { version = "0.4.2", default-features = false }
miniz_oxide = { version = "0.9.0", default-features = false }
chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-derive = "0.1.1"
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.20.0" }
@@ -613,7 +607,7 @@ quote = "1.0"
# tokio
tokio = { version = "1.44.2", default-features = false }
tokio-stream = "0.1.11"
tokio-tungstenite = "0.26.2"
tokio-tungstenite = "0.28.0"
tokio-util = { version = "0.7.4", features = ["codec"] }
# async
@@ -626,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
hyper = "1.3"
hyper-util = "0.1.5"
pin-project = "1.0.12"
reqwest = { version = "0.12", default-features = false }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
tracing-futures = "0.2"
tower = "0.5"
tower-http = "0.6"
@@ -646,7 +640,6 @@ jsonrpsee-types = "0.26.0"
http = "1.0"
http-body = "1.0"
http-body-util = "0.1.2"
jsonwebtoken = "9"
proptest-arbitrary-interop = "0.1.0"
# crypto
@@ -660,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
c-kzg = "2.1.5"
# config
toml = "0.8"
toml = "0.9"
# rocksdb
rocksdb = { version = "0.24" }
@@ -676,19 +669,19 @@ tracing-opentelemetry = "0.32"
# misc-testing
arbitrary = "1.3"
assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.5"
proptest-derive = "0.7"
similar-asserts = { version = "1.5.0", features = ["serde"] }
tempfile = "3.20"
test-fuzz = "7"
rstest = "0.24.0"
rstest = "0.26.1"
test-case = "3"
# ssz encoding
ethereum_ssz = "0.9.0"
ethereum_ssz_derive = "0.9.0"
ethereum_ssz = "0.10.1"
ethereum_ssz_derive = "0.10.1"
# allocators
jemalloc_pprof = { version = "0.8", default-features = false }
@@ -700,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
aes = "0.8.1"
ahash = "0.8"
anyhow = "1.0"
bindgen = { version = "0.71", default-features = false }
block-padding = "0.3.2"
bindgen = { version = "0.72", default-features = false }
block-padding = "0.3"
cc = "1.2.15"
cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossterm = "0.28.0"
crossterm = "0.29.0"
csv = "1.3.0"
ctrlc = "3.4"
ctr = "0.9.2"
@@ -720,7 +713,7 @@ hmac = "0.12.1"
human_bytes = "0.4.1"
indexmap = "2"
interprocess = "2.2.0"
lz4_flex = { version = "0.11", default-features = false }
lz4_flex = { version = "0.12", default-features = false }
memmap2 = "0.9.4"
mev-share-sse = { version = "0.5.0", default-features = false }
num-traits = "0.2.15"
@@ -728,70 +721,66 @@ page_size = "0.6.0"
parity-scale-codec = "3.2.1"
plain_hasher = "0.2"
pretty_assertions = "1.4"
ratatui = { version = "0.29", default-features = false }
ringbuffer = "0.15.0"
ratatui = { version = "0.30", default-features = false }
ringbuffer = "0.16.0"
rmp-serde = "1.3"
roaring = "0.10.2"
roaring = "0.11.3"
rolling-file = "0.2.0"
sha3 = "0.10.5"
snap = "1.1.1"
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
socket2 = { version = "0.6", default-features = false }
sysinfo = { version = "0.38", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "0.3.3"
tracing-logfmt = "=0.3.5"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
triehash = "0.8"
typenum = "1.15.0"
vergen = "9.0.4"
vergen = "9.1.0"
visibility = "0.1.1"
walkdir = "2.3.3"
vergen-git2 = "1.0.5"
vergen-git2 = "9.1.0"
# networking
ipnet = "2.11"
[patch.crates-io]
alloy-consensus = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-contract = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-eips = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-genesis = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-json-rpc = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-network = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-network-primitives = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-provider = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-pubsub = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-client = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-admin = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-anvil = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-beacon = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-debug = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-engine = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-eth = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-mev = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-trace = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-txpool = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-serde = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-signer = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-signer-local = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport-http = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport-ipc = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport-ws = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
# alloy-hardforks = { git = "https://github.com/Rimeeeeee/hardforks", branch = "amsterdam" }
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-serde = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-op-hardforks = { git = "https://github.com/Rimeeeeee/hardforks", branch = "amsterdam" }
# op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-jsonrpsee = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
#
# alloy-evm with BAL support for revm v34/v103
alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "staging-revm" }
alloy-op-evm = { git = "https://github.com/alloy-rs/evm", branch = "staging-revm" }
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", branch = "staging-revm" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "1207e33" }
#
# jsonrpsee = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-core = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
@@ -799,5 +788,10 @@ revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", bran
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }

View File

@@ -1,15 +0,0 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth"]

84
Dockerfile.depot Normal file
View File

@@ -0,0 +1,84 @@
# syntax=docker/dockerfile:1
# Unified Dockerfile for reth and op-reth, optimized for Depot builds
# Usage:
# reth: --build-arg BINARY=reth
# op-reth: --build-arg BINARY=op-reth --build-arg MANIFEST_PATH=crates/optimism/bin
FROM rust:1 AS builder
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
RUN apt-get update && apt-get install -y libclang-dev pkg-config
# Install sccache for compilation caching
RUN cargo install sccache --locked
ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
# Binary to build (reth or op-reth)
ARG BINARY=reth
# Manifest path for the binary
ARG MANIFEST_PATH=bin/reth
# Build profile, release by default
ARG BUILD_PROFILE=release
ENV BUILD_PROFILE=$BUILD_PROFILE
# Extra Cargo flags
ARG RUSTFLAGS=""
ENV RUSTFLAGS="$RUSTFLAGS"
# Extra Cargo features
ARG FEATURES=""
ENV FEATURES=$FEATURES
# Git info for vergen (since .git is excluded from Docker context)
ARG VERGEN_GIT_SHA=""
ARG VERGEN_GIT_DESCRIBE=""
ARG VERGEN_GIT_DIRTY="false"
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Build application
COPY --exclude=.git . .
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml
RUN sccache --show-stats || true
# Copy binary to a known location (ARG not resolved in COPY)
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
cp /app/target/release/$BINARY /app/binary
FROM ubuntu:24.04 AS runtime
WORKDIR /app
# Binary name for entrypoint
ARG BINARY=reth
# Install runtime dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Copy binary from build stage and create canonical symlink for entrypoint
COPY --from=builder /app/binary /usr/local/bin/
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
chmod +x /usr/local/bin/$BINARY
# Copy licenses
COPY LICENSE-* ./
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth-binary"]

View File

@@ -1,15 +0,0 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/op-reth"]

134
Makefile
View File

@@ -35,9 +35,6 @@ EEST_TESTS_TAG := v4.5.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
# The docker image name
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
##@ Help
.PHONY: help
@@ -242,137 +239,6 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
--features "$(FEATURES)" \
--profile "$(PROFILE)"
##@ Docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-git-sha
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-latest
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call docker_build_push,nightly,nightly)
.PHONY: docker-build-push-nightly-edge-profiling
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Create a cross-arch Docker image with the given tags and push it
define docker_build_push
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
docker buildx build --file ./Dockerfile.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Optimism docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-git-sha
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-latest
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call op_docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly,nightly)
.PHONY: op-docker-build-push-nightly-edge-profiling
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly-profiling
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
$(call docker_build_push,nightly-profiling,nightly-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly-profiling
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
# Create a cross-arch Docker image with the given tags and push it
define op_docker_build_push
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
docker buildx build --file ./DockerfileOp.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Other
.PHONY: clean

View File

@@ -56,7 +56,7 @@ ctrlc.workspace = true
shlex.workspace = true
[target.'cfg(unix)'.dependencies]
nix = { version = "0.29", features = ["signal", "process"] }
nix = { version = "0.31", features = ["signal", "process"] }
[features]
default = ["jemalloc"]

View File

@@ -274,10 +274,10 @@ impl Args {
/// Get the default RPC URL for a given chain
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
match chain.id() {
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
8453 => "https://base.reth.rs/rpc", // base
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
}
}

View File

@@ -163,6 +163,7 @@ impl NodeManager {
"eth,reth".to_string(),
"--disable-discovery".to_string(),
"--trusted-only".to_string(),
"--disable-tx-gossip".to_string(),
]);
// Add tracing arguments if OTLP endpoint is configured

View File

@@ -32,7 +32,7 @@ alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
# reqwest
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
# tower
tower.workspace = true

View File

@@ -3,7 +3,7 @@
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
@@ -25,9 +25,16 @@ use tracing::info;
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate.
#[arg(long, value_name = "BLOCKS")]
blocks: u64,
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
@@ -42,12 +49,37 @@ pub struct Command {
output: PathBuf,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
if self.blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
// Ensure output directory exists
if self.output.is_file() {
@@ -84,14 +116,31 @@ impl Command {
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
let end_block = start_block + self.blocks - 1;
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
match mode {
RampMode::Blocks(blocks) => {
info!(
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
let mut next_block_number = start_block;
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
while next_block_number <= end_block {
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
@@ -140,13 +189,13 @@ impl Command {
parent_header = block.header;
parent_hash = block_hash;
next_block_number += 1;
blocks_processed += 1;
}
let final_gas_limit = parent_header.gas_limit;
info!(
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed = self.blocks,
blocks_processed,
final_gas_limit,
"Benchmark complete"
);
@@ -158,3 +207,10 @@ impl Command {
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}

View File

@@ -3,7 +3,9 @@
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::authenticated_transport::AuthenticatedTransportConnect;
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
@@ -130,13 +132,24 @@ impl<S: TransactionSource> TransactionCollector<S> {
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
/// Returns a `CollectionResult` with transactions, gas info, and next block.
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
self.collect_gas(start_block, self.target_gas).await
}
/// Collect transactions up to a specific gas target.
///
/// This is used both for initial collection and for retry top-ups.
pub async fn collect_gas(
&self,
start_block: u64,
gas_target: u64,
) -> eyre::Result<CollectionResult> {
let mut transactions: Vec<RawTransaction> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < self.target_gas {
while total_gas < gas_target {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(block = current_block, "Block not found, stopping");
@@ -149,12 +162,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
continue;
}
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
if total_gas + tx.gas_used <= gas_target {
total_gas += tx.gas_used;
transactions.push(tx);
}
if total_gas >= self.target_gas {
if total_gas >= gas_target {
break;
}
}
@@ -162,7 +175,7 @@ impl<S: TransactionSource> TransactionCollector<S> {
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = self.target_gas.saturating_sub(total_gas);
let remaining_gas = gas_target.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
@@ -170,12 +183,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
info!(
total_txs = transactions.len(),
total_gas,
gas_sent = total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok((transactions, total_gas, current_block))
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
}
}
@@ -202,13 +215,26 @@ pub struct Command {
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
target_gas: u64,
/// Starting block number to fetch transactions from.
/// If not specified, starts from the engine's latest block.
/// Block number to start fetching transactions from (required).
///
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
/// The command collects transactions from historical blocks starting at this number
/// to pack into large blocks.
///
/// How to determine this value:
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
/// - If gas limit ramping has already been performed: use the block number that was the chain
/// tip BEFORE ramping began (you must track this yourself)
///
/// Using a block after ramping started will cause transaction collection to fail
/// because those blocks contain synthetic transactions that cannot be replayed.
#[arg(long, value_name = "FROM_BLOCK")]
from_block: Option<u64>,
from_block: u64,
/// Execute the payload (call newPayload + forkchoiceUpdated).
/// If false, only builds the payload and prints it.
@@ -237,6 +263,80 @@ struct BuiltPayload {
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
/// The actual gas used in the built block.
gas_used: u64,
}
/// Result of collecting transactions from blocks.
#[derive(Debug)]
pub struct CollectionResult {
/// Collected transactions with their gas info.
pub transactions: Vec<RawTransaction>,
/// Total gas sent (sum of historical `gas_used` for all collected txs).
pub gas_sent: u64,
/// Next block number to continue collecting from.
pub next_block: u64,
}
/// Constants for retry logic.
const MAX_BUILD_RETRIES: u32 = 5;
/// Maximum retries for fetching a transaction batch.
const MAX_FETCH_RETRIES: u32 = 5;
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
const MIN_TARGET_SLACK: u64 = 1_000_000;
/// Maximum gas to request in retries (10x target as safety cap).
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
/// Fetches a batch of transactions with retry logic.
///
/// Returns `None` if all retries are exhausted.
async fn fetch_batch_with_retry<S: TransactionSource>(
collector: &TransactionCollector<S>,
block: u64,
) -> Option<CollectionResult> {
for attempt in 1..=MAX_FETCH_RETRIES {
match collector.collect(block).await {
Ok(result) => return Some(result),
Err(e) => {
if attempt == MAX_FETCH_RETRIES {
warn!(attempt, error = %e, "Failed to fetch transactions after max retries");
return None;
}
warn!(attempt, error = %e, "Failed to fetch transactions, retrying...");
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
}
None
}
/// Outcome of a build attempt check.
enum RetryOutcome {
/// Payload is close enough to target gas.
Success,
/// Max retries reached, accept what we have.
MaxRetries,
/// Need more transactions with the specified gas amount.
NeedMore(u64),
}
/// Buffer for receiving transaction batches from the fetcher.
///
/// This abstracts over the channel to allow the main loop to request
/// batches on demand, including for retries.
struct TxBuffer {
receiver: mpsc::Receiver<CollectionResult>,
}
impl TxBuffer {
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
Self { receiver }
}
/// Take the next available batch from the fetcher.
async fn take_batch(&mut self) -> Option<CollectionResult> {
self.receiver.recv().await
}
}
impl Command {
@@ -284,7 +384,7 @@ impl Command {
format!("Failed to create output directory: {:?}", self.output_dir)
})?;
let start_block = self.from_block.unwrap_or(parent_number);
let start_block = self.from_block;
// Use pipelined execution when generating multiple payloads
if self.count > 1 {
@@ -297,19 +397,20 @@ impl Command {
)
.await?;
} else {
// Single payload - collect transactions and build
// Single payload - collect transactions and build with retry
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
let result = collector.collect(start_block).await?;
if transactions.is_empty() {
if result.transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential(
self.execute_sequential_with_retry(
&auth_provider,
&testing_provider,
transactions,
&collector,
result,
parent_hash,
parent_timestamp,
)
@@ -320,32 +421,34 @@ impl Command {
Ok(())
}
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
/// Sequential execution path with retry logic for underfilled payloads.
async fn execute_sequential_with_retry<S: TransactionSource>(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
transactions: Vec<Bytes>,
collector: &TransactionCollector<S>,
initial_result: CollectionResult,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let mut current_result = initial_result;
for i in 0..self.count {
let built = self
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.build_with_retry(
testing_provider,
collector,
&mut current_result,
i,
parent_hash,
parent_timestamp,
)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
info!(payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
}
@@ -356,7 +459,62 @@ impl Command {
Ok(())
}
/// Pipelined execution - fetches transactions and builds payloads in background.
/// Build a payload with retry logic when `gas_used` is below target.
///
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
/// are needed to hit the target gas.
async fn build_with_retry<S: TransactionSource>(
&self,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let additional =
collector.collect_gas(result.next_block, additional_gas).await?;
result.transactions.extend(additional.transactions);
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
result.next_block = additional.next_block;
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
}
/// Pipelined execution - fetches transactions in background, builds with retry.
///
/// The fetcher continuously produces transaction batches. The main loop consumes them,
/// builds payloads with retry logic (requesting more transactions if underfilled),
/// and executes each payload before moving to the next.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
@@ -365,167 +523,229 @@ impl Command {
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Create channel for transaction batches - fetcher sends CollectionResult
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(error = %e, "Failed to create transaction source");
return;
return None;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
});
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
let is_last = i == self.count - 1;
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
if batch.transactions.is_empty() {
info!(block = current_block, "Reached chain tip, stopping fetcher");
break;
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
tx_count = batch.transactions.len(),
gas_sent = batch.gas_sent,
blocks = format!("{}..{}", current_block, batch.next_block),
"Fetched transaction batch"
);
self.build_payload(
current_block = batch.next_block;
if tx_sender.send(batch).await.is_err() {
break;
}
}
Some(current_block)
});
// Transaction buffer: holds transactions from batches + any extras from retries
let mut tx_buffer = TxBuffer::new(tx_receiver);
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
for i in 0..self.count {
// Get initial batch of transactions for this payload
let Some(mut result) = tx_buffer.take_batch().await else {
info!(
payloads_built = i,
payloads_requested = self.count,
"Transaction source exhausted, stopping"
);
break;
};
if result.transactions.is_empty() {
info!(
payloads_built = i,
payloads_requested = self.count,
"No more transactions available, stopping"
);
break;
}
// Build with retry - may need to request more transactions
let built = self
.build_with_retry_buffered(
testing_provider,
&transactions,
&mut tx_buffer,
&mut result,
i,
parent_hash,
parent_timestamp,
)
.await?
};
.await?;
self.save_payload(&current_payload)?;
self.save_payload(&built)?;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
let current_block_hash = built.block_hash;
let current_timestamp = built.timestamp;
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
// Execute payload
info!(payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_receiver);
drop(tx_buffer);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
/// Build a payload with retry logic, using the buffered transaction source.
async fn build_with_retry_buffered(
&self,
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
tx_buffer: &mut TxBuffer,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let mut collected_gas = 0u64;
while collected_gas < additional_gas {
if let Some(batch) = tx_buffer.take_batch().await {
collected_gas += batch.gas_sent;
result.transactions.extend(batch.transactions);
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
result.next_block = batch.next_block;
} else {
warn!("Transaction fetcher exhausted, proceeding with available transactions");
break;
}
}
}
}
}
warn!(payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
}
/// Static version for use in spawned tasks.
/// Determines the outcome of a build attempt.
fn check_retry_outcome(
&self,
built: &BuiltPayload,
index: u64,
attempt: u32,
gas_sent: u64,
) -> RetryOutcome {
let gas_used = built.gas_used;
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
attempts = attempt,
"Payload built successfully"
);
return RetryOutcome::Success;
}
if attempt == MAX_BUILD_RETRIES {
warn!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
gas_sent,
"Underfilled after max retries, accepting payload"
);
return RetryOutcome::MaxRetries;
}
if gas_used == 0 {
warn!(
payload = index + 1,
"Zero gas used in payload, requesting fixed chunk of additional transactions"
);
return RetryOutcome::NeedMore(self.target_gas);
}
let gas_sent_needed_total =
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
if additional == 0 {
info!(
payload = index + 1,
gas_used,
target_gas = self.target_gas,
"No additional transactions needed based on ratio"
);
return RetryOutcome::Success;
}
let ratio = gas_used as f64 / gas_sent as f64;
info!(
payload = index + 1,
gas_used,
gas_sent,
ratio = format!("{:.4}", ratio),
additional_gas = additional,
"Underfilled, collecting more transactions for retry"
);
RetryOutcome::NeedMore(additional)
}
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
@@ -541,7 +761,6 @@ impl Command {
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
},
transactions: transactions.to_vec(),
extra_data: None,
@@ -564,8 +783,9 @@ impl Command {
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
let gas_used = inner.gas_used;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
}
/// Save a payload to disk.

View File

@@ -1,6 +1,56 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use eyre::Result;
use std::io::{BufReader, Read};
/// Read input from either a file path or stdin.
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
Ok(match path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly.
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
match jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.to_string())),
}
}
None => Ok(None),
}
}
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
@@ -69,7 +119,6 @@ pub(crate) fn prepare_payload_request(
suggested_fee_recipient: Address::ZERO,
withdrawals: shanghai_active.then(Vec::new),
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
slot_number: None,
},
forkchoice_state: ForkchoiceState {
head_block_hash: parent_hash,
@@ -195,3 +244,50 @@ pub(crate) async fn get_payload_with_sidecar(
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -15,7 +15,9 @@ pub use generate_big_block::{
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod persistence_waiter;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -74,6 +76,18 @@ pub enum Subcommands {
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -89,6 +103,7 @@ impl BenchmarkCommand {
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}

View File

@@ -15,28 +15,23 @@ use crate::{
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
};
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_provider::Provider;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use url::Url;
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -105,7 +100,11 @@ impl Command {
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let sub = self.setup_persistence_subscription().await?;
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
@@ -245,293 +244,22 @@ impl Command {
results.into_iter().unzip();
if let Some(ref path) = self.benchmark.output {
write_benchmark_results(path, &gas_output_results, combined_results)?;
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
let gas_output = TotalGasOutput::new(gas_output_results)?;
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
blocks_processed=?gas_output.blocks_processed,
"Total Ggas/s: {:.4}",
gas_output.total_gigagas_per_second()
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
);
Ok(())
}
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `--ws-rpc-url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %self.benchmark.engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
let ws_url = self.derive_ws_rpc_url()?;
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -6,7 +6,7 @@ use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{path::Path, time::Duration};
use std::{fs, path::Path, time::Duration};
use tracing::info;
/// This is the suffix for gas output csv files.
@@ -158,29 +158,58 @@ pub(crate) struct TotalGasRow {
pub(crate) struct TotalGasOutput {
/// The total gas used in the benchmark.
pub(crate) total_gas_used: u64,
/// The total duration of the benchmark.
/// The total wall-clock duration of the benchmark (includes wait times).
pub(crate) total_duration: Duration,
/// The total gas used per second.
pub(crate) total_gas_per_second: f64,
/// The total execution-only duration (excludes wait times).
pub(crate) execution_duration: Duration,
/// The number of blocks processed.
pub(crate) blocks_processed: u64,
}
impl TotalGasOutput {
/// Create a new [`TotalGasOutput`] from a list of [`TotalGasRow`].
/// Create a new [`TotalGasOutput`] from gas rows only.
///
/// Use this when execution-only timing is not available (e.g., `new_payload_only`).
/// `execution_duration` will equal `total_duration`.
pub(crate) fn new(rows: Vec<TotalGasRow>) -> eyre::Result<Self> {
// the duration is obtained from the last row
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
let total_gas_per_second = total_gas_used as f64 / total_duration.as_secs_f64();
Ok(Self { total_gas_used, total_duration, total_gas_per_second, blocks_processed })
Ok(Self {
total_gas_used,
total_duration,
execution_duration: total_duration,
blocks_processed,
})
}
/// Return the total gigagas per second.
/// Create a new [`TotalGasOutput`] from gas rows and combined results.
///
/// - `rows`: Used for total gas and wall-clock duration
/// - `combined_results`: Used for execution-only duration (sum of `total_latency`)
pub(crate) fn with_combined_results(
rows: Vec<TotalGasRow>,
combined_results: &[CombinedResult],
) -> eyre::Result<Self> {
let total_duration = rows.last().map(|row| row.time).ok_or_eyre("empty results")?;
let blocks_processed = rows.len() as u64;
let total_gas_used: u64 = rows.into_iter().map(|row| row.gas_used).sum();
// Sum execution-only time from combined results
let execution_duration: Duration = combined_results.iter().map(|r| r.total_latency).sum();
Ok(Self { total_gas_used, total_duration, execution_duration, blocks_processed })
}
/// Return the total gigagas per second based on wall-clock time.
pub(crate) fn total_gigagas_per_second(&self) -> f64 {
self.total_gas_per_second / GIGAGAS as f64
self.total_gas_used as f64 / self.total_duration.as_secs_f64() / GIGAGAS as f64
}
/// Return the execution-only gigagas per second (excludes wait times).
pub(crate) fn execution_gigagas_per_second(&self) -> f64 {
self.total_gas_used as f64 / self.execution_duration.as_secs_f64() / GIGAGAS as f64
}
}
@@ -192,8 +221,10 @@ impl TotalGasOutput {
pub(crate) fn write_benchmark_results(
output_dir: &Path,
gas_results: &[TotalGasRow],
combined_results: Vec<CombinedResult>,
combined_results: &[CombinedResult],
) -> eyre::Result<()> {
fs::create_dir_all(output_dir)?;
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;

View File

@@ -0,0 +1,304 @@
//! Persistence waiting utilities for benchmarks.
//!
//! Provides waiting behavior to control benchmark pacing:
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
//! - **Persistence-based waits**: Wait for blocks to be persisted using
//! `reth_subscribePersistedBlock` subscription
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_transport_ws::WsConnect;
use eyre::Context;
use futures::StreamExt;
use std::time::Duration;
use tracing::{debug, info};
/// Default `WebSocket` RPC port for reth.
const DEFAULT_WS_RPC_PORT: u16 = 8546;
use url::Url;
/// Default timeout for waiting on persistence.
pub(crate) const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `ws_rpc_url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since we may only have the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
pub(crate) fn derive_ws_rpc_url(
ws_rpc_url: Option<&str>,
engine_rpc_url: &str,
) -> eyre::Result<Url> {
if let Some(ws_url) = ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url
.set_port(Some(DEFAULT_WS_RPC_PORT))
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
pub(crate) struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
pub(crate) async fn setup_persistence_subscription(
ws_url: Url,
) -> eyre::Result<PersistenceSubscription> {
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
pub(crate) struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
pub(crate) const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Instant;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -2,10 +2,27 @@
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::output::GasRampPayloadFile,
bench::{
output::{
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
PERSISTENCE_CHECKPOINT_TIMEOUT,
},
},
valid_payload::{call_forkchoice_updated, call_new_payload},
};
use alloy_primitives::B256;
@@ -14,11 +31,16 @@ use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use clap::Parser;
use eyre::Context;
use reqwest::Url;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_api::EngineApiMessageVersion;
use std::path::PathBuf;
use std::{
path::PathBuf,
time::{Duration, Instant},
};
use tracing::{debug, info};
use url::Url;
/// `reth bench replay-payloads` command
///
@@ -51,6 +73,42 @@ pub struct Command {
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
/// Optional output directory for benchmark results (CSV files).
#[arg(long, value_name = "OUTPUT")]
output: Option<PathBuf>,
/// How long to wait after a forkchoice update before sending the next payload.
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(
long = "persistence-threshold",
value_name = "PERSISTENCE_THRESHOLD",
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
verbatim_doc_comment
)]
persistence_threshold: u64,
/// Optional `WebSocket` RPC URL for persistence subscription.
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
ws_rpc_url: Option<String>,
}
/// A loaded payload ready for execution.
@@ -78,6 +136,33 @@ impl Command {
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
// Log mode configuration
if let Some(duration) = self.wait_time {
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options (duration takes precedence)
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
let sub = setup_persistence_subscription(ws_url).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
PERSISTENCE_CHECKPOINT_TIMEOUT,
))
}
(None, false) => None,
};
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
@@ -144,6 +229,11 @@ impl Command {
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
if let Some(w) = &mut waiter {
w.on_block(payload.block_number).await?;
}
parent_hash = payload.file.block_hash;
}
@@ -151,22 +241,112 @@ impl Command {
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
let mut results = Vec::new();
let total_benchmark_duration = Instant::now();
for (i, payload) in payloads.iter().enumerate() {
info!(
let envelope = &payload.envelope;
let block_hash = payload.block_hash;
let execution_payload = &envelope.envelope_inner.execution_payload;
let inner_payload = &execution_payload.payload_inner.payload_inner;
let gas_used = inner_payload.gas_used;
let gas_limit = inner_payload.gas_limit;
let block_number = inner_payload.block_number;
let transaction_count =
execution_payload.payload_inner.payload_inner.transactions.len() as u64;
debug!(
payload = i + 1,
total = payloads.len(),
index = payload.index,
block_hash = %payload.block_hash,
block_hash = %block_hash,
"Executing payload (newPayload + FCU)"
);
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
let start = Instant::now();
info!(payload = i + 1, "Payload executed successfully");
parent_hash = payload.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = auth_provider
.new_payload_v4(
execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let combined_result = CombinedResult {
block_number,
gas_limit,
transaction_count,
new_payload_result,
fcu_latency,
total_latency,
};
let current_duration = total_benchmark_duration.elapsed();
info!(%combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
let gas_row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
debug!(?status, ?fcu_result, "Payload executed successfully");
parent_hash = block_hash;
}
info!(count = payloads.len(), "All payloads replayed successfully");
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
if let Some(ref path) = self.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
);
Ok(())
}
@@ -180,7 +360,7 @@ impl Command {
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_")
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
@@ -191,7 +371,7 @@ impl Command {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract index from "payload_NNN.json"
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let index: u64 = index_str.parse().ok()?;
Some((index, e.path()))
})
@@ -284,49 +464,4 @@ impl Command {
Ok(payloads)
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: &ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
info!(?status, "newPayloadV4 response");
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
info!(?fcu_result, "forkchoiceUpdatedV3 response");
Ok(())
}
}

View File

@@ -0,0 +1,219 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
pub(super) struct InvalidationConfig {
// Explicit value overrides (Option<T>)
pub(super) parent_hash: Option<B256>,
pub(super) fee_recipient: Option<Address>,
pub(super) state_root: Option<B256>,
pub(super) receipts_root: Option<B256>,
pub(super) logs_bloom: Option<Bloom>,
pub(super) prev_randao: Option<B256>,
pub(super) block_number: Option<u64>,
pub(super) gas_limit: Option<u64>,
pub(super) gas_used: Option<u64>,
pub(super) timestamp: Option<u64>,
pub(super) extra_data: Option<Bytes>,
pub(super) base_fee_per_gas: Option<u64>,
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
pub(super) invalidate_state_root: bool,
pub(super) invalidate_receipts_root: bool,
pub(super) invalidate_gas_used: bool,
pub(super) invalidate_block_number: bool,
pub(super) invalidate_timestamp: bool,
pub(super) invalidate_base_fee: bool,
pub(super) invalidate_transactions: bool,
pub(super) invalidate_block_hash: bool,
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
}
impl InvalidationConfig {
/// Returns true if `block_hash` is being explicitly set or auto-invalidated.
/// When true, the caller should skip recalculating the block hash since it will be overwritten.
pub(super) const fn should_skip_hash_recalc(&self) -> bool {
self.block_hash.is_some() || self.invalidate_block_hash
}
/// Applies invalidations to a V1 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v1(&self, payload: &mut ExecutionPayloadV1) -> Vec<String> {
let mut changes = Vec::new();
// Explicit value overrides
if let Some(parent_hash) = self.parent_hash {
payload.parent_hash = parent_hash;
changes.push(format!("parent_hash = {parent_hash}"));
}
if let Some(fee_recipient) = self.fee_recipient {
payload.fee_recipient = fee_recipient;
changes.push(format!("fee_recipient = {fee_recipient}"));
}
if let Some(state_root) = self.state_root {
payload.state_root = state_root;
changes.push(format!("state_root = {state_root}"));
}
if let Some(receipts_root) = self.receipts_root {
payload.receipts_root = receipts_root;
changes.push(format!("receipts_root = {receipts_root}"));
}
if let Some(logs_bloom) = self.logs_bloom {
payload.logs_bloom = logs_bloom;
changes.push("logs_bloom = <custom>".to_string());
}
if let Some(prev_randao) = self.prev_randao {
payload.prev_randao = prev_randao;
changes.push(format!("prev_randao = {prev_randao}"));
}
if let Some(block_number) = self.block_number {
payload.block_number = block_number;
changes.push(format!("block_number = {block_number}"));
}
if let Some(gas_limit) = self.gas_limit {
payload.gas_limit = gas_limit;
changes.push(format!("gas_limit = {gas_limit}"));
}
if let Some(gas_used) = self.gas_used {
payload.gas_used = gas_used;
changes.push(format!("gas_used = {gas_used}"));
}
if let Some(timestamp) = self.timestamp {
payload.timestamp = timestamp;
changes.push(format!("timestamp = {timestamp}"));
}
if let Some(ref extra_data) = self.extra_data {
payload.extra_data = extra_data.clone();
changes.push(format!("extra_data = {} bytes", extra_data.len()));
}
if let Some(base_fee_per_gas) = self.base_fee_per_gas {
payload.base_fee_per_gas = U256::from_limbs([base_fee_per_gas, 0, 0, 0]);
changes.push(format!("base_fee_per_gas = {base_fee_per_gas}"));
}
if let Some(block_hash) = self.block_hash {
payload.block_hash = block_hash;
changes.push(format!("block_hash = {block_hash}"));
}
// Auto-invalidation flags
if self.invalidate_parent_hash {
let random_hash = B256::random();
payload.parent_hash = random_hash;
changes.push(format!("parent_hash = {random_hash} (auto-invalidated: random)"));
}
if self.invalidate_state_root {
payload.state_root = B256::ZERO;
changes.push("state_root = ZERO (auto-invalidated: empty trie root)".to_string());
}
if self.invalidate_receipts_root {
payload.receipts_root = B256::ZERO;
changes.push("receipts_root = ZERO (auto-invalidated)".to_string());
}
if self.invalidate_gas_used {
let invalid_gas = payload.gas_limit + 1;
payload.gas_used = invalid_gas;
changes.push(format!("gas_used = {invalid_gas} (auto-invalidated: exceeds gas_limit)"));
}
if self.invalidate_block_number {
let invalid_number = payload.block_number + 999;
payload.block_number = invalid_number;
changes.push(format!("block_number = {invalid_number} (auto-invalidated: huge gap)"));
}
if self.invalidate_timestamp {
payload.timestamp = 0;
changes.push("timestamp = 0 (auto-invalidated: impossibly old)".to_string());
}
if self.invalidate_base_fee {
payload.base_fee_per_gas = U256::ZERO;
changes
.push("base_fee_per_gas = 0 (auto-invalidated: invalid post-London)".to_string());
}
if self.invalidate_transactions {
let invalid_tx = Bytes::from_static(&[0xff, 0xff, 0xff]);
payload.transactions.insert(0, invalid_tx);
changes.push("transactions = prepended invalid RLP (auto-invalidated)".to_string());
}
if self.invalidate_block_hash {
let random_hash = B256::random();
payload.block_hash = random_hash;
changes.push(format!("block_hash = {random_hash} (auto-invalidated: random)"));
}
changes
}
/// Applies invalidations to a V2 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v2(&self, payload: &mut ExecutionPayloadV2) -> Vec<String> {
let mut changes = self.apply_to_payload_v1(&mut payload.payload_inner);
// Handle withdrawals invalidation (V2+)
if self.invalidate_withdrawals {
let fake_withdrawal = Withdrawal {
index: u64::MAX,
validator_index: u64::MAX,
address: Address::ZERO,
amount: u64::MAX,
};
payload.withdrawals.push(fake_withdrawal);
changes.push("withdrawals = added fake withdrawal (auto-invalidated)".to_string());
}
changes
}
/// Applies invalidations to a V3 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v3(&self, payload: &mut ExecutionPayloadV3) -> Vec<String> {
let mut changes = self.apply_to_payload_v2(&mut payload.payload_inner);
// Explicit overrides for V3 fields
if let Some(blob_gas_used) = self.blob_gas_used {
payload.blob_gas_used = blob_gas_used;
changes.push(format!("blob_gas_used = {blob_gas_used}"));
}
if let Some(excess_blob_gas) = self.excess_blob_gas {
payload.excess_blob_gas = excess_blob_gas;
changes.push(format!("excess_blob_gas = {excess_blob_gas}"));
}
// Auto-invalidation for V3 fields
if self.invalidate_blob_gas_used {
payload.blob_gas_used = u64::MAX;
changes.push("blob_gas_used = MAX (auto-invalidated)".to_string());
}
if self.invalidate_excess_blob_gas {
payload.excess_blob_gas = u64::MAX;
changes.push("excess_blob_gas = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -0,0 +1,347 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use invalidation::InvalidationConfig;
use super::helpers::{load_jwt_secret, read_input};
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::Write;
/// Command for generating and sending an invalid `engine_newPayload` request.
///
/// Takes a valid block and modifies fields to make it invalid for testing
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalidate-block-hash` or `--skip-hash-recalc` is used.
#[derive(Debug, Parser)]
pub struct Command {
// ==================== Input Options ====================
/// Path to the JSON file containing the block. If not specified, stdin will be used.
#[arg(short, long, help_heading = "Input Options")]
path: Option<String>,
/// The engine RPC URL to use.
#[arg(
short,
long,
help_heading = "Input Options",
required_if_eq_any([("mode", "execute"), ("mode", "cast")]),
required_unless_present("mode")
)]
rpc_url: Option<String>,
/// The JWT secret to use. Can be either a path to a file containing the secret or the secret
/// itself.
#[arg(short, long, help_heading = "Input Options")]
jwt_secret: Option<String>,
/// The newPayload version to use (3 or 4).
#[arg(long, default_value_t = 3, help_heading = "Input Options")]
new_payload_version: u8,
/// The output mode to use.
#[arg(long, value_enum, default_value = "execute", help_heading = "Input Options")]
mode: Mode,
// ==================== Explicit Value Overrides ====================
/// Override the parent hash with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_hash: Option<B256>,
/// Override the fee recipient (coinbase) with a specific address.
#[arg(long, value_name = "ADDR", help_heading = "Explicit Value Overrides")]
fee_recipient: Option<Address>,
/// Override the state root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
state_root: Option<B256>,
/// Override the receipts root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
receipts_root: Option<B256>,
/// Override the block number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
block_number: Option<u64>,
/// Override the gas limit with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_limit: Option<u64>,
/// Override the gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
gas_used: Option<u64>,
/// Override the timestamp with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
timestamp: Option<u64>,
/// Override the base fee per gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
base_fee_per_gas: Option<u64>,
/// Override the block hash with a specific value (skips hash recalculation).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
block_hash: Option<B256>,
/// Override the blob gas used with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
blob_gas_used: Option<u64>,
/// Override the excess blob gas with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
excess_blob_gas: Option<u64>,
/// Override the parent beacon block root with a specific value.
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
parent_beacon_block_root: Option<B256>,
/// Override the requests hash with a specific value (EIP-7685).
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_parent_hash: bool,
/// Invalidate the state root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_state_root: bool,
/// Invalidate the receipts root by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_receipts_root: bool,
/// Invalidate the gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_gas_used: bool,
/// Invalidate the block number by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_number: bool,
/// Invalidate the timestamp by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_timestamp: bool,
/// Invalidate the base fee by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_base_fee: bool,
/// Invalidate the transactions by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_transactions: bool,
/// Invalidate the block hash by not recalculating it after modifications.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_hash: bool,
/// Invalidate the withdrawals by modifying them.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_withdrawals: bool,
/// Invalidate the blob gas used by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_blob_gas_used: bool,
/// Invalidate the excess blob gas by setting it to an incorrect value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_excess_blob_gas: bool,
/// Invalidate the requests hash by setting it to a random value (EIP-7685).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
skip_hash_recalc: bool,
/// Print what would be done without actually sending the payload.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
dry_run: bool,
}
#[derive(Debug, Clone, clap::ValueEnum)]
enum Mode {
/// Execute the `cast` command. This works with blocks of any size, because it pipes the
/// payload into the `cast` command.
Execute,
/// Print the `cast` command. Caution: this may not work with large blocks because of the
/// command length limit.
Cast,
/// Print the JSON payload. Can be piped into `cast` command if the block is small enough.
Json,
}
impl Command {
/// Build `InvalidationConfig` from command flags
const fn build_invalidation_config(&self) -> InvalidationConfig {
InvalidationConfig {
parent_hash: self.parent_hash,
fee_recipient: self.fee_recipient,
state_root: self.state_root,
receipts_root: self.receipts_root,
logs_bloom: None,
prev_randao: None,
block_number: self.block_number,
gas_limit: self.gas_limit,
gas_used: self.gas_used,
timestamp: self.timestamp,
extra_data: None,
base_fee_per_gas: self.base_fee_per_gas,
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
invalidate_gas_used: self.invalidate_gas_used,
invalidate_block_number: self.invalidate_block_number,
invalidate_timestamp: self.invalidate_timestamp,
invalidate_base_fee: self.invalidate_base_fee,
invalidate_transactions: self.invalidate_transactions,
invalidate_block_hash: self.invalidate_block_hash,
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
}
}
/// Execute the command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
let block_json = read_input(self.path.as_deref())?;
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
.try_map_transactions(|tx| tx.try_into_either::<OpTxEnvelope>())?
.into_consensus();
let config = self.build_invalidation_config();
let parent_beacon_block_root =
self.parent_beacon_block_root.or(block.header.parent_beacon_block_root);
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
if !skip_recalc {
let new_hash = match execution_payload.clone().into_block_raw() {
Ok(block) => block.header.hash_slow(),
Err(e) => {
eprintln!(
"Warning: Could not recalculate block hash: {e}. Using original hash."
);
match &execution_payload {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
}
}
};
match &mut execution_payload {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
}
}
if self.dry_run {
println!("=== Dry Run ===");
println!("Changes that would be applied:");
for change in &changes {
println!(" - {}", change);
}
if changes.is_empty() {
println!(" (no changes)");
}
if skip_recalc {
println!(" - Block hash recalculation: SKIPPED");
} else {
println!(" - Block hash recalculation: PERFORMED");
}
println!("\nResulting payload JSON:");
let json = serde_json::to_string_pretty(&execution_payload)?;
println!("{}", json);
return Ok(());
}
let json_request = if use_v4 {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
requests_hash.unwrap_or_default(),
))?
} else {
serde_json::to_string(&(
execution_payload,
blob_versioned_hashes,
parent_beacon_block_root,
))?
};
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
}
if let Some(secret) = &jwt_secret {
command.arg("--jwt-secret").arg(secret);
}
let mut process = command.stdin(std::process::Stdio::piped()).spawn()?;
process
.stdin
.take()
.ok_or_eyre("stdin not available")?
.write_all(json_request.as_bytes())?;
process.wait()?;
}
Mode::Cast => {
let mut cmd = format!(
"cast rpc engine_newPayloadV{} --raw '{}'",
self.new_payload_version, json_request
);
if let Some(rpc_url) = self.rpc_url {
cmd += &format!(" --rpc-url {rpc_url}");
}
if let Some(secret) = &jwt_secret {
cmd += &format!(" --jwt-secret {secret}");
}
println!("{cmd}");
}
Mode::Json => {
println!("{json_request}");
}
}
Ok(())
}
}

View File

@@ -1,10 +1,11 @@
use super::helpers::{load_jwt_secret, read_input};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
use op_alloy_consensus::OpTxEnvelope;
use reth_cli_runner::CliContext;
use std::io::{BufReader, Read, Write};
use std::io::Write;
/// Command for generating and sending an `engine_newPayload` request constructed from an RPC
/// block.
@@ -51,38 +52,13 @@ enum Mode {
}
impl Command {
/// Read input from either a file or stdin
fn read_input(&self) -> Result<String> {
Ok(match &self.path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly
fn load_jwt_secret(&self) -> Result<Option<String>> {
match &self.jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.clone())),
}
}
None => Ok(None),
}
}
/// Execute the generate payload command
pub async fn execute(self, _ctx: CliContext) -> Result<()> {
// Load block
let block_json = self.read_input()?;
let block_json = read_input(self.path.as_deref())?;
// Load JWT secret
let jwt_secret = self.load_jwt_secret()?;
let jwt_secret = load_jwt_secret(self.jwt_secret.as_deref())?;
// Parse the block
let block = serde_json::from_str::<AnyRpcBlock>(&block_json)?

View File

@@ -181,45 +181,6 @@ pub(crate) fn payload_to_new_payload(
target_version: Option<EngineApiMessageVersion>,
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
let (version, params) = match payload {
ExecutionPayload::V4(payload) => {
let cancun = sidecar.cancun().unwrap();
if let Some(prague) = sidecar.prague() {
if is_optimism {
(
EngineApiMessageVersion::V4,
serde_json::to_value((
OpExecutionPayloadV4 {
payload_inner: payload.payload_inner,
withdrawals_root: withdrawals_root.unwrap(),
},
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
Requests::default(),
))?,
)
} else {
(
EngineApiMessageVersion::V4,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
prague.requests.requests_hash(),
))?,
)
}
} else {
(
EngineApiMessageVersion::V3,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
))?,
)
}
}
ExecutionPayload::V3(payload) => {
let cancun = sidecar.cancun().unwrap();
@@ -242,11 +203,7 @@ pub(crate) fn payload_to_new_payload(
)
} else {
// Extract actual Requests from RequestsOrHash
let requests = prague
.requests
.requests()
.cloned()
.ok_or_else(|| eyre::eyre!("Prague sidecar has hash, not requests"))?;
let requests = prague.requests.requests_hash();
(
version,
serde_json::to_value((
@@ -303,7 +260,9 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
while !status.is_valid() {
if status.is_invalid() {
error!(?status, ?params, "Invalid {method}",);
panic!("Invalid {method}: {status:?}");
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
format!("Invalid {method}: {status:?}"),
))))
}
if status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
@@ -328,10 +287,7 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 |
EngineApiMessageVersion::V6 => {
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V2 => {

View File

@@ -25,8 +25,8 @@
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.
//! Use `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!

View File

@@ -41,6 +41,7 @@ derive_more.workspace = true
metrics.workspace = true
parking_lot.workspace = true
pin-project.workspace = true
rayon = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
# optional deps for test-utils
@@ -84,6 +85,7 @@ test-utils = [
"reth-trie/test-utils",
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"

View File

@@ -163,14 +163,29 @@ impl DeferredTrieData {
anchor_hash: B256,
ancestors: &[Self],
) -> ComputedTrieData {
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
#[cfg(feature = "rayon")]
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|| match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
|| match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
#[cfg(not(feature = "rayon"))]
let (sorted_hashed_state, sorted_trie_updates) = (
match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
// Reuse parent's overlay if available and anchors match.
// We can only reuse the parent's overlay if it was built on top of the same
@@ -191,11 +206,33 @@ impl DeferredTrieData {
Default::default(), // prefix_sets are per-block, not cumulative
);
// Only trigger COW clone if there's actually data to add.
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
#[cfg(feature = "rayon")]
{
rayon::join(
|| {
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
},
|| {
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
},
);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
#[cfg(not(feature = "rayon"))]
{
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state)
.extend_ref_and_sort(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes)
.extend_ref_and_sort(&sorted_trie_updates);
}
}
overlay
}
@@ -242,13 +279,22 @@ impl DeferredTrieData {
for ancestor in ancestors {
let ancestor_data = ancestor.wait_cloned();
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
}
// Extend with current block's sorted data last (takes precedence)
state_mut.extend_ref(sorted_hashed_state);
nodes_mut.extend_ref(sorted_trie_updates);
#[cfg(feature = "rayon")]
rayon::join(
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
);
#[cfg(not(feature = "rayon"))]
{
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
}
overlay
}
@@ -287,6 +333,11 @@ impl DeferredTrieData {
&inputs.ancestors,
);
*state = DeferredState::Ready(computed.clone());
// Release lock before inputs (and its ancestors) drop to avoid holding it
// while their potential last Arc refs drop (which could trigger recursive locking)
drop(state);
computed
}
}
@@ -516,7 +567,7 @@ mod tests {
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
let trie_updates = Arc::default();
let mut overlay = TrieInputSorted::default();
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
DeferredTrieData::ready(ComputedTrieData {
hashed_state,

View File

@@ -10,15 +10,18 @@ use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256};
use parking_lot::RwLock;
use reth_chainspec::ChainInfo;
use reth_ethereum_primitives::EthPrimitives;
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_metrics::{metrics::Gauge, Metrics};
use reth_primitives_traits::{
BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
SignedTransaction,
};
use reth_storage_api::StateProviderBox;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::{collections::BTreeMap, ops::Deref, sync::Arc, time::Instant};
use reth_trie::{
updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, SortedTrieData,
TrieInputSorted,
};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tokio::sync::{broadcast, watch};
/// Size of the broadcast channel used to notify canonical state events.
@@ -648,7 +651,7 @@ impl<N: NodePrimitives> BlockState<N> {
}
/// Returns the `Receipts` of executed block that determines the state.
pub fn receipts(&self) -> &Vec<Vec<N::Receipt>> {
pub fn receipts(&self) -> &Vec<N::Receipt> {
&self.block.execution_outcome().receipts
}
@@ -659,15 +662,7 @@ impl<N: NodePrimitives> BlockState<N> {
///
/// This clones the vector of receipts. To avoid it, use [`Self::executed_block_receipts_ref`].
pub fn executed_block_receipts(&self) -> Vec<N::Receipt> {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().cloned().unwrap_or_default()
self.receipts().clone()
}
/// Returns a slice of `Receipt` of executed block that determines the state.
@@ -675,15 +670,7 @@ impl<N: NodePrimitives> BlockState<N> {
/// has only one element corresponding to the executed block associated to
/// the state.
pub fn executed_block_receipts_ref(&self) -> &[N::Receipt] {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().map(|receipts| receipts.deref()).unwrap_or_default()
self.receipts()
}
/// Returns an iterator over __parent__ `BlockStates`.
@@ -767,7 +754,7 @@ pub struct ExecutedBlock<N: NodePrimitives = EthPrimitives> {
/// Recovered Block
pub recovered_block: Arc<RecoveredBlock<N::Block>>,
/// Block's execution outcome.
pub execution_output: Arc<ExecutionOutcome<N::Receipt>>,
pub execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
/// Deferred trie data produced by execution.
///
/// This allows deferring the computation of the trie data which can be expensive.
@@ -779,7 +766,15 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
fn default() -> Self {
Self {
recovered_block: Default::default(),
execution_output: Default::default(),
execution_output: Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: Default::default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: Default::default(),
}),
trie_data: DeferredTrieData::ready(ComputedTrieData::default()),
}
}
@@ -800,7 +795,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// payload builders). This is the safe default path.
pub fn new(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: ComputedTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data: DeferredTrieData::ready(trie_data) }
@@ -822,7 +817,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Use [`Self::new()`] instead when trie data is already computed and available immediately.
pub const fn with_deferred_trie_data(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: DeferredTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data }
@@ -842,7 +837,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Returns a reference to the block's execution outcome
#[inline]
pub fn execution_outcome(&self) -> &ExecutionOutcome<N::Receipt> {
pub fn execution_outcome(&self) -> &BlockExecutionOutput<N::Receipt> {
&self.execution_output
}
@@ -942,37 +937,53 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
pub fn to_chain_notification(&self) -> CanonStateNotification<N> {
match self {
Self::Commit { new } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
);
chain
}));
CanonStateNotification::Commit { new }
CanonStateNotification::Commit { new: Arc::new(Self::blocks_to_chain(new)) }
}
Self::Reorg { new, old } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
Self::Reorg { new, old } => CanonStateNotification::Reorg {
new: Arc::new(Self::blocks_to_chain(new)),
old: Arc::new(Self::blocks_to_chain(old)),
},
}
}
/// Converts a slice of executed blocks into a [`Chain`].
fn blocks_to_chain(blocks: &[ExecutedBlock<N>]) -> Chain<N> {
match blocks {
[] => Chain::default(),
[first, rest @ ..] => {
let trie_data_handle = first.trie_data_handle();
let mut chain = Chain::from_block(
first.recovered_block().clone(),
ExecutionOutcome::from((
first.execution_outcome().clone(),
first.block_number(),
)),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
);
for exec in rest {
let trie_data_handle = exec.trie_data_handle();
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
ExecutionOutcome::from((
exec.execution_outcome().clone(),
exec.block_number(),
)),
LazyTrieData::deferred(move || {
let trie_data = trie_data_handle.wait_cloned();
SortedTrieData {
hashed_state: trie_data.hashed_state,
trie_updates: trie_data.trie_updates,
}
}),
);
chain
}));
let old = Arc::new(old.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
);
chain
}));
CanonStateNotification::Reorg { new, old }
}
chain
}
}
}
@@ -1266,7 +1277,7 @@ mod tests {
let state = BlockState::new(block);
assert_eq!(state.receipts(), &receipts);
assert_eq!(state.receipts(), receipts.first().unwrap());
}
#[test]
@@ -1543,33 +1554,31 @@ mod tests {
let block2a =
test_block_builder.get_executed_block_with_number(2, block1.recovered_block.hash());
let sample_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
..Default::default()
};
// Test commit notification
let chain_commit = NewCanonicalChain::Commit { new: vec![block0.clone(), block1.clone()] };
// Build expected trie updates map
let mut expected_trie_updates = BTreeMap::new();
expected_trie_updates.insert(0, block0.trie_updates());
expected_trie_updates.insert(1, block1.trie_updates());
// Build expected trie data map
let mut expected_trie_data = BTreeMap::new();
expected_trie_data
.insert(0, LazyTrieData::ready(block0.hashed_state(), block0.trie_updates()));
expected_trie_data
.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
// Build expected hashed state map
let mut expected_hashed_state = BTreeMap::new();
expected_hashed_state.insert(0, block0.hashed_state());
expected_hashed_state.insert(1, block1.hashed_state());
// Build expected execution outcome (first_block matches first block number)
let commit_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 0,
..Default::default()
};
assert_eq!(
chain_commit.to_chain_notification(),
CanonStateNotification::Commit {
new: Arc::new(Chain::new(
vec![block0.recovered_block().clone(), block1.recovered_block().clone()],
sample_execution_outcome.clone(),
expected_trie_updates,
expected_hashed_state
commit_execution_outcome,
expected_trie_data,
))
}
);
@@ -1580,40 +1589,39 @@ mod tests {
old: vec![block1.clone(), block2.clone()],
};
// Build expected trie updates for old chain
let mut old_trie_updates = BTreeMap::new();
old_trie_updates.insert(1, block1.trie_updates());
old_trie_updates.insert(2, block2.trie_updates());
// Build expected trie data for old chain
let mut old_trie_data = BTreeMap::new();
old_trie_data.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
old_trie_data.insert(2, LazyTrieData::ready(block2.hashed_state(), block2.trie_updates()));
// Build expected trie updates for new chain
let mut new_trie_updates = BTreeMap::new();
new_trie_updates.insert(1, block1a.trie_updates());
new_trie_updates.insert(2, block2a.trie_updates());
// Build expected trie data for new chain
let mut new_trie_data = BTreeMap::new();
new_trie_data
.insert(1, LazyTrieData::ready(block1a.hashed_state(), block1a.trie_updates()));
new_trie_data
.insert(2, LazyTrieData::ready(block2a.hashed_state(), block2a.trie_updates()));
// Build expected hashed state for old chain
let mut old_hashed_state = BTreeMap::new();
old_hashed_state.insert(1, block1.hashed_state());
old_hashed_state.insert(2, block2.hashed_state());
// Build expected hashed state for new chain
let mut new_hashed_state = BTreeMap::new();
new_hashed_state.insert(1, block1a.hashed_state());
new_hashed_state.insert(2, block2a.hashed_state());
// Build expected execution outcome for reorg chains (first_block matches first block
// number)
let reorg_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 1,
..Default::default()
};
assert_eq!(
chain_reorg.to_chain_notification(),
CanonStateNotification::Reorg {
old: Arc::new(Chain::new(
vec![block1.recovered_block().clone(), block2.recovered_block().clone()],
sample_execution_outcome.clone(),
old_trie_updates,
old_hashed_state
reorg_execution_outcome.clone(),
old_trie_data,
)),
new: Arc::new(Chain::new(
vec![block1a.recovered_block().clone(), block2a.recovered_block().clone()],
sample_execution_outcome,
new_trie_updates,
new_hashed_state
reorg_execution_outcome,
new_trie_data,
))
}
);

View File

@@ -0,0 +1,186 @@
//! Lazy overlay computation for trie input.
//!
//! This module provides [`LazyOverlay`], a type that computes the [`TrieInputSorted`]
//! lazily on first access. This allows execution to start before the trie overlay
//! is fully computed.
use crate::DeferredTrieData;
use alloy_primitives::B256;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::sync::{Arc, OnceLock};
use tracing::{debug, trace};
/// Inputs captured for lazy overlay computation.
#[derive(Clone)]
struct LazyOverlayInputs {
/// The persisted ancestor hash (anchor) this overlay should be built on.
anchor_hash: B256,
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
blocks: Vec<DeferredTrieData>,
}
/// Lazily computed trie overlay.
///
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
/// but for overlay computation.
///
/// # Fast Path vs Slow Path
///
/// - **Fast path**: If the tip block's cached `anchored_trie_input` is ready and its `anchor_hash`
/// matches our expected anchor, we can reuse it directly (O(1)).
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
#[derive(Clone)]
pub struct LazyOverlay {
/// Computed result, cached after first access.
inner: Arc<OnceLock<TrieInputSorted>>,
/// Inputs for lazy computation.
inputs: LazyOverlayInputs,
}
impl std::fmt::Debug for LazyOverlay {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LazyOverlay")
.field("anchor_hash", &self.inputs.anchor_hash)
.field("num_blocks", &self.inputs.blocks.len())
.field("computed", &self.inner.get().is_some())
.finish()
}
}
impl LazyOverlay {
/// Create a new lazy overlay with the given anchor hash and block handles.
///
/// # Arguments
///
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
}
/// Returns the anchor hash this overlay is built on.
pub const fn anchor_hash(&self) -> B256 {
self.inputs.anchor_hash
}
/// Returns the number of in-memory blocks this overlay covers.
pub const fn num_blocks(&self) -> usize {
self.inputs.blocks.len()
}
/// Returns true if the overlay has already been computed.
pub fn is_computed(&self) -> bool {
self.inner.get().is_some()
}
/// Returns the computed trie input, computing it if necessary.
///
/// The first call triggers computation (which may block waiting for deferred data).
/// Subsequent calls return the cached result immediately.
pub fn get(&self) -> &TrieInputSorted {
self.inner.get_or_init(|| self.compute())
}
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
let input = self.get();
(Arc::clone(&input.nodes), Arc::clone(&input.state))
}
/// Compute the trie input overlay.
fn compute(&self) -> TrieInputSorted {
let anchor_hash = self.inputs.anchor_hash;
let blocks = &self.inputs.blocks;
if blocks.is_empty() {
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
return TrieInputSorted::default();
}
// Fast path: Check if tip block's overlay is ready and anchor matches.
// The tip block (first in list) has the cumulative overlay from all ancestors.
if let Some(tip) = blocks.first() {
let data = tip.wait_cloned();
if let Some(anchored) = &data.anchored_trie_input {
if anchored.anchor_hash == anchor_hash {
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
return (*anchored.trie_input).clone();
}
debug!(
target: "chain_state::lazy_overlay",
computed_anchor = %anchored.anchor_hash,
%anchor_hash,
"Anchor mismatch, falling back to merge"
);
}
}
// Slow path: Merge all blocks' trie data into a new overlay.
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
Self::merge_blocks(blocks)
}
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
///
/// Blocks are ordered newest to oldest.
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
if blocks.is_empty() {
return TrieInputSorted::default();
}
let state =
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
let nodes =
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
}
}
#[cfg(test)]
mod tests {
use super::*;
use reth_trie::{updates::TrieUpdates, HashedPostState};
fn empty_deferred(anchor: B256) -> DeferredTrieData {
DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
anchor,
Vec::new(),
)
}
#[test]
fn empty_blocks_returns_default() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
let result = overlay.get();
assert!(result.state.is_empty());
assert!(result.nodes.is_empty());
}
#[test]
fn single_block_uses_data_directly() {
let anchor = B256::random();
let deferred = empty_deferred(anchor);
let overlay = LazyOverlay::new(anchor, vec![deferred]);
assert!(!overlay.is_computed());
let _ = overlay.get();
assert!(overlay.is_computed());
}
#[test]
fn cached_after_first_access() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
// First access computes
let _ = overlay.get();
assert!(overlay.is_computed());
// Second access uses cache
let _ = overlay.get();
assert!(overlay.is_computed());
}
}

View File

@@ -14,6 +14,9 @@ pub use in_memory::*;
mod deferred_trie;
pub use deferred_trie::*;
mod lazy_overlay;
pub use lazy_overlay::*;
mod noop;
mod chain_info;

View File

@@ -280,7 +280,6 @@ mod tests {
vec![block1.clone(), block2.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
// Create a commit notification
@@ -319,13 +318,11 @@ mod tests {
vec![block1.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
let new_chain = Arc::new(Chain::new(
vec![block2.clone(), block3.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
// Create a reorg notification
@@ -391,7 +388,6 @@ mod tests {
vec![block1.clone(), block2.clone()],
execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
// Create a commit notification containing the new chain segment.
@@ -449,12 +445,8 @@ mod tests {
ExecutionOutcome { receipts: old_receipts, ..Default::default() };
// Create an old chain segment to be reverted, containing `old_block1`.
let old_chain: Arc<Chain> = Arc::new(Chain::new(
vec![old_block1.clone()],
old_execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
let old_chain: Arc<Chain> =
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, BTreeMap::new()));
// Define block2 for the new chain segment, which will be committed.
let mut body = BlockBody::<TransactionSigned>::default();
@@ -482,12 +474,8 @@ mod tests {
ExecutionOutcome { receipts: new_receipts, ..Default::default() };
// Create a new chain segment to be committed, containing `new_block1`.
let new_chain = Arc::new(Chain::new(
vec![new_block1.clone()],
new_execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
let new_chain =
Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, BTreeMap::new()));
// Create a reorg notification with both reverted (old) and committed (new) chain segments.
let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain };

View File

@@ -3,10 +3,7 @@ use crate::{
CanonStateSubscriptions, ComputedTrieData,
};
use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH};
use alloy_eips::{
eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE},
eip7685::Requests,
};
use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE};
use alloy_primitives::{Address, BlockNumber, B256, U256};
use alloy_signer::SignerSync;
use alloy_signer_local::PrivateKeySigner;
@@ -16,7 +13,7 @@ use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS};
use reth_ethereum_primitives::{
Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned,
};
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_primitives_traits::{
proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root},
Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader,
@@ -173,7 +170,6 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
transactions: transactions.into_iter().map(|tx| tx.into_inner()).collect(),
ommers: Vec::new(),
withdrawals: Some(vec![].into()),
block_access_list: None,
},
)
}
@@ -202,7 +198,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
fn get_executed_block(
&mut self,
block_number: BlockNumber,
receipts: Vec<Vec<Receipt>>,
mut receipts: Vec<Vec<Receipt>>,
parent_hash: B256,
) -> ExecutedBlock {
let block = self.generate_random_block(block_number, parent_hash);
@@ -210,12 +206,15 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
let trie_data = ComputedTrieData::default();
ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(ExecutionOutcome::new(
BundleState::default(),
receipts,
block_number,
vec![Requests::default()],
)),
Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: receipts.pop().unwrap_or_default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: BundleState::default(),
}),
trie_data,
)
}

View File

@@ -25,6 +25,7 @@ pub use alloy_chains::{Chain, ChainKind, NamedChain};
/// Re-export for convenience
pub use reth_ethereum_forks::*;
pub use alloy_evm::EvmLimitParams;
pub use api::EthChainSpec;
pub use info::ChainInfo;
#[cfg(any(test, feature = "test-utils"))]

View File

@@ -45,10 +45,6 @@ use reth_network_peers::{
};
use reth_primitives_traits::{sync::LazyLock, BlockHeader, SealedHeader};
/// The hash of an empty block access list.
const EMPTY_BLOCK_ACCESS_LIST_HASH: B256 =
b256!("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347");
/// Helper method building a [`Header`] given [`Genesis`] and [`ChainHardforks`].
pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Header {
// If London is activated at genesis, we set the initial base fee as per EIP-1559.
@@ -83,12 +79,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
.active_at_timestamp(genesis.timestamp)
.then_some(EMPTY_REQUESTS_HASH);
// If Amsterdam is activated at genesis we set block access list hash empty hash.
let block_access_list_hash = hardforks
.fork(EthereumHardfork::Amsterdam)
.active_at_timestamp(genesis.timestamp)
.then_some(EMPTY_BLOCK_ACCESS_LIST_HASH);
Header {
number: genesis.number.unwrap_or_default(),
parent_hash: genesis.parent_hash.unwrap_or_default(),
@@ -106,7 +96,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
blob_gas_used,
excess_blob_gas,
requests_hash,
block_access_list_hash,
..Default::default()
}
}
@@ -289,6 +278,7 @@ pub fn create_chain_config(
// Check if DAO fork is supported (it has an activation block)
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
#[allow(clippy::needless_update)]
ChainConfig {
chain_id: chain.map(|c| c.id()).unwrap_or(0),
homestead_block: block_num(EthereumHardfork::Homestead),
@@ -311,7 +301,6 @@ pub fn create_chain_config(
cancun_time: timestamp(EthereumHardfork::Cancun),
prague_time: timestamp(EthereumHardfork::Prague),
osaka_time: timestamp(EthereumHardfork::Osaka),
amsterdam_time: timestamp(EthereumHardfork::Amsterdam),
bpo1_time: timestamp(EthereumHardfork::Bpo1),
bpo2_time: timestamp(EthereumHardfork::Bpo2),
bpo3_time: timestamp(EthereumHardfork::Bpo3),
@@ -325,6 +314,7 @@ pub fn create_chain_config(
extra_fields: Default::default(),
deposit_contract_address,
blob_schedule,
..Default::default()
}
}
@@ -551,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
}
}
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
}
}
}
@@ -913,7 +903,6 @@ impl From<Genesis> for ChainSpec {
(EthereumHardfork::Bpo3.boxed(), genesis.config.bpo3_time),
(EthereumHardfork::Bpo4.boxed(), genesis.config.bpo4_time),
(EthereumHardfork::Bpo5.boxed(), genesis.config.bpo5_time),
(EthereumHardfork::Amsterdam.boxed(), genesis.config.amsterdam_time),
];
let mut time_hardforks = time_hardfork_opts
@@ -1220,19 +1209,6 @@ impl ChainSpecBuilder {
self
}
/// Enable Amsterdam at genesis.
pub fn amsterdam_activated(mut self) -> Self {
self = self.osaka_activated();
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(0));
self
}
/// Enable Amsterdam at the given timestamp.
pub fn with_amsterdam_at(mut self, timestamp: u64) -> Self {
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(timestamp));
self
}
/// Build the resulting [`ChainSpec`].
///
/// # Panics

View File

@@ -50,6 +50,7 @@ reth-stages-types = { workspace = true, optional = true }
reth-static-file-types = { workspace = true, features = ["clap"] }
reth-static-file.workspace = true
reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true
@@ -77,6 +78,7 @@ lz4.workspace = true
zstd.workspace = true
serde.workspace = true
serde_json.workspace = true
parking_lot.workspace = true
tar.workspace = true
tracing.workspace = true
backon.workspace = true
@@ -131,4 +133,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -27,7 +27,7 @@ use reth_provider::{
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
ProviderFactory, StaticFileProviderFactory,
ProviderFactory, StaticFileProviderFactory, StorageSettings,
};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
/// All static files related arguments
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
@@ -106,13 +121,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
let (db, sfp) = match access {
AccessRights::RW => (
Arc::new(init_db(db_path, self.db.database_args())?),
init_db(db_path, self.db.database_args())?,
StaticFileProviderBuilder::read_write(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?,
),
AccessRights::RO | AccessRights::RoInconsistent => {
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
(open_db_read_only(&db_path, self.db.database_args())?, {
let provider = StaticFileProviderBuilder::read_only(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?;
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
// TransactionDB only support read-write mode
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
if access.is_read_write() {
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
}
Ok(Environment { config, provider_factory, data_dir })
@@ -145,16 +160,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
fn create_provider_factory<N: CliNodeTypes>(
&self,
config: &Config,
db: Arc<DatabaseEnv>,
db: DatabaseEnv,
static_file_provider: StaticFileProvider<N::Primitives>,
rocksdb_provider: RocksDBProvider,
access: AccessRights,
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let prune_modes = config.prune.segments.clone();
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
db,
self.chain.clone(),
static_file_provider,
@@ -185,7 +200,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
// Builds and executes an unwind-only pipeline
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
.add_stages(DefaultStages::new(
factory.clone(),
tip_rx,
@@ -214,7 +229,7 @@ pub struct Environment<N: NodeTypes> {
/// Configuration for reth node
pub config: Config,
/// Provider factory.
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
/// Datadir path.
pub data_dir: ChainPath<DataDirPath>,
}
@@ -246,8 +261,8 @@ impl AccessRights {
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
type FullTypesAdapter<T> = FullNodeTypesAdapter<
T,
Arc<DatabaseEnv>,
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
DatabaseEnv,
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
>;
/// Helper trait with a common set of requirements for the

View File

@@ -1,145 +0,0 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use reth_chainspec::EthereumHardforks;
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
self.table.view(&ChecksumViewer {
tool,
start_key: self.start_key,
end_key: self.end_key,
limit: self.limit,
})?;
Ok(())
}
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(100_000) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,287 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use itertools::Itertools;
use reth_chainspec::EthereumHardforks;
use reth_db::{static_file::iter_static_files, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
time::{Duration, Instant},
};
use tracing::{info, warn};
#[cfg(all(unix, feature = "edge"))]
mod rocksdb;
/// Interval for logging progress during checksum computation.
const PROGRESS_LOG_INTERVAL: usize = 100_000;
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
#[command(subcommand)]
subcommand: Subcommand,
}
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Calculates the checksum of a database table
Mdbx {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a static file segment
StaticFile {
/// The static file segment
#[arg(value_enum)]
segment: StaticFileSegment,
/// The block number to start from (inclusive).
#[arg(long)]
start_block: Option<u64>,
/// The block number to end at (inclusive).
#[arg(long)]
end_block: Option<u64>,
/// The maximum number of rows to checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a RocksDB table
#[cfg(all(unix, feature = "edge"))]
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: rocksdb::RocksDbTable,
/// The maximum number of records to checksum.
#[arg(long)]
limit: Option<usize>,
},
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
match self.subcommand {
Subcommand::Mdbx { table, start_key, end_key, limit } => {
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
}
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
checksum_static_file(tool, segment, start_block, end_block, limit)?;
}
#[cfg(all(unix, feature = "edge"))]
Subcommand::Rocksdb { table, limit } => {
rocksdb::checksum_rocksdb(tool, table, limit)?;
}
}
Ok(())
}
}
/// Creates a new hasher with the standard seed used for checksum computation.
fn checksum_hasher() -> impl Hasher {
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,
limit: Option<usize>,
) -> eyre::Result<()> {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
warn!("Error checking consistency of static files: {err}");
}
let static_files = iter_static_files(static_file_provider.directory())?;
let ranges = static_files
.get(segment)
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
segment,
start_block,
end_block,
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
let jar_provider = static_file_provider
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
.ok_or_else(|| {
eyre::eyre!(
"Failed to get segment provider for segment {} at range {}",
segment,
block_range
)
})?;
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!(
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
segment, checksum, total, elapsed
);
Ok(())
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,106 @@
//! RocksDB checksum implementation.
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
use crate::common::CliNodeTypes;
use clap::ValueEnum;
use reth_chainspec::EthereumHardforks;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl RocksDbTable {
/// Returns the table name as a string
const fn name(&self) -> &'static str {
match self {
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
Self::AccountsHistory => tables::AccountsHistory::NAME,
Self::StoragesHistory => tables::StoragesHistory::NAME,
}
}
}
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
let start_time = Instant::now();
let limit = limit.unwrap_or(usize::MAX);
info!(
"Computing checksum for RocksDB table `{}`, limit={:?}",
table.name(),
if limit == usize::MAX { None } else { Some(limit) }
);
let (checksum, total) = match table {
RocksDbTable::TransactionHashNumbers => {
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
}
RocksDbTable::AccountsHistory => {
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
}
RocksDbTable::StoragesHistory => {
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
}
};
let elapsed = start_time.elapsed();
info!(
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
table.name(),
checksum,
total,
elapsed
);
Ok(())
}
/// Computes checksum for a specific RocksDB table by iterating over rows.
fn checksum_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
limit: usize,
) -> eyre::Result<(u64, usize)> {
let iter = rocksdb.raw_iter::<T>()?;
let mut hasher = checksum_hasher();
let mut total = 0usize;
for entry in iter {
let (key_bytes, value_bytes) = entry?;
hasher.write(&key_bytes);
hasher.write(&value_bytes);
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break;
}
}
Ok((hasher.finish(), total))
}

View File

@@ -16,7 +16,6 @@ use std::{
hash::Hash,
io::Write,
path::{Path, PathBuf},
sync::Arc,
};
use tracing::{info, warn};
@@ -56,7 +55,7 @@ impl Command {
/// then written to a file in the output directory.
pub fn execute<T: NodeTypes>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("Make sure the node is not running when running `reth db diff`!");
// open second db

View File

@@ -21,6 +21,7 @@ use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
/// The arguments for the `reth db get` command
@@ -82,6 +83,33 @@ impl Command {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
table_subkey::<tables::StorageChangeSets>(subkey.as_deref()).ok();
let key = table_key::<tables::StorageChangeSets>(&key)?;
let provider = tool.provider_factory.static_file_provider();
if let Some(storage_key) = storage_key {
let entry = provider.get_storage_before_block(
key.block_number(),
key.address(),
storage_key,
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(());
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(());
}
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
@@ -112,6 +140,9 @@ impl Command {
AccountChangesetMask::MASK,
)
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changesets handled above");
}
};
// handle account changesets differently if a subkey is provided.
@@ -190,6 +221,9 @@ impl Command {
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
StaticFileSegment::StorageChangeSets => {
unreachable!("storage changeset static files are special cased before this match")
}
}
}
}

View File

@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
use reth_db_common::{DbTool, ListFilter};
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
use std::{cell::RefCell, sync::Arc};
use std::cell::RefCell;
use tracing::error;
#[derive(Parser, Debug)]
@@ -55,25 +55,27 @@ impl Command {
/// Execute `db list` command
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
self.table.view(&ListTableViewer { tool, args: &self })
}
/// Generate [`ListFilter`] from command.
pub fn list_filter(&self) -> ListFilter {
let search = self
.search
.as_ref()
.map(|search| {
pub fn list_filter(&self) -> eyre::Result<ListFilter> {
let search = match self.search.as_deref() {
Some(search) => {
if let Some(search) = search.strip_prefix("0x") {
return hex::decode(search).unwrap()
hex::decode(search).wrap_err(
"Invalid hex content after 0x prefix in --search (expected valid hex like 0xdeadbeef).",
)?
} else {
search.as_bytes().to_vec()
}
search.as_bytes().to_vec()
})
.unwrap_or_default();
}
None => Vec::new(),
};
ListFilter {
Ok(ListFilter {
skip: self.skip,
len: self.len,
search,
@@ -82,12 +84,12 @@ impl Command {
min_value_size: self.min_value_size,
reverse: self.reverse,
only_count: self.count,
}
})
}
}
struct ListTableViewer<'a, N: NodeTypes> {
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
args: &'a Command,
}
@@ -99,8 +101,8 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
// We may be using the tui for a long time
tx.disable_long_read_transaction_safety();
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let table_db = tx.inner().open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner().db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let total_entries = stats.entries();
let final_entry_idx = total_entries.saturating_sub(1);
if self.args.skip > final_entry_idx {
@@ -115,7 +117,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
}
let list_filter = self.args.list_filter();
let list_filter = self.args.list_filter()?;
if self.args.json || self.args.count {
let (list, count) = self.tool.list::<T>(&list_filter)?;

View File

@@ -17,6 +17,7 @@ mod get;
mod list;
mod repair_trie;
mod settings;
mod state;
mod static_file_header;
mod stats;
/// DB List TUI
@@ -39,7 +40,7 @@ pub enum Subcommands {
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Calculates the content checksum of a table
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),
@@ -65,6 +66,8 @@ pub enum Subcommands {
Settings(settings::Command),
/// Gets storage size information for an account
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
State(state::Command),
}
/// Initializes a provider factory with specified access rights, and then execute with the provided
@@ -162,7 +165,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
command.execute(&tool, ctx.task_executor, &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {
@@ -198,6 +201,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::State(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(&tool)?;
});
}
}
Ok(())

View File

@@ -69,6 +69,11 @@ pub enum SetCommand {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
}
impl Command {
@@ -115,6 +120,7 @@ impl Command {
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
@@ -167,6 +173,14 @@ impl Command {
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
}
// Write updated settings

View File

@@ -0,0 +1,412 @@
use alloy_primitives::{Address, BlockNumber, B256, U256};
use clap::Parser;
use parking_lot::Mutex;
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRO},
database::Database,
tables,
transaction::DbTx,
};
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_provider::providers::ProviderNodeTypes;
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
use std::{
collections::BTreeSet,
thread,
time::{Duration, Instant},
};
use tracing::{error, info};
/// Log progress every 5 seconds
const LOG_INTERVAL: Duration = Duration::from_secs(30);
/// The arguments for the `reth db state` command
#[derive(Parser, Debug)]
pub struct Command {
/// The account address to get state for
address: Address,
/// Block number to query state at (uses current state if not provided)
#[arg(long, short)]
block: Option<BlockNumber>,
/// Maximum number of storage slots to display
#[arg(long, short, default_value = "100")]
limit: usize,
/// Output format (table, json, csv)
#[arg(long, short, default_value = "table")]
format: OutputFormat,
}
impl Command {
/// Execute `db state` command
pub fn execute<N: NodeTypesWithDB + ProviderNodeTypes>(
self,
tool: &DbTool<N>,
) -> eyre::Result<()> {
let address = self.address;
let limit = self.limit;
if let Some(block) = self.block {
self.execute_historical(tool, address, block, limit)
} else {
self.execute_current(tool, address, limit)
}
}
fn execute_current<N: NodeTypesWithDB + ProviderNodeTypes>(
&self,
tool: &DbTool<N>,
address: Address,
limit: usize,
) -> eyre::Result<()> {
let entries = tool.provider_factory.db_ref().view(|tx| {
// Get account info
let account = tx.get::<tables::PlainAccountState>(address)?;
// Get storage entries
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
let walker = cursor.walk_dup(Some(address), None)?;
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
Ok::<_, eyre::Report>((account, entries))
})??;
let (account, storage_entries) = entries;
self.print_results(address, None, account, &storage_entries);
Ok(())
}
fn execute_historical<N: NodeTypesWithDB + ProviderNodeTypes>(
&self,
tool: &DbTool<N>,
address: Address,
block: BlockNumber,
limit: usize,
) -> eyre::Result<()> {
let provider = tool.provider_factory.history_by_block_number(block)?;
// Get account info at that block
let account = provider.basic_account(&address)?;
// Check storage settings to determine where history is stored
let storage_settings = tool.provider_factory.cached_storage_settings();
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
// For historical queries, enumerate keys from history indices only
// (not PlainStorageState, which reflects current state)
let mut storage_keys = BTreeSet::new();
if history_in_rocksdb {
error!(
target: "reth::cli",
"Historical storage queries with RocksDB backend are not yet supported. \
Use MDBX for storage history or query current state without --block."
);
return Ok(());
}
// Collect keys from MDBX StorageChangeSets using parallel scanning
self.collect_mdbx_storage_keys_parallel(tool, address, &mut storage_keys)?;
info!(
target: "reth::cli",
address = %address,
block = block,
total_keys = storage_keys.len(),
"Found storage keys to query"
);
// Now query each key at the historical block using the StateProvider
// This handles both MDBX and RocksDB backends transparently
let mut entries = Vec::new();
let mut last_log = Instant::now();
for (idx, key) in storage_keys.iter().enumerate() {
match provider.storage(address, *key) {
Ok(Some(value)) if value != U256::ZERO => {
entries.push((*key, value));
}
_ => {}
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
block = block,
keys_total = storage_keys.len(),
slots_scanned = idx,
slots_found = entries.len(),
"Scanning historical storage slots"
);
last_log = Instant::now();
}
}
self.print_results(address, Some(block), account, &entries);
Ok(())
}
/// Collects storage keys from MDBX StorageChangeSets using parallel block range scanning.
fn collect_mdbx_storage_keys_parallel<N: NodeTypesWithDB + ProviderNodeTypes>(
&self,
tool: &DbTool<N>,
address: Address,
keys: &mut BTreeSet<B256>,
) -> eyre::Result<()> {
const CHUNK_SIZE: u64 = 500_000; // 500k blocks per thread
let num_threads = std::thread::available_parallelism()
.map(|p| p.get().saturating_sub(1).max(1))
.unwrap_or(4);
// Get the current tip block
let tip = tool.provider_factory.provider()?.best_block_number()?;
if tip == 0 {
return Ok(());
}
info!(
target: "reth::cli",
address = %address,
tip,
chunk_size = CHUNK_SIZE,
num_threads,
"Starting parallel MDBX changeset scan"
);
// Shared state for collecting keys
let collected_keys: Mutex<BTreeSet<B256>> = Mutex::new(BTreeSet::new());
let total_entries_scanned = Mutex::new(0usize);
// Create chunk ranges
let mut chunks: Vec<(u64, u64)> = Vec::new();
let mut start = 0u64;
while start <= tip {
let end = (start + CHUNK_SIZE - 1).min(tip);
chunks.push((start, end));
start = end + 1;
}
let chunks_ref = &chunks;
let next_chunk = Mutex::new(0usize);
let next_chunk_ref = &next_chunk;
let collected_keys_ref = &collected_keys;
let total_entries_ref = &total_entries_scanned;
thread::scope(|s| {
let handles: Vec<_> = (0..num_threads)
.map(|thread_id| {
s.spawn(move || {
loop {
// Get next chunk to process
let chunk_idx = {
let mut idx = next_chunk_ref.lock();
if *idx >= chunks_ref.len() {
return Ok::<_, eyre::Report>(());
}
let current = *idx;
*idx += 1;
current
};
let (chunk_start, chunk_end) = chunks_ref[chunk_idx];
// Open a new read transaction for this chunk
tool.provider_factory.db_ref().view(|tx| {
tx.disable_long_read_transaction_safety();
let mut changeset_cursor =
tx.cursor_read::<tables::StorageChangeSets>()?;
let start_key =
reth_db_api::models::BlockNumberAddress((chunk_start, address));
let end_key =
reth_db_api::models::BlockNumberAddress((chunk_end, address));
let mut local_keys = BTreeSet::new();
let mut entries_in_chunk = 0usize;
if let Ok(walker) = changeset_cursor.walk_range(start_key..=end_key)
{
for (block_addr, storage_entry) in walker.flatten() {
if block_addr.address() == address {
local_keys.insert(storage_entry.key);
}
entries_in_chunk += 1;
}
}
// Merge into global state
collected_keys_ref.lock().extend(local_keys);
*total_entries_ref.lock() += entries_in_chunk;
info!(
target: "reth::cli",
thread_id,
chunk_start,
chunk_end,
entries_in_chunk,
"Thread completed chunk"
);
Ok::<_, eyre::Report>(())
})??;
}
})
})
.collect();
for handle in handles {
handle.join().map_err(|_| eyre::eyre!("Thread panicked"))??;
}
Ok::<_, eyre::Report>(())
})?;
let final_keys = collected_keys.into_inner();
let total = *total_entries_scanned.lock();
info!(
target: "reth::cli",
address = %address,
total_entries = total,
unique_keys = final_keys.len(),
"Finished parallel MDBX changeset scan"
);
keys.extend(final_keys);
Ok(())
}
fn print_results(
&self,
address: Address,
block: Option<BlockNumber>,
account: Option<reth_primitives_traits::Account>,
storage: &[(alloy_primitives::B256, U256)],
) {
match self.format {
OutputFormat::Table => {
println!("Account: {address}");
if let Some(b) = block {
println!("Block: {b}");
} else {
println!("Block: latest");
}
println!();
if let Some(acc) = account {
println!("Nonce: {}", acc.nonce);
println!("Balance: {} wei", acc.balance);
if let Some(code_hash) = acc.bytecode_hash {
println!("Code hash: {code_hash}");
}
} else {
println!("Account not found");
}
println!();
println!("Storage ({} slots):", storage.len());
println!("{:-<130}", "");
println!("{:<66} | {:<64}", "Slot", "Value");
println!("{:-<130}", "");
for (key, value) in storage {
println!("{key} | {value:#066x}");
}
}
OutputFormat::Json => {
let output = serde_json::json!({
"address": address.to_string(),
"block": block,
"account": account.map(|a| serde_json::json!({
"nonce": a.nonce,
"balance": a.balance.to_string(),
"code_hash": a.bytecode_hash.map(|h| h.to_string()),
})),
"storage": storage.iter().map(|(k, v)| {
serde_json::json!({
"key": k.to_string(),
"value": format!("{v:#066x}"),
})
}).collect::<Vec<_>>(),
});
println!("{}", serde_json::to_string_pretty(&output).unwrap());
}
OutputFormat::Csv => {
println!("slot,value");
for (key, value) in storage {
println!("{key},{value:#066x}");
}
}
}
}
}
#[derive(Debug, Clone, Default, clap::ValueEnum)]
pub enum OutputFormat {
#[default]
Table,
Json,
Csv,
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn parse_state_args() {
let cmd = Command::try_parse_from([
"state",
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045",
"--block",
"1000000",
])
.unwrap();
assert_eq!(
cmd.address,
"0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045".parse::<Address>().unwrap()
);
assert_eq!(cmd.block, Some(1000000));
}
#[test]
fn parse_state_args_no_block() {
let cmd = Command::try_parse_from(["state", "0xd8dA6BF26964aF9D7eEd9e03E53415D37aA96045"])
.unwrap();
assert_eq!(cmd.block, None);
}
}

View File

@@ -11,9 +11,12 @@ use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
use reth_provider::{
providers::{ProviderNodeTypes, StaticFileProvider},
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
use std::time::Duration;
#[derive(Parser, Debug)]
/// The arguments for the `reth db stats` command
@@ -45,7 +48,7 @@ impl Command {
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
data_dir: ChainPath<DataDirPath>,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
if self.checksum {
let checksum_report = self.checksum_report(tool)?;
@@ -61,10 +64,15 @@ impl Command {
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
println!("\n");
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
println!("{rocksdb_stats_table}");
Ok(())
}
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
&self,
tool: &DbTool<N>,
) -> eyre::Result<ComfyTable> {
@@ -84,10 +92,10 @@ impl Command {
db_tables.sort();
let mut total_size = 0;
for db_table in db_tables {
let table_db = tx.inner.open_db(Some(db_table)).wrap_err("Could not open db.")?;
let table_db = tx.inner().open_db(Some(db_table)).wrap_err("Could not open db.")?;
let stats = tx
.inner
.inner()
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {db_table}"))?;
@@ -128,9 +136,9 @@ impl Command {
.add_cell(Cell::new(human_bytes(total_size as f64)));
table.add_row(row);
let freelist = tx.inner.env().freelist()?;
let freelist = tx.inner().env().freelist()?;
let pagesize =
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
tx.inner().db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
let freelist_size = freelist * pagesize;
let mut row = Row::new();
@@ -148,6 +156,70 @@ impl Command {
Ok(table)
}
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"RocksDB Table Name",
"# Entries",
"SST Size",
"Memtable Size",
"Total Size",
"Pending Compaction",
]);
let stats = tool.provider_factory.rocksdb_provider().table_stats();
let mut total_sst: u64 = 0;
let mut total_memtable: u64 = 0;
let mut total_size: u64 = 0;
let mut total_pending: u64 = 0;
for stat in &stats {
total_sst += stat.sst_size_bytes;
total_memtable += stat.memtable_size_bytes;
total_size += stat.estimated_size_bytes;
total_pending += stat.pending_compaction_bytes;
let mut row = Row::new();
row.add_cell(Cell::new(&stat.name))
.add_cell(Cell::new(stat.estimated_num_keys))
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
table.add_row(row);
}
if !stats.is_empty() {
let max_widths = table.column_max_content_widths();
let mut separator = Row::new();
for width in max_widths {
separator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(separator);
let mut row = Row::new();
row.add_cell(Cell::new("RocksDB Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_sst as f64)))
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
let wal_size = tool.provider_factory.rocksdb_provider().wal_size_bytes();
let mut row = Row::new();
row.add_cell(Cell::new("WAL"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(wal_size as f64)))
.add_cell(Cell::new(""));
table.add_row(row);
}
table
}
fn static_files_stats_table<N: NodePrimitives>(
&self,
data_dir: ChainPath<DataDirPath>,

View File

@@ -227,8 +227,9 @@ where
// Handle errors
if let Err(err) = res {
error!("{:?}", err)
error!("{err}");
}
Ok(())
}
}
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
) -> io::Result<()>
where
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
io::Error: From<B::Error>,
{
let mut last_tick = Instant::now();
let mut running = true;

View File

@@ -2,14 +2,15 @@ use crate::common::EnvironmentArgs;
use clap::Parser;
use eyre::Result;
use lz4::Decoder;
use reqwest::Client;
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_fs_util as fs;
use std::{
borrow::Cow,
io::{self, Read, Write},
path::Path,
fs::OpenOptions,
io::{self, BufWriter, Read, Write},
path::{Path, PathBuf},
sync::{Arc, OnceLock},
time::{Duration, Instant},
};
@@ -327,18 +328,158 @@ fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path)
extract_archive(file, total_size, format, target_dir)
}
/// Fetches the snapshot from a remote URL, uncompressing it in a streaming fashion.
const MAX_DOWNLOAD_RETRIES: u32 = 10;
const RETRY_BACKOFF_SECS: u64 = 5;
/// Wrapper that tracks download progress while writing data.
/// Used with [`io::copy`] to display progress during downloads.
struct ProgressWriter<W> {
inner: W,
progress: DownloadProgress,
}
impl<W: Write> Write for ProgressWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
let _ = self.progress.update(n as u64);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Downloads a file with resume support using HTTP Range requests.
/// Automatically retries on failure, resuming from where it left off.
/// Returns the path to the downloaded file and its total size.
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
let file_name = Url::parse(url)
.ok()
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
.unwrap_or_else(|| "snapshot.tar".to_string());
let final_path = target_dir.join(&file_name);
let part_path = target_dir.join(format!("{file_name}.part"));
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
let mut total_size: Option<u64> = None;
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
if let Some(total) = total_size &&
existing_size >= total
{
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, total));
}
if attempt > 1 {
info!(target: "reth::cli",
"Retry attempt {}/{} - resuming from {} bytes",
attempt, MAX_DOWNLOAD_RETRIES, existing_size
);
}
let mut request = client.get(url);
if existing_size > 0 {
request = request.header(RANGE, format!("bytes={existing_size}-"));
if attempt == 1 {
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
}
}
let response = match request.send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(e) => {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
let size = if is_partial {
response
.headers()
.get("Content-Range")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.split('/').next_back())
.and_then(|v| v.parse().ok())
} else {
response.content_length()
};
if total_size.is_none() {
total_size = size;
}
let current_total = total_size.ok_or_else(|| {
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
})?;
let file = if is_partial && existing_size > 0 {
OpenOptions::new()
.append(true)
.open(&part_path)
.map_err(|e| fs::FsPathError::open(e, &part_path))?
} else {
fs::create_file(&part_path)?
};
let start_offset = if is_partial { existing_size } else { 0 };
let mut progress = DownloadProgress::new(current_total);
progress.downloaded = start_offset;
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
let mut reader = response;
let copy_result = io::copy(&mut reader, &mut writer);
let flush_result = writer.inner.flush();
println!();
if let Err(e) = copy_result.and(flush_result) {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, current_total));
}
Err(last_error
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let client = reqwest::blocking::Client::builder().build()?;
let response = client.get(url).send()?.error_for_status()?;
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
let total_size = response.content_length().ok_or_else(|| {
eyre::eyre!(
"Server did not provide Content-Length header. This is required for snapshot downloads"
)
})?;
info!(target: "reth::cli", "Extracting snapshot...");
let file = fs::open(&downloaded_path)?;
extract_archive(file, total_size, format, target_dir)?;
extract_archive(response, total_size, format, target_dir)
fs::remove_file(&downloaded_path)?;
info!(target: "reth::cli", "Removed downloaded archive");
Ok(())
}
/// Downloads and extracts a snapshot, blocking until finished.

View File

@@ -26,6 +26,14 @@ pub struct ImportCommand<C: ChainSpecParser> {
#[arg(long, value_name = "CHUNK_LEN", verbatim_doc_comment)]
chunk_len: Option<u64>,
/// Fail immediately when an invalid block is encountered.
///
/// By default, the import will stop at the last valid block if an invalid block is
/// encountered during execution or validation, leaving the database at the last valid
/// block state. When this flag is set, the import will instead fail with an error.
#[arg(long, verbatim_doc_comment)]
fail_on_invalid_block: bool,
/// The path(s) to block file(s) for import.
///
/// The online stages (headers and bodies) are replaced by a file import, after which the
@@ -52,7 +60,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
info!(target: "reth::cli", "Starting import of {} file(s)", self.paths.len());
let import_config = ImportConfig { no_state: self.no_state, chunk_len: self.chunk_len };
let import_config = ImportConfig {
no_state: self.no_state,
chunk_len: self.chunk_len,
fail_on_invalid_block: self.fail_on_invalid_block,
};
let executor = components.evm_config().clone();
let consensus = Arc::new(components.consensus().clone());
@@ -81,7 +93,20 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
total_decoded_blocks += result.total_decoded_blocks;
total_decoded_txns += result.total_decoded_txns;
if !result.is_complete() {
// Check if we stopped due to an invalid block
if result.stopped_on_invalid_block {
info!(target: "reth::cli",
"Stopped at last valid block {} due to invalid block {} in file: {}. Imported {} blocks, {} transactions",
result.last_valid_block.unwrap_or(0),
result.bad_block.unwrap_or(0),
path.display(),
result.total_imported_blocks,
result.total_imported_txns);
// Stop importing further files and exit successfully
break;
}
if !result.is_successful() {
return Err(eyre::eyre!(
"Chain was partially imported from file: {}. Imported {}/{} blocks, {}/{} transactions",
path.display(),
@@ -98,7 +123,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> ImportComm
}
info!(target: "reth::cli",
"All files imported successfully. Total: {}/{} blocks, {}/{} transactions",
"Import complete. Total: {}/{} blocks, {}/{} transactions",
total_imported_blocks, total_decoded_blocks, total_imported_txns, total_decoded_txns);
Ok(())
@@ -139,4 +164,20 @@ mod tests {
assert_eq!(args.paths[1], PathBuf::from("file2.rlp"));
assert_eq!(args.paths[2], PathBuf::from("file3.rlp"));
}
#[test]
fn parse_import_command_with_fail_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "--fail-on-invalid-block", "chain.rlp"]);
assert!(args.fail_on_invalid_block);
assert_eq!(args.paths.len(), 1);
assert_eq!(args.paths[0], PathBuf::from("chain.rlp"));
}
#[test]
fn parse_import_command_default_stops_on_invalid_block() {
let args: ImportCommand<EthereumChainSpecParser> =
ImportCommand::parse_from(["reth", "chain.rlp"]);
assert!(!args.fail_on_invalid_block);
}
}

View File

@@ -22,11 +22,11 @@ use reth_provider::{
StageCheckpointReader,
};
use reth_prune::PruneModes;
use reth_stages::{prelude::*, Pipeline, StageId, StageSet};
use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
use reth_static_file::StaticFileProducer;
use std::{path::Path, sync::Arc};
use tokio::sync::watch;
use tracing::{debug, error, info};
use tracing::{debug, error, info, warn};
/// Configuration for importing blocks from RLP files.
#[derive(Debug, Clone, Default)]
@@ -35,6 +35,9 @@ pub struct ImportConfig {
pub no_state: bool,
/// Chunk byte length to read from file.
pub chunk_len: Option<u64>,
/// If true, fail immediately when an invalid block is encountered.
/// By default (false), the import stops at the last valid block and exits successfully.
pub fail_on_invalid_block: bool,
}
/// Result of an import operation.
@@ -48,6 +51,12 @@ pub struct ImportResult {
pub total_imported_blocks: usize,
/// Total number of transactions imported into the database.
pub total_imported_txns: usize,
/// Whether the import was stopped due to an invalid block.
pub stopped_on_invalid_block: bool,
/// The block number that was invalid, if any.
pub bad_block: Option<u64>,
/// The last valid block number when stopped due to invalid block.
pub last_valid_block: Option<u64>,
}
impl ImportResult {
@@ -56,6 +65,14 @@ impl ImportResult {
self.total_decoded_blocks == self.total_imported_blocks &&
self.total_decoded_txns == self.total_imported_txns
}
/// Returns true if the import was successful, considering stop-on-invalid-block mode.
///
/// In stop-on-invalid-block mode, a partial import is considered successful if we
/// stopped due to an invalid block (leaving the DB at the last valid block).
pub fn is_successful(&self) -> bool {
self.is_complete() || self.stopped_on_invalid_block
}
}
/// Imports blocks from an RLP-encoded file into the database.
@@ -103,6 +120,11 @@ where
let static_file_producer =
StaticFileProducer::new(provider_factory.clone(), PruneModes::default());
// Track if we stopped due to an invalid block
let mut stopped_on_invalid_block = false;
let mut bad_block_number: Option<u64> = None;
let mut last_valid_block_number: Option<u64> = None;
while let Some(file_client) =
reader.next_chunk::<BlockTy<N>>(consensus.clone(), Some(sealed_header)).await?
{
@@ -137,12 +159,51 @@ where
// Run pipeline
info!(target: "reth::import", "Starting sync pipeline");
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
if import_config.fail_on_invalid_block {
// Original behavior: fail on unwind
tokio::select! {
res = pipeline.run() => res?,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
}
} else {
// Default behavior: Use run_loop() to handle unwinds gracefully
let result = tokio::select! {
res = pipeline.run_loop() => res,
_ = tokio::signal::ctrl_c() => {
info!(target: "reth::import", "Import interrupted by user");
break;
},
};
match result {
Ok(ControlFlow::Unwind { target, bad_block }) => {
// An invalid block was encountered; stop at last valid block
let bad = bad_block.block.number;
warn!(
target: "reth::import",
bad_block = bad,
last_valid_block = target,
"Invalid block encountered during import; stopping at last valid block"
);
stopped_on_invalid_block = true;
bad_block_number = Some(bad);
last_valid_block_number = Some(target);
break;
}
Ok(ControlFlow::Continue { block_number }) => {
debug!(target: "reth::import", block_number, "Pipeline chunk completed");
}
Ok(ControlFlow::NoProgress { block_number }) => {
debug!(target: "reth::import", ?block_number, "Pipeline made no progress");
}
Err(e) => {
// Propagate other pipeline errors
return Err(e.into());
}
}
}
sealed_header = provider_factory
@@ -160,9 +221,20 @@ where
total_decoded_txns,
total_imported_blocks,
total_imported_txns,
stopped_on_invalid_block,
bad_block: bad_block_number,
last_valid_block: last_valid_block_number,
};
if !result.is_complete() {
if result.stopped_on_invalid_block {
info!(target: "reth::import",
total_imported_blocks,
total_imported_txns,
bad_block = ?result.bad_block,
last_valid_block = ?result.last_valid_block,
"Import stopped at last valid block due to invalid block"
);
} else if !result.is_complete() {
error!(target: "reth::import",
total_decoded_blocks,
total_imported_blocks,

View File

@@ -2,7 +2,7 @@ use futures::Future;
use reth_cli::chainspec::ChainSpecParser;
use reth_db::DatabaseEnv;
use reth_node_builder::{NodeBuilder, WithLaunchContext};
use std::{fmt, sync::Arc};
use std::fmt;
/// A trait for launching a reth node with custom configuration strategies.
///
@@ -30,7 +30,7 @@ where
/// * `builder_args` - Extension arguments for configuration
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>>;
}
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
where
C: ChainSpecParser,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
@@ -77,13 +77,13 @@ where
C: ChainSpecParser,
Ext: clap::Args + fmt::Debug,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>> {
(self.func)(builder, builder_args)

View File

@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -166,12 +171,16 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -187,6 +196,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
@@ -196,7 +206,7 @@ where
let db_path = data_dir.db();
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
if with_unused_ports {
node_config = node_config.with_unused_ports();

View File

@@ -0,0 +1,34 @@
//! Enode identifier command
use clap::Parser;
use reth_cli_util::get_secret_key;
use reth_network_peers::NodeRecord;
use std::{
net::{IpAddr, Ipv4Addr, SocketAddr},
path::PathBuf,
};
/// Print the enode identifier for a given secret key.
#[derive(Parser, Debug)]
pub struct Command {
/// Path to the secret key file for discovery.
pub discovery_secret: PathBuf,
/// Optional IP address to include in the enode URL.
///
/// If not provided, defaults to 0.0.0.0.
#[arg(long)]
pub ip: Option<IpAddr>,
}
impl Command {
/// Execute the enode command.
pub fn execute(self) -> eyre::Result<()> {
let sk = get_secret_key(&self.discovery_secret)?;
let ip = self.ip.unwrap_or(IpAddr::V4(Ipv4Addr::UNSPECIFIED));
let addr = SocketAddr::new(ip, 30303);
let enr = NodeRecord::from_secret_key(addr, &sk);
println!("{enr}");
Ok(())
}
}

View File

@@ -18,6 +18,7 @@ use reth_node_core::{
};
pub mod bootnode;
pub mod enode;
pub mod rlpx;
/// `reth p2p` command
@@ -85,6 +86,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Subcommands::Bootnode(command) => {
command.execute().await?;
}
Subcommands::Enode(command) => {
command.execute()?;
}
}
Ok(())
@@ -99,6 +103,7 @@ impl<C: ChainSpecParser> Command<C> {
Subcommands::Body { args, .. } => Some(&args.chain),
Subcommands::Rlpx(_) => None,
Subcommands::Bootnode(_) => None,
Subcommands::Enode(_) => None,
}
}
}
@@ -126,6 +131,8 @@ pub enum Subcommands<C: ChainSpecParser> {
Rlpx(rlpx::Command),
/// Bootnode command
Bootnode(bootnode::Command),
/// Print enode identifier
Enode(enode::Command),
}
#[derive(Debug, Clone, Parser)]
@@ -225,4 +232,16 @@ mod tests {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "body", "--chain", "mainnet", "1000"]);
}
#[test]
fn parse_enode_cmd() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret"]);
}
#[test]
fn parse_enode_cmd_with_ip() {
let _args: Command<EthereumChainSpecParser> =
Command::parse_from(["reth", "enode", "/tmp/secret", "--ip", "192.168.1.1"]);
}
}

View File

@@ -1,26 +1,62 @@
//! Command that runs pruning without any limits.
//! Command that runs pruning.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{args::MetricArgs, version::version_metadata};
use reth_node_metrics::{
chain::ChainSpecInfo,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
use tracing::info;
/// Prunes according to the configuration without any limits
/// Prunes according to the configuration
#[derive(Debug, Parser)]
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
env: EnvironmentArgs<C>,
/// Prometheus metrics configuration.
#[command(flatten)]
metrics: MetricArgs,
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneCommand<C> {
/// Execute the `prune` command
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(self) -> eyre::Result<()> {
pub async fn execute<N: CliNodeTypes<ChainSpec = C::ChainSpec>>(
self,
ctx: CliContext,
) -> eyre::Result<()> {
let env = self.env.init::<N>(AccessRights::RW)?;
let provider_factory = env.provider_factory;
let config = env.config.prune;
let data_dir = env.data_dir;
if let Some(listen_addr) = self.metrics.prometheus {
let config = MetricServerConfig::new(
listen_addr,
VersionInfo {
version: version_metadata().cargo_pkg_version.as_ref(),
build_timestamp: version_metadata().vergen_build_timestamp.as_ref(),
cargo_features: version_metadata().vergen_cargo_features.as_ref(),
git_sha: version_metadata().vergen_git_sha.as_ref(),
target_triple: version_metadata().vergen_cargo_target_triple.as_ref(),
build_profile: version_metadata().build_profile_name.as_ref(),
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
MetricServer::new(config).serve().await?;
}
// Copy data from database to static files
info!(target: "reth::cli", "Copying data from database to static files...");
@@ -33,13 +69,43 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
// Delete data which has been copied to static files.
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
// Use batched pruning with a limit to bound memory, running in a loop until complete.
const DELETE_LIMIT: usize = 200_000;
let mut pruner = PrunerBuilder::new(config)
.delete_limit(usize::MAX)
.delete_limit(DELETE_LIMIT)
.build_with_provider_factory(provider_factory);
pruner.run(prune_tip)?;
info!(target: "reth::cli", "Pruned data from database");
let mut total_pruned = 0usize;
loop {
let output = pruner.run(prune_tip)?;
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
total_pruned = total_pruned.saturating_add(batch_pruned);
// Check if all segments are finished (not just the overall progress,
// since the pruner sets overall progress from the last segment only)
let all_segments_finished =
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
if all_segments_finished {
break;
}
if batch_pruned == 0 {
return Err(eyre::eyre!(
"pruner made no progress but reported more data remaining; \
aborting to prevent infinite loop"
));
}
info!(
target: "reth::cli",
batch_pruned,
total_pruned,
"Pruning batch complete, continuing..."
);
}
info!(target: "reth::cli", total_pruned, "Pruned data from database");
}
Ok(())

View File

@@ -42,9 +42,9 @@ pub struct Command<C: ChainSpecParser> {
#[arg(long)]
to: Option<u64>,
/// Number of tasks to run in parallel
#[arg(long, default_value = "10")]
num_tasks: u64,
/// Number of tasks to run in parallel. Defaults to the number of available CPUs.
#[arg(long)]
num_tasks: Option<u64>,
/// Continues with execution when an invalid block is encountered and collects these blocks.
#[arg(long)]
@@ -84,12 +84,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
}
};
let num_tasks = self.num_tasks.unwrap_or_else(|| {
std::thread::available_parallelism().map(|n| n.get() as u64).unwrap_or(10)
});
let total_blocks = max_block - min_block;
let total_gas = calculate_gas_used_from_headers(
&provider_factory.static_file_provider(),
min_block..=max_block,
)?;
let blocks_per_task = total_blocks / self.num_tasks;
let blocks_per_task = total_blocks / num_tasks;
let db_at = {
let provider_factory = provider_factory.clone();
@@ -107,10 +111,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let _guard = cancellation.drop_guard();
let mut tasks = JoinSet::new();
for i in 0..self.num_tasks {
for i in 0..num_tasks {
let start_block = min_block + i * blocks_per_task;
let end_block =
if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task };
if i == num_tasks - 1 { max_block } else { start_block + blocks_per_task };
// Spawn thread executing blocks
let provider_factory = provider_factory.clone();
@@ -148,7 +152,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
};
if let Err(err) = consensus
.validate_block_post_execution(&block, &result)
.validate_block_post_execution(&block, &result, None)
.wrap_err_with(|| {
format!("Failed to validate block {} {}", block.number(), block.hash())
})

View File

@@ -15,7 +15,8 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, TrieWriter,
DBProvider, RocksDBProviderFactory, StaticFileProviderFactory, StaticFileWriter,
StorageSettingsCache,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -90,11 +91,14 @@ impl<C: ChainSpecParser> Command<C> {
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
StaticFileSegment::StorageChangeSets => {
writer.prune_storage_changesets(highest_block)?;
}
}
}
}
let provider_rw = tool.provider_factory.database_provider_rw()?;
let provider_rw = tool.provider_factory.unwind_provider_rw()?;
let tx = provider_rw.tx_ref();
match self.stage {
@@ -113,7 +117,6 @@ impl<C: ChainSpecParser> Command<C> {
tx.clear::<tables::TransactionBlocks>()?;
tx.clear::<tables::BlockOmmers<HeaderTy<N>>>()?;
tx.clear::<tables::BlockWithdrawals>()?;
tx.clear::<tables::BlockAccessLists>()?;
reset_stage_checkpoint(tx, StageId::Bodies)?;
insert_genesis_header(&provider_rw, &self.env.chain)?;
@@ -168,13 +171,21 @@ impl<C: ChainSpecParser> Command<C> {
None,
)?;
}
StageEnum::MerkleChangeSets => {
provider_rw.clear_trie_changesets()?;
reset_stage_checkpoint(tx, StageId::MerkleChangeSets)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;
let settings = provider_rw.cached_storage_settings();
let rocksdb = tool.provider_factory.rocksdb_provider();
if settings.account_history_in_rocksdb {
rocksdb.clear::<tables::AccountsHistory>()?;
} else {
tx.clear::<tables::AccountsHistory>()?;
}
if settings.storages_history_in_rocksdb {
rocksdb.clear::<tables::StoragesHistory>()?;
} else {
tx.clear::<tables::StoragesHistory>()?;
}
reset_stage_checkpoint(tx, StageId::IndexAccountHistory)?;
reset_stage_checkpoint(tx, StageId::IndexStorageHistory)?;
@@ -182,7 +193,14 @@ impl<C: ChainSpecParser> Command<C> {
insert_genesis_history(&provider_rw, self.env.chain.genesis().alloc.iter())?;
}
StageEnum::TxLookup => {
tx.clear::<tables::TransactionHashNumbers>()?;
if provider_rw.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
tool.provider_factory
.rocksdb_provider()
.clear::<tables::TransactionHashNumbers>()?;
} else {
tx.clear::<tables::TransactionHashNumbers>()?;
}
reset_prune_checkpoint(tx, PruneSegment::TransactionLookup)?;
reset_stage_checkpoint(tx, StageId::TransactionLookup)?;

View File

@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
consensus: C,
) -> eyre::Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
C: FullConsensus<E::Primitives> + 'static,
{
@@ -39,7 +39,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -10,10 +10,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: BlockNumber,
to: BlockNumber,
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -9,10 +9,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: u64,
to: u64,
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
consensus: impl FullConsensus<N::Primitives> + 'static,
) -> Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
{
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
@@ -59,7 +59,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -158,7 +158,7 @@ enum Subcommands {
impl Subcommands {
/// Returns the block to unwind to. The returned block will stay in database.
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
&self,
factory: ProviderFactory<N>,
) -> eyre::Result<u64> {

View File

@@ -121,7 +121,16 @@ pub fn install() {
unsafe {
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
let mut alt_stack: libc::stack_t = mem::zeroed();
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
// generous here and just use a size of a page.
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
let page_sz = if raw_page_sz == -1 {
// Fallback alignment in case sysconf fails.
4096_usize
} else {
raw_page_sz as usize
};
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
alt_stack.ss_size = alt_stack_size;
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());

View File

@@ -438,6 +438,8 @@ pub struct BlocksPerFileConfig {
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
/// Number of blocks per file for the storage changesets segment.
pub storage_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -451,6 +453,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
@@ -469,6 +472,10 @@ impl StaticFilesConfig {
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
eyre::ensure!(
storage_change_sets != Some(0),
"Storage changesets segment blocks per file must be greater than 0"
);
Ok(())
}
@@ -480,6 +487,7 @@ impl StaticFilesConfig {
receipts,
transaction_senders,
account_change_sets,
storage_change_sets,
} = self.blocks_per_file;
let mut map = StaticFileMap::default();
@@ -492,6 +500,7 @@ impl StaticFilesConfig {
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
StaticFileSegment::StorageChangeSets => storage_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {
@@ -550,7 +559,6 @@ impl PruneConfig {
/// - `Option<PruneMode>` fields: set from `other` only if `self` is `None`.
/// - `block_interval`: set from `other` only if `self.block_interval ==
/// DEFAULT_BLOCK_INTERVAL`.
/// - `merkle_changesets`: always set from `other`.
/// - `receipts_log_filter`: set from `other` only if `self` is empty and `other` is non-empty.
pub fn merge(&mut self, other: Self) {
let Self {
@@ -563,7 +571,6 @@ impl PruneConfig {
account_history,
storage_history,
bodies_history,
merkle_changesets,
receipts_log_filter,
},
} = other;
@@ -580,8 +587,6 @@ impl PruneConfig {
self.segments.account_history = self.segments.account_history.or(account_history);
self.segments.storage_history = self.segments.storage_history.or(storage_history);
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
// Merkle changesets is not optional; always take the value from `other`
self.segments.merkle_changesets = merkle_changesets;
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
self.segments.receipts_log_filter = receipts_log_filter;
@@ -1091,7 +1096,6 @@ receipts = { distance = 16384 }
account_history: None,
storage_history: Some(PruneMode::Before(5000)),
bodies_history: None,
merkle_changesets: PruneMode::Before(0),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Full,
@@ -1108,7 +1112,6 @@ receipts = { distance = 16384 }
account_history: Some(PruneMode::Distance(2000)),
storage_history: Some(PruneMode::Distance(3000)),
bodies_history: None,
merkle_changesets: PruneMode::Distance(10000),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
(Address::random(), PruneMode::Distance(1000)),
(Address::random(), PruneMode::Before(2000)),
@@ -1127,7 +1130,6 @@ receipts = { distance = 16384 }
assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000));
assert_eq!(config1.segments.receipts_log_filter, original_filter);
}

View File

@@ -14,14 +14,11 @@ workspace = true
# reth
reth-chainspec.workspace = true
reth-consensus.workspace = true
tracing.workspace = true
# ethereum
reth-primitives-traits.workspace = true
alloy-consensus.workspace = true
alloy-primitives.workspace = true
alloy-eips.workspace = true
alloy-rlp.workspace = true
[dev-dependencies]
alloy-primitives = { workspace = true, features = ["rand"] }
@@ -38,6 +35,4 @@ std = [
"reth-primitives-traits/std",
"reth-ethereum-primitives/std",
"alloy-primitives/std",
"alloy-rlp/std",
"tracing/std",
]

Some files were not shown because too many files have changed in this diff Show More