Compare commits

...

329 Commits

Author SHA1 Message Date
Emma Jamieson-Hoare
c408bcd1dc chore: fix hive tests 2026-02-27 14:32:53 +00:00
Emma Jamieson-Hoare
9b50bb7c8d chore: fix hive failures 2026-02-27 11:38:34 +00:00
Emma Jamieson-Hoare
a66d49c190 Merge branch 'main' into bal-devnet-2 2026-02-27 11:32:19 +00:00
Emma Jamieson-Hoare
ab2252c33d fix merge issues 2026-02-27 11:22:03 +00:00
Alexey Shekhirin
f82d143d0c refactor(engine): PayloadProcessor::spawn_state_root (#22604) 2026-02-27 11:13:31 +00:00
Emma Jamieson-Hoare
8dbb015770 Merge remote-tracking branch 'origin/main' into bal-devnet-2 2026-02-27 11:07:57 +00:00
Ishika Choudhury
96be836679 chore: expected failing test for devnet 2 bal (#22453)
Co-authored-by: Soubhik Singha Mahapatra <soubhiksmp2004@gmail.com>
Co-authored-by: Soubhik Singha Mahapatra <160333583+Soubhik-10@users.noreply.github.com>
2026-02-27 10:56:20 +00:00
Derek Cofausper
bebc532e0e ci: match release binary RUSTFLAGS with Depot Docker builds (#22640)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-27 10:55:14 +00:00
DaniPopes
0df9791bea chore: bump alloy-evm to 0.28.0 (#22636)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-02-27 10:22:58 +00:00
Delweng
09adb83922 fix(engine/tree): continue sync-target progression for already-seen downloaded blocks (#22628)
Signed-off-by: Delweng <delweng@gmail.com>
2026-02-27 08:12:06 +00:00
Delweng
c12b6d4c90 fix(rpc): return -38003 for FCU beacon-root payloadAttributes mismatches (#22634)
Signed-off-by: Delweng <delweng@gmail.com>
2026-02-27 07:54:20 +00:00
Derek Cofausper
7a78044587 chore(libmdbx): fix MDB_ -> MDBX_ typos (#22630)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-27 06:06:07 +00:00
figtracer
f88538e033 refactor(net): add peers() accessors on Swarm to flatten accessor chains (#22616)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-27 05:35:14 +00:00
DaniPopes
63dff64b8a chore: simplify tx iterator (#22365) 2026-02-27 05:09:13 +00:00
DaniPopes
233590cefd chore: use better hasher for precompile cache (#22360) 2026-02-27 05:09:12 +00:00
Derek Cofausper
40962ef6fc chore(hive): remove engine-withdrawals from ignored tests (#22625)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-27 03:57:43 +00:00
github-actions[bot]
2f121b099b chore(deps): weekly cargo update (#22624)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-27 03:36:42 +00:00
Delweng
0470050c05 fix(engine): continue downloading head block after making non-head sync target canonical (#22613)
Signed-off-by: Delweng <delweng@gmail.com>
2026-02-27 03:15:52 +00:00
MagicJoshh
cbc416b82a fix(rpc-provider): state_root delegates to stub that always returns zero (#22610) 2026-02-27 02:53:57 +00:00
MagicJoshh
3fddefbd38 fix(rpc): prevent u64 underflow when re-executing genesis block (#22532) 2026-02-27 02:48:59 +00:00
Julian Meyer
f97a6530c1 chore: make cached overlay fetch public (#22619) 2026-02-27 02:47:50 +00:00
Derek Cofausper
80e3e1c79d docs: add storage v2 guide (#22620)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
2026-02-26 20:22:52 +00:00
Arsenii Kulikov
ee37c25a4b perf: use more multiproof workers (#22615) 2026-02-26 19:59:06 +00:00
Derek Cofausper
c01f9688e2 feat: add transaction iterator helpers to Chain (#22618)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-26 19:39:34 +00:00
bigbear
815a75833e refactor(exex): remove redundant update_capacity call (#22603) 2026-02-26 13:09:41 +00:00
cui
59c4e24296 fix(downloaders): reset metrics on clear (#21858) 2026-02-26 12:38:55 +00:00
Derek Cofausper
d5b5caa439 docs: add PR title and description guidelines to CLAUDE.md (#22602)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-26 12:20:54 +00:00
Julio
47f1999654 fix(net): abort discv4 and DNS discovery tasks on Discovery drop (#22590)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-26 10:37:57 +00:00
MergeBot
3ac5637bd1 chore(ci): fix collapsible_match clippy lint in chainspec (#22594)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-26 10:04:19 +00:00
Derek Cofausper
4cec99ed13 chore(bench): include core count in Slack notification when non-default (#22584)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 21:58:39 +00:00
Arsenii Kulikov
2f73835483 feat(reth-bench): support benchmarking via rlp blocks (#22581) 2026-02-25 20:28:47 +00:00
stevencartavia
ed20a40649 refactor(rpc): fetch block before tracing to avoid double lookups (#22503) 2026-02-25 20:17:45 +00:00
MergeBot
080a9cfc10 fix(rpc): add missing apply_pre_execution_changes in spawn_replay_transaction (#22575) 2026-02-25 20:04:02 +00:00
MergeBot
c4cd5c9b7b fix(rpc): add missing apply_pre_execution_changes in debug_traceCallMany (#22577) 2026-02-25 20:00:12 +00:00
Dan Cline
ce2a194fb7 feat(cli): add db stage-checkpoints command (#22579)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 19:58:59 +00:00
Vitalyr
6dcab51c97 fix(rpc): respect pending-block=none for provider blocks (#22556) 2026-02-25 19:45:42 +00:00
Derek Cofausper
4db23809cc fix(storage): return early in RocksDB healing when checkpoint is 0 (#22576)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-25 19:29:45 +00:00
Dan Cline
f84d5e6d7f chore: add Rjected as crates/cli codeowner (#22580) 2026-02-25 20:44:00 +01:00
Arsenii Kulikov
e63b6239d7 ci(bench): support configuring number of cores (#22573) 2026-02-25 17:28:35 +00:00
Matthias Seitz
660a0dee90 feat(net): persist richer peer metadata to peers file (#22557)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 17:03:25 +00:00
Arsenii Kulikov
f92c9b4370 perf: delay branch masks updates (#22565) 2026-02-25 15:35:12 +00:00
Brian Picciano
f0e2522294 perf: Remove unnecessary single-target storage proofs (#22539)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-02-25 14:35:23 +00:00
Matthias Seitz
7103088adc feat(txpool): support additional custom validation checks in EthTransactionValidator (#22559)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 13:32:21 +00:00
Derek Cofausper
663765af5c ci(bench): skip DM when results are posted to channel (#22563)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 13:18:25 +00:00
Zac Holme
20cfb2d517 fix: compute hashed post state in RpcBlockchainStateProvider (#22546)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-25 10:48:26 +00:00
Georgios Konstantopoulos
0bdf6e2f2e chore(engine): add debug log in spawned tx iterator after yielding tx index (#22558)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 08:49:33 +00:00
Georgios Konstantopoulos
85abd41824 perf: add thread-priority utils and boost engine/sparse-trie priority (#22541)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 08:33:35 +00:00
James Niken
70fb03a530 refactor(chainspec): use existing paris difficulty getter (#22474)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-25 05:39:12 +00:00
Georgios Konstantopoulos
96fce4dc4f chore: remove unmaintained shellexpand dependency (#22514)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-25 05:38:39 +00:00
Elaela Solis
728c7acd08 feat(exex): expose ExExManager buffer capacity in ExExLauncher (#22553) 2026-02-25 05:15:55 +00:00
stevencartavia
626c82db33 refactor(rpc): use replay_transactions_until in debug_trace_call_at_tx_index (#22542) 2026-02-25 05:00:48 +00:00
stevencartavia
624fcbd345 refactor(rpc): extract proof window validation into reusable helper (#22552) 2026-02-25 04:55:05 +00:00
Georgios Konstantopoulos
aed47bc3f8 fix(ci): add fallback for BENCH_JOB_URL in bench failure step (#22550)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 21:21:41 +00:00
Dan Cline
7680c1e4f6 fix: detect and remove stale CLI doc pages (#22433)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-24 19:34:55 +00:00
Dan Cline
93cb4068d2 fix: handle payload builder stream termination gracefully (#21710) 2026-02-24 19:24:24 +00:00
Georgios Konstantopoulos
2fba05dc67 feat(rpc): add reth_forkchoiceUpdated endpoint (#22536)
Co-authored-by: Arsenii Kulikov <klkvr@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 18:29:38 +00:00
Alexey Shekhirin
ea143d4d31 ci(bench): report panics and error logs in comments (#22544)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-24 18:19:22 +00:00
Matthias Seitz
fddb7dad10 feat(net): use fork_id as tiebreaker in peer selection (#22545)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 18:12:25 +00:00
Arsenii Kulikov
af6d674cac perf: decrease chunk size (#22527)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 17:56:39 +00:00
Georgios Konstantopoulos
de5688a76e perf(engine): remove spawn for prewarm pool init (#22543)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 17:35:20 +00:00
figtracer
d4cb91f0a5 perf(txpool): use BTree range queries in pending_txs_by_sender/queued_txs_by_sender (#22528)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-24 17:01:59 +00:00
Georgios Konstantopoulos
d122c7b49c chore(tasks): remove quanta upkeep from runtime (#22540)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 16:39:10 +00:00
Arsenii Kulikov
aed9014e1e chore: don't include spans for noops (#22538) 2026-02-24 16:21:38 +00:00
Arsenii Kulikov
d340114d52 refactor: don't return hashes for blinded nodes (#22535) 2026-02-24 16:08:40 +00:00
Georgios Konstantopoulos
7fc22f7b5b feat(rpc): accept RLP-encoded blocks in reth_newPayload (#22533)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-02-24 15:46:02 +00:00
Georgios Konstantopoulos
c8c5f8886d perf(engine): use rayon par_iter for tx prewarming instead of manual workers (#22521)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-24 15:42:06 +00:00
Arsenii Kulikov
2f3c8d7d03 feat(bench): enable --log.samply when samply is configred (#22526) 2026-02-24 13:03:19 +00:00
Georgios Konstantopoulos
a90f8be67b revert: "perf(trie): replace Box clone with unsafe reborrow in prune (#22516)" (#22525) 2026-02-24 12:14:22 +00:00
Georgios Konstantopoulos
7faca05344 refactor(engine): use spawn_blocking_named for tx_iterator thread (#22522)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 11:07:19 +00:00
Matthias Seitz
2827b0aca0 refactor: simplify uncle block fetching in RPC (#22523)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 12:11:33 +01:00
Georgios Konstantopoulos
d3bb2faf28 refactor(rpc): extract RethEngineApi into standalone struct (#22504)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-24 10:57:13 +00:00
Arsenii Kulikov
ef292ffa00 fix: don't produce both updates and removals for trie nodes (#22507)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-02-24 10:36:34 +00:00
Georgios Konstantopoulos
ea98d37bb3 ci: use native ARM runner for aarch64 linux release builds (#22519)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 09:16:38 +00:00
Emma Jamieson-Hoare
f2b3201187 chore(release-builds): remove the riscv builds from release pipeline (#22499) 2026-02-24 09:00:48 +00:00
Georgios Konstantopoulos
d1cbf6ca5a perf(trie): reserve capacity in apply_subtrie_update_actions (#22517)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 08:38:09 +00:00
Georgios Konstantopoulos
56bb47709c perf(trie): replace Box clone with unsafe reborrow in prune (#22516)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-24 08:22:52 +00:00
Elaela Solis
3703255d5d fix: make SerdeBincodeCompat generic for EthereumTxEnvelope (#22513) 2026-02-24 08:11:15 +00:00
DaniPopes
b431caf806 fix: avoid duplicate runtime initialization on startup (#22515) 2026-02-24 07:49:48 +00:00
Matthias Seitz
21dadb71c3 fix: update shellexpand to 3.1.2 and unpin nightly (#22506)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-23 21:42:58 +01:00
Arsenii Kulikov
98c45a4245 fix: remove debug_asssert! (#22505) 2026-02-23 20:13:51 +00:00
Arsenii Kulikov
ac2cc7b4e2 fix: proper SerdeBincodeCompat for EthereumReceipt (#22461) 2026-02-23 19:31:24 +00:00
Arsenii Kulikov
3931affcf2 revert: feat(rpc): move reth_newPayload from EngineApi to RethApi (#22500) 2026-02-23 18:33:59 +00:00
Alexey Shekhirin
93b7ae9286 chore(storage): propagate span context across rayon thread boundaries (#22497)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-23 18:17:18 +00:00
Emma Jamieson-Hoare
7e7717bdaa chore: release 1.11.1 (#22496)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-23 18:07:50 +00:00
Georgios Konstantopoulos
815037e27d feat(storage): slot preimage DB for plain changeset keys in v2 (#22379)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-23 18:01:44 +00:00
Georgios Konstantopoulos
80bf5532ac perf(trie): pack StoredNibblesSubKey from 65→33 bytes, generic cursor factory (#22158)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-23 17:02:43 +00:00
Arsenii Kulikov
028e99191a perf: optimize sparse trie (#22418)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-02-23 16:18:45 +00:00
Emma Jamieson-Hoare
ab5f2db594 Merge branch 'main' into bal-devnet-2 2026-02-23 15:46:05 +00:00
Georgios Konstantopoulos
dc35fc8251 feat(rpc): move reth_newPayload from EngineApi to RethApi (#22425)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-02-23 15:43:20 +00:00
Georgios Konstantopoulos
285c325d71 feat(re-execute): work-stealing parallelization (#22242)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-02-23 15:39:24 +00:00
Arsenii Kulikov
ca47a7e9f9 fix: overlay preparation on tokio (#22492) 2026-02-23 15:37:55 +00:00
MergeBot
6d718d0c21 fix(rpc): use actual configured limit in trace_filter (#22477) 2026-02-23 13:14:19 +00:00
Emma Jamieson-Hoare
6a633a42f0 fix: handle EIP-7778 gas accounting mismatch in payload builder (#22490)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-23 13:51:48 +01:00
YK
949111c953 perf(engine): precompute tx root during payload validation (#22489) 2026-02-23 10:35:22 +00:00
Georgios Konstantopoulos
742eb56949 perf(engine): add tracing spans for post-execution validation wait times (#22483)
Co-authored-by: Yong Kang <yongkangc@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-02-23 09:56:09 +00:00
Matthias Seitz
4af4836ec1 ci: pin nightly to 2026-02-21 (#22485)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-23 10:26:08 +01:00
figtracer
3bc71e7ec0 chore: use ValidPoolTransaction methods instead of reaching into inner field (#22475)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-22 18:15:49 +01:00
VolodymyrBg
03fbb6cafe fix(rpc): stop IPC handle in AuthServerHandle::stop() (#22467) 2026-02-22 07:56:52 +01:00
Alexey Shekhirin
b09b097a0b chore(ci): enhance benchmark artifact collection (#22457)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-21 13:43:59 +00:00
MergeBot
0fffdcdd23 fix(tracing): handle file_writer in LogFmt format (#22429) 2026-02-21 09:12:32 +00:00
strmfos
bc33eb764a fix(txpool): prevent underflow in blobstore versioned hash lookup (#22454)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-21 08:54:09 +00:00
Georgios Konstantopoulos
190157636e chore: remove unused Default impl for ExecutionEnv (#22451)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-21 08:29:59 +00:00
figtracer
8e3bc6567c chore(txpool): use to_consensus helper instead of reaching into inner field (#22426)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-21 08:28:57 +00:00
Georgios Konstantopoulos
45b961c7b3 chore: deprecate reth-primitives crate (#22450)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-21 08:16:34 +00:00
stevencartavia
94818d7676 feat(rpc): add reth_getBlockExecutionOutcome endpoint (#22397)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-21 05:47:05 +00:00
Alexey Shekhirin
4c2a9a9b4a feat(bench): add Slack notifications with Block Kit (#22447)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-21 00:09:07 +00:00
Alexey Shekhirin
76c37f0f80 ci(bench): install all runner dependencies from job (#22445)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 21:54:45 +00:00
figtracer
0275ff35fd refactor(net): add methods to PropagatedTransactions instead of exposing .0 (#22441)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-20 20:51:20 +00:00
Alexey Shekhirin
3f011c8328 ci(bench): add median lines to benchmark charts (#22439)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 20:40:32 +00:00
figtracer
beac28dbb2 chore(payload): use Transaction::blob_versioned_hashes() directly (#22440)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-20 18:59:06 +00:00
Alexey Shekhirin
bce100c6c8 ci(bench): add samply profiling support (#22432)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 18:16:28 +00:00
Alexey Shekhirin
40e99a4a4f ci(bench): switch to @decofe bot and new secret names (#22434)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 17:55:16 +00:00
Dan Cline
1ff88e43cd fix: handle missing rocksdb gracefully in read-only db commands (#22394)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 17:18:43 +00:00
joshieDo
d23c244cd1 fix: align static-file changeset checksum with MDBX semantics (#22389)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-20 17:11:08 +00:00
Dan Cline
3de9259026 docs: add CLI docs regeneration guide (#22395)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 16:46:18 +00:00
Alexey Shekhirin
d24f0b1e05 fix: update PR comment when bench workflow is cancelled (#22430)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-20 15:20:10 +00:00
iPLAY888
bb1b9ec611 fix(net): log transaction count instead of bool in broadcast debug log (#22417) 2026-02-20 12:15:40 +00:00
Emma Jamieson-Hoare
08fd55d8c9 ci: use larger runner for hive reth builds
Amp-Thread-ID: https://ampcode.com/threads/T-019c7acf-3e9a-7459-8100-dae97ec43d52
Co-authored-by: Amp <amp@ampcode.com>
2026-02-20 11:59:27 +00:00
Arsenii Kulikov
70cab0d163 fix: properly reveal trie nodes (#22415) 2026-02-20 11:51:04 +00:00
Alexey Shekhirin
e530b1f6a1 refactor(bench): push charts to external repo instead of bench-charts branch (#22414)
Co-authored-by: Alexey Shekhirin <shekhirin@shekhirin-tempo.tail388b2e.ts.net>
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-20 11:42:15 +00:00
Emma Jamieson-Hoare
ff5d375526 docs(hive): add comments explaining why flaky tests are ignored (#22383)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-20 11:18:20 +00:00
Emma Jamieson-Hoare
fab95c1f3a Merge branch 'main' into bal-devnet-2 2026-02-20 11:13:52 +00:00
YK
d1a92afb57 feat(engine): add sub-phase timing histograms for sparse trie event loop (#22368)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-20 06:56:04 +00:00
Dan Cline
0517c12c90 docs: remove stale db settings subcommands (#22396)
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-20 05:34:01 +00:00
YK
237eb1675c perf(trie): cache pending sparse trie target count (#22355) 2026-02-20 04:38:13 +00:00
Arsenii Kulikov
b6bcd7e6bd fix: catch panics of named tasks (#22386) 2026-02-19 22:32:02 +00:00
Alexey Shekhirin
48122300d7 fix(bench): validate cached binaries match expected commit SHA (#22392)
Co-authored-by: Alexey Shekhirin <shekhirin@shekhirin-tempo.tail388b2e.ts.net>
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-19 21:35:11 +00:00
Alexey Shekhirin
13f214f160 ci(bench): use schelk promote instead of recover when updating snapshot (#22391)
Co-authored-by: Alexey Shekhirin <shekhirin@shekhirin-tempo.tail388b2e.ts.net>
Co-authored-by: Claude Haiku 4.5 <noreply@anthropic.com>
2026-02-19 21:29:38 +00:00
Georgios Konstantopoulos
f17592670d fix(bench): checkout feature source to correct ref instead of symlinking (#22390)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 20:42:21 +00:00
Alexey Shekhirin
c225132b81 ci(bench): drop root privileges for reth-bench (#22380) 2026-02-19 14:46:52 +00:00
radik878
dcc5d9ec30 fix(events): handle PipelineEvent::Unwound to clean up current_stage (#22340) 2026-02-19 13:48:57 +00:00
Alexey Shekhirin
6cd56b645b ci(bench): support running benchmarks on closed/merged PRs (#22378) 2026-02-19 13:16:03 +00:00
Emma Jamieson-Hoare
794dbff26e ci(hive): remove EIP-6110 deposit tests from expected failures (now passing) (#22377)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 12:56:36 +00:00
Emma Jamieson-Hoare
fcfbed0bbc ci(hive): ignore flaky reorg and sync timeout tests (#22376)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 12:55:20 +00:00
Alexey Shekhirin
70bcd475fe ci(bench): ABBA run order (#22335) 2026-02-19 12:40:44 +00:00
Emma Jamieson-Hoare
cd6e895a97 fix(rpc): return -32602 for PayloadAttributes structure validation errors (#22374)
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-19 12:32:31 +00:00
Emma Jamieson-Hoare
6552a3a9ab ci(hive): fix eels runner OOM crashes (#22373)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 12:04:22 +00:00
Derek Cofausper
6a91089542 ci(bench): fix cleanup to use sudo pkill and lazy unmount (#22372)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-02-19 11:23:03 +00:00
YK
a9a1e504b4 refactor(trie): simplify encode_account_leaf_value (#22366) 2026-02-19 10:36:44 +00:00
YK
e280f25885 feat(trie): expose storage_wait_time as dedicated Prometheus metric (#22359) 2026-02-19 10:36:26 +00:00
Emma Jamieson-Hoare
e4191ccea8 Merge branch 'main' into bal-devnet-2 2026-02-19 10:27:42 +00:00
Arsenii Kulikov
37c4f908fa perf: store blinded node hashes on SparseNode::Branch (#22290)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-02-19 09:34:42 +00:00
Georgios Konstantopoulos
a157be3f3b perf(tasks): add LazyHandle<T>, use for hash post state (#22347)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 08:48:24 +00:00
Georgios Konstantopoulos
e0eb306b2b chore(engine): rename finish span to BlockExecutor::finish (#22356)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 07:29:18 +00:00
Micke
7f4f3f1eb9 fix(prune): correct deleted entries count when skip_filter is used (#22312) 2026-02-19 06:19:02 +00:00
Georgios Konstantopoulos
8970f82aaf perf(engine): prefetch first txs sequentially to avoid rayon scheduling stall (#22305)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-19 05:53:52 +00:00
Georgios Konstantopoulos
8529da976f fix(cli): store extradata as Bytes, decode hex in parser (#22344)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 04:38:16 +00:00
stevencartavia
8fa539225b refactor: remove duplicate apply_pre_execution_changes from Trace trait (#22333) 2026-02-19 04:32:42 +00:00
Doohyun Cho
93d546a36d perf(trie): preserve allocations in sparse trie wipe() (#21089) 2026-02-19 04:02:20 +00:00
zhygis
5c83eb0b06 feat(log): disable file logging by default for non-node commands (#21521)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-02-19 03:16:47 +00:00
Georgios Konstantopoulos
cd32e3cc05 feat(reth-bench): add prometheus metrics scraper (#22244)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-19 03:13:40 +00:00
MergeBot
26470cadfc perf(trie): remove redundant HashMap lookup in sparse trie account state query (#22328)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 03:09:04 +00:00
Brian Picciano
506ab806e4 fix: propagate trie update diff result to trigger debug recorder writes (#22331)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 02:52:43 +00:00
Forostovec
c2e846093e fix(net): use continue instead of return in buffer_hashes loop (#22337) 2026-02-19 02:46:33 +00:00
dependabot[bot]
5df22b12d8 chore(deps): bump actions/upload-artifact from 4 to 6 (#22338)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-02-19 02:33:00 +00:00
dependabot[bot]
ff9700bb3b chore(deps): bump actions/github-script from 7 to 8 (#22339)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-02-19 02:32:57 +00:00
Georgios Konstantopoulos
85d35fa6c0 feat(tasks): add WorkerMap for named single-thread workers (#22262)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-19 02:27:02 +00:00
Georgios Konstantopoulos
47544d9a7e fix(txpool): ensure transactions are added to pending subpool in nonce order (#22308)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-18 20:31:04 +00:00
Alexey Shekhirin
ef33961aff ci(bench): download snapshot in parallel with builds (#22332) 2026-02-18 17:40:17 +00:00
Ishika Choudhury
080ff004e3 chore: fixed bal devnet error (#22325)
Co-authored-by: Soubhik Singha Mahapatra <soubhiksmp2004@gmail.com>
2026-02-18 18:36:22 +01:00
Georgios Konstantopoulos
0e01a694a7 fix(storage): clarify storage settings mismatch warning (#22330)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 16:35:28 +00:00
Alexey Shekhirin
ee19320ee8 ci(bench): use ABBA run order to reduce variance (#22321) 2026-02-18 15:33:31 +00:00
Alexey Shekhirin
9251997c1f ci(bench): build baseline and feature binaries in parallel (#22323) 2026-02-18 14:30:58 +00:00
Brian Picciano
302993b45a feat(trie-debug): record SetRoot op in ParallelSparseTrie::set_root (#22324)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 14:03:42 +00:00
Brian Picciano
8d97ab63c6 perf: use stack-allocated [u8; 65] for StoredNibblesSubKey encoding (#22314)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-18 13:35:20 +00:00
Matthias Seitz
251f83ab0b refactor: replace TryFrom*Response traits with unified RpcResponseConverter (#22320)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 13:32:47 +00:00
Alexey Shekhirin
e6e0dde903 ci(bench): queue reth-bench jobs and report queue position in PR comment (#22318) 2026-02-18 12:53:12 +00:00
Emma Jamieson-Hoare
18599f1732 Merge branch 'main' into bal-devnet-2 2026-02-18 12:47:57 +00:00
Emma Jamieson-Hoare
9fd35e2917 Revert "chore: merge main into devnet-2 branch (#22316)"
This reverts commit c1a5e20b50.
2026-02-18 12:40:23 +00:00
Emma Jamieson-Hoare
c1a5e20b50 chore: merge main into devnet-2 branch (#22316) 2026-02-18 12:39:07 +00:00
Georgios Konstantopoulos
b1b51261af feat(ci): granular status updates for reth-bench workflow (#22297)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 12:05:59 +00:00
Georgios Konstantopoulos
2ae5ef475e feat(ci): add workflow_dispatch trigger for reth-bench (#22298)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 11:22:22 +00:00
Ishika Choudhury
0ff16ea053 chore: fix failing tests(hive) for devnet 2 BAL (#22259) 2026-02-18 11:17:01 +01:00
drhgencer
8861e2724f fix(txpool): notify subscribers when set_block_info promotes transaction (#22243)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 09:08:16 +00:00
Georgios Konstantopoulos
734ec4ffe6 feat(engine): add tracing spans to execute_block setup (#22304)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 06:23:24 +00:00
Georgios Konstantopoulos
cbcdf8dac0 chore(tracing): use underscores instead of spaces in span names (#22307)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 06:21:04 +00:00
Georgios Konstantopoulos
826e387c87 refactor(rpc): use ..Default::default() for SimCallResult initialization (#22309)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 05:42:22 +00:00
Forostovec
1c40188993 fix: correct message ID in NodeData version error (#22291) 2026-02-18 05:02:33 +00:00
Matthias Seitz
49a2df0d7a chore: bump alloy deps 1.7.1 -> 1.7.3 (#22296)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 06:02:04 +01:00
DaniPopes
a1d1b6def6 fix: prevent ANSI escape codes leaking into Tracy zone text (#22306) 2026-02-18 03:49:34 +00:00
Georgios Konstantopoulos
56bbb3ce2c feat(cli): add reth db prune-checkpoints command (#22288)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 01:25:53 +00:00
Georgios Konstantopoulos
5b1010322c docs: clarify StateWriteConfig is about database (MDBX) writes vs static files (#22299)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-18 01:13:31 +00:00
Georgios Konstantopoulos
a195b777eb perf(storage): skip plain state conversion in write_state for storage v2 (#22294)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-18 00:48:06 +00:00
Georgios Konstantopoulos
5045e6ef8b feat(bench): add wait time breakdown tables to CI report (#22293) 2026-02-17 23:44:03 +00:00
Alexey Shekhirin
b49cadb346 ci(bench): rename main/branch to baseline/feature, add ref args (#22284)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 23:00:01 +00:00
Georgios Konstantopoulos
aeb2c6e731 chore(primitives): remove legacy transaction roundtrip tests (#22292)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 21:15:14 +00:00
stevencartavia
477fed7a11 refactor(primitives): use alloy's EthereumReceipt type (#22254)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 20:30:52 +00:00
MergeBot
59993b974a fix(rpc): resolve AtBlockHash to single block in eth_getFilterChanges (#22283)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 20:15:33 +00:00
Georgios Konstantopoulos
9ecef47aff fix(provider): skip sender pruning during reorg when sender_recovery is full (#22271)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 19:15:03 +00:00
DaniPopes
0ba685386d refactor: dedup runtime initializations (#22263)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-02-17 17:35:31 +00:00
Georgios Konstantopoulos
6ff4f947c8 fix(trie): propagate parent span to proof workers (#22279)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 16:48:19 +00:00
Alexey Shekhirin
719bbc2543 ci: reth-bench (#22134) 2026-02-17 16:47:47 +00:00
Emma Jamieson-Hoare
a9a6044bc5 chore: fix the rust version for docker (#22278) 2026-02-17 15:19:03 +00:00
Brian Picciano
6f9a3242ef chore: remove legacy proof code paths and simplify to V2-only (#22270)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 14:42:13 +00:00
Matthias Seitz
e89bf483bc feat(rpc): add query methods to ActiveFilters (#22275)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 14:37:04 +00:00
Georgios Konstantopoulos
61038449c8 fix(rpc): update eth_simulateV1 revert error code to 3 (#22272)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 14:36:34 +00:00
Georgios Konstantopoulos
48b2cd970f docs: fix default jwt.hex path in cli args (#22269)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 14:35:32 +00:00
Brian Picciano
fb90051010 fix(trie): subtrie root node too small to have hash (#22114)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 13:30:47 +00:00
Georgios Konstantopoulos
a0a622a155 ci: use normal Docker builds for fork PRs instead of Depot (#22268)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 13:27:38 +00:00
Emma Jamieson-Hoare
3541bd7f65 fix rust issue 2026-02-17 12:52:09 +00:00
Emma Jamieson-Hoare
a3aec0c662 merge: resolve conflict with main in payload_validator.rs
Amp-Thread-ID: https://ampcode.com/threads/T-019c6b9f-1ef5-76f8-bc77-7547d5460bf8
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 12:42:29 +00:00
Brian Picciano
8db352dfd2 feat(trie): add trie-debug feature for recording sparse trie mutations (#22234)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 11:59:11 +00:00
Emma Jamieson-Hoare
0dfdaca3f0 Merge branch 'main' into bal-devnet-2 2026-02-17 11:20:26 +00:00
Brian Picciano
117b212e2e feat(trie): Combine extension and branch nodes in output from proof v2 (#22021)
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-02-17 11:12:48 +00:00
Brian Picciano
df9e3669aa chore: Update nix flake (#22237) 2026-02-17 11:06:11 +00:00
Georgios Konstantopoulos
0464cddfb0 ci: fall back to GitHub-hosted runners for forks (#22266)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 10:34:43 +00:00
Tomass
e21a174737 refactor(net): use VecDeque::pop_front_if from Rust 1.93 (#22260) 2026-02-17 08:46:56 +00:00
DaniPopes
e972d9d8c7 chore: rm transact_batch span (#22258) 2026-02-17 04:45:05 +00:00
Georgios Konstantopoulos
7f00ebfafe chore: elide lifetimes in iter_sub_trie_targets (#22256)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 02:11:09 +00:00
Matthias Seitz
883e9ae8cc feat(node-core): add with_dev_block_time helper to NodeConfig (#22251)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-17 02:40:46 +01:00
DaniPopes
a1e4132c2d chore: reduce size of common spans (#22253) 2026-02-17 01:29:32 +00:00
DaniPopes
4ecb0d5680 perf: use mutex in for_each_ordered (#22252) 2026-02-17 01:19:56 +00:00
Georgios Konstantopoulos
5b8808e5fd feat(engine): add trigger-based MiningMode variant (#22250)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-17 00:28:25 +00:00
Georgios Konstantopoulos
2eec519bf9 feat(tasks): add WorkerPool with per-thread Worker state (#22154)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 23:46:51 +00:00
Georgios Konstantopoulos
02513ecf3b perf(engine): overlap block conversion with execution in payload validation (#21957)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-16 23:46:45 +00:00
Emma Jamieson-Hoare
c535a7fb5b Merge branch 'main' into bal-devnet-2 2026-02-16 12:26:52 -05:00
Soubhik Singha Mahapatra
bf6270b8a3 chore: try validation of bal after execution (#22165)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmail.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 18:25:20 +01:00
jenpaff
1e78685a6c fix: resolve clippy and build issues from revm hashmap changes
- Fix doc-markdown warnings in validation.rs (backtick identifiers)
- Fix StorageKeyMap type mismatch from revm specialized hashmaps
- Remove unused HashMap import

Amp-Thread-ID: https://ampcode.com/threads/T-019c66b1-1100-777b-98e4-42a7e5d2be57
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 15:53:25 +00:00
jenpaff
1f1e320643 fix: gate BytecodeKind import behind reth-codec feature
Fixes cargo hack --no-default-features check for reth-primitives-traits.

Amp-Thread-ID: https://ampcode.com/threads/T-019c66b1-1100-777b-98e4-42a7e5d2be57
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 14:38:44 +00:00
jenpaff
74ea20400e fix: add block_access_list field to BlockExecutionResult initializers
Required by alloy-evm #287 (EIP-7928). Set to None/default for now.

Amp-Thread-ID: https://ampcode.com/threads/T-019c66b1-1100-777b-98e4-42a7e5d2be57
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 14:32:20 +00:00
Alexey Shekhirin
10c6bdb5ff fix(engine): wait for persistence to complete in reth_newPayload (#22239) 2026-02-16 14:08:36 +00:00
jenpaff
ffff5fbce2 Merge remote-tracking branch 'origin/main' into bal-devnet-2
Amp-Thread-ID: https://ampcode.com/threads/T-019c66b1-1100-777b-98e4-42a7e5d2be57
Co-authored-by: Amp <amp@ampcode.com>

# Conflicts:
#	Cargo.lock
#	crates/rpc/rpc-engine-api/src/engine_api.rs
2026-02-16 13:51:39 +00:00
jenpaff
f514892b41 Merge remote-tracking branch 'origin/bal-devnet-2' into bal-devnet-2
Amp-Thread-ID: https://ampcode.com/threads/T-019c66b1-1100-777b-98e4-42a7e5d2be57
Co-authored-by: Amp <amp@ampcode.com>

# Conflicts:
#	Cargo.lock
2026-02-16 13:50:56 +00:00
Matthias Seitz
20ae9ac405 docs: add type ordering style guide to CLAUDE.md (#22236)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 13:38:53 +01:00
Alexey Shekhirin
881500e592 feat(rpc, reth-bench): reth_newPayload methods for reth-bench (#22133)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-02-16 11:11:13 +00:00
pepes
8db125daff fix(engine-primitives): delegate block_to_payload to T (#22180) 2026-02-16 10:09:58 +00:00
James Niken
bf2071f773 fix(primitives-traits): handle KECCAK_EMPTY in From<TrieAccount> (#22200) 2026-02-16 10:02:56 +00:00
Alvarez
ee5ec069cd refactor(tracing): use Option::transpose() for file_guard (#22181) 2026-02-16 11:08:59 +01:00
YK
8722277d6e perf: adaptive multiproof chunk size based on block gas usage (#22233) 2026-02-16 09:49:56 +00:00
Georgios Konstantopoulos
57148eac9f refactor(tasks): remove TaskSpawner trait in favor of concrete Runtime (#22052)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 08:51:10 +00:00
YK
74abad29ad perf: reduce update_leaves key cloning (#22228) 2026-02-16 08:34:21 +00:00
drhgencer
997af404a5 fix(rpc): trim spaces in CORS domain parsing (#22192)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-16 06:51:34 +00:00
bobtajson
314a92e93c refactor(cli): deduplicate download finalization logic (#22164) 2026-02-16 06:41:47 +00:00
Georgios Konstantopoulos
f0c4be108b fix(engine): send correct transaction index in prewarm task (#22223)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 06:21:02 +00:00
Georgios Konstantopoulos
9265e8e46c chore: remove reserved_cpu_cores from rayon thread pools (#22221)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 06:13:24 +00:00
Georgios Konstantopoulos
7594e1513a perf: replace some std::time::Instant with quanta::Instant (#22211)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-16 05:15:06 +00:00
Georgios Konstantopoulos
7f5acc2723 fix(net): use test backoff durations in Testnet PeerConfig (#22222)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 04:45:47 +00:00
DaniPopes
60d0430c2b chore(trie): add level=debug to sparse trie state spans (#22220) 2026-02-16 04:31:26 +00:00
Georgios Konstantopoulos
d49f828998 test: speed up slow integration tests (#22216)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 03:53:15 +00:00
Georgios Konstantopoulos
2f78bcd7b5 fix(test): activate prague for sparse trie reuse e2e test (#22215)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 03:50:20 +00:00
Georgios Konstantopoulos
f60febfa62 chore(ci): reduce default test timeout to 60s (#22212)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 03:43:58 +00:00
Georgios Konstantopoulos
317f858bd4 feat(engine): add gas-bucketed sub-phase metrics for new_payload (#22210)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-02-16 03:35:59 +00:00
Georgios Konstantopoulos
11acd97982 chore: use --locked for all cargo install invocations (#22214)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 03:35:17 +00:00
Georgios Konstantopoulos
f5cf90227b fix(net): fix flaky test_trusted_peer_only test (#22213)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 03:30:37 +00:00
DaniPopes
0dd47af250 perf: add dedicated prewarming rayon pool (#22108) 2026-02-16 03:05:36 +00:00
Georgios Konstantopoulos
0142769191 fix(engine): fix flaky test_prefetch_proofs_batching test (#22209) 2026-02-16 02:35:42 +00:00
DaniPopes
e1dc93e24f chore: add some more spans to validation setup (#22208) 2026-02-16 02:35:24 +00:00
Georgios Konstantopoulos
33ac869a85 perf(engine): replace channel+BTreeMap reorder with lock-free for_each_ordered (#22144)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-02-16 02:06:10 +00:00
Georgios Konstantopoulos
ec982f8686 perf: bound more channels with known upper limits (#22206)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 02:05:43 +00:00
Georgios Konstantopoulos
47cef33a0d fix: record bare tracing instrument fields (#22207)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 01:41:05 +00:00
Georgios Konstantopoulos
9529de4cf2 perf(engine): bound channels in spawn_tx_iterator by transaction count (#22205)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 01:07:26 +00:00
Georgios Konstantopoulos
5a9dd02301 chore: bump MSRV to 1.93 (#22204)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-16 00:31:05 +00:00
Georgios Konstantopoulos
d71a0c0c7b feat(txpool): add PoolTransaction::consensus_ref (#22182)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-15 12:23:37 +00:00
0xMars42
2be3788481 fix(exex): drain notification channel during backfill to prevent stall (#22168)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-15 10:48:11 +00:00
github-actions[bot]
adbec3218d chore(deps): weekly cargo update (#22197)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-02-15 08:53:11 +00:00
Georgios Konstantopoulos
2e5560b444 feat(rpc): add eth_getStorageValues batch storage slot retrieval (#22186)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-14 15:57:56 +00:00
Georgios Konstantopoulos
1f3fd5da2e refactor(engine): remove reth-engine-service crate (#22187)
Co-authored-by: mattsse <mattsse@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-14 15:40:16 +00:00
Georgios Konstantopoulos
3ab7cb98aa fix(storage): add back Arc auto_impl for storage-api traits (#22178)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-14 11:16:31 +00:00
Georgios Konstantopoulos
d3088e171c feat(execution-types): add account_state helper to BlockExecutionOutput (#22177)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-14 11:08:05 +00:00
Matthias Seitz
2c443a3dcb fix: remove unused RangeBounds import in storage-api (#22176)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-14 12:03:20 +01:00
andrewshab
4b444069a5 perf(cli): remove clone in trie repair (#22152) 2026-02-14 09:14:35 +00:00
drhgencer
25d371817a fix(pruning): trim spaces in receipts log filter parsing (#22172) 2026-02-14 09:13:40 +00:00
Karl Yu
4b0fa8a330 feat: implement variants for BAL devp2p variants (#22024)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-14 08:22:26 +00:00
James Niken
df22d38224 fix(era): encode TotalDifficulty as SSZ uint256 (little-endian) (#22160) 2026-02-14 07:57:57 +00:00
Georgios Konstantopoulos
e4ec836a46 perf(engine): reduce proof worker count for small blocks (#22074)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Ubuntu <ubuntu@dev-yk.tail388b2e.ts.net>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-02-13 21:33:25 +00:00
0xsensei
d3c42fc718 perf(reth-engine-tree): sparse trie bulk move new storage update (#22116) 2026-02-13 15:55:13 +00:00
Arsenii Kulikov
8171cee927 fix: change add_transactions_with_origins to take Vec (#22161) 2026-02-13 12:34:24 +00:00
Soubhik Singha Mahapatra
d0ad4b0e18 chore: gas traces for failing tests (#21943)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
2026-02-13 13:28:55 +01:00
Dan Cline
61cfcd8195 chore: fix riscv build for rocksdb (#22153) 2026-02-13 00:09:14 +00:00
YK
b646f4559c perf: skip dispatch pipeline when all proof targets already fetched (#22147)
Co-authored-by: Ubuntu <ubuntu@dev-yk.tail388b2e.ts.net>
2026-02-12 22:35:33 +00:00
Georgios Konstantopoulos
564ffa5868 fix(ci): pass docker tags as separate set entries in bake action (#22151)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 22:10:35 +00:00
Dan Cline
12891dd171 chore: allow invalid storage metadata (#22150) 2026-02-12 22:02:26 +00:00
Emma Jamieson-Hoare
c1015022f5 chore: release reth v1.11.0 (#22148) 2026-02-12 21:39:30 +00:00
Dan Cline
e3fe6326bc chore(storage): rm storage settings, use only one (#22042)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-12 21:17:05 +00:00
Dan Cline
e3d520b24f feat(network): add inbound / outbound scopes for disconnect reasons (#22070) 2026-02-12 20:54:03 +00:00
Dan Cline
9f29939ea1 feat: bundle mdbx_copy as reth db copy subcommand (#22061)
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
2026-02-12 20:39:56 +00:00
Matthias Seitz
10881d1c73 chore: fix book (#22142) 2026-02-12 21:44:53 +01:00
John Letey
408593467b feat(download): optional chain-aware snapshot url (#22119) 2026-02-12 21:42:19 +01:00
Emma Jamieson-Hoare
8caf8cdf11 docs: improve reth.rs/overview page (#22131)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 20:10:34 +00:00
Georgios Konstantopoulos
1e8030ef28 fix(engine): return error on updates channel disconnect in sparse trie task (#22139)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 20:00:36 +00:00
YK
f72c503d6f feat(metrics): use 5M first gas bucket for finer-grained newPayload metrics (#22136)
Co-authored-by: Ubuntu <ubuntu@dev-yk.tail388b2e.ts.net>
2026-02-12 19:03:21 +00:00
Emma Jamieson-Hoare
42890e6e7f fix: improve nightly Docker build failure Slack notification (#22130)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 18:58:55 +00:00
jenpaff
cb6ed16485 chore: merge main into bal-devnet-2
Merge main into bal-devnet-2 to resolve conflicts. Key resolutions:
- Removed deleted optimism, stateless, and custom-node files (removed on main)
- Downgraded alloy workspace versions to 1.5.2 to match bal-devnet2 patches
- Added comprehensive alloy bal-devnet2 patches for all alloy crates
- Added ExecutionPayloadBodiesV2/BodyV2 type aliases (pending alloy support)
- Adapted reth-bench V4/V5 payload handling for alloy 1.5.2 API
- Kept BAL-specific changes (EIP-7778, EIP-7928)

Amp-Thread-ID: https://ampcode.com/threads/T-019c5311-f28a-7584-8224-29e16e5095c1
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 13:40:29 -05:00
Dan Cline
e30e441ada fix: stage drop prunes account/storage changeset static files (#22062) 2026-02-12 18:34:46 +00:00
Georgios Konstantopoulos
121160d248 refactor(db): use hashed state as canonical state representation (#21115)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-12 18:02:02 +00:00
Georgios Konstantopoulos
7ff78ca082 perf(engine): use transaction count threshold for prewarm skip (#22094)
Co-authored-by: yk <yongkang@tempo.xyz>
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Ubuntu <ubuntu@dev-yk.tail388b2e.ts.net>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-02-12 17:07:52 +00:00
Georgios Konstantopoulos
d7f56d509c chore: add DaniPopes as codeowner for tasks crate (#22128)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 12:08:02 -05:00
Georgios Konstantopoulos
3300e404cf feat(engine): add --engine.disable-sparse-trie-cache-pruning flag (#21967)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: mattsse <19890894+mattsse@users.noreply.github.com>
Co-authored-by: alexey <17802178+shekhirin@users.noreply.github.com>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-02-12 16:36:31 +00:00
Georgios Konstantopoulos
77cb99fc78 chore(node): update misleading consensus engine log message (#22124)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
2026-02-12 16:14:03 +00:00
Georgios Konstantopoulos
66169c7e7c feat(reth-bench): add progress field to per-block benchmark logs (#22016)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 16:03:32 +00:00
Georgios Konstantopoulos
4f5fafc8f3 fix(net): correct EthMessageID::max for eth70 and later versions (#22076)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-12 15:53:11 +00:00
Georgios Konstantopoulos
0b8e6c6ed3 feat(net): enforce EIP-868 fork ID for discovered peers (#22013)
Co-authored-by: Emma <emma@tempo.xyz>
Co-authored-by: Matthias Seitz <mattsse@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmail.com>
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
2026-02-12 15:29:37 +00:00
Stefan
a88eef91f4 fix: pass slot_number in next_evm_env for block building (#21945)
Co-authored-by: Claude Opus 4.6 <noreply@anthropic.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-07 18:24:57 +01:00
Ishika Choudhury
f7e7afd51f chore: slot num fixes and error mapping (#21940)
Co-authored-by: Soubhik Singha Mahapatra <soubhiksmp2004@gmail.com>
2026-02-07 14:35:49 +01:00
Matthias Seitz
102764285b chore: update alloy-evm
Amp-Thread-ID: https://ampcode.com/threads/T-019c2ed0-8d62-7649-b718-257fd74ce3ce
Co-authored-by: Amp <amp@ampcode.com>
2026-02-05 18:21:00 +01:00
Matthias Seitz
4679c86003 bump evm 2026-02-04 19:41:37 +01:00
Stefan
7671838c61 fix: EIP-7778 gas accounting in receipts and block validation (#21821)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-04 19:18:59 +01:00
Soubhik Singha Mahapatra
8f4461c060 chore: update fixture for bal test (#21787)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
2026-02-04 18:08:15 +01:00
Matthias Seitz
0119f3c612 bump lock 2026-02-04 13:41:10 +01:00
Stefan
094aaef5a1 fix: add engine_forkchoiceUpdatedV4 to capabilities list (#21799)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-02-04 13:29:24 +01:00
Matthias Seitz
32d03ff4d7 chore: update revm to 6aa06829d2caa2aa38606ed22b83354a7a7ff98e
Amp-Thread-ID: https://ampcode.com/threads/T-019c2389-3d07-76c9-a09f-e21b588a135c
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 13:48:25 +01:00
Matthias Seitz
3368ce6485 chore: update alloy-evm to 394f0ecf (EIP-7778 is_amsterdam fix)
Amp-Thread-ID: https://ampcode.com/threads/T-019c1e41-4c08-74bd-8901-5037d92e6fac
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 13:18:21 +01:00
Jennifer
9bc2388871 Fix bal fcu validation (#21679)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 20:49:50 +01:00
Matthias Seitz
1728fa97c0 bump revs 2026-01-30 15:26:49 +01:00
Ishika Choudhury
868248ec54 feat: add fcu and fixes (#21567)
Co-authored-by: Soubhik Singha Mahapatra <soubhiksmp2004@gmail.com>
2026-01-29 13:29:33 +01:00
Matthias Seitz
16ab4b8518 chore: update revm to f3b74d4ff0c6c88a09ca281323a100257fa61ebf
Amp-Thread-ID: https://ampcode.com/threads/T-019c05af-57ea-72ab-b16a-57cfaf907a9e
Co-authored-by: Amp <amp@ampcode.com>
2026-01-28 18:40:26 +01:00
Matthias Seitz
ce74466b93 chore: update EEST fixtures to bal@v5.0.0
Amp-Thread-ID: https://ampcode.com/threads/T-019bff17-51e0-775f-840e-e67a55fc347c
Co-authored-by: Amp <amp@ampcode.com>
2026-01-27 11:56:00 +01:00
Matthias Seitz
992fc30ff5 refactor: use GotExpectedBoxed for BlockAccessListHashMismatch
Amp-Thread-ID: https://ampcode.com/threads/T-019bfc38-6e25-7093-8775-7764904c7e88
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 23:10:21 +01:00
Matthias Seitz
3ece6b6047 refactor: use with_bal_builder_if instead of conditional
Amp-Thread-ID: https://ampcode.com/threads/T-019bfc38-6e25-7093-8775-7764904c7e88
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 22:53:36 +01:00
Matthias Seitz
dec9f93ad1 fix: use block_hashes.lowest() to get lowest block number
Amp-Thread-ID: https://ampcode.com/threads/T-019bfc38-6e25-7093-8775-7764904c7e88
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 22:51:48 +01:00
Matthias Seitz
03484f76ec chore: update revm to 300efbf3e391e1796f5210cd4506508e385a55d2
Amp-Thread-ID: https://ampcode.com/threads/T-019bfc38-6e25-7093-8775-7764904c7e88
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 22:50:08 +01:00
Matthias Seitz
b870f04509 chore: update alloy to fix ExecutionPayload V4 deserializer
Amp-Thread-ID: https://ampcode.com/threads/T-019bfbfc-d6f6-73ec-b044-919aa35326fc
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 21:28:18 +01:00
Matthias Seitz
5277e59cc4 feat: validate BAL hash after block execution
- Add BlockAccessListHashMismatch variant to ConsensusError
- After execution, compute hash of built BAL and compare with expected BAL hash
- Return consensus error if hashes don't match

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb9c-5974-732a-8101-32f6711e7d31
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 19:57:30 +01:00
Matthias Seitz
6271c2702f feat: add BAL support to engine tree payload validator
- Enable with_bal_builder() in execute_block when payload contains block access list
- Decode BlockAccessList from payload bytes in BlockOrPayload::block_access_list()
- Bump BAL index after pre-execution changes and after each transaction
- Add ExecutionPayload trait bound for block_access_list() method

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb9c-5974-732a-8101-32f6711e7d31
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 19:52:17 +01:00
Matthias Seitz
ce15ab9f55 chore: update hive build_simulators.sh for BAL devnet
- Use bal@v4.0.0 fixtures for ethereum/eels simulator
- Add branch=eips/amsterdam/eip-7928 buildarg

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb5b-9488-7388-95a2-5b93e1b3e9eb
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 19:40:55 +01:00
Matthias Seitz
0b1ec2dc89 docs: add missing docs for Amsterdam engine API endpoints
- Add doc comments for new_payload_v5 and get_payload_v6 methods
- Include links to the Amsterdam spec

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb5b-9488-7388-95a2-5b93e1b3e9eb
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 19:34:59 +01:00
Matthias Seitz
5862c72880 chore: add block_access_list_hash and slot_number to HeaderExt
- Update HeaderExt to include new Amsterdam fields
- Fix EthBuiltPayload doctest to pass 5th argument
- Update Compact impl for AlloyHeader to handle new fields

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb5b-9488-7388-95a2-5b93e1b3e9eb
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 19:23:11 +01:00
Matthias Seitz
8a7655ca5d chore: docs 2026-01-26 19:06:13 +01:00
Matthias Seitz
cd20adc1d4 chore: alloc 2026-01-26 19:05:13 +01:00
Matthias Seitz
f0fe45d6bf chore: fix compilation issues after BAL revert
- Remove block_access_list from BlockExecutionResult (handled in reth)
- Add block_access_list_hash and slot_number to Header initializations
- Add slot_number to PayloadAttributes initializations
- Fix clippy doc markdown warnings
- Remove unused alloy-rlp dependency from reth-evm-ethereum

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb5b-9488-7388-95a2-5b93e1b3e9eb
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 18:40:46 +01:00
Matthias Seitz
00422207f4 feat: add slot_number support and enable BAL builder for Amsterdam
- Forward slot_number from PayloadAttributes to EthPayloadBuilderAttributes
- Enable BAL builder in State when Amsterdam is active
- Pin alloy-evm to rev 3df0a06 (before block_access_list in BlockExecutionResult)
- Set block_access_list_hash to None temporarily until BAL extraction is implemented

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb3d-e550-767e-9df1-f6987376dbc1
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 18:05:00 +01:00
Matthias Seitz
28a94829e9 feat(consensus): add Amsterdam header field validation
- Add BlockAccessListHashMissing, BlockAccessListHashUnexpected,
  SlotNumberMissing, SlotNumberUnexpected to ConsensusError
- Add validate_amsterdam_header_fields() to consensus-common
- Clean up amsterdam.rs in payload-validator

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb00-a1a7-7601-9dd6-d26171b03370
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 17:37:02 +01:00
Matthias Seitz
e081249f65 fix(engine-api): restore is_critical_method and add V5/V6 capabilities
Amp-Thread-ID: https://ampcode.com/threads/T-019bfb00-a1a7-7601-9dd6-d26171b03370
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 17:10:27 +01:00
Matthias Seitz
99fedf01f8 feat(chainspec): add Amsterdam hardfork support
- Add EMPTY_BLOCK_ACCESS_LIST_HASH constant
- Set block_access_list_hash in genesis header when Amsterdam is active
- Add amsterdam_time to create_chain_config
- Add Amsterdam to time_hardfork_opts in From<Genesis> for ChainSpec
- Add amsterdam_activated() and with_amsterdam_at() builder methods

Amp-Thread-ID: https://ampcode.com/threads/T-019bfb00-a1a7-7601-9dd6-d26171b03370
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 17:07:55 +01:00
Matthias Seitz
179e1bfc34 feat: integrate BAL devnet2 changes
- Add dependency patches for revm staging, alloy/op-alloy bal-devnet2 branches
- Cherry-pick engine API changes from PR 21203 (capabilities, engine_api, metrics, payload primitives)
- Add block_access_list_hash and slot_number fields to Header initializations
- Add slot_number field to PayloadAttributes initializations
- Update EthBuiltPayload::new() signature with block_access_list parameter
- Handle ExecutionPayload::V4 and EngineApiMessageVersion::V6 variants
- Add amsterdam payload validator

Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Amp-Thread-ID: https://ampcode.com/threads/T-019bfb00-a1a7-7601-9dd6-d26171b03370
Co-authored-by: Amp <amp@ampcode.com>
2026-01-26 17:01:51 +01:00
Matthias Seitz
3adb5b9e58 Merge remote-tracking branch 'origin/staging' into bal-devnet-2 2026-01-26 16:42:55 +01:00
rakita
57d7c98f66 chore: merge main and update alloy-evm staging patch 2026-01-26 12:39:58 +01:00
rakita
5d9a43f2d4 Merge remote-tracking branch 'origin/main' into staging 2026-01-26 12:36:44 +01:00
rakita
defd0e8e5c Bump revm to staging and fix breaking changes
- Patch revm and all sub-crates to staging commit 0dc217a9
- Patch revm-inspectors to staging commit fccc4ac5
- Patch alloy-evm to staging commit 625ccc0f
- Add slot_num field to BlockEnv initializers
- Update BlockHashCache usage (no longer has keys method)
2026-01-26 02:41:26 +01:00
605 changed files with 24529 additions and 17463 deletions

View File

@@ -0,0 +1,10 @@
---
reth-engine-primitives: patch
reth-engine-tree: patch
reth-node-core: patch
reth-trie-parallel: minor
---
Removed legacy proof calculation system and V2-specific configuration flags.
Removed the legacy (non-V2) proof calculation code paths, simplified multiproof task architecture by removing the dual-mode system, and cleaned up V2-specific CLI flags (`--engine.disable-proof-v2`, `--engine.disable-trie-cache`) that are no longer needed. The codebase now exclusively uses V2 proofs with the sparse trie cache.

View File

@@ -0,0 +1,5 @@
---
reth-trie-sparse: patch
---
Refactored sparse trie node state tracking to use RLP nodes instead of hashes. Replaced `Option<B256>` hash fields with `SparseNodeState` enum that tracks either dirty nodes or cached RLP nodes with optional database storage flags. Added debug assertions to validate leaf path lengths and improved pruning logic to use node paths directly instead of path-hash tuples.

View File

@@ -0,0 +1,5 @@
---
reth-transaction-pool: minor
---
Added support for optional custom stateless and stateful validation hooks in `EthTransactionValidator` via `set_additional_stateless_validation` and `set_additional_stateful_validation` methods. Also implemented a manual `Debug` impl to handle the non-`Debug` function pointer fields.

View File

@@ -0,0 +1,5 @@
---
reth-trie-sparse: patch
---
Added recording of `SetRoot` operation in `ParallelSparseTrie::set_root` when the `trie-debug` feature is enabled.

View File

@@ -0,0 +1,6 @@
---
reth-rpc-convert: minor
reth-storage-rpc-provider: minor
---
Replaced the separate `TryFromBlockResponse`, `TryFromReceiptResponse`, and `TryFromTransactionResponse` traits with a unified `RpcResponseConverter` trait and default `EthRpcConverter` implementation. Removed the `op-alloy-network` dependency and refactored `RpcBlockchainProvider` to store a dynamic converter instance instead of relying on per-type trait bounds.

View File

@@ -0,0 +1,5 @@
---
reth-transaction-pool: minor
---
Added `consensus_ref` method to `PoolTransaction` trait for borrowing consensus transactions without cloning.

View File

@@ -0,0 +1,4 @@
---
---
Improved nightly Docker build failure Slack notification with more detailed formatting and context.

View File

@@ -0,0 +1,5 @@
---
reth-rpc-eth-types: patch
---
Updated `eth_simulateV1` revert error code from `-32000` to `3` to be consistent with `eth_call`, per [execution-apis#748](https://github.com/ethereum/execution-apis/pull/748).

View File

@@ -0,0 +1,5 @@
---
reth-engine-tree: patch
---
Added sub-phase timing histograms to the sparse trie event loop, tracking channel wait, proof coalescing, multiproof reveal, and trie update durations separately.

View File

@@ -0,0 +1,4 @@
---
---
Improved documentation overview page with better structure and clarity.

View File

@@ -0,0 +1,5 @@
---
reth-node-events: patch
---
Updated consensus engine log message to be more accurate about received updates.

View File

@@ -0,0 +1,6 @@
---
reth-rpc-eth-api: minor
reth-rpc-server-types: minor
---
Added `eth_getStorageValues` RPC method for batch storage slot retrieval across multiple addresses.

View File

@@ -0,0 +1,9 @@
---
reth-network-api: minor
reth-network-types: minor
reth-network: minor
reth-node-core: minor
reth: minor
---
Added optional ENR fork ID enforcement to filter out peers from incompatible networks during peer discovery, controlled by the `--enforce-enr-fork-id` CLI flag.

View File

@@ -0,0 +1,5 @@
---
reth-engine-tree: patch
---
Fixed `compare_trie_updates` to return `bool` indicating whether differences were found, and updated the caller to properly use the return value instead of treating all successful comparisons as having no differences.

View File

@@ -0,0 +1,5 @@
---
reth-node-core: minor
---
Added `with_dev_block_time` helper method to `NodeConfig` for configuring dev miner block production interval.

View File

@@ -0,0 +1,5 @@
---
reth-db-api: patch
---
Changed `StoredNibblesSubKey` encoding to use a stack-allocated `[u8; 65]` array instead of a heap-allocated `Vec<u8>`, avoiding unnecessary heap allocation.

View File

@@ -0,0 +1,5 @@
---
reth-storage-api: patch
---
Added `Arc` to `auto_impl` derive for storage-api traits to support automatic `Arc` wrapper implementations.

View File

@@ -0,0 +1,8 @@
---
reth: patch
reth-engine-tree: patch
reth-node-builder: patch
reth-trie-sparse: minor
---
Added `trie-debug` feature for recording sparse trie mutations to aid in debugging state root mismatches.

View File

@@ -0,0 +1,5 @@
---
reth-provider: patch
---
Fixed sender pruning during block reorg to skip when sender_recovery is fully pruned, preventing a fatal crash when no sender data exists in static files.

View File

@@ -0,0 +1,7 @@
---
reth-network-types: minor
reth-network: minor
reth-node-core: patch
---
Added `PersistedPeerInfo` struct to persist richer peer metadata (kind, fork ID, reputation) to disk. Updated `PeersConfig::with_basic_nodes_from_file` to support both the new `PersistedPeerInfo` format and the legacy `Vec<NodeRecord>` format with automatic conversion, and updated `write_peers_to_file` to exclude backed-off and banned peers.

View File

@@ -0,0 +1,5 @@
---
reth-network: minor
---
Added `fork_id` as a tiebreaker in peer selection when reputations are equal, preferring peers with a discovered `fork_id` as it indicates fork compatibility. Added a test to verify the tiebreaker behavior.

View File

@@ -0,0 +1,5 @@
---
reth-trie-sparse: patch
---
Fixed a bug where trie nodes could appear in both `updated_nodes` and `removed_nodes` simultaneously by removing entries from `removed_nodes` when a node is inserted as updated.

View File

@@ -0,0 +1,6 @@
---
reth-engine-local: minor
reth-node-builder: minor
---
Added trigger-based `MiningMode` variant that allows blocks to be built on-demand via custom streams, and exposed `with_mining_mode` method on `DebugNodeLauncherFuture` to override default mining configuration.

View File

@@ -0,0 +1,5 @@
---
reth-transaction-pool: patch
---
Fixed a bug where transactions from the same sender were added to the pending subpool out of nonce order. Ensured `process_updates` runs before `add_new_transaction` so that lower-nonce promotions are enqueued before the newly inserted higher-nonce transaction, preserving correct ordering for live `BestTransactions` iterators.

View File

@@ -1,6 +1,6 @@
[profile.default]
retries = { backoff = "exponential", count = 2, delay = "2s", jitter = true }
slow-timeout = { period = "30s", terminate-after = 4 }
slow-timeout = { period = "30s", terminate-after = 2 }
[[profile.default.overrides]]
filter = "test(general_state_tests)"

4
.github/CODEOWNERS vendored
View File

@@ -1,7 +1,7 @@
* @gakonst
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/cli/ @mattsse @Rjected
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
@@ -38,7 +38,7 @@ crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tasks/ @mattsse @DaniPopes
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected

99
.github/scripts/bench-reth-build.sh vendored Executable file
View File

@@ -0,0 +1,99 @@
#!/usr/bin/env bash
#
# Builds (or fetches from cache) reth binaries for benchmarking.
#
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
#
# baseline — build/fetch the baseline binary at <commit> (merge-base)
# source-dir must be checked out at <commit>
# feature — build/fetch the candidate binary + reth-bench at <commit>
# source-dir must be checked out at <commit>
# optional branch-sha is the PR head commit for cache key
#
# Outputs:
# baseline: <source-dir>/target/profiling/reth
# feature: <source-dir>/target/profiling/reth, reth-bench installed to cargo bin
#
# Required: mc (MinIO client) with a configured alias
set -euo pipefail
MC="mc"
MODE="$1"
SOURCE_DIR="$2"
COMMIT="$3"
# Verify a cached reth binary was built from the expected commit.
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
verify_binary() {
local binary="$1" expected_commit="$2"
local version binary_sha
version=$("$binary" --version 2>/dev/null) || return 1
binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
if [ -z "$binary_sha" ]; then
echo "Warning: could not extract commit SHA from version output"
return 1
fi
if [ "$binary_sha" = "$expected_commit" ]; then
return 0
fi
echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
return 1
}
case "$MODE" in
baseline|main)
BUCKET="minio/reth-binaries/${COMMIT}"
mkdir -p "${SOURCE_DIR}/target/profiling"
CACHE_VALID=false
if $MC stat "${BUCKET}/reth" &>/dev/null; then
echo "Cache hit for baseline (${COMMIT}), downloading binary..."
$MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
chmod +x "${SOURCE_DIR}/target/profiling/reth"
if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
CACHE_VALID=true
else
echo "Cached baseline binary is stale, rebuilding..."
fi
fi
if [ "$CACHE_VALID" = false ]; then
echo "Building baseline (${COMMIT}) from source..."
cd "${SOURCE_DIR}"
cargo build --profile profiling --bin reth
$MC cp target/profiling/reth "${BUCKET}/reth"
fi
;;
feature|branch)
BRANCH_SHA="${4:-$COMMIT}"
BUCKET="minio/reth-binaries/${BRANCH_SHA}"
CACHE_VALID=false
if $MC stat "${BUCKET}/reth" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
mkdir -p "${SOURCE_DIR}/target/profiling"
$MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench
chmod +x "${SOURCE_DIR}/target/profiling/reth" /home/ubuntu/.cargo/bin/reth-bench
if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
CACHE_VALID=true
else
echo "Cached feature binary is stale, rebuilding..."
fi
fi
if [ "$CACHE_VALID" = false ]; then
echo "Building feature (${COMMIT}) from source..."
cd "${SOURCE_DIR}"
rustup show active-toolchain || rustup default stable
make profiling
make install-reth-bench
$MC cp target/profiling/reth "${BUCKET}/reth"
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
fi
;;
*)
echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
exit 1
;;
esac

260
.github/scripts/bench-reth-charts.py vendored Normal file
View File

@@ -0,0 +1,260 @@
#!/usr/bin/env python3
"""Generate benchmark charts from reth-bench CSV output.
Usage:
bench-engine-charts.py <combined_csv> --output-dir <dir> [--baseline <baseline_csv>]
Generates three PNG charts:
1. newPayload latency + Ggas/s per block (+ latency diff when baseline present)
2. Wait breakdown (persistence, execution cache, sparse trie) per block
3. Scatter plot of gas used vs latency
When --baseline is provided, charts overlay both datasets for comparison.
"""
import argparse
import csv
import sys
from pathlib import Path
import numpy as np
try:
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
except ImportError:
print("matplotlib is required: pip install matplotlib", file=sys.stderr)
sys.exit(1)
GIGAGAS = 1_000_000_000
def parse_combined_csv(path: str) -> list[dict]:
rows = []
with open(path) as f:
reader = csv.DictReader(f)
for row in reader:
rows.append(
{
"block_number": int(row["block_number"]),
"gas_used": int(row["gas_used"]),
"new_payload_latency_us": int(row["new_payload_latency"]),
"persistence_wait_us": int(row["persistence_wait"])
if row.get("persistence_wait")
else None,
"execution_cache_wait_us": int(row.get("execution_cache_wait", 0)),
"sparse_trie_wait_us": int(row.get("sparse_trie_wait", 0)),
}
)
return rows
def plot_latency_and_throughput(
feature: list[dict], baseline: list[dict] | None, out: Path,
baseline_name: str = "baseline", feature_name: str = "feature",
):
num_plots = 3 if baseline else 2
fig, axes = plt.subplots(num_plots, 1, figsize=(12, 4 * num_plots), sharex=True)
ax1, ax2 = axes[0], axes[1]
feat_x = [r["block_number"] for r in feature]
feat_lat = [r["new_payload_latency_us"] / 1_000 for r in feature]
feat_ggas = []
for r in feature:
lat_s = r["new_payload_latency_us"] / 1_000_000
feat_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
if baseline:
base_x = [r["block_number"] for r in baseline]
base_lat = [r["new_payload_latency_us"] / 1_000 for r in baseline]
base_ggas = []
for r in baseline:
lat_s = r["new_payload_latency_us"] / 1_000_000
base_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
l, = ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
ax1.axhline(np.median(base_lat), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
l, = ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
ax2.axhline(np.median(base_ggas), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
l, = ax1.plot(feat_x, feat_lat, linewidth=0.8, label=feature_name)
ax1.axhline(np.median(feat_lat), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
ax1.set_ylabel("Latency (ms)")
ax1.set_title("newPayload Latency per Block")
ax1.grid(True, alpha=0.3)
ax1.legend()
l, = ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=feature_name)
ax2.axhline(np.median(feat_ggas), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
ax2.set_ylabel("Ggas/s")
ax2.set_title("Execution Throughput per Block")
ax2.grid(True, alpha=0.3)
ax2.legend()
if baseline:
ax3 = axes[2]
base_by_block = {r["block_number"]: r["new_payload_latency_us"] for r in baseline}
blocks, diffs = [], []
for r in feature:
bn = r["block_number"]
if bn in base_by_block and base_by_block[bn] > 0:
pct = (r["new_payload_latency_us"] - base_by_block[bn]) / base_by_block[bn] * 100
blocks.append(bn)
diffs.append(pct)
if blocks:
colors = ["green" if d <= 0 else "red" for d in diffs]
ax3.bar(blocks, diffs, width=1.0, color=colors, alpha=0.7, edgecolor="none")
ax3.axhline(0, color="black", linewidth=0.5)
ax3.set_ylabel("Δ Latency (%)")
ax3.set_title("Per-Block newPayload Latency Change (feature vs baseline)")
ax3.grid(True, alpha=0.3, axis="y")
axes[-1].set_xlabel("Block Number")
fig.tight_layout()
fig.savefig(out, dpi=150)
plt.close(fig)
def plot_wait_breakdown(
feature: list[dict], baseline: list[dict] | None, out: Path,
baseline_name: str = "baseline", feature_name: str = "feature",
):
series = [
("Persistence Wait", "persistence_wait_us"),
("State Cache Wait", "execution_cache_wait_us"),
("Trie Cache Wait", "sparse_trie_wait_us"),
]
fig, axes = plt.subplots(len(series), 1, figsize=(12, 3 * len(series)), sharex=True)
for ax, (label, key) in zip(axes, series):
if baseline:
bx = [r["block_number"] for r in baseline if r[key] is not None]
by = [r[key] / 1_000 for r in baseline if r[key] is not None]
if bx:
ax.plot(bx, by, linewidth=0.8, label=baseline_name, alpha=0.7)
fx = [r["block_number"] for r in feature if r[key] is not None]
fy = [r[key] / 1_000 for r in feature if r[key] is not None]
if fx:
ax.plot(fx, fy, linewidth=0.8, label=feature_name)
ax.set_ylabel("ms")
ax.set_title(label)
ax.grid(True, alpha=0.3)
if baseline:
ax.legend()
axes[-1].set_xlabel("Block Number")
fig.suptitle("Wait Time Breakdown per Block", fontsize=14, y=1.01)
fig.tight_layout()
fig.savefig(out, dpi=150, bbox_inches="tight")
plt.close(fig)
def _add_regression(ax, x, y, color, label):
"""Add a linear regression line to the axes."""
if len(x) < 2:
return
xa, ya = np.array(x), np.array(y)
m, b = np.polyfit(xa, ya, 1)
x_range = np.linspace(xa.min(), xa.max(), 100)
ax.plot(x_range, m * x_range + b, color=color, linewidth=1.5, alpha=0.8,
label=label)
def plot_gas_vs_latency(
feature: list[dict], baseline: list[dict] | None, out: Path,
baseline_name: str = "baseline", feature_name: str = "feature",
):
fig, ax = plt.subplots(figsize=(8, 6))
if baseline:
bgas = [r["gas_used"] / 1_000_000 for r in baseline]
blat = [r["new_payload_latency_us"] / 1_000 for r in baseline]
ax.scatter(bgas, blat, s=8, alpha=0.5)
_add_regression(ax, bgas, blat, "tab:blue", baseline_name)
fgas = [r["gas_used"] / 1_000_000 for r in feature]
flat = [r["new_payload_latency_us"] / 1_000 for r in feature]
ax.scatter(fgas, flat, s=8, alpha=0.6)
_add_regression(ax, fgas, flat, "tab:orange", feature_name)
ax.set_xlabel("Gas Used (Mgas)")
ax.set_ylabel("newPayload Latency (ms)")
ax.set_title("Gas Used vs Latency")
ax.grid(True, alpha=0.3)
ax.legend()
fig.tight_layout()
fig.savefig(out, dpi=150)
plt.close(fig)
def merge_csvs(paths: list[str]) -> list[dict]:
"""Parse and merge multiple CSVs, averaging values for duplicate blocks."""
by_block: dict[int, list[dict]] = {}
for path in paths:
for row in parse_combined_csv(path):
by_block.setdefault(row["block_number"], []).append(row)
merged = []
for bn in sorted(by_block):
rows = by_block[bn]
if len(rows) == 1:
merged.append(rows[0])
else:
avg = {"block_number": bn}
for key in ("gas_used", "new_payload_latency_us"):
avg[key] = int(sum(r[key] for r in rows) / len(rows))
for key in ("persistence_wait_us", "execution_cache_wait_us", "sparse_trie_wait_us"):
vals = [r[key] for r in rows if r[key] is not None]
avg[key] = int(sum(vals) / len(vals)) if vals else None
merged.append(avg)
return merged
def main():
parser = argparse.ArgumentParser(description="Generate benchmark charts")
parser.add_argument(
"--feature", nargs="+", required=True,
help="Path(s) to feature combined_latency.csv",
)
parser.add_argument(
"--output-dir", required=True, help="Output directory for PNG charts"
)
parser.add_argument(
"--baseline", nargs="+", help="Path(s) to baseline combined_latency.csv"
)
parser.add_argument("--baseline-name", default="baseline", help="Label for baseline")
parser.add_argument("--feature-name", "--branch-name", default="feature", help="Label for feature")
args = parser.parse_args()
feature = merge_csvs(args.feature)
if not feature:
print("No results found in feature CSV(s)", file=sys.stderr)
sys.exit(1)
baseline = None
if args.baseline:
baseline = merge_csvs(args.baseline)
if not baseline:
print(
"Warning: no results in baseline CSV(s), skipping comparison",
file=sys.stderr,
)
baseline = None
out_dir = Path(args.output_dir)
out_dir.mkdir(parents=True, exist_ok=True)
bname = args.baseline_name
fname = args.feature_name
plot_latency_and_throughput(feature, baseline, out_dir / "latency_throughput.png", bname, fname)
plot_wait_breakdown(feature, baseline, out_dir / "wait_breakdown.png", bname, fname)
plot_gas_vs_latency(feature, baseline, out_dir / "gas_vs_latency.png", bname, fname)
print(f"Charts written to {out_dir}")
if __name__ == "__main__":
main()

144
.github/scripts/bench-reth-run.sh vendored Executable file
View File

@@ -0,0 +1,144 @@
#!/usr/bin/env bash
#
# Runs a single reth-bench cycle: mount snapshot → start node → warmup →
# benchmark → stop node → recover snapshot.
#
# Usage: bench-reth-run.sh <label> <binary> <output-dir>
#
# Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
set -euo pipefail
LABEL="$1"
BINARY="$2"
OUTPUT_DIR="$3"
DATADIR="$SCHELK_MOUNT/datadir"
mkdir -p "$OUTPUT_DIR"
LOG="${OUTPUT_DIR}/node.log"
cleanup() {
kill "$TAIL_PID" 2>/dev/null || true
if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
# Send SIGINT to the inner reth process by exact name (not -f which
# would also match samply's cmdline containing "reth"). Samply will
# capture reth's exit and save the profile.
sudo pkill -INT -x reth 2>/dev/null || true
# Wait for samply to finish writing the profile and exit
for i in $(seq 1 120); do
sudo pgrep -x samply > /dev/null 2>&1 || break
if [ $((i % 10)) -eq 0 ]; then
echo "Waiting for samply to finish writing profile... (${i}s)"
fi
sleep 1
done
if sudo pgrep -x samply > /dev/null 2>&1; then
echo "Samply still running after 120s, sending SIGTERM..."
sudo pkill -x samply 2>/dev/null || true
fi
else
sudo kill "$RETH_PID"
for i in $(seq 1 30); do
sudo kill -0 "$RETH_PID" 2>/dev/null || break
sleep 1
done
fi
sudo kill -9 "$RETH_PID" 2>/dev/null || true
sleep 1
fi
# Fix ownership of reth-created files (reth runs as root)
sudo chown -R "$(id -un):$(id -gn)" "$OUTPUT_DIR" 2>/dev/null || true
if mountpoint -q "$SCHELK_MOUNT"; then
sudo umount -l "$SCHELK_MOUNT" || true
sudo schelk recover -y || true
fi
}
TAIL_PID=
trap cleanup EXIT
# Mount
sudo schelk mount -y
sync
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
echo "=== Cache state after drop ==="
free -h
grep Cached /proc/meminfo
# Start reth
# CPU layout: core 0 = OS/IRQs/reth-bench/aux, cores 1+ = reth node
RETH_BENCH="$(which reth-bench)"
ONLINE=$(nproc --all)
MAX_RETH=$(( ONLINE - 1 ))
if [ "${BENCH_CORES:-0}" -gt 0 ] && [ "$BENCH_CORES" -lt "$MAX_RETH" ]; then
MAX_RETH=$BENCH_CORES
fi
RETH_CPUS="1-${MAX_RETH}"
RETH_ARGS=(
node
--datadir "$DATADIR"
--log.file.directory "$OUTPUT_DIR/reth-logs"
--engine.accept-execution-requests-hash
--http
--http.port 8545
--ws
--ws.api all
--authrpc.port 8551
--disable-discovery
--no-persist-peers
)
if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
RETH_ARGS+=(--log.samply)
SAMPLY="$(which samply)"
sudo taskset -c "$RETH_CPUS" nice -n -20 \
"$SAMPLY" record --save-only --presymbolicate --rate 10000 \
--output "$OUTPUT_DIR/samply-profile.json.gz" \
-- "$BINARY" "${RETH_ARGS[@]}" \
> "$LOG" 2>&1 &
else
sudo taskset -c "$RETH_CPUS" nice -n -20 "$BINARY" "${RETH_ARGS[@]}" \
> "$LOG" 2>&1 &
fi
RETH_PID=$!
stdbuf -oL tail -f "$LOG" | sed -u "s/^/[reth] /" &
TAIL_PID=$!
for i in $(seq 1 60); do
if curl -sf http://127.0.0.1:8545 -X POST \
-H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
> /dev/null 2>&1; then
echo "reth (${LABEL}) is ready after ${i}s"
break
fi
if [ "$i" -eq 60 ]; then
echo "::error::reth (${LABEL}) failed to start within 60s"
cat "$LOG"
exit 1
fi
sleep 1
done
# Run reth-bench with high priority but as the current user so output
# files are not root-owned (avoids EACCES on next checkout).
BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
# Warmup
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
--rpc-url "$BENCH_RPC_URL" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
--reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
# Benchmark
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
--rpc-url "$BENCH_RPC_URL" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "$BENCH_BLOCKS" \
--reth-new-payload \
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
# cleanup runs via trap

127
.github/scripts/bench-reth-snapshot.sh vendored Executable file
View File

@@ -0,0 +1,127 @@
#!/usr/bin/env bash
#
# Downloads the latest nightly snapshot into the schelk volume with
# progress reporting to the GitHub PR comment.
#
# Skips the download if the local ETag marker matches the remote one.
#
# Usage: bench-reth-snapshot.sh [--check]
# --check Only check if a download is needed; exits 0 if up-to-date, 1 if not.
#
# Required env:
# SCHELK_MOUNT schelk mount point (e.g. /reth-bench)
# GITHUB_TOKEN token for GitHub API calls (only for download)
# BENCH_COMMENT_ID PR comment ID to update (optional)
# BENCH_REPO owner/repo (e.g. paradigmxyz/reth)
# BENCH_JOB_URL link to the Actions job
# BENCH_ACTOR user who triggered the benchmark
# BENCH_CONFIG config summary line
set -euo pipefail
BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous.tar.zst"
DATADIR="$SCHELK_MOUNT/datadir"
ETAG_FILE="$HOME/.reth-bench-snapshot-etag"
# Get remote metadata via JSON for reliable parsing
MC_STAT=$(mc stat --json "$BUCKET" 2>/dev/null || true)
REMOTE_ETAG=$(echo "$MC_STAT" | jq -r '.etag // empty')
if [ -z "$REMOTE_ETAG" ]; then
echo "::warning::Failed to get ETag from mc stat, will re-download"
REMOTE_ETAG="unknown-$(date +%s)"
fi
LOCAL_ETAG=""
[ -f "$ETAG_FILE" ] && LOCAL_ETAG=$(cat "$ETAG_FILE")
if [ "$REMOTE_ETAG" = "$LOCAL_ETAG" ]; then
echo "Snapshot is up-to-date (ETag: ${REMOTE_ETAG})"
if [ "${1:-}" = "--check" ]; then
exit 0
fi
exit 0
fi
echo "Snapshot needs update (local: ${LOCAL_ETAG:-<none>}, remote: ${REMOTE_ETAG})"
if [ "${1:-}" = "--check" ]; then
exit 1
fi
# Get compressed size for progress tracking
TOTAL_BYTES=$(echo "$MC_STAT" | jq -r '.size // empty')
if [ -z "$TOTAL_BYTES" ] || [ "$TOTAL_BYTES" = "0" ]; then
echo "::error::Failed to get snapshot size from mc stat"
exit 1
fi
echo "Snapshot size: $TOTAL_BYTES bytes ($(numfmt --to=iec "$TOTAL_BYTES"))"
# Prepare mount
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
sudo schelk mount -y
sudo rm -rf "$DATADIR"
sudo mkdir -p "$DATADIR"
update_comment() {
local pct="$1"
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
local status="Building binaries & downloading snapshot… ${pct}%"
local body
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
curl -sf -X PATCH \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
-d "$(jq -nc --arg body "$body" '{body: $body}')" \
> /dev/null 2>&1 || true
}
# Track compressed bytes flowing through the pipe
DL_BYTES_FILE=$(mktemp)
echo 0 > "$DL_BYTES_FILE"
# Start progress reporter in background
(
while true; do
sleep 10
CURRENT=$(cat "$DL_BYTES_FILE" 2>/dev/null || echo 0)
if [ "$TOTAL_BYTES" -gt 0 ]; then
PCT=$(( CURRENT * 100 / TOTAL_BYTES ))
[ "$PCT" -gt 100 ] && PCT=100
echo "Snapshot download: $(numfmt --to=iec "$CURRENT") / $(numfmt --to=iec "$TOTAL_BYTES") (${PCT}%)"
update_comment "$PCT"
fi
done
) &
PROGRESS_PID=$!
trap 'kill $PROGRESS_PID 2>/dev/null || true; rm -f "$DL_BYTES_FILE"' EXIT
# Download and extract; python byte counter tracks compressed bytes received
mc cat "$BUCKET" | python3 -c "
import sys
count = 0
while True:
data = sys.stdin.buffer.read(1048576)
if not data:
break
count += len(data)
sys.stdout.buffer.write(data)
with open('$DL_BYTES_FILE', 'w') as f:
f.write(str(count))
" | pzstd -d -p 6 | sudo tar -xf - -C "$DATADIR"
# Stop progress reporter
kill $PROGRESS_PID 2>/dev/null || true
wait $PROGRESS_PID 2>/dev/null || true
update_comment "100"
echo "Snapshot download complete"
# Promote the new snapshot to become the schelk baseline (virgin volume).
# This copies changed blocks from scratch → virgin so that future
# `schelk recover` calls restore to this new state.
sync
sudo schelk promote -y
# Save ETag marker
echo "$REMOTE_ETAG" > "$ETAG_FILE"
echo "Snapshot promoted to schelk baseline (ETag: ${REMOTE_ETAG})"

553
.github/scripts/bench-reth-summary.py vendored Executable file
View File

@@ -0,0 +1,553 @@
#!/usr/bin/env python3
"""Parse reth-bench CSV output and generate a summary JSON + markdown comparison.
Usage:
bench-reth-summary.py <combined_csv> <gas_csv> \
--output-summary <summary.json> \
--output-markdown <comment.md> \
--baseline-csv <baseline_combined.csv> \
[--repo <owner/repo>] \
[--baseline-ref <sha>] \
[--feature-name <name>] \
[--feature-sha <sha>]
Generates a paired statistical comparison between baseline and feature.
Matches blocks by number and computes per-block diffs to cancel out gas
variance. Fails if baseline or feature CSV is missing or empty.
"""
import argparse
import csv
import json
import math
import random
import sys
GIGAGAS = 1_000_000_000
T_CRITICAL = 1.96 # two-tailed 95% confidence
BOOTSTRAP_ITERATIONS = 10_000
def _opt_int(row: dict, key: str) -> int | None:
"""Return int value for a CSV field, or None if missing/empty."""
v = row.get(key)
if v is None or v == "":
return None
return int(v)
def parse_combined_csv(path: str) -> list[dict]:
"""Parse combined_latency.csv into a list of per-block dicts."""
rows = []
with open(path) as f:
reader = csv.DictReader(f)
for row in reader:
rows.append(
{
"block_number": int(row["block_number"]),
"gas_used": int(row["gas_used"]),
"gas_limit": int(row["gas_limit"]),
"transaction_count": int(row["transaction_count"]),
"new_payload_latency_us": int(row["new_payload_latency"]),
"fcu_latency_us": int(row["fcu_latency"]),
"total_latency_us": int(row["total_latency"]),
"persistence_wait_us": _opt_int(row, "persistence_wait"),
"execution_cache_wait_us": _opt_int(row, "execution_cache_wait"),
"sparse_trie_wait_us": _opt_int(row, "sparse_trie_wait"),
}
)
return rows
def parse_gas_csv(path: str) -> list[dict]:
"""Parse total_gas.csv into a list of per-block dicts."""
rows = []
with open(path) as f:
reader = csv.DictReader(f)
for row in reader:
rows.append(
{
"block_number": int(row["block_number"]),
"gas_used": int(row["gas_used"]),
"time_us": int(row["time"]),
}
)
return rows
def stddev(values: list[float], mean: float) -> float:
if len(values) < 2:
return 0.0
return math.sqrt(sum((v - mean) ** 2 for v in values) / (len(values) - 1))
def percentile(sorted_vals: list[float], pct: int) -> float:
if not sorted_vals:
return 0.0
idx = int(len(sorted_vals) * pct / 100)
idx = min(idx, len(sorted_vals) - 1)
return sorted_vals[idx]
def compute_stats(combined: list[dict]) -> dict:
"""Compute per-run statistics from parsed CSV data."""
n = len(combined)
if n == 0:
return {}
latencies_ms = [r["new_payload_latency_us"] / 1_000 for r in combined]
sorted_lat = sorted(latencies_ms)
mean_lat = sum(latencies_ms) / n
std_lat = stddev(latencies_ms, mean_lat)
mgas_s_values = []
for r in combined:
lat_s = r["new_payload_latency_us"] / 1_000_000
if lat_s > 0:
mgas_s_values.append(r["gas_used"] / lat_s / 1_000_000)
mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0
return {
"n": n,
"mean_ms": mean_lat,
"stddev_ms": std_lat,
"p50_ms": percentile(sorted_lat, 50),
"p90_ms": percentile(sorted_lat, 90),
"p99_ms": percentile(sorted_lat, 99),
"mean_mgas_s": mean_mgas_s,
}
def compute_wait_stats(combined: list[dict], field: str) -> dict:
"""Compute mean/p50/p95 for a wait time field (in ms)."""
values_ms = []
for r in combined:
v = r.get(field)
if v is not None:
values_ms.append(v / 1_000)
if not values_ms:
return {}
n = len(values_ms)
mean_val = sum(values_ms) / n
sorted_vals = sorted(values_ms)
return {
"mean_ms": mean_val,
"p50_ms": percentile(sorted_vals, 50),
"p95_ms": percentile(sorted_vals, 95),
}
def _paired_data(
baseline: list[dict], feature: list[dict]
) -> tuple[list[tuple[float, float]], list[float], list[float]]:
"""Match blocks and return paired latencies and per-block diffs.
Returns:
pairs: list of (baseline_ms, feature_ms) tuples
lat_diffs_ms: list of feature baseline latency diffs in ms
mgas_diffs: list of feature baseline Mgas/s diffs
"""
baseline_by_block = {r["block_number"]: r for r in baseline}
feature_by_block = {r["block_number"]: r for r in feature}
common_blocks = sorted(set(baseline_by_block) & set(feature_by_block))
pairs = []
lat_diffs_ms = []
mgas_diffs = []
for bn in common_blocks:
b = baseline_by_block[bn]
f = feature_by_block[bn]
b_ms = b["new_payload_latency_us"] / 1_000
f_ms = f["new_payload_latency_us"] / 1_000
pairs.append((b_ms, f_ms))
lat_diffs_ms.append(f_ms - b_ms)
b_lat_s = b["new_payload_latency_us"] / 1_000_000
f_lat_s = f["new_payload_latency_us"] / 1_000_000
if b_lat_s > 0 and f_lat_s > 0:
mgas_diffs.append(
f["gas_used"] / f_lat_s / 1_000_000
- b["gas_used"] / b_lat_s / 1_000_000
)
return pairs, lat_diffs_ms, mgas_diffs
def compute_paired_stats(
baseline_runs: list[list[dict]],
feature_runs: list[list[dict]],
) -> dict:
"""Compute paired statistics between baseline and feature runs.
Each pair (baseline_runs[i], feature_runs[i]) produces per-block diffs.
All diffs are pooled for the final CI.
"""
all_pairs = []
all_lat_diffs = []
all_mgas_diffs = []
blocks_per_pair = []
for baseline, feature in zip(baseline_runs, feature_runs):
pairs, lat_diffs, mgas_diffs = _paired_data(baseline, feature)
all_pairs.extend(pairs)
all_lat_diffs.extend(lat_diffs)
all_mgas_diffs.extend(mgas_diffs)
blocks_per_pair.append(len(pairs))
if not all_lat_diffs:
return {}
n = len(all_lat_diffs)
mean_diff = sum(all_lat_diffs) / n
std_diff = stddev(all_lat_diffs, mean_diff)
se = std_diff / math.sqrt(n) if n > 0 else 0.0
ci = T_CRITICAL * se
# Bootstrap CI on difference-of-percentiles (resample paired blocks)
base_lats = sorted([p[0] for p in all_pairs])
feature_lats = sorted([p[1] for p in all_pairs])
p50_diff = percentile(feature_lats, 50) - percentile(base_lats, 50)
p90_diff = percentile(feature_lats, 90) - percentile(base_lats, 90)
p99_diff = percentile(feature_lats, 99) - percentile(base_lats, 99)
rng = random.Random(42)
p50_boot, p90_boot, p99_boot = [], [], []
for _ in range(BOOTSTRAP_ITERATIONS):
sample = rng.choices(all_pairs, k=n)
b_sorted = sorted(p[0] for p in sample)
f_sorted = sorted(p[1] for p in sample)
p50_boot.append(percentile(f_sorted, 50) - percentile(b_sorted, 50))
p90_boot.append(percentile(f_sorted, 90) - percentile(b_sorted, 90))
p99_boot.append(percentile(f_sorted, 99) - percentile(b_sorted, 99))
p50_boot.sort()
p90_boot.sort()
p99_boot.sort()
lo = int(BOOTSTRAP_ITERATIONS * 0.025)
hi = int(BOOTSTRAP_ITERATIONS * 0.975)
mean_mgas_diff = sum(all_mgas_diffs) / len(all_mgas_diffs) if all_mgas_diffs else 0.0
std_mgas_diff = stddev(all_mgas_diffs, mean_mgas_diff) if len(all_mgas_diffs) > 1 else 0.0
mgas_se = std_mgas_diff / math.sqrt(len(all_mgas_diffs)) if all_mgas_diffs else 0.0
mgas_ci = T_CRITICAL * mgas_se
return {
"n": n,
"mean_diff_ms": mean_diff,
"ci_ms": ci,
"p50_diff_ms": p50_diff,
"p50_ci_ms": (p50_boot[hi] - p50_boot[lo]) / 2,
"p90_diff_ms": p90_diff,
"p90_ci_ms": (p90_boot[hi] - p90_boot[lo]) / 2,
"p99_diff_ms": p99_diff,
"p99_ci_ms": (p99_boot[hi] - p99_boot[lo]) / 2,
"mean_mgas_diff": mean_mgas_diff,
"mgas_ci": mgas_ci,
"blocks": max(blocks_per_pair),
}
def format_duration(seconds: float) -> str:
if seconds >= 60:
return f"{seconds / 60:.1f}min"
return f"{seconds}s"
def format_gas(gas: int) -> str:
if gas >= GIGAGAS:
return f"{gas / GIGAGAS:.1f}G"
if gas >= 1_000_000:
return f"{gas / 1_000_000:.1f}M"
return f"{gas:,}"
def fmt_ms(v: float) -> str:
return f"{v:.2f}ms"
def fmt_mgas(v: float) -> str:
return f"{v:.2f}"
def significance(pct: float, ci_pct: float, lower_is_better: bool) -> str:
"""Return significance label: 'good', 'bad', or 'neutral'."""
significant = abs(pct) > ci_pct
if not significant:
return "neutral"
elif (pct < 0) == lower_is_better:
return "good"
else:
return "bad"
def change_str(pct: float, ci_pct: float, lower_is_better: bool) -> str:
"""Format change% with paired CI significance.
Significant if the CI doesn't cross zero (i.e. |pct| > ci_pct).
"""
sig = significance(pct, ci_pct, lower_is_better)
emoji = {"good": "", "bad": "", "neutral": ""}[sig]
return f"{pct:+.2f}% {emoji}{ci_pct:.2f}%)"
def compute_changes(
baseline_stats: dict, feature_stats: dict, paired_stats: dict
) -> dict:
"""Pre-compute change percentages and significance for each metric."""
def pct(base: float, feat: float) -> float:
return (feat - base) / base * 100.0 if base > 0 else 0.0
def ci_pct(ci_ms: float, base_ms: float) -> float:
return ci_ms / base_ms * 100.0 if base_ms > 0 else 0.0
metrics = [
("mean", "mean_ms", "ci_ms", "mean_ms", True),
("p50", "p50_ms", "p50_ci_ms", "p50_ms", True),
("p90", "p90_ms", "p90_ci_ms", "p90_ms", True),
("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
]
changes = {}
for name, stat_key, ci_key, base_key, lower_is_better in metrics:
p = pct(baseline_stats[stat_key], feature_stats[stat_key])
c = ci_pct(paired_stats[ci_key], baseline_stats[base_key])
changes[name] = {
"pct": round(p, 4),
"ci_pct": round(c, 4),
"sig": significance(p, c, lower_is_better),
}
return changes
def generate_comparison_table(
run1: dict,
run2: dict,
paired: dict,
repo: str,
baseline_ref: str,
baseline_name: str,
feature_name: str,
feature_sha: str,
) -> str:
"""Generate a markdown comparison table between baseline and feature."""
n = paired["blocks"]
def pct(base: float, feat: float) -> float:
return (feat - base) / base * 100.0 if base > 0 else 0.0
mean_pct = pct(run1["mean_ms"], run2["mean_ms"])
gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"])
p50_pct = pct(run1["p50_ms"], run2["p50_ms"])
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
p99_pct = pct(run1["p99_ms"], run2["p99_ms"])
# Bootstrap CIs as % of baseline percentile
p50_ci_pct = paired["p50_ci_ms"] / run1["p50_ms"] * 100.0 if run1["p50_ms"] > 0 else 0.0
p90_ci_pct = paired["p90_ci_ms"] / run1["p90_ms"] * 100.0 if run1["p90_ms"] > 0 else 0.0
p99_ci_pct = paired["p99_ci_ms"] / run1["p99_ms"] * 100.0 if run1["p99_ms"] > 0 else 0.0
# CI as a percentage of baseline mean
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
base_url = f"https://github.com/{repo}/commit"
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
feature_label = f"[`{feature_name}`]({base_url}/{feature_sha})"
lines = [
f"| Metric | {baseline_label} | {feature_label} | Change |",
"|--------|------|--------|--------|",
f"| Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, lat_ci_pct, lower_is_better=True)} |",
f"| StdDev | {fmt_ms(run1['stddev_ms'])} | {fmt_ms(run2['stddev_ms'])} | |",
f"| P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, lower_is_better=True)} |",
f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, lower_is_better=True)} |",
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
"",
f"*{n} blocks*",
]
return "\n".join(lines)
def generate_wait_time_table(
title: str,
baseline_stats: dict,
feature_stats: dict,
baseline_label: str,
feature_label: str,
) -> str:
"""Generate a markdown table for a wait time metric."""
if not baseline_stats or not feature_stats:
return ""
lines = [
f"### {title}",
"",
f"| Metric | {baseline_label} | {feature_label} |",
"|--------|------|--------|",
f"| Mean | {fmt_ms(baseline_stats['mean_ms'])} | {fmt_ms(feature_stats['mean_ms'])} |",
f"| P50 | {fmt_ms(baseline_stats['p50_ms'])} | {fmt_ms(feature_stats['p50_ms'])} |",
f"| P95 | {fmt_ms(baseline_stats['p95_ms'])} | {fmt_ms(feature_stats['p95_ms'])} |",
]
return "\n".join(lines)
def generate_markdown(
summary: dict, comparison_table: str,
wait_time_tables: list[str] | None = None,
behind_baseline: int = 0, repo: str = "", baseline_ref: str = "", baseline_name: str = "",
) -> str:
"""Generate a markdown comment body."""
lines = ["## Benchmark Results", ""]
if behind_baseline > 0:
s = "s" if behind_baseline > 1 else ""
diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...{baseline_name}"
lines.append(f"> ⚠️ Feature is [**{behind_baseline} commit{s} behind `{baseline_name}`**]({diff_link}). Consider rebasing for accurate results.")
lines.append("")
lines.append(comparison_table)
if wait_time_tables:
lines.append("")
lines.append("<details>")
lines.append("<summary>Wait Time Breakdown</summary>")
lines.append("")
for table in wait_time_tables:
if table:
lines.append(table)
lines.append("")
lines.append("</details>")
return "\n".join(lines)
def main():
parser = argparse.ArgumentParser(description="Parse reth-bench ABBA results")
parser.add_argument(
"--baseline-csv", nargs="+", required=True,
help="Baseline combined_latency.csv files (A1, A2)",
)
parser.add_argument(
"--feature-csv", "--branch-csv", nargs="+", required=True,
help="Feature combined_latency.csv files (B1, B2)",
)
parser.add_argument("--gas-csv", required=True, help="Path to total_gas.csv")
parser.add_argument(
"--output-summary", required=True, help="Output JSON summary path"
)
parser.add_argument("--output-markdown", required=True, help="Output markdown path")
parser.add_argument(
"--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)"
)
parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA")
parser.add_argument("--baseline-name", default=None, help="Baseline display name")
parser.add_argument("--feature-name", "--branch-name", default=None, help="Feature branch name")
parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
args = parser.parse_args()
if len(args.baseline_csv) != len(args.feature_csv):
print("Must provide equal number of baseline and feature CSVs", file=sys.stderr)
sys.exit(1)
baseline_runs = []
feature_runs = []
for path in args.baseline_csv:
data = parse_combined_csv(path)
if not data:
print(f"No results in {path}", file=sys.stderr)
sys.exit(1)
baseline_runs.append(data)
for path in args.feature_csv:
data = parse_combined_csv(path)
if not data:
print(f"No results in {path}", file=sys.stderr)
sys.exit(1)
feature_runs.append(data)
gas = parse_gas_csv(args.gas_csv)
all_baseline = [r for run in baseline_runs for r in run]
all_feature = [r for run in feature_runs for r in run]
baseline_stats = compute_stats(all_baseline)
feature_stats = compute_stats(all_feature)
paired_stats = compute_paired_stats(baseline_runs, feature_runs)
if not paired_stats:
print("No common blocks between baseline and feature runs", file=sys.stderr)
sys.exit(1)
baseline_ref = args.baseline_ref or "main"
baseline_name = args.baseline_name or "baseline"
feature_name = args.feature_name or "feature"
feature_sha = args.feature_ref or "unknown"
comparison_table = generate_comparison_table(
baseline_stats,
feature_stats,
paired_stats,
repo=args.repo,
baseline_ref=baseline_ref,
baseline_name=baseline_name,
feature_name=feature_name,
feature_sha=feature_sha,
)
print(f"Generated comparison ({paired_stats['n']} paired blocks, "
f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")
base_url = f"https://github.com/{args.repo}/commit"
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
feature_label = f"[`{feature_name}`]({base_url}/{feature_sha})"
wait_fields = [
("persistence_wait_us", "Persistence Wait"),
("sparse_trie_wait_us", "Trie Cache Update Wait"),
("execution_cache_wait_us", "Execution Cache Update Wait"),
]
wait_time_tables = []
wait_time_data = {}
for field, title in wait_fields:
b_stats = compute_wait_stats(all_baseline, field)
f_stats = compute_wait_stats(all_feature, field)
if b_stats and f_stats:
wait_time_data[field] = {
"title": title,
"baseline": b_stats,
"feature": f_stats,
}
table = generate_wait_time_table(title, b_stats, f_stats, baseline_label, feature_label)
if table:
wait_time_tables.append(table)
summary = {
"blocks": paired_stats["blocks"],
"baseline": {
"name": baseline_name,
"ref": baseline_ref,
"stats": baseline_stats,
},
"feature": {
"name": feature_name,
"ref": feature_sha,
"stats": feature_stats,
},
"paired": paired_stats,
"changes": compute_changes(baseline_stats, feature_stats, paired_stats),
"wait_times": wait_time_data,
}
with open(args.output_summary, "w") as f:
json.dump(summary, f, indent=2)
print(f"Summary written to {args.output_summary}")
markdown = generate_markdown(
summary, comparison_table,
wait_time_tables=wait_time_tables,
behind_baseline=args.behind_baseline,
repo=args.repo,
baseline_ref=baseline_ref,
baseline_name=baseline_name,
)
with open(args.output_markdown, "w") as f:
f.write(markdown)
print(f"Markdown written to {args.output_markdown}")
if __name__ == "__main__":
main()

342
.github/scripts/bench-slack-notify.js vendored Normal file
View File

@@ -0,0 +1,342 @@
// Sends Slack notifications for reth-bench results.
//
// Reads from environment:
// SLACK_BENCH_BOT_TOKEN Slack Bot User OAuth Token (xoxb-...)
// SLACK_BENCH_CHANNEL Public channel ID for significant improvements
// BENCH_WORK_DIR Directory containing summary.json
// BENCH_PR PR number (may be empty)
// BENCH_ACTOR GitHub user who triggered the bench
// BENCH_JOB_URL URL to the Actions job page
// BENCH_SAMPLY 'true' if samply profiling was enabled
//
// Usage from actions/github-script:
// const notify = require('./.github/scripts/bench-slack-notify.js');
// await notify.success({ core, context });
// await notify.failure({ core, context, failedStep: '...' });
const fs = require('fs');
const path = require('path');
const SLACK_API = 'https://slack.com/api/chat.postMessage';
function loadSlackUsers(repoRoot) {
try {
const raw = fs.readFileSync(path.join(repoRoot, '.github', 'scripts', 'bench-slack-users.json'), 'utf8');
const data = JSON.parse(raw);
// Filter out non-user-ID entries (like _comment)
const users = {};
for (const [k, v] of Object.entries(data)) {
if (!k.startsWith('_') && typeof v === 'string' && v.startsWith('U')) {
users[k] = v;
}
}
return users;
} catch {
return {};
}
}
async function postToSlack(token, channel, blocks, text, core, threadTs) {
const payload = { channel, blocks, text, unfurl_links: false };
if (threadTs) payload.thread_ts = threadTs;
const resp = await fetch(SLACK_API, {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
});
const data = await resp.json();
if (!data.ok) {
core.warning(`Slack API error (channel ${channel}): ${JSON.stringify(data)}`);
}
return data;
}
function cell(text) {
const s = String(text);
return { type: 'raw_text', text: s || ' ' };
}
function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls }) {
const b = summary.baseline.stats;
const f = summary.feature.stats;
const c = summary.changes;
const sigEmoji = { good: '\u2705', bad: '\u274c', neutral: '\u26aa' };
function fmtMs(v) { return v.toFixed(2) + 'ms'; }
function fmtMgas(v) { return v.toFixed(2); }
function fmtChange(ch) {
if (!ch.pct && !ch.ci_pct) return ' ';
const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
const ciStr = ch.ci_pct ? ` (\u00b1${ch.ci_pct.toFixed(2)}%)` : '';
return `${pctStr}${ciStr} ${sigEmoji[ch.sig]}`;
}
// Overall result for header
const vals = Object.values(c);
const hasBad = vals.some(v => v.sig === 'bad');
const hasGood = vals.some(v => v.sig === 'good');
let headerEmoji, headerResult;
if (hasBad && hasGood) {
headerEmoji = ':warning:';
headerResult = 'Mixed Results';
} else if (hasBad) {
headerEmoji = ':x:';
headerResult = 'Regression';
} else if (hasGood) {
headerEmoji = ':white_check_mark:';
headerResult = 'Improvement';
} else {
headerEmoji = ':white_circle:';
headerResult = 'No Difference';
}
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
const commitUrl = `https://github.com/${repo}/commit`;
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
// Meta line
const metaParts = [];
if (prNumber) metaParts.push(`*<${prUrl}|PR #${prNumber}>*`);
metaParts.push(`triggered by ${actorSlackId ? `<@${actorSlackId}>` : `@${actor}`}`);
// Baseline/feature lines with samply profile links
let baselineLine = `*Baseline:* ${baselineLink}`;
const bl1 = samplyUrls['baseline-1'];
const bl2 = samplyUrls['baseline-2'];
if (bl1) baselineLine += ` | <${bl1}|Samply 1>`;
if (bl2) baselineLine += ` | <${bl2}|Samply 2>`;
let featureLine = `*Feature:* ${featureLink}`;
const fl1 = samplyUrls['feature-1'];
const fl2 = samplyUrls['feature-2'];
if (fl1) featureLine += ` | <${fl1}|Samply 1>`;
if (fl2) featureLine += ` | <${fl2}|Samply 2>`;
const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
const cores = process.env.BENCH_CORES || '0';
const countsParts = [];
if (warmup) countsParts.push(`*Warmup:* ${warmup}`);
countsParts.push(`*Blocks:* ${summary.blocks}`);
if (cores !== '0') countsParts.push(`*Cores:* ${cores}`);
const countsLine = countsParts.join(' | ');
const sectionText = [metaParts.join(' | '), '', baselineLine, featureLine, countsLine].join('\n');
// Action buttons
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
const buttons = [
{
type: 'button',
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
},
{
type: 'button',
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
url: diffUrl,
action_id: 'diff_button',
},
];
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: `${headerEmoji} ${headerResult}`, emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: sectionText },
},
{
type: 'table',
column_settings: [
{ align: 'left' },
{ align: 'right' },
{ align: 'right' },
{ align: 'right' },
],
rows: [
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
[cell('Mean'), cell(fmtMs(b.mean_ms)), cell(fmtMs(f.mean_ms)), cell(fmtChange(c.mean))],
[cell('StdDev'), cell(fmtMs(b.stddev_ms)), cell(fmtMs(f.stddev_ms)), cell(' ')],
[cell('P50'), cell(fmtMs(b.p50_ms)), cell(fmtMs(f.p50_ms)), cell(fmtChange(c.p50))],
[cell('P90'), cell(fmtMs(b.p90_ms)), cell(fmtMs(f.p90_ms)), cell(fmtChange(c.p90))],
[cell('P99'), cell(fmtMs(b.p99_ms)), cell(fmtMs(f.p99_ms)), cell(fmtChange(c.p99))],
[cell('Mgas/s'), cell(fmtMgas(b.mean_mgas_s)), cell(fmtMgas(f.mean_mgas_s)), cell(fmtChange(c.mgas_s))],
],
},
{
type: 'actions',
elements: buttons,
},
];
// Wait times as a separate table block (sent as threaded reply due to Slack one-table limit)
const threadBlocks = [];
const waitTimes = summary.wait_times || {};
const waitKeys = Object.keys(waitTimes);
if (waitKeys.length > 0) {
const waitRows = [
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
];
for (const key of waitKeys) {
const wt = waitTimes[key];
waitRows.push([cell(wt.title), cell(fmtMs(wt.baseline.mean_ms)), cell(fmtMs(wt.feature.mean_ms))]);
}
threadBlocks.push({
type: 'table',
column_settings: [
{ align: 'left' },
{ align: 'right' },
{ align: 'right' },
],
rows: waitRows,
});
}
return { blocks, threadBlocks };
}
function buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep }) {
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
const actorMention = actorSlackId ? `<@${actorSlackId}>` : `@${actor}`;
const parts = [
prNumber ? `*<${prUrl}|PR #${prNumber}>*` : '',
`by ${actorMention}`,
`failed while *${failedStep}*`,
].filter(Boolean);
const buttons = [
{
type: 'button',
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
},
];
return [
{
type: 'header',
text: { type: 'plain_text', text: ':rotating_light: Bench Failed', emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: parts.join(' | ') },
},
{
type: 'actions',
elements: buttons,
},
];
}
async function success({ core, context }) {
const token = process.env.SLACK_BENCH_BOT_TOKEN;
if (!token) {
core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
return;
}
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
core.warning('Could not read summary.json for Slack notification');
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const prNumber = process.env.BENCH_PR;
const actor = process.env.BENCH_ACTOR;
const jobUrl = process.env.BENCH_JOB_URL ||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
// Load samply profile URLs (files exist when samply profiling was enabled)
const samplyUrls = {};
for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
try {
const url = fs.readFileSync(
path.join(process.env.BENCH_WORK_DIR, run, 'samply-profile-url.txt'), 'utf8'
).trim();
if (url) samplyUrls[run] = url;
} catch {}
}
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
const actorSlackId = slackUsers[actor];
const { blocks, threadBlocks } = buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls });
const text = `Bench: ${summary.baseline.name} vs ${summary.feature.name}`;
async function sendWithThread(ch) {
const res = await postToSlack(token, ch, blocks, text, core);
if (res.ok && res.ts && threadBlocks.length > 0) {
for (const tb of threadBlocks) {
await postToSlack(token, ch, [tb], 'Wait time breakdown', core, res.ts);
}
}
}
// Post to public channel if any metric shows significant improvement or regression
const channel = process.env.SLACK_BENCH_CHANNEL;
let postedToChannel = false;
if (channel) {
const changes = summary.changes || {};
const hasImprovement = Object.values(changes).some(c => c.sig === 'good');
if (hasImprovement) {
await sendWithThread(channel);
postedToChannel = true;
} else {
core.info('No significant improvement, skipping public channel notification');
}
}
// DM the actor only when results were not posted to the public channel
if (!postedToChannel) {
if (actorSlackId) {
await sendWithThread(actorSlackId);
} else {
core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
}
} else {
core.info(`Results posted to channel, skipping DM to ${actor}`);
}
}
async function failure({ core, context, failedStep }) {
const token = process.env.SLACK_BENCH_BOT_TOKEN;
if (!token) {
core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const prNumber = process.env.BENCH_PR;
const actor = process.env.BENCH_ACTOR;
const jobUrl = process.env.BENCH_JOB_URL ||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
const actorSlackId = slackUsers[actor];
const blocks = buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep });
const text = `Bench failed while ${failedStep}`;
// Always DM the actor
if (actorSlackId) {
await postToSlack(token, actorSlackId, blocks, text, core);
} else {
core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
}
// Only DM for failures, don't post to public channel
}
module.exports = { success, failure };

13
.github/scripts/bench-slack-users.json vendored Normal file
View File

@@ -0,0 +1,13 @@
{
"_comment": "Maps GitHub usernames to Slack user IDs. Find yours: Slack profile > ··· > Copy member ID.",
"shekhirin": "U09FAL2UMLJ",
"mattsse": "U09FQNPMRT3",
"klkvr": "U09FAK95FC2",
"joshieDo": "U09LHN6GYAU",
"mediocregopher": "U09FF75KMQU",
"yongkangc": "U09FB0ECTD4",
"gakonst": "U092SEPDM40",
"Rjected": "U09F6SCKRGT",
"DaniPopes": "U09FAT8EK2A",
"emmajam": "U0A34UN92HW"
}

27
.github/scripts/bench-update-status.js vendored Normal file
View File

@@ -0,0 +1,27 @@
// Updates the reth-bench PR comment with current status.
//
// Reads from environment:
// BENCH_COMMENT_ID GitHub comment ID to update
// BENCH_JOB_URL URL to the Actions job page
// BENCH_CONFIG Config line (blocks, warmup, refs)
// BENCH_ACTOR User who triggered the benchmark
//
// Usage from actions/github-script:
// const s = require('./.github/scripts/bench-update-status.js');
// await s({github, context, status: 'Building baseline binary...'});
function buildBody(status) {
return `cc @${process.env.BENCH_ACTOR}\n\n🚀 Benchmark started! [View job](${process.env.BENCH_JOB_URL})\n\n⏳ **Status:** ${status}\n\n${process.env.BENCH_CONFIG}`;
}
async function updateStatus({ github, context, status }) {
await github.rest.issues.updateComment({
owner: context.repo.owner,
repo: context.repo.repo,
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
body: buildBody(status),
});
}
updateStatus.buildBody = buildBody;
module.exports = updateStatus;

View File

@@ -3,7 +3,7 @@
#
# We'll use cargo-chef to speed up the build
#
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
FROM lukemathwalker/cargo-chef:latest-rust-trixie AS chef
WORKDIR /app
# Install system dependencies

View File

@@ -9,10 +9,13 @@ go build .
./hive -client reth # first builds and caches the client
# Run each hive command in the background for each simulator and wait
# Run each hive command in the background for each simulator and wait
echo "Building images"
# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/eels" \
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
--sim.buildarg branch=err-map-3 \
--sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &

View File

@@ -18,30 +18,18 @@ rpc-compat:
# no fix due to https://github.com/paradigmxyz/reth/issues/8732
engine-withdrawals:
- Withdrawals Fork On Genesis (Paris) (reth)
- Withdrawals Fork on Block 1 (Paris) (reth)
- Withdrawals Fork on Block 2 (Paris) (reth)
- Withdrawals Fork on Block 3 (Paris) (reth)
- Withdraw to a single account (Paris) (reth)
- Withdraw to two accounts (Paris) (reth)
- Withdraw many accounts (Paris) (reth)
- Withdraw zero amount (Paris) (reth)
- Empty Withdrawals (Paris) (reth)
- Corrupted Block Hash Payload (INVALID) (Paris) (reth)
- Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
engine-api: [ ]
engine-api: []
# no fix due to https://github.com/paradigmxyz/reth/issues/8732
engine-cancun:
- Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
# the test fails with older versions of the code for which it passed before, probably related to changes
# in hive or its dependencies
- Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
sync: [ ]
sync: []
engine-auth: [ ]
engine-auth: []
# EIP-7610 related tests (Revert creation in case of non-empty storage):
#
@@ -49,7 +37,7 @@ engine-auth: [ ]
# The test artificially creates an empty account with storage, then tests EIP-7610's behavior.
# On mainnet, ~25 such accounts exist as contract addresses (derived from keccak(prefix, caller,
# nonce/salt), not from public keys). No private key exists for contract addresses. To trigger
# this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible.
# this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible.
#
# tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_*
# Requires hash collision on create2 address to target already deployed accounts with storage.
@@ -97,6 +85,18 @@ eels/consume-engine:
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_True]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_False]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
@@ -147,6 +147,20 @@ eels/consume-engine:
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_engine_from_state_test]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-empty-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-sstore-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
# Blob limit tests:
#
@@ -241,3 +255,17 @@ eels/consume-rlp:
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-initcode-with-deploy]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-sstore-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_from_state_test]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-empty-initcode]-reth

View File

@@ -11,17 +11,16 @@
#
# When a test should no longer be ignored, remove it from this list.
# flaky
engine-withdrawals:
- Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
- Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
- Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
- Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
engine-cancun:
- Transaction Re-Org, New Payload on Revert Back (Cancun) (reth)
- Transaction Re-Org, Re-Org to Different Block (Cancun) (reth)
- Transaction Re-Org, Re-Org Out (Cancun) (reth)
- Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth)
# Hive test infra bug: geth sidecar switched to PathScheme for state storage, which has
# strict trie integrity requirements incompatible with inserting intentionally invalid blocks.
# Affects all clients, not just reth. Tracked: https://github.com/ethereum/hive/issues/1382
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
- Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Cancun) (reth)
engine-api:
- Transaction Re-Org, Re-Org Out (Paris) (reth)

View File

@@ -6,8 +6,14 @@ cd hivetests/
sim="${1}"
limit="${2}"
# Use lower parallelism for eels tests to avoid OOM-killing the runner
parallelism=16
if [[ "${sim}" == *"eels"* ]]; then
parallelism=4
fi
run_hive() {
hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 16 --client reth 2>&1 | tee /tmp/log || true
hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism "${parallelism}" --client reth 2>&1 | tee /tmp/log || true
}
check_log() {

File diff suppressed because it is too large Load Diff

View File

@@ -15,7 +15,7 @@ env:
jobs:
build:
runs-on: depot-ubuntu-latest-8
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
timeout-minutes: 90
steps:
- name: Checkout

View File

@@ -25,7 +25,7 @@ env:
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
timeout-minutes: 60
steps:
- uses: actions/checkout@v4

View File

@@ -18,7 +18,7 @@ env:
name: compact-codec
jobs:
compact-codec:
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
strategy:
matrix:
bin:

View File

@@ -15,7 +15,6 @@ on:
jobs:
build:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: ubuntu-latest
permissions:
@@ -31,10 +30,22 @@ jobs:
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
- name: Detect fork
id: fork
run: |
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
echo "is_fork=true" >> "$GITHUB_OUTPUT"
else
echo "is_fork=false" >> "$GITHUB_OUTPUT"
fi
# Depot build (upstream only)
- name: Set up Depot CLI
if: steps.fork.outputs.is_fork == 'false'
uses: depot/setup-action@v1
- name: Build reth image
- name: Build reth image (Depot)
if: steps.fork.outputs.is_fork == 'false'
uses: depot/bake-action@v1
env:
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
@@ -46,8 +57,26 @@ jobs:
targets: ${{ inputs.hive_target }}
push: false
# Docker build (forks)
- name: Set up Docker Buildx
if: steps.fork.outputs.is_fork == 'true'
uses: docker/setup-buildx-action@v3
- name: Build reth image (Docker)
if: steps.fork.outputs.is_fork == 'true'
uses: docker/bake-action@v6
env:
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
with:
files: docker-bake.hcl
targets: ${{ inputs.hive_target }}
push: false
set: |
*.dockerfile=Dockerfile
- name: Upload reth image
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./artifacts
path: ./artifacts

View File

@@ -70,18 +70,27 @@ jobs:
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
{
echo "ethereum_set<<EOF"
echo "ethereum.tags=${REGISTRY}/reth:${VERSION}"
echo "ethereum.tags=${REGISTRY}/reth:latest"
echo "EOF"
} >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
@@ -97,7 +106,7 @@ jobs:
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
${{ steps.params.outputs.ethereum_set }}
- name: Verify image architectures
env:
@@ -117,6 +126,18 @@ jobs:
- name: Slack Webhook Action
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
SLACK_COLOR: danger
SLACK_ICON_EMOJI: ":rotating_light:"
SLACK_USERNAME: "GitHub Actions"
SLACK_TITLE: ":rotating_light: Nightly Docker Build Failed"
SLACK_MESSAGE: |
The scheduled nightly Docker build failed.
*Commit:* `${{ github.sha }}`
*Branch:* `${{ github.ref_name }}`
*Run:* <https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}|View logs>
*Action required:* Re-run the workflow or investigate the build failure.
SLACK_FOOTER: "paradigmxyz/reth · docker.yml"
MSG_MINIMAL: true
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -20,7 +20,7 @@ concurrency:
jobs:
test:
name: e2e-testsuite
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_BACKTRACE: 1
timeout-minutes: 90
@@ -47,7 +47,7 @@ jobs:
rocksdb:
name: e2e-rocksdb
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_BACKTRACE: 1
timeout-minutes: 60

View File

@@ -5,7 +5,10 @@ name: hive
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
- cron: "0 */6 * * *"
pull_request:
branches:
- "**"
env:
CARGO_TERM_COLOR: always
@@ -15,30 +18,34 @@ concurrency:
cancel-in-progress: true
jobs:
build-reth-stable:
uses: ./.github/workflows/docker-test.yml
prepare-reth-stable:
uses: ./.github/workflows/prepare-reth.yml
with:
hive_target: hive-stable
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak"
artifact_name: "reth-stable"
secrets: inherit
build-reth-edge:
uses: ./.github/workflows/docker-test.yml
prepare-reth-edge:
uses: ./.github/workflows/prepare-reth.yml
with:
hive_target: hive-edge
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak edge"
artifact_name: "reth-edge"
secrets: inherit
prepare-hive:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: depot-ubuntu-latest-4
runs-on:
group: Reth
steps:
- uses: actions/checkout@v6
- name: Checkout hive tests
uses: actions/checkout@v6
with:
repository: ethereum/hive
repository: Soubhik-10/hive
ref: master
path: hivetests
- name: Get hive commit hash
@@ -124,27 +131,29 @@ jobs:
# eth_ rpc methods
- sim: ethereum/rpc-compat
include:
- eth_blockNumber
# - eth_blockNumber
- eth_call
- eth_chainId
- eth_createAccessList
- eth_estimateGas
- eth_feeHistory
- eth_getBalance
- eth_getBlockBy
- eth_getBlockTransactionCountBy
- eth_getCode
- eth_getProof
- eth_getStorage
- eth_getTransactionBy
- eth_getTransactionCount
- eth_getTransactionReceipt
- eth_sendRawTransaction
- eth_syncing
# debug_ rpc methods
- debug_
# - eth_chainId
# - eth_createAccessList
# - eth_estimateGas
# - eth_feeHistory
# - eth_getBalance
# - eth_getBlockBy
# - eth_getBlockTransactionCountBy
# - eth_getCode
# - eth_getProof
# - eth_getStorage
# - eth_getTransactionBy
# - eth_getTransactionCount
# - eth_getTransactionReceipt
# - eth_sendRawTransaction
# - eth_syncing
# # debug_ rpc methods
# - debug_
# consume-engine
- sim: ethereum/eels/consume-engine
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-engine
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-engine
@@ -165,6 +174,8 @@ jobs:
limit: .*tests/paris.*
# consume-rlp
- sim: ethereum/eels/consume-rlp
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-rlp
@@ -184,11 +195,12 @@ jobs:
- sim: ethereum/eels/consume-rlp
limit: .*tests/paris.*
needs:
- build-reth-stable
- build-reth-edge
- prepare-reth-stable
- prepare-reth-edge
- prepare-hive
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
runs-on: depot-ubuntu-latest-4
# Use larger runners for eels tests to avoid OOM runner crashes
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
permissions:
issues: write
steps:
@@ -219,7 +231,7 @@ jobs:
- name: Checkout hive tests
uses: actions/checkout@v6
with:
repository: ethereum/hive
repository: Soubhik-10/hive
ref: master
path: hivetests

View File

@@ -24,7 +24,7 @@ jobs:
test:
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
if: github.event_name != 'schedule'
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_BACKTRACE: 1
strategy:

View File

@@ -31,7 +31,7 @@ jobs:
strategy:
fail-fast: false
name: run kurtosis
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
needs:
- build-reth
steps:

View File

@@ -13,7 +13,7 @@ env:
jobs:
clippy-binaries:
name: clippy binaries / ${{ matrix.type }}
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
strategy:
matrix:
@@ -42,7 +42,7 @@ jobs:
clippy:
name: clippy
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -59,7 +59,7 @@ jobs:
RUSTFLAGS: -D warnings
wasm:
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -79,7 +79,7 @@ jobs:
.github/scripts/check_wasm.sh
riscv:
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
@@ -98,7 +98,7 @@ jobs:
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
strategy:
matrix:
partition: [1, 2, 3]
@@ -117,14 +117,14 @@ jobs:
msrv:
name: MSRV
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@master
with:
toolchain: "1.88" # MSRV
toolchain: "1.93" # MSRV
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
with:
@@ -135,7 +135,7 @@ jobs:
docs:
name: docs
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -153,7 +153,7 @@ jobs:
fmt:
name: fmt
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -167,7 +167,7 @@ jobs:
udeps:
name: udeps
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -182,7 +182,7 @@ jobs:
book:
name: book
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -240,7 +240,7 @@ jobs:
# Checks that selected crates can compile with power set of features
features:
name: features
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -264,7 +264,7 @@ jobs:
# Check crates correctly propagate features
feature-propagation:
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
timeout-minutes: 20
steps:
- uses: actions/checkout@v6

61
.github/workflows/prepare-reth.yml vendored Normal file
View File

@@ -0,0 +1,61 @@
name: Prepare Reth Image
on:
workflow_call:
inputs:
image_tag:
required: true
type: string
description: "Docker image tag to use"
binary_name:
required: false
type: string
default: "reth"
description: "Binary name to build (reth or op-reth)"
cargo_features:
required: false
type: string
default: "asm-keccak"
description: "Cargo features to enable"
cargo_package:
required: false
type: string
description: "Optional cargo package path"
artifact_name:
required: false
type: string
default: "artifacts"
description: "Name for the uploaded artifact"
jobs:
prepare-reth:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: depot-ubuntu-latest-4
steps:
- uses: actions/checkout@v6
- run: mkdir artifacts
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and export reth image
uses: docker/build-push-action@v6
with:
context: .
file: .github/scripts/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |
CARGO_BIN=${{ inputs.binary_name }}
MANIFEST_PATH=${{ inputs.cargo_package }}
FEATURES=${{ inputs.cargo_features }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Upload reth image
id: upload
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./artifacts

View File

@@ -73,22 +73,22 @@ jobs:
os: ubuntu-24.04
profile: maxperf
allow_fail: false
rustflags: "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
- target: aarch64-unknown-linux-gnu
os: ubuntu-24.04
os: ubuntu-24.04-arm
profile: maxperf
allow_fail: false
rustflags: ""
- target: x86_64-apple-darwin
os: macos-14
profile: maxperf
allow_fail: false
rustflags: "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
- target: aarch64-apple-darwin
os: macos-14
profile: maxperf
allow_fail: false
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-24.04
profile: maxperf
allow_fail: true
rustflags: ""
build:
- command: build
binary: reth
@@ -102,7 +102,7 @@ jobs:
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
cargo install cross --locked --git https://github.com/cross-rs/cross
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
@@ -114,7 +114,7 @@ jobs:
echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
- name: Build Reth
run: make PROFILE=${{ matrix.configs.profile }} ${{ matrix.build.command }}-${{ matrix.configs.target }}
run: make PROFILE=${{ matrix.configs.profile }} EXTRA_RUSTFLAGS="${{ matrix.configs.rustflags }}" ${{ matrix.build.command }}-${{ matrix.configs.target }}
- name: Move binary
run: |
mkdir artifacts

View File

@@ -23,7 +23,7 @@ jobs:
name: stage-run-test
# Only run stage commands test in merge groups
if: github.event_name == 'merge_group'
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
env:
RUST_LOG: info,sync=error
RUST_BACKTRACE: 1
@@ -38,7 +38,7 @@ jobs:
cache-on-failure: true
- name: Build reth
run: |
cargo install --path bin/reth
cargo install --locked --path bin/reth
- name: Run headers stage
run: |
reth stage run headers --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints

View File

@@ -19,7 +19,7 @@ jobs:
sync:
if: github.repository == 'paradigmxyz/reth'
name: sync (${{ matrix.chain.bin }})
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
env:
RUST_LOG: info,sync=error
RUST_BACKTRACE: 1

View File

@@ -19,7 +19,7 @@ jobs:
sync:
if: github.repository == 'paradigmxyz/reth'
name: sync (${{ matrix.chain.bin }})
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
env:
RUST_LOG: info,sync=error
RUST_BACKTRACE: 1

View File

@@ -20,7 +20,7 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.type }} / ${{ matrix.storage }}
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_BACKTRACE: 1
EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
@@ -57,7 +57,7 @@ jobs:
state:
name: Ethereum state tests
runs-on: depot-ubuntu-latest-4
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_LOG: info,sync=error
RUST_BACKTRACE: 1
@@ -92,7 +92,7 @@ jobs:
doc:
name: doc tests
runs-on: depot-ubuntu-latest
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
env:
RUST_BACKTRACE: 1
timeout-minutes: 30

162
CLAUDE.md
View File

@@ -172,10 +172,97 @@ Before submitting changes, ensure:
2. **Clippy**: No warnings
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below)
6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### CLI Reference Docs (`book` CI Job)
The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless.
When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running:
```bash
make update-book-cli
```
This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes.
The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`.
### Opening PRs against <https://github.com/paradigmxyz/reth>
#### Titles
Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope:
```
<type>(<scope>): <short description>
```
**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore`
**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net`
Examples:
- `fix(rpc): correct gas estimation for ERC-20 transfers`
- `perf: batch trie updates to reduce cursor overhead`
- `feat(engine): add new_payload_interval metric`
#### Descriptions
Keep it short. Say what changed and why — nothing more.
**Do:**
- Write 13 sentences summarizing the change
- Explain _why_ if the diff doesn't make it obvious
- Link related issues or EIPs
- Include benchmark numbers for perf changes
**Don't:**
- List every file changed — that's what the diff is for
- Repeat the title in the body
- Add "Files changed" or "Changes" sections
- Write walls of text that go stale when the diff is updated
- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage"
**Template:**
```
Closes #<issue>
<what changed, 1-3 sentences>
<why, if not obvious from the diff>
```
**Good example:**
```
Closes #16800
Adds fallback for external IP resolution so node startup doesn't fail
when STUN is unreachable. Falls back to the configured default.
```
**Bad example:**
```
## Summary
This PR introduces comprehensive improvements to the IP resolution system.
## Changes
- Modified `crates/net/discv4/src/lib.rs` to add fallback
- Modified `crates/net/discv4/src/config.rs` to add default IP
- Added tests in `crates/net/discv4/src/tests/ip.rs`
## Files Changed
- crates/net/discv4/src/lib.rs
- crates/net/discv4/src/config.rs
- crates/net/discv4/src/tests/ip.rs
```
#### Labels and CI
Label PRs appropriately, first check the available labels and then apply the relevant ones:
* when changes are RPC related, add A-rpc label
* when changes are docs related, add C-docs label
@@ -313,6 +400,74 @@ GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst);
Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful?
#### Rust Style Guides
##### Type Ordering in Files
When defining structs, traits, and functions in a file, follow this ordering convention. The file's primary type (matching the file name) comes first, followed by supporting public types, then private types and helpers.
```rust
use ...;
/// The primary type of this file (matches filename).
pub struct PayloadProcessor { ... }
impl PayloadProcessor { ... }
// Followed by public auxiliary types that support the primary type
/// Configuration for the processor.
pub struct PayloadProcessorConfig { ... }
/// Result type returned by processor operations.
pub struct ProcessorResult { ... }
// Followed by public traits related to the primary type
pub trait ProcessorExt { ... }
// Followed by private helper types
struct InternalState { ... }
// Followed by private helper functions
fn validate_input() { ... }
```
❌ **Bad**: Adding new traits and auxiliary types **above** the file's primary type (see [#22133](https://github.com/paradigmxyz/reth/pull/22133)):
```rust
use ...;
// ❌ BAD - new auxiliary struct added before the file's main type
pub struct CacheWaitDurations { ... }
// ❌ BAD - new trait added before the file's main type
pub trait WaitForCaches { ... }
// The file's primary type is buried below unrelated additions
pub struct PayloadProcessor { ... }
```
✅ **Good**: New types go **after** the primary type:
```rust
use ...;
// ✅ The file's primary type stays at the top
pub struct PayloadProcessor { ... }
impl PayloadProcessor { ... }
// ✅ Auxiliary types follow the primary type
pub struct CacheWaitDurations { ... }
pub trait WaitForCaches { ... }
impl WaitForCaches for PayloadProcessor { ... }
```
### Example Contribution Workflow
Let's say you want to fix a bug where external IP resolution fails on startup:
@@ -387,5 +542,8 @@ cargo build --release
cargo check --workspace --all-features
# Check documentation
cargo docs --document-private-items
cargo docs --document-private-items
# Regenerate CLI reference docs (after CLI changes)
make update-book-cli
```

1105
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,7 +1,7 @@
[workspace.package]
version = "1.10.2"
version = "1.11.0"
edition = "2024"
rust-version = "1.88"
rust-version = "1.93"
license = "MIT OR Apache-2.0"
homepage = "https://paradigmxyz.github.io/reth"
repository = "https://github.com/paradigmxyz/reth"
@@ -27,7 +27,6 @@ members = [
"crates/engine/invalid-block-hooks/",
"crates/engine/local",
"crates/engine/primitives/",
"crates/engine/service",
"crates/engine/tree/",
"crates/engine/util/",
"crates/era",
@@ -349,7 +348,6 @@ reth-ecies = { path = "crates/net/ecies" }
reth-engine-local = { path = "crates/engine/local" }
reth-engine-primitives = { path = "crates/engine/primitives", default-features = false }
reth-engine-tree = { path = "crates/engine/tree" }
reth-engine-service = { path = "crates/engine/service" }
reth-engine-util = { path = "crates/engine/util" }
reth-era = { path = "crates/era" }
reth-era-downloader = { path = "crates/era-downloader" }
@@ -399,7 +397,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
reth-payload-primitives = { path = "crates/payload/primitives" }
reth-payload-validator = { path = "crates/payload/validator" }
reth-payload-util = { path = "crates/payload/util" }
reth-primitives = { path = "crates/primitives", default-features = false }
reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
reth-provider = { path = "crates/storage/provider" }
reth-prune = { path = "crates/prune/prune" }
@@ -448,49 +446,48 @@ op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.2"
# eth
alloy-dyn-abi = "1.5.6"
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.6", default-features = false }
alloy-dyn-abi = "1.5.0"
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false, features = ["rlp"] }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.6.3", default-features = false }
alloy-contract = { version = "1.6.3", default-features = false }
alloy-eips = { version = "1.6.3", default-features = false }
alloy-genesis = { version = "1.6.3", default-features = false }
alloy-json-rpc = { version = "1.6.3", default-features = false }
alloy-network = { version = "1.6.3", default-features = false }
alloy-network-primitives = { version = "1.6.3", default-features = false }
alloy-provider = { version = "1.6.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.6.3", default-features = false }
alloy-rpc-client = { version = "1.6.3", default-features = false }
alloy-rpc-types = { version = "1.6.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.6.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.6.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.6.3", default-features = false }
alloy-rpc-types-debug = { version = "1.6.3", default-features = false }
alloy-rpc-types-engine = { version = "1.6.3", default-features = false }
alloy-rpc-types-eth = { version = "1.6.3", default-features = false }
alloy-rpc-types-mev = { version = "1.6.3", default-features = false }
alloy-rpc-types-trace = { version = "1.6.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.6.3", default-features = false }
alloy-serde = { version = "1.6.3", default-features = false }
alloy-signer = { version = "1.6.3", default-features = false }
alloy-signer-local = { version = "1.6.3", default-features = false }
alloy-transport = { version = "1.6.3" }
alloy-transport-http = { version = "1.6.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.6.3", default-features = false }
alloy-transport-ws = { version = "1.6.3", default-features = false }
alloy-consensus = { version = "1.7.3", default-features = false }
alloy-contract = { version = "1.7.3", default-features = false }
alloy-eips = { version = "1.7.3", default-features = false }
alloy-genesis = { version = "1.7.3", default-features = false }
alloy-json-rpc = { version = "1.7.3", default-features = false }
alloy-network = { version = "1.7.3", default-features = false }
alloy-network-primitives = { version = "1.7.3", default-features = false }
alloy-provider = { version = "1.7.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.7.3", default-features = false }
alloy-rpc-client = { version = "1.7.3", default-features = false }
alloy-rpc-types = { version = "1.7.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.7.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.7.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.7.3", default-features = false }
alloy-rpc-types-debug = { version = "1.7.3", default-features = false }
alloy-rpc-types-engine = { version = "1.7.3", default-features = false }
alloy-rpc-types-eth = { version = "1.7.3", default-features = false }
alloy-rpc-types-mev = { version = "1.7.3", default-features = false }
alloy-rpc-types-trace = { version = "1.7.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.7.3", default-features = false }
alloy-serde = { version = "1.7.3", default-features = false }
alloy-signer = { version = "1.7.3", default-features = false }
alloy-signer-local = { version = "1.7.3", default-features = false }
alloy-transport = { version = "1.7.3" }
alloy-transport-http = { version = "1.7.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.7.3", default-features = false }
alloy-transport-ws = { version = "1.7.3", default-features = false }
# op
alloy-op-evm = { version = "0.27.2", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -530,6 +527,7 @@ notify = { version = "8.0.0", default-features = false, features = ["macos_fseve
nybbles = { version = "0.4.8", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
parking_lot = "0.12"
quanta = "0.12"
paste = "1.0"
rand = "0.9"
rayon = "1.7"
@@ -546,6 +544,7 @@ strum = { version = "0.27", default-features = false }
strum_macros = "0.27"
syn = "2.0"
thiserror = { version = "2.0.0", default-features = false }
thread-priority = "3.0.0"
tar = "0.4.44"
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
tracing-appender = "0.2"
@@ -652,7 +651,7 @@ ethereum_ssz_derive = "0.10.1"
jemalloc_pprof = { version = "0.8", default-features = false }
tikv-jemalloc-ctl = "0.6"
tikv-jemallocator = "0.6"
tracy-client = "0.18.0"
tracy-client = { version = "0.18.0", features = ["demangle"] }
snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
aes = "0.8.1"
@@ -665,6 +664,7 @@ cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossbeam-utils = "0.8"
crossterm = "0.29.0"
csv = "1.3.0"
ctrlc = "3.4"
@@ -760,3 +760,65 @@ ipnet = "2.11"
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# =============================================================================
# BAL devnet2 patches (EIP-7778, EIP-7928 block access lists)
# =============================================================================
# revm staging patches
revm = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-database = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-state = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-primitives = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-precompile = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-context = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-context-interface = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-handler = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-inspector = { git = "https://github.com/bluealloy/revm", branch = "main" }
op-revm = { git = "https://github.com/bluealloy/revm", branch = "main" }
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", branch = "staging" }
# alloy-evm bal-devnet2 patches
alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "bal-devnet2" }
# alloy bal-devnet2 patches
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-consensus-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-tx-macros = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
# op-alloy bal-devnet2 patches
op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }

View File

@@ -19,10 +19,11 @@ pre-build = [
image = "ubuntu:24.04"
pre-build = [
"apt update",
"apt install --yes gcc gcc-riscv64-linux-gnu libclang-dev make",
"apt install --yes gcc gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libclang-dev make",
]
env.passthrough = [
"CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER=riscv64-linux-gnu-gcc",
"CXX_riscv64gc_unknown_linux_gnu=riscv64-linux-gnu-g++",
]
[build.env]

View File

@@ -1,6 +1,6 @@
# syntax=docker.io/docker/dockerfile:1.7-labs
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
FROM lukemathwalker/cargo-chef:latest-rust-1.93 AS chef
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth

View File

@@ -4,7 +4,7 @@
# Usage:
# reth: --build-arg BINARY=reth
FROM rust:1 AS builder
FROM rust:1.93 AS builder
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth

View File

@@ -17,6 +17,9 @@ FEATURES ?=
# Cargo profile for builds. Default is for local builds, CI uses an override.
PROFILE ?= release
# Extra RUSTFLAGS to append to build targets (e.g., "-C target-cpu=x86-64-v3")
EXTRA_RUSTFLAGS ?=
# Extra flags for Cargo
CARGO_INSTALL_EXTRA_FLAGS ?=
@@ -26,7 +29,7 @@ EF_TESTS_URL := https://github.com/ethereum/tests/archive/refs/tags/$(EF_TESTS_T
EF_TESTS_DIR := ./testing/ef-tests/ethereum-tests
# The release tag of https://github.com/ethereum/execution-spec-tests to use for EEST tests
EEST_TESTS_TAG := v4.5.0
EEST_TESTS_TAG := bal@v5.0.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
@@ -74,13 +77,13 @@ build-debug: ## Build the reth binary into `target/debug` directory.
cargo build --bin reth --features "$(FEATURES)"
# Builds the reth binary natively.
build-native-%:
cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
$(if $(EXTRA_RUSTFLAGS),RUSTFLAGS="$(EXTRA_RUSTFLAGS)") cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
# The following commands use `cross` to build a cross-compile.
#
# These commands require that:
#
# - `cross` is installed (`cargo install cross`).
# - `cross` is installed (`cargo install --locked cross`).
# - Docker is running.
# - The current user is in the `docker` group.
#
@@ -96,7 +99,7 @@ build-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
# Note: The additional rustc compiler flags are for intrinsics needed by MDBX.
# See: https://github.com/cross-rs/cross/wiki/FAQ#undefined-reference-with-build-std
build-%:
RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \
RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc $(EXTRA_RUSTFLAGS)" \
cross build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
# Unfortunately we can't easily use cross to build for Darwin because of licensing issues.
@@ -261,7 +264,7 @@ lint-typos: ensure-typos
ensure-typos:
@if ! command -v typos &> /dev/null; then \
echo "typos not found. Please install it by running the command 'cargo install typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \
echo "typos not found. Please install it by running the command 'cargo install --locked typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \
exit 1; \
fi

View File

@@ -93,7 +93,7 @@ When updating this, also update:
- .github/workflows/lint.yml
-->
The Minimum Supported Rust Version (MSRV) of this project is [1.88.0](https://blog.rust-lang.org/2025/06/26/Rust-1.88.0/).
The Minimum Supported Rust Version (MSRV) of this project is [1.93.0](https://blog.rust-lang.org/2026/01/22/Rust-1.93.0/).
See the docs for detailed instructions on how to [build from source](https://reth.rs/installation/source/).

View File

@@ -45,9 +45,6 @@ serde_json.workspace = true
# Time handling
chrono = { workspace = true, features = ["serde"] }
# Path manipulation
shellexpand.workspace = true
# CSV handling
csv.workspace = true

View File

@@ -289,11 +289,7 @@ impl Args {
/// Get the JWT secret path - either provided or derived from datadir
pub(crate) fn jwt_secret_path(&self) -> PathBuf {
match &self.jwt_secret {
Some(path) => {
let jwt_secret_str = path.to_string_lossy();
let expanded = shellexpand::tilde(&jwt_secret_str);
PathBuf::from(expanded.as_ref())
}
Some(path) => path.clone(),
None => {
// Use the same logic as reth: <datadir>/<chain>/jwt.hex
let chain_path = self.datadir.clone().resolve_datadir(self.chain);
@@ -308,10 +304,9 @@ impl Args {
chain_path.data_dir().to_path_buf()
}
/// Get the expanded output directory path
/// Get the output directory path
pub(crate) fn output_dir_path(&self) -> PathBuf {
let expanded = shellexpand::tilde(&self.output_dir);
PathBuf::from(expanded.as_ref())
PathBuf::from(&self.output_dir)
}
/// Get the effective warmup blocks value - either specified or defaults to blocks

View File

@@ -29,6 +29,10 @@ pub(crate) struct BenchContext {
pub(crate) next_block: u64,
/// Whether the chain is an OP rollup.
pub(crate) is_optimism: bool,
/// Whether to use `reth_newPayload` endpoint instead of `engine_newPayload*`.
pub(crate) use_reth_namespace: bool,
/// Whether to fetch and replay RLP-encoded blocks.
pub(crate) rlp_blocks: bool,
}
impl BenchContext {
@@ -140,6 +144,16 @@ impl BenchContext {
};
let next_block = first_block.header.number + 1;
Ok(Self { auth_provider, block_provider, benchmark_mode, next_block, is_optimism })
let rlp_blocks = bench_args.rlp_blocks;
let use_reth_namespace = bench_args.reth_new_payload || rlp_blocks;
Ok(Self {
auth_provider,
block_provider,
benchmark_mode,
next_block,
is_optimism,
use_reth_namespace,
rlp_blocks,
})
}
}

View File

@@ -6,7 +6,9 @@ use crate::{
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
valid_payload::{
call_forkchoice_updated_with_reth, call_new_payload_with_reth, payload_to_new_payload,
},
};
use alloy_eips::BlockNumberOrTag;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
@@ -19,6 +21,7 @@ use reth_chainspec::ChainSpec;
use reth_cli_runner::CliContext;
use reth_ethereum_primitives::TransactionSigned;
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
use reth_rpc_api::RethNewPayloadInput;
use std::{path::PathBuf, time::Instant};
use tracing::info;
@@ -47,6 +50,14 @@ pub struct Command {
/// Output directory for benchmark results and generated payloads.
#[arg(long, value_name = "OUTPUT")]
output: PathBuf,
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
///
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
/// directly, waits for persistence and cache updates to complete before processing,
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
#[arg(long, default_value = "false", verbatim_doc_comment)]
reth_new_payload: bool,
}
/// Mode for determining when to stop ramping.
@@ -138,6 +149,9 @@ impl Command {
);
}
}
if self.reth_new_payload {
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
}
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
@@ -163,7 +177,7 @@ impl Command {
// Regenerate the payload from the modified block, but keep the original sidecar
// which contains the actual execution requests data (not just the hash)
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
let (version, params) = payload_to_new_payload(
let (version, params, execution_data) = payload_to_new_payload(
payload,
sidecar,
false,
@@ -171,27 +185,45 @@ impl Command {
Some(new_payload_version),
)?;
let (version, params) = if self.reth_new_payload {
(None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?)
} else {
(Some(version), params)
};
// Save payload to file with version info for replay
let payload_path =
self.output.join(format!("payload_block_{}.json", block.header.number));
let file =
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
let file = GasRampPayloadFile {
version: version.map(|v| v as u8),
block_hash,
params: params.clone(),
};
let payload_json = serde_json::to_string_pretty(&file)?;
std::fs::write(&payload_path, &payload_json)?;
info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
call_new_payload(&provider, version, params).await?;
let _ = call_new_payload_with_reth(&provider, version, params).await?;
let forkchoice_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: block_hash,
finalized_block_hash: block_hash,
};
call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
call_forkchoice_updated_with_reth(&provider, version, forkchoice_state).await?;
parent_header = block.header;
parent_hash = block_hash;
blocks_processed += 1;
let progress = match mode {
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
RampMode::TargetGasLimit(target) => {
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
format!("{pct:.1}%")
}
};
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
}
let final_gas_limit = parent_header.gas_limit;

View File

@@ -773,6 +773,7 @@ impl Command {
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
},
transactions: transactions.to_vec(),
extra_data: None,

View File

@@ -76,7 +76,7 @@ use alloy_primitives::{Address, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
PayloadAttributes, PayloadId,
PayloadAttributes, PayloadId, PraguePayloadFields,
};
use eyre::OptionExt;
use reth_chainspec::{ChainSpec, EthereumHardforks};
@@ -138,6 +138,7 @@ pub(crate) fn prepare_payload_request(
suggested_fee_recipient: Address::ZERO,
withdrawals: shanghai_active.then(Vec::new),
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
slot_number: None,
},
forkchoice_state: ForkchoiceState {
head_block_hash: parent_hash,
@@ -230,14 +231,33 @@ pub(crate) async fn get_payload_with_sidecar(
}
4 => {
let envelope = provider.get_payload_v4(payload_id).await?;
Ok(envelope.into_payload_and_sidecar(
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V4")?,
let versioned_hashes = versioned_hashes_from_commitments(
&envelope.envelope_inner.blobs_bundle.commitments,
);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V4")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.envelope_inner.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
5 => {
let envelope = provider.get_payload_v5(payload_id).await?;
Ok(envelope.into_payload_and_sidecar(
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V5")?,
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V5")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
_ => panic!("This tool does not support getPayload versions past v5"),

View File

@@ -0,0 +1,156 @@
//! Prometheus metrics scraper for reth-bench.
//!
//! Scrapes a node's Prometheus metrics endpoint after each block to record
//! execution and state root durations with block-level granularity.
use csv::Writer;
use eyre::Context;
use reqwest::Client;
use serde::Serialize;
use std::{path::Path, time::Duration};
use tracing::info;
/// Suffix for the metrics CSV output file.
pub(crate) const METRICS_OUTPUT_SUFFIX: &str = "metrics.csv";
/// A single row of scraped prometheus metrics for one block.
#[derive(Debug, Clone, Serialize)]
pub(crate) struct MetricsRow {
/// The block number.
pub(crate) block_number: u64,
/// EVM execution duration in seconds (from `sync_execution_execution_duration` gauge).
pub(crate) execution_duration_secs: Option<f64>,
/// State root computation duration in seconds (from
/// `sync_block_validation_state_root_duration` gauge).
pub(crate) state_root_duration_secs: Option<f64>,
}
/// Scrapes a Prometheus metrics endpoint after each block to collect
/// execution and state root durations.
pub(crate) struct MetricsScraper {
/// The full URL of the Prometheus metrics endpoint.
url: String,
/// Reusable HTTP client.
client: Client,
/// Collected metrics rows, one per block.
rows: Vec<MetricsRow>,
}
impl MetricsScraper {
/// Creates a new scraper if a URL is provided.
pub(crate) fn maybe_new(url: Option<String>) -> Option<Self> {
url.map(|url| {
info!(target: "reth-bench", %url, "Prometheus metrics scraping enabled");
let client = Client::builder()
.timeout(Duration::from_secs(5))
.build()
.expect("failed to build reqwest client");
Self { url, client, rows: Vec::new() }
})
}
/// Scrapes the metrics endpoint and records values for the given block.
pub(crate) async fn scrape_after_block(&mut self, block_number: u64) -> eyre::Result<()> {
let text = self
.client
.get(&self.url)
.send()
.await
.wrap_err("failed to fetch metrics endpoint")?
.error_for_status()
.wrap_err("metrics endpoint returned error status")?
.text()
.await
.wrap_err("failed to read metrics response body")?;
let execution = parse_gauge(&text, "sync_execution_execution_duration");
let state_root = parse_gauge(&text, "sync_block_validation_state_root_duration");
self.rows.push(MetricsRow {
block_number,
execution_duration_secs: execution,
state_root_duration_secs: state_root,
});
Ok(())
}
/// Writes collected metrics to a CSV file in the output directory.
pub(crate) fn write_csv(&self, output_dir: &Path) -> eyre::Result<()> {
let path = output_dir.join(METRICS_OUTPUT_SUFFIX);
info!(target: "reth-bench", "Writing scraped metrics to file: {:?}", path);
let mut writer = Writer::from_path(&path)?;
for row in &self.rows {
writer.serialize(row)?;
}
writer.flush()?;
Ok(())
}
}
/// Parses a Prometheus gauge value from exposition-format text.
///
/// Searches for lines starting with `name` followed by either a space or `{`
/// (for labeled metrics), then parses the numeric value. Returns the last
/// matching sample to handle metrics emitted with multiple label sets.
fn parse_gauge(text: &str, name: &str) -> Option<f64> {
let mut result = None;
for line in text.lines() {
let line = line.trim();
if line.is_empty() || line.starts_with('#') {
continue;
}
if !line.starts_with(name) {
continue;
}
// Ensure we match the full metric name, not a prefix of another metric.
let rest = &line[name.len()..];
if !rest.starts_with(' ') && !rest.starts_with('{') {
continue;
}
// Format: `metric_name{labels} value [timestamp]` or `metric_name value [timestamp]`
// Value is always the second whitespace-separated token.
let mut parts = line.split_whitespace();
if let Some(value_str) = parts.nth(1) &&
let Ok(v) = value_str.parse::<f64>()
{
result = Some(v);
}
}
result
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gauge_simple() {
let text = r#"# HELP sync_execution_execution_duration Duration of execution
# TYPE sync_execution_execution_duration gauge
sync_execution_execution_duration 0.123456
"#;
assert_eq!(parse_gauge(text, "sync_execution_execution_duration"), Some(0.123456));
}
#[test]
fn test_parse_gauge_missing() {
let text = "some_other_metric 1.0\n";
assert_eq!(parse_gauge(text, "sync_execution_execution_duration"), None);
}
#[test]
fn test_parse_gauge_with_labels() {
let text = "sync_block_validation_state_root_duration{instance=\"node1\"} 0.5\n";
assert_eq!(parse_gauge(text, "sync_block_validation_state_root_duration"), Some(0.5));
}
#[test]
fn test_parse_gauge_prefix_no_false_match() {
let text =
"sync_execution_execution_duration_total 99.0\nsync_execution_execution_duration 0.5\n";
assert_eq!(parse_gauge(text, "sync_execution_execution_duration"), Some(0.5));
}
}

View File

@@ -12,6 +12,7 @@ pub(crate) mod helpers;
pub use generate_big_block::{
RawTransaction, RpcTransactionSource, TransactionCollector, TransactionSource,
};
pub(crate) mod metrics_scraper;
mod new_payload_fcu;
mod new_payload_only;
mod output;

View File

@@ -13,6 +13,7 @@ use crate::{
bench::{
context::BenchContext,
helpers::parse_duration,
metrics_scraper::MetricsScraper,
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
@@ -20,9 +21,11 @@ use crate::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
valid_payload::{
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
},
};
use alloy_provider::Provider;
use alloy_provider::{ext::DebugApi, Provider};
use alloy_rpc_types_engine::ForkchoiceState;
use clap::Parser;
use eyre::{Context, OptionExt};
@@ -30,7 +33,7 @@ use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use tracing::{debug, info, warn};
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -150,9 +153,18 @@ impl Command {
auth_provider,
mut next_block,
is_optimism,
..
use_reth_namespace,
rlp_blocks,
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
let mut metrics_scraper = MetricsScraper::maybe_new(self.benchmark.metrics_url.clone());
if use_reth_namespace {
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
}
let buffer_size = self.rpc_block_buffer_size;
// Use a oneshot channel to propagate errors from the spawned task
@@ -175,6 +187,21 @@ impl Command {
}
};
let rlp = if rlp_blocks {
let rlp = match block_provider.debug_get_raw_block(next_block.into()).await {
Ok(rlp) => rlp,
Err(e) => {
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
let _ = error_sender
.send(eyre::eyre!("Failed to fetch raw block {next_block}: {e}"));
break;
}
};
Some(rlp)
} else {
None
};
let head_block_hash = block.header.hash;
let safe_block_hash = block_provider
.get_block_by_number(block.header.number.saturating_sub(32).into());
@@ -196,7 +223,7 @@ impl Command {
next_block += 1;
if let Err(e) = sender
.send((block, head_block_hash, safe_block_hash, finalized_block_hash))
.send((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
.await
{
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
@@ -206,10 +233,11 @@ impl Command {
});
let mut results = Vec::new();
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
while let Some((block, head, safe, finalized)) = {
while let Some((block, head, safe, finalized, rlp)) = {
let wait_start = Instant::now();
let result = receiver.recv().await;
total_wait_time += wait_start.elapsed();
@@ -228,16 +256,40 @@ impl Command {
finalized_block_hash: finalized,
};
let (version, params) = block_to_new_payload(block, is_optimism)?;
let (version, params) =
block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
let start = Instant::now();
call_new_payload(&auth_provider, version, params).await?;
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
let np_latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
let new_payload_result = NewPayloadResult {
gas_used,
latency: np_latency,
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
execution_cache_wait: server_timings
.as_ref()
.map(|t| t.execution_cache_wait)
.unwrap_or_default(),
sparse_trie_wait: server_timings
.as_ref()
.map(|t| t.sparse_trie_wait)
.unwrap_or_default(),
};
call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?;
let fcu_start = Instant::now();
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
let fcu_latency = fcu_start.elapsed();
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let total_latency = if server_timings.is_some() {
// When using server-side latency for newPayload, derive total from the
// independently measured components to avoid mixing server-side and
// client-side (network-inclusive) timings.
np_latency + fcu_latency
} else {
start.elapsed()
};
let combined_result = CombinedResult {
block_number,
gas_limit,
@@ -249,8 +301,19 @@ impl Command {
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
// We want to measure engine throughput, not RPC fetch latency.
blocks_processed += 1;
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
info!(target: "reth-bench", %combined_result);
let progress = match total_blocks {
Some(total) => format!("{blocks_processed}/{total}"),
None => format!("{blocks_processed}"),
};
info!(target: "reth-bench", progress, %combined_result);
if let Some(scraper) = metrics_scraper.as_mut() &&
let Err(err) = scraper.scrape_after_block(block_number).await
{
warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
}
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
@@ -277,6 +340,10 @@ impl Command {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
if let (Some(path), Some(scraper)) = (&self.benchmark.output, &metrics_scraper) {
scraper.write_csv(path)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;

View File

@@ -3,14 +3,15 @@
use crate::{
bench::{
context::BenchContext,
metrics_scraper::MetricsScraper,
output::{
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
NEW_PAYLOAD_OUTPUT_SUFFIX,
},
},
valid_payload::{block_to_new_payload, call_new_payload},
valid_payload::{block_to_new_payload, call_new_payload_with_reth},
};
use alloy_provider::Provider;
use alloy_provider::{ext::DebugApi, Provider};
use clap::Parser;
use csv::Writer;
use eyre::{Context, OptionExt};
@@ -49,9 +50,18 @@ impl Command {
auth_provider,
mut next_block,
is_optimism,
..
use_reth_namespace,
rlp_blocks,
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
let mut metrics_scraper = MetricsScraper::maybe_new(self.benchmark.metrics_url.clone());
if use_reth_namespace {
info!("Using reth_newPayload endpoint");
}
let buffer_size = self.rpc_block_buffer_size;
// Use a oneshot channel to propagate errors from the spawned task
@@ -74,20 +84,33 @@ impl Command {
}
};
let rlp = if rlp_blocks {
let Ok(rlp) = block_provider.debug_get_raw_block(next_block.into()).await
else {
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}");
let _ = error_sender
.send(eyre::eyre!("Failed to fetch raw block {next_block}"));
break;
};
Some(rlp)
} else {
None
};
next_block += 1;
if let Err(e) = sender.send(block).await {
if let Err(e) = sender.send((block, rlp)).await {
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
break;
}
}
});
// put results in a summary vec so they can be printed at the end
let mut results = Vec::new();
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
while let Some(block) = {
while let Some((block, rlp)) = {
let wait_start = Instant::now();
let result = receiver.recv().await;
total_wait_time += wait_start.elapsed();
@@ -99,13 +122,34 @@ impl Command {
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
let (version, params) = block_to_new_payload(block, is_optimism)?;
let (version, params) =
block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
let start = Instant::now();
call_new_payload(&auth_provider, version, params).await?;
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
info!(target: "reth-bench", %new_payload_result);
let latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
let new_payload_result = NewPayloadResult {
gas_used,
latency,
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
execution_cache_wait: server_timings
.as_ref()
.map(|t| t.execution_cache_wait)
.unwrap_or_default(),
sparse_trie_wait: server_timings
.as_ref()
.map(|t| t.sparse_trie_wait)
.unwrap_or_default(),
};
blocks_processed += 1;
let progress = match total_blocks {
Some(total) => format!("{blocks_processed}/{total}"),
None => format!("{blocks_processed}"),
};
info!(target: "reth-bench", progress, %new_payload_result);
// current duration since the start of the benchmark minus the time
// waiting for blocks
@@ -115,6 +159,12 @@ impl Command {
let row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((row, new_payload_result));
if let Some(scraper) = metrics_scraper.as_mut() &&
let Err(err) = scraper.scrape_after_block(block_number).await
{
tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
}
}
// Check if the spawned task encountered an error
@@ -145,6 +195,10 @@ impl Command {
}
writer.flush()?;
if let Some(scraper) = &metrics_scraper {
scraper.write_csv(&path)?;
}
info!(target: "reth-bench", "Finished writing benchmark output files to {:?}.", path);
}

View File

@@ -22,7 +22,10 @@ pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
#[derive(Debug, Serialize, Deserialize)]
pub(crate) struct GasRampPayloadFile {
/// Engine API version (1-5).
pub(crate) version: u8,
///
/// `None` indicates that `reth_newPayload` should be used.
#[serde(skip_serializing_if = "Option::is_none")]
pub(crate) version: Option<u8>,
/// The block hash for FCU.
pub(crate) block_hash: B256,
/// The params to pass to newPayload.
@@ -37,6 +40,12 @@ pub(crate) struct NewPayloadResult {
pub(crate) gas_used: u64,
/// The latency of the `newPayload` call.
pub(crate) latency: Duration,
/// Time spent waiting for persistence. `None` when no persistence was in-flight.
pub(crate) persistence_wait: Option<Duration>,
/// Time spent waiting for execution cache lock.
pub(crate) execution_cache_wait: Duration,
/// Time spent waiting for sparse trie lock.
pub(crate) sparse_trie_wait: Duration,
}
impl NewPayloadResult {
@@ -67,9 +76,12 @@ impl Serialize for NewPayloadResult {
{
// convert the time to microseconds
let time = self.latency.as_micros();
let mut state = serializer.serialize_struct("NewPayloadResult", 2)?;
let mut state = serializer.serialize_struct("NewPayloadResult", 5)?;
state.serialize_field("gas_used", &self.gas_used)?;
state.serialize_field("latency", &time)?;
state.serialize_field("persistence_wait", &self.persistence_wait.map(|d| d.as_micros()))?;
state.serialize_field("execution_cache_wait", &self.execution_cache_wait.as_micros())?;
state.serialize_field("sparse_trie_wait", &self.sparse_trie_wait.as_micros())?;
state.end()
}
}
@@ -126,7 +138,7 @@ impl Serialize for CombinedResult {
let fcu_latency = self.fcu_latency.as_micros();
let new_payload_latency = self.new_payload_result.latency.as_micros();
let total_latency = self.total_latency.as_micros();
let mut state = serializer.serialize_struct("CombinedResult", 7)?;
let mut state = serializer.serialize_struct("CombinedResult", 10)?;
// flatten the new payload result because this is meant for CSV writing
state.serialize_field("block_number", &self.block_number)?;
@@ -136,6 +148,18 @@ impl Serialize for CombinedResult {
state.serialize_field("new_payload_latency", &new_payload_latency)?;
state.serialize_field("fcu_latency", &fcu_latency)?;
state.serialize_field("total_latency", &total_latency)?;
state.serialize_field(
"persistence_wait",
&self.new_payload_result.persistence_wait.map(|d| d.as_micros()),
)?;
state.serialize_field(
"execution_cache_wait",
&self.new_payload_result.execution_cache_wait.as_micros(),
)?;
state.serialize_field(
"sparse_trie_wait",
&self.new_payload_result.sparse_trie_wait.as_micros(),
)?;
state.end()
}
}

View File

@@ -160,12 +160,11 @@ impl PersistenceSubscription {
/// to pings.
pub(crate) async fn setup_persistence_subscription(
ws_url: Url,
persistence_timeout: Duration,
_persistence_timeout: Duration,
) -> eyre::Result<PersistenceSubscription> {
info!(target: "reth-bench", "Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect =
WsConnect::new(ws_url.to_string()).with_keepalive_interval(persistence_timeout);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;

View File

@@ -15,6 +15,7 @@ use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::parse_duration,
metrics_scraper::MetricsScraper,
output::{
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
TotalGasOutput, TotalGasRow,
@@ -23,17 +24,21 @@ use crate::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
},
},
valid_payload::{call_forkchoice_updated, call_new_payload},
valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
};
use alloy_primitives::B256;
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV4, ExecutionPayloadSidecar,
ForkchoiceState, JwtSecret, PraguePayloadFields,
};
use clap::Parser;
use eyre::Context;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_api::EngineApiMessageVersion;
use reth_rpc_api::RethNewPayloadInput;
use std::{
path::PathBuf,
time::{Duration, Instant},
@@ -124,6 +129,22 @@ pub struct Command {
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
ws_rpc_url: Option<String>,
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
///
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
/// directly, waits for persistence and cache updates to complete before processing,
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
#[arg(long, default_value = "false", verbatim_doc_comment)]
reth_new_payload: bool,
/// Optional Prometheus metrics endpoint to scrape after each block.
///
/// When provided, reth-bench will fetch metrics from this URL after each
/// payload, recording per-block execution and state root durations.
/// Results are written to `metrics.csv` in the output directory.
#[arg(long = "metrics-url", value_name = "URL", verbatim_doc_comment)]
metrics_url: Option<String>,
}
/// A loaded payload ready for execution.
@@ -141,7 +162,9 @@ struct GasRampPayload {
/// Block number from filename.
block_number: u64,
/// Engine API version for newPayload.
version: EngineApiMessageVersion,
///
/// `None` indicates that `reth_newPayload` should be used.
version: Option<EngineApiMessageVersion>,
/// The file contents.
file: GasRampPayloadFile,
}
@@ -163,6 +186,9 @@ impl Command {
self.persistence_threshold
);
}
if self.reth_new_payload {
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
}
// Set up waiter based on configured options
// When both are set: wait at least wait_time, and also wait for persistence if needed
@@ -190,6 +216,8 @@ impl Command {
(None, false) => None,
};
let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
@@ -248,14 +276,19 @@ impl Command {
"Executing gas ramp payload (newPayload + FCU)"
);
call_new_payload(&auth_provider, payload.version, payload.file.params.clone()).await?;
let _ = call_new_payload_with_reth(
&auth_provider,
payload.version,
payload.file.params.clone(),
)
.await?;
let fcu_state = ForkchoiceState {
head_block_hash: payload.file.block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
call_forkchoice_updated_with_reth(&auth_provider, payload.version, fcu_state).await?;
info!(target: "reth-bench", gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
@@ -303,20 +336,50 @@ impl Command {
"Sending newPayload"
);
let status = auth_provider
.new_payload_v4(
execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
let (version, params) = if self.reth_new_payload {
let reth_data = ExecutionData {
payload: execution_payload.clone().into(),
sidecar: ExecutionPayloadSidecar::v4(
CancunPayloadFields {
versioned_hashes: Vec::new(),
parent_beacon_block_root: B256::ZERO,
},
PraguePayloadFields {
requests: envelope.execution_requests.clone().into(),
},
),
};
(None, serde_json::to_value((RethNewPayloadInput::ExecutionData(reth_data),))?)
} else {
(
Some(EngineApiMessageVersion::V4),
serde_json::to_value((
execution_payload.clone(),
Vec::<B256>::new(),
B256::ZERO,
envelope.execution_requests.to_vec(),
))?,
)
.await?;
};
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let np_latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
let new_payload_result = NewPayloadResult {
gas_used,
latency: np_latency,
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
execution_cache_wait: server_timings
.as_ref()
.map(|t| t.execution_cache_wait)
.unwrap_or_default(),
sparse_trie_wait: server_timings
.as_ref()
.map(|t| t.sparse_trie_wait)
.unwrap_or_default(),
};
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
@@ -324,12 +387,12 @@ impl Command {
finalized_block_hash: parent_hash,
};
debug!(target: "reth-bench", method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_start = Instant::now();
call_forkchoice_updated_with_reth(&auth_provider, version, fcu_state).await?;
let fcu_latency = fcu_start.elapsed();
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let total_latency =
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
let combined_result = CombinedResult {
block_number,
@@ -341,7 +404,14 @@ impl Command {
};
let current_duration = total_benchmark_duration.elapsed();
info!(target: "reth-bench", %combined_result);
let progress = format!("{}/{}", i + 1, payloads.len());
info!(target: "reth-bench", progress, %combined_result);
if let Some(scraper) = metrics_scraper.as_mut() &&
let Err(err) = scraper.scrape_after_block(block_number).await
{
tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
}
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
@@ -351,7 +421,6 @@ impl Command {
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
debug!(target: "reth-bench", ?status, ?fcu_result, "Payload executed successfully");
parent_hash = block_hash;
}
@@ -366,6 +435,10 @@ impl Command {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
}
if let (Some(path), Some(scraper)) = (&self.output, &metrics_scraper) {
scraper.write_csv(path)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
info!(
@@ -476,13 +549,18 @@ impl Command {
let file: GasRampPayloadFile = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let version = match file.version {
1 => EngineApiMessageVersion::V1,
2 => EngineApiMessageVersion::V2,
3 => EngineApiMessageVersion::V3,
4 => EngineApiMessageVersion::V4,
5 => EngineApiMessageVersion::V5,
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
let version = if let Some(version) = file.version {
match version {
1 => EngineApiMessageVersion::V1,
2 => EngineApiMessageVersion::V2,
3 => EngineApiMessageVersion::V3,
4 => EngineApiMessageVersion::V4,
5 => EngineApiMessageVersion::V5,
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
}
.into()
} else {
None
};
info!(

View File

@@ -240,6 +240,7 @@ impl Command {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
@@ -254,6 +255,9 @@ impl Command {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
ExecutionPayload::V4(p) => {
p.payload_inner.payload_inner.payload_inner.block_hash
}
}
}
};
@@ -262,6 +266,9 @@ impl Command {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
ExecutionPayload::V4(p) => {
p.payload_inner.payload_inner.payload_inner.block_hash = new_hash
}
}
}

View File

@@ -20,6 +20,19 @@ impl BenchMode {
}
}
/// Returns the total number of blocks in the benchmark, if known.
///
/// For [`BenchMode::Range`] this is the length of the range.
/// For [`BenchMode::Continuous`] the total is unbounded, so `None` is returned.
pub const fn total_blocks(&self) -> Option<u64> {
match self {
Self::Continuous(_) => None,
Self::Range(range) => {
Some(range.end().saturating_sub(*range.start()).saturating_add(1))
}
}
}
/// Create a [`BenchMode`] from optional `from` and `to` fields.
pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
// If neither `--from` nor `--to` are provided, we will run the benchmark continuously,

View File

@@ -3,15 +3,18 @@
//! before sending additional calls.
use alloy_eips::eip7685::Requests;
use alloy_primitives::B256;
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
use alloy_rpc_types_engine::{
ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ForkchoiceState,
ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
ExecutionData, ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar,
ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
};
use alloy_transport::TransportResult;
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
use reth_node_api::EngineApiMessageVersion;
use reth_rpc_api::RethNewPayloadInput;
use serde::Deserialize;
use std::time::Duration;
use tracing::{debug, error};
/// An extension trait for providers that implement the engine API, to wait for a VALID response.
@@ -161,10 +164,21 @@ where
}
}
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
///
/// Returns `(version, versioned_params, execution_data)`.
pub(crate) fn block_to_new_payload(
block: AnyRpcBlock,
is_optimism: bool,
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
rlp: Option<Bytes>,
reth_new_payload: bool,
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
if let Some(rlp) = rlp {
return Ok((
None,
serde_json::to_value((RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),))?,
));
}
let block = block
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
@@ -176,17 +190,44 @@ pub(crate) fn block_to_new_payload(
// Convert to execution payload
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
let (version, params, execution_data) =
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)?;
if reth_new_payload {
Ok((None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?))
} else {
Ok((Some(version), params))
}
}
/// Converts an execution payload and sidecar into versioned engine API params and an
/// [`ExecutionData`].
///
/// Returns `(version, versioned_params, execution_data)`.
pub(crate) fn payload_to_new_payload(
payload: ExecutionPayload,
sidecar: ExecutionPayloadSidecar,
is_optimism: bool,
withdrawals_root: Option<B256>,
target_version: Option<EngineApiMessageVersion>,
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value, ExecutionData)> {
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
let (version, params) = match payload {
ExecutionPayload::V4(payload) => {
let cancun = sidecar.cancun().unwrap();
let prague = sidecar.prague().unwrap();
let requests = prague.requests.requests_hash();
(
EngineApiMessageVersion::V5,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
requests,
))?,
)
}
ExecutionPayload::V3(payload) => {
let cancun = sidecar.cancun().unwrap();
@@ -244,7 +285,7 @@ pub(crate) fn payload_to_new_payload(
}
};
Ok((version, params))
Ok((version, params, execution_data))
}
/// Calls the correct `engine_newPayload` method depending on the given [`ExecutionPayload`] and its
@@ -252,32 +293,85 @@ pub(crate) fn payload_to_new_payload(
///
/// # Panics
/// If the given payload is a V3 payload, but a parent beacon block root is provided as `None`.
#[allow(dead_code)]
pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
provider: P,
version: EngineApiMessageVersion,
version: Option<EngineApiMessageVersion>,
params: serde_json::Value,
) -> TransportResult<()> {
let method = version.method_name();
) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
call_new_payload_with_reth(provider, version, params).await
}
/// Response from `reth_newPayload` endpoint, which includes server-measured latency.
#[derive(Debug, Deserialize)]
struct RethPayloadStatus {
latency_us: u64,
#[serde(default)]
persistence_wait_us: Option<u64>,
#[serde(default)]
execution_cache_wait_us: u64,
#[serde(default)]
sparse_trie_wait_us: u64,
}
/// Server-side timing breakdown from `reth_newPayload` endpoint.
#[derive(Debug, Clone, Copy, Default)]
pub(crate) struct NewPayloadTimingBreakdown {
/// Server-side execution latency.
pub(crate) latency: Duration,
/// Time spent waiting for persistence. `None` when no persistence was in-flight.
pub(crate) persistence_wait: Option<Duration>,
/// Time spent waiting for execution cache lock.
pub(crate) execution_cache_wait: Duration,
/// Time spent waiting for sparse trie lock.
pub(crate) sparse_trie_wait: Duration,
}
/// Calls either `engine_newPayload*` or `reth_newPayload` depending on whether
/// `version` is provided.
///
/// When `version` is `None`, uses `reth_newPayload` endpoint with provided params.
///
/// Returns the server-reported timing breakdown when using the reth namespace, or `None` for
/// the standard engine namespace.
pub(crate) async fn call_new_payload_with_reth<N: Network, P: Provider<N>>(
provider: P,
version: Option<EngineApiMessageVersion>,
params: serde_json::Value,
) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
let method = version.map(|v| v.method_name()).unwrap_or("reth_newPayload");
debug!(target: "reth-bench", method, "Sending newPayload");
let mut status: PayloadStatus = provider.client().request(method, &params).await?;
let resp = loop {
let resp: serde_json::Value = provider.client().request(method, &params).await?;
let status = PayloadStatus::deserialize(&resp)?;
while !status.is_valid() {
if status.is_valid() {
break resp;
}
if status.is_invalid() {
error!(target: "reth-bench", ?status, ?params, "Invalid {method}",);
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
format!("Invalid {method}: {status:?}"),
))))
return Err(eyre::eyre!("Invalid {method}: {status:?}"));
}
if status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
"invalid range: no canonical state found for parent of requested block",
))
return Err(eyre::eyre!(
"invalid range: no canonical state found for parent of requested block"
));
}
status = provider.client().request(method, &params).await?;
};
if version.is_some() {
return Ok(None);
}
Ok(())
let resp: RethPayloadStatus = serde_json::from_value(resp)?;
Ok(Some(NewPayloadTimingBreakdown {
latency: Duration::from_micros(resp.latency_us),
persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
}))
}
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
@@ -291,9 +385,12 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
forkchoice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
// FCU V3 is used for Cancun, Prague, and Amsterdam (there is no FCU V4-V6)
match message_version {
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 |
EngineApiMessageVersion::V6 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V2 => {
@@ -304,3 +401,47 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
}
}
}
/// Calls either `reth_forkchoiceUpdated` or the standard `engine_forkchoiceUpdated*` depending
/// on `use_reth`.
///
/// When `use_reth` is true, uses the `reth_forkchoiceUpdated` endpoint which sends a regular FCU
/// with no payload attributes.
pub(crate) async fn call_forkchoice_updated_with_reth<
N: Network,
P: Provider<N> + EngineApiValidWaitExt<N>,
>(
provider: P,
message_version: Option<EngineApiMessageVersion>,
forkchoice_state: ForkchoiceState,
) -> TransportResult<ForkchoiceUpdated> {
if let Some(message_version) = message_version {
call_forkchoice_updated(provider, message_version, forkchoice_state, None).await
} else {
let method = "reth_forkchoiceUpdated";
let reth_params = serde_json::to_value((forkchoice_state,))
.expect("ForkchoiceState serialization cannot fail");
debug!(target: "reth-bench", method, "Sending forkchoiceUpdated");
loop {
let resp: ForkchoiceUpdated = provider.client().request(method, &reth_params).await?;
if resp.is_valid() {
break Ok(resp)
}
if resp.is_invalid() {
error!(target: "reth-bench", ?resp, "Invalid {method}");
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
std::io::Error::other(format!("Invalid {method}: {resp:?}")),
)))
}
if resp.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
"invalid range: no canonical state found for parent of requested block",
))
}
}
}
}

View File

@@ -30,7 +30,8 @@ workspace = true
# reth
reth-ethereum-cli.workspace = true
reth-chainspec.workspace = true
reth-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-ethereum-primitives.workspace = true
reth-db = { workspace = true, features = ["mdbx"] }
reth-provider.workspace = true
reth-revm.workspace = true
@@ -69,7 +70,7 @@ aquamarine.workspace = true
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
alloy-node-bindings = "1.6.3"
alloy-node-bindings = "1.5.2"
alloy-provider = { workspace = true, features = ["reqwest"] }
alloy-rpc-types-eth.workspace = true
backon.workspace = true
@@ -110,7 +111,6 @@ dev = ["reth-ethereum-cli/dev"]
asm-keccak = [
"reth-node-core/asm-keccak",
"reth-primitives/asm-keccak",
"reth-ethereum-cli/asm-keccak",
"reth-node-ethereum/asm-keccak",
"alloy-primitives/asm-keccak",
@@ -190,6 +190,7 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"]
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
edge = ["rocksdb"]

View File

@@ -124,9 +124,11 @@ pub mod providers {
pub use reth_provider::*;
}
/// Re-exported from `reth_primitives`.
/// Re-exported primitives.
#[allow(ambiguous_glob_reexports)]
pub mod primitives {
pub use reth_primitives::*;
pub use reth_ethereum_primitives::*;
pub use reth_primitives_traits::*;
}
/// Re-exported from `reth_ethereum_consensus`.

View File

@@ -772,6 +772,7 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
block_access_list: Default::default(),
},
state: Default::default(),
}),

View File

@@ -212,6 +212,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
block_access_list: Default::default(),
},
state: BundleState::default(),
}),

View File

@@ -132,6 +132,6 @@ impl<H: BlockHeader> EthChainSpec for ChainSpec<H> {
}
fn final_paris_total_difficulty(&self) -> Option<U256> {
self.paris_block_and_final_difficulty.map(|(_, final_difficulty)| final_difficulty)
self.get_final_paris_total_difficulty()
}
}

View File

@@ -28,7 +28,7 @@ use alloy_consensus::{
};
use alloy_eips::{
eip1559::INITIAL_BASE_FEE, eip7685::EMPTY_REQUESTS_HASH, eip7840::BlobParams,
eip7892::BlobScheduleBlobParams,
eip7892::BlobScheduleBlobParams, eip7928::EMPTY_BLOCK_ACCESS_LIST_HASH,
};
use alloy_genesis::{ChainConfig, Genesis};
use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256};
@@ -76,6 +76,18 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
.active_at_timestamp(genesis.timestamp)
.then_some(EMPTY_REQUESTS_HASH);
// If Amsterdam is activated at genesis we set block access list hash to empty hash.
let block_access_list_hash = hardforks
.fork(EthereumHardfork::Amsterdam)
.active_at_timestamp(genesis.timestamp)
.then_some(EMPTY_BLOCK_ACCESS_LIST_HASH);
// If Amsterdam is activated at genesis we set slot number to 0.
let slot_number = hardforks
.fork(EthereumHardfork::Amsterdam)
.active_at_timestamp(genesis.timestamp)
.then_some(0);
Header {
number: genesis.number.unwrap_or_default(),
parent_hash: genesis.parent_hash.unwrap_or_default(),
@@ -93,6 +105,8 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
blob_gas_used,
excess_blob_gas,
requests_hash,
block_access_list_hash,
slot_number,
..Default::default()
}
}
@@ -298,6 +312,7 @@ pub fn create_chain_config(
cancun_time: timestamp(EthereumHardfork::Cancun),
prague_time: timestamp(EthereumHardfork::Prague),
osaka_time: timestamp(EthereumHardfork::Osaka),
amsterdam_time: timestamp(EthereumHardfork::Amsterdam),
bpo1_time: timestamp(EthereumHardfork::Bpo1),
bpo2_time: timestamp(EthereumHardfork::Bpo2),
bpo3_time: timestamp(EthereumHardfork::Bpo3),
@@ -855,15 +870,9 @@ impl From<Genesis> for ChainSpec {
// those networks we use the activation
// blocks of those networks
match genesis.config.chain_id {
1 => {
if ttd == MAINNET_PARIS_TTD {
return Some(MAINNET_PARIS_BLOCK)
}
}
11155111 => {
if ttd == SEPOLIA_PARIS_TTD {
return Some(SEPOLIA_PARIS_BLOCK)
}
1 if ttd == MAINNET_PARIS_TTD => return Some(MAINNET_PARIS_BLOCK),
11155111 if ttd == SEPOLIA_PARIS_TTD => {
return Some(SEPOLIA_PARIS_BLOCK)
}
_ => {}
};
@@ -886,6 +895,7 @@ impl From<Genesis> for ChainSpec {
(EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time),
(EthereumHardfork::Prague.boxed(), genesis.config.prague_time),
(EthereumHardfork::Osaka.boxed(), genesis.config.osaka_time),
(EthereumHardfork::Amsterdam.boxed(), genesis.config.amsterdam_time),
(EthereumHardfork::Bpo1.boxed(), genesis.config.bpo1_time),
(EthereumHardfork::Bpo2.boxed(), genesis.config.bpo2_time),
(EthereumHardfork::Bpo3.boxed(), genesis.config.bpo3_time),
@@ -1197,6 +1207,19 @@ impl ChainSpecBuilder {
self
}
/// Enable Amsterdam at genesis.
pub fn amsterdam_activated(mut self) -> Self {
self = self.osaka_activated();
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(0));
self
}
/// Enable Amsterdam at the given timestamp.
pub fn with_amsterdam_at(mut self, timestamp: u64) -> Self {
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(timestamp));
self
}
/// Build the resulting [`ChainSpec`].
///
/// # Panics

View File

@@ -18,6 +18,5 @@ alloy-genesis.workspace = true
# misc
clap.workspace = true
shellexpand.workspace = true
eyre.workspace = true
serde_json.workspace = true

View File

@@ -73,7 +73,7 @@ pub trait ChainSpecParser: Clone + Send + Sync + 'static {
/// A helper to parse a [`Genesis`](alloy_genesis::Genesis) as argument or from disk.
pub fn parse_genesis(s: &str) -> eyre::Result<alloy_genesis::Genesis> {
// try to read json from path first
let raw = match fs::read_to_string(PathBuf::from(shellexpand::full(s)?.into_owned())) {
let raw = match fs::read_to_string(PathBuf::from(s)) {
Ok(raw) => raw,
Err(io_err) => {
// valid json may start with "\n", but must contain "{"

View File

@@ -43,7 +43,7 @@ reth-node-metrics.workspace = true
reth-ethereum-primitives = { workspace = true, optional = true }
reth-provider.workspace = true
reth-prune.workspace = true
reth-prune-types = { workspace = true, optional = true }
reth-prune-types.workspace = true
reth-revm.workspace = true
reth-stages.workspace = true
reth-stages-types = { workspace = true, optional = true }
@@ -53,7 +53,7 @@ reth-tasks.workspace = true
reth-storage-api.workspace = true
reth-trie = { workspace = true, features = ["metrics"] }
reth-trie-db = { workspace = true, features = ["metrics"] }
reth-trie-common.workspace = true
reth-trie-common = { workspace = true, optional = true }
reth-primitives-traits.workspace = true
reth-discv4.workspace = true
reth-discv5.workspace = true
@@ -113,6 +113,7 @@ arbitrary = [
"dep:proptest",
"dep:arbitrary",
"dep:proptest-arbitrary-interop",
"dep:reth-trie-common",
"reth-db-api/arbitrary",
"reth-eth-wire/arbitrary",
"reth-db/arbitrary",
@@ -123,11 +124,11 @@ arbitrary = [
"reth-codecs/test-utils",
"reth-prune-types/test-utils",
"reth-stages-types/test-utils",
"reth-trie-common/test-utils",
"reth-trie-common?/test-utils",
"reth-codecs/arbitrary",
"reth-prune-types?/arbitrary",
"reth-prune-types/arbitrary",
"reth-stages-types?/arbitrary",
"reth-trie-common/arbitrary",
"reth-trie-common?/arbitrary",
"alloy-consensus/arbitrary",
"reth-primitives-traits/arbitrary",
"reth-ethereum-primitives/arbitrary",

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs, StorageArgs},
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs, StorageArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -67,74 +67,37 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Storage mode configuration (v2 vs v1/legacy)
#[command(flatten)]
pub storage: StorageArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from `--storage.v2`, static-file, and
/// `RocksDB` CLI args.
/// Returns the effective storage settings derived from `--storage.v2`.
///
/// The base storage mode is determined by `--storage.v2`:
/// - When `--storage.v2` is set: uses [`StorageSettings::v2()`] defaults
/// - Otherwise: uses [`StorageSettings::v1()`] defaults
///
/// Individual `--static-files.*` and `--rocksdb.*` flags override the base when explicitly set.
/// - Otherwise: uses [`StorageSettings::base()`] defaults
pub fn storage_settings(&self) -> StorageSettings {
let mut s = if self.storage.v2 { StorageSettings::v2() } else { StorageSettings::base() };
// Apply static files overrides (only when explicitly set)
if let Some(v) = self.static_files.receipts {
s = s.with_receipts_in_static_files(v);
if self.storage.v2 {
StorageSettings::v2()
} else {
StorageSettings::base()
}
if let Some(v) = self.static_files.transaction_senders {
s = s.with_transaction_senders_in_static_files(v);
}
if let Some(v) = self.static_files.account_changesets {
s = s.with_account_changesets_in_static_files(v);
}
if let Some(v) = self.static_files.storage_changesets {
s = s.with_storage_changesets_in_static_files(v);
}
// Apply rocksdb overrides
// --rocksdb.all sets all rocksdb flags to true
if self.rocksdb.all {
s = s
.with_transaction_hash_numbers_in_rocksdb(true)
.with_storages_history_in_rocksdb(true)
.with_account_history_in_rocksdb(true);
}
// Individual rocksdb flags override --rocksdb.all when explicitly set
if let Some(v) = self.rocksdb.tx_hash {
s = s.with_transaction_hash_numbers_in_rocksdb(v);
}
if let Some(v) = self.rocksdb.storages_history {
s = s.with_storages_history_in_rocksdb(v);
}
if let Some(v) = self.rocksdb.account_history {
s = s.with_account_history_in_rocksdb(v);
}
s
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
///
/// Internally builds a [`reth_tasks::Runtime`] attached to the current tokio handle for
/// parallel storage I/O.
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
/// The provided `runtime` is used for parallel storage I/O.
pub fn init<N: CliNodeTypes>(
&self,
access: AccessRights,
runtime: reth_tasks::Runtime,
) -> eyre::Result<Environment<N>>
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let runtime = reth_tasks::Runtime::with_existing_handle(tokio::runtime::Handle::current())?;
let data_dir = self.datadir.clone().resolve_datadir(self.chain.chain());
let db_path = data_dir.db();
let sf_path = data_dir.static_files();
@@ -183,11 +146,24 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let rocksdb_provider = if !access.is_read_write() && !RocksDBProvider::exists(&rocksdb_path)
{
// RocksDB database doesn't exist yet (e.g. datadir restored from a snapshot
// or created before RocksDB storage). Create an empty one so read-only
// commands can proceed.
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
reth_fs_util::create_dir_all(&rocksdb_path)?;
RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.build()?
} else {
RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?
};
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access, runtime)?;

View File

@@ -5,6 +5,7 @@ use reth_codecs::Compact;
use reth_db_api::{cursor::DbDupCursorRO, database::Database, tables, transaction::DbTx};
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_storage_api::StorageSettingsCache;
use std::time::{Duration, Instant};
use tracing::info;
@@ -22,52 +23,94 @@ impl Command {
/// Execute `db account-storage` command
pub fn execute<N: NodeTypesWithDB>(self, tool: &DbTool<N>) -> eyre::Result<()> {
let address = self.address;
let (slot_count, plain_size) = tool.provider_factory.db_ref().view(|tx| {
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut count = 0usize;
let mut total_value_bytes = 0usize;
let mut last_log = Instant::now();
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
// Walk all storage entries for this address
let walker = cursor.walk_dup(Some(address), None)?;
for entry in walker {
let (_, storage_entry) = entry?;
count += 1;
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
let mut buf = Vec::new();
let entry_len = storage_entry.to_compact(&mut buf);
total_value_bytes += entry_len;
let (slot_count, storage_size) = if use_hashed_state {
let hashed_address = keccak256(address);
tool.provider_factory.db_ref().view(|tx| {
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
let mut count = 0usize;
let mut total_value_bytes = 0usize;
let mut last_log = Instant::now();
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots = count,
key = %storage_entry.key,
"Processing storage slots"
);
last_log = Instant::now();
let walker = cursor.walk_dup(Some(hashed_address), None)?;
for entry in walker {
let (_, storage_entry) = entry?;
count += 1;
let mut buf = Vec::new();
let entry_len = storage_entry.to_compact(&mut buf);
total_value_bytes += entry_len;
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots = count,
key = %storage_entry.key,
"Processing hashed storage slots"
);
last_log = Instant::now();
}
}
}
// Add 20 bytes for the Address key (stored once per account in dupsort)
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
let total_size = if count > 0 { 32 + total_value_bytes } else { 0 };
Ok::<_, eyre::Report>((count, total_size))
})??;
Ok::<_, eyre::Report>((count, total_size))
})??
} else {
tool.provider_factory.db_ref().view(|tx| {
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut count = 0usize;
let mut total_value_bytes = 0usize;
let mut last_log = Instant::now();
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
let hashed_size_estimate = if slot_count > 0 { plain_size + 12 } else { 0 };
let total_estimate = plain_size + hashed_size_estimate;
// Walk all storage entries for this address
let walker = cursor.walk_dup(Some(address), None)?;
for entry in walker {
let (_, storage_entry) = entry?;
count += 1;
let mut buf = Vec::new();
// StorageEntry encodes as: 32 bytes (key/subkey uncompressed) + compressed U256
let entry_len = storage_entry.to_compact(&mut buf);
total_value_bytes += entry_len;
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots = count,
key = %storage_entry.key,
"Processing storage slots"
);
last_log = Instant::now();
}
}
// Add 20 bytes for the Address key (stored once per account in dupsort)
let total_size = if count > 0 { 20 + total_value_bytes } else { 0 };
Ok::<_, eyre::Report>((count, total_size))
})??
};
let hashed_address = keccak256(address);
println!("Account: {address}");
println!("Hashed address: {hashed_address}");
println!("Storage slots: {slot_count}");
println!("Plain storage size: {} (estimated)", human_bytes(plain_size as f64));
println!("Hashed storage size: {} (estimated)", human_bytes(hashed_size_estimate as f64));
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
if use_hashed_state {
println!("Hashed storage size: {} (estimated)", human_bytes(storage_size as f64));
} else {
// Estimate hashed storage size: 32-byte B256 key instead of 20-byte Address
let hashed_size_estimate = if slot_count > 0 { storage_size + 12 } else { 0 };
let total_estimate = storage_size + hashed_size_estimate;
println!("Plain storage size: {} (estimated)", human_bytes(storage_size as f64));
println!(
"Hashed storage size: {} (estimated)",
human_bytes(hashed_size_estimate as f64)
);
println!("Total estimated size: {}", human_bytes(total_estimate as f64));
}
Ok(())
}

View File

@@ -14,7 +14,7 @@ use reth_db_api::{
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_static_file_types::{ChangesetOffset, StaticFileSegment};
use std::{
hash::{BuildHasher, Hasher},
time::{Duration, Instant},
@@ -134,12 +134,12 @@ fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let mut checksummer = Checksummer::new(checksum_hasher(), limit);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
let is_change_based = segment.is_change_based();
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
@@ -149,7 +149,8 @@ fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
let mut reached_limit = false;
for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
@@ -167,28 +168,42 @@ fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
if is_change_based {
let offsets = jar_provider.read_changeset_offsets()?.ok_or_else(|| {
eyre::eyre!(
"Missing changeset offsets sidecar for segment {} at range {}",
segment,
block_range
)
})?;
let input = ChangeBasedChecksumInput {
segment,
block_range_start: block_range.start(),
start_block,
end_block,
offsets: &offsets,
};
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
reached_limit = checksum_change_based_segment(&mut checksummer, input, &mut cursor)?;
} else {
while let Some(row) = cursor.next_row()? {
if checksummer.write_row(&row) {
reached_limit = true;
break;
}
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
if reached_limit {
break;
}
}
let checksum = hasher.finish();
let (checksum, total) = checksummer.finish();
let elapsed = start_time.elapsed();
info!(
@@ -267,7 +282,7 @@ impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N
total = index + 1;
if total >= limit {
break
break;
}
}
@@ -285,3 +300,139 @@ impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N
Ok((checksum, elapsed))
}
}
/// Accumulates a checksum over key-value entries, tracking count and limit.
struct Checksummer<H> {
hasher: H,
total: usize,
limit: usize,
}
impl<H: Hasher> Checksummer<H> {
fn new(hasher: H, limit: usize) -> Self {
Self { hasher, total: 0, limit }
}
/// Hash a row's columns (non-changeset segments). Returns `true` if the limit is reached.
fn write_row(&mut self, row: &[&[u8]]) -> bool {
for col in row {
self.hasher.write(col);
}
self.advance()
}
/// Hash a key + value as two separate writes, matching MDBX raw entry semantics.
/// Write boundaries matter: foldhash rotates its accumulator by `len` on each `write`.
fn write_entry(&mut self, key: &[u8], value: &[u8]) -> bool {
self.hasher.write(key);
self.hasher.write(value);
self.advance()
}
fn advance(&mut self) -> bool {
self.total += 1;
if self.total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {} entries.", self.total);
}
self.total >= self.limit
}
fn finish(self) -> (u64, usize) {
(self.hasher.finish(), self.total)
}
}
/// Reconstruct MDBX `StorageChangeSets` key/value boundaries from a static-file row.
///
/// MDBX layout:
/// - key: `BlockNumberAddress` => `[8B block_number][20B address]`
/// - value: `StorageEntry` => `[32B storage_key][compact U256 value]`
///
/// Static-file row layout for `StorageBeforeTx`:
/// - `[20B address][32B storage_key][compact U256 value]`
fn split_storage_changeset_row(block_number: u64, row: &[u8]) -> eyre::Result<([u8; 28], &[u8])> {
if row.len() < 20 {
return Err(eyre::eyre!(
"Storage changeset row too short: expected at least 20 bytes, got {}",
row.len()
));
}
let mut key_buf = [0u8; 28];
key_buf[..8].copy_from_slice(&block_number.to_be_bytes());
key_buf[8..].copy_from_slice(&row[..20]);
Ok((key_buf, &row[20..]))
}
struct ChangeBasedChecksumInput<'a> {
segment: StaticFileSegment,
block_range_start: u64,
start_block: u64,
end_block: u64,
offsets: &'a [ChangesetOffset],
}
fn checksum_change_based_segment<H: Hasher>(
checksummer: &mut Checksummer<H>,
input: ChangeBasedChecksumInput<'_>,
cursor: &mut reth_db::static_file::StaticFileCursor<'_>,
) -> eyre::Result<bool> {
let ChangeBasedChecksumInput { segment, block_range_start, start_block, end_block, offsets } =
input;
let is_storage = segment.is_storage_change_sets();
let mut reached_limit = false;
for (offset_index, offset) in offsets.iter().enumerate() {
let block_number = block_range_start + offset_index as u64;
let include = block_number >= start_block && block_number <= end_block;
for _ in 0..offset.num_changes() {
let row = cursor.next_row()?.ok_or_else(|| {
eyre::eyre!(
"Unexpected EOF while checksumming {} static file at range starting {}",
segment,
block_range_start
)
})?;
if !include {
continue;
}
// Reconstruct MDBX key/value write boundaries. foldhash rotates
// its accumulator by `len` on each write(), so boundaries must
// match exactly.
let done = if is_storage {
// StorageChangeSets: MDBX key = BlockNumberAddress (28B),
// value = compact StorageEntry. Column 0 is compact
// StorageBeforeTx = [20B address][32B key][compact U256].
let col = row[0];
let (key, value) = split_storage_changeset_row(block_number, col)?;
checksummer.write_entry(&key, value)
} else {
// AccountChangeSets: MDBX key = BlockNumber (8B),
// value = compact AccountBeforeTx (= column 0).
checksummer.write_entry(&block_number.to_be_bytes(), row[0])
};
if done {
reached_limit = true;
break;
}
}
if reached_limit {
break;
}
}
if !reached_limit && cursor.next_row()?.is_some() {
return Err(eyre::eyre!(
"Changeset offsets do not cover all rows for {} at range starting {}",
segment,
block_range_start
));
}
Ok(reached_limit)
}

View File

@@ -0,0 +1,61 @@
use clap::Parser;
use reth_db::mdbx::{self, ffi};
use std::path::PathBuf;
/// Copies the MDBX database to a new location.
///
/// Equivalent to the standalone `mdbx_copy` tool but bundled into reth.
#[derive(Parser, Debug)]
pub struct Command {
/// Destination path for the database copy.
dest: PathBuf,
/// Compact the database while copying (reclaims free space).
#[arg(short, long)]
compact: bool,
/// Force dynamic size for the destination database.
#[arg(short = 'd', long)]
force_dynamic_size: bool,
/// Throttle to avoid MVCC pressure on writers.
#[arg(short = 'p', long)]
throttle_mvcc: bool,
}
impl Command {
/// Execute `db copy` command
pub fn execute(self, db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
let mut flags: ffi::MDBX_copy_flags_t = ffi::MDBX_CP_DEFAULTS;
if self.compact {
flags |= ffi::MDBX_CP_COMPACT;
}
if self.force_dynamic_size {
flags |= ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
}
if self.throttle_mvcc {
flags |= ffi::MDBX_CP_THROTTLE_MVCC;
}
let dest = self
.dest
.to_str()
.ok_or_else(|| eyre::eyre!("destination path must be valid UTF-8"))?;
let dest_cstr = std::ffi::CString::new(dest)?;
println!("Copying database to {} ...", self.dest.display());
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
});
if rc != 0 {
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
});
}
println!("Done.");
Ok(())
}
}

View File

@@ -98,7 +98,8 @@ impl Command {
)?;
if let Some(entry) = entry {
println!("{}", serde_json::to_string_pretty(&entry)?);
let se: reth_primitives_traits::StorageEntry = entry;
println!("{}", serde_json::to_string_pretty(&se)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
@@ -106,7 +107,14 @@ impl Command {
}
let changesets = provider.storage_changeset(key.block_number())?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
let serializable: Vec<_> = changesets
.into_iter()
.map(|(addr, entry)| {
let se: reth_primitives_traits::StorageEntry = entry;
(addr, se)
})
.collect();
println!("{}", serde_json::to_string_pretty(&serializable)?);
return Ok(());
}

View File

@@ -12,11 +12,14 @@ use std::{
mod account_storage;
mod checksum;
mod clear;
mod copy;
mod diff;
mod get;
mod list;
mod prune_checkpoints;
mod repair_trie;
mod settings;
mod stage_checkpoints;
mod state;
mod static_file_header;
mod stats;
@@ -42,6 +45,8 @@ pub enum Subcommands {
List(list::Command),
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Copies the MDBX database to a new location (bundled mdbx_copy)
Copy(copy::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),
/// Gets the content of a table for the given key
@@ -64,6 +69,10 @@ pub enum Subcommands {
Path,
/// Manage storage settings
Settings(settings::Command),
/// View or set prune checkpoints
PruneCheckpoints(prune_checkpoints::Command),
// View or set stage checkpoints
StageCheckpoints(stage_checkpoints::Command),
/// Gets storage size information for an account
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
@@ -80,7 +89,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
/// provided command.
macro_rules! db_exec {
($env:expr, $tool:ident, $N:ident, $access_rights:expr, $command:block) => {
let Environment { provider_factory, .. } = $env.init::<$N>($access_rights)?;
let Environment { provider_factory, .. } =
$env.init::<$N>($access_rights, ctx.task_executor.clone())?;
let $tool = DbTool::new(provider_factory)?;
$command;
@@ -124,6 +134,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::Copy(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(tool.provider_factory.db_ref())?;
});
}
Subcommands::Diff(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(&tool)?;
@@ -196,6 +211,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::PruneCheckpoints(command) => {
db_exec!(self.env, tool, N, command.access_rights(), {
command.execute(&tool)?;
});
}
Subcommands::StageCheckpoints(command) => {
db_exec!(self.env, tool, N, command.access_rights(), {
command.execute(&tool)?;
});
}
Subcommands::AccountStorage(command) => {
db_exec!(self.env, tool, N, AccessRights::RO, {
command.execute(&tool)?;

View File

@@ -0,0 +1,221 @@
//! `reth db prune-checkpoints` command for viewing and setting prune checkpoint values.
use clap::{Args, Parser, Subcommand, ValueEnum};
use reth_db_common::DbTool;
use reth_provider::{providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory};
use reth_prune_types::{PruneCheckpoint, PruneMode, PruneSegment};
use reth_storage_api::{PruneCheckpointReader, PruneCheckpointWriter};
use crate::common::AccessRights;
/// `reth db prune-checkpoints` subcommand
#[derive(Debug, Parser)]
pub struct Command {
#[command(subcommand)]
command: Subcommands,
}
impl Command {
/// Returns database access rights required for the command.
pub fn access_rights(&self) -> AccessRights {
match &self.command {
Subcommands::Get { .. } => AccessRights::RO,
Subcommands::Set(_) => AccessRights::RW,
}
}
}
#[derive(Debug, Subcommand)]
enum Subcommands {
/// Get prune checkpoint(s) from database.
///
/// Shows the current prune progress for each segment, including the highest
/// pruned block/tx number and the active prune mode.
Get {
/// Specific segment to query. If omitted, shows all segments.
#[arg(long, value_enum)]
segment: Option<SegmentArg>,
},
/// Set a prune checkpoint for a segment.
///
/// WARNING: Manually setting checkpoints can cause data inconsistencies.
/// Only use this if you know what you're doing (e.g., recovering from a
/// corrupted checkpoint or forcing a re-prune from a specific block).
Set(SetArgs),
}
/// Arguments for the `set` subcommand
#[derive(Debug, Args)]
pub struct SetArgs {
/// The prune segment to update
#[arg(long, value_enum)]
segment: SegmentArg,
/// Highest pruned block number
#[arg(long)]
block_number: Option<u64>,
/// Highest pruned transaction number
#[arg(long)]
tx_number: Option<u64>,
/// Prune mode to write: full, distance, or before
#[arg(long, value_enum)]
mode: PruneModeArg,
/// Value for distance or before mode (required unless mode is full)
#[arg(long, required_if_eq_any([("mode", "distance"), ("mode", "before")]))]
mode_value: Option<u64>,
}
/// CLI-friendly prune segment names (excludes deprecated variants)
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "kebab-case")]
pub enum SegmentArg {
SenderRecovery,
TransactionLookup,
Receipts,
ContractLogs,
AccountHistory,
StorageHistory,
Bodies,
}
impl From<SegmentArg> for PruneSegment {
fn from(arg: SegmentArg) -> Self {
match arg {
SegmentArg::SenderRecovery => Self::SenderRecovery,
SegmentArg::TransactionLookup => Self::TransactionLookup,
SegmentArg::Receipts => Self::Receipts,
SegmentArg::ContractLogs => Self::ContractLogs,
SegmentArg::AccountHistory => Self::AccountHistory,
SegmentArg::StorageHistory => Self::StorageHistory,
SegmentArg::Bodies => Self::Bodies,
}
}
}
/// CLI-friendly prune mode
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "kebab-case")]
pub enum PruneModeArg {
/// Prune all blocks
Full,
/// Keep the last N blocks (requires --mode-value)
Distance,
/// Prune blocks before a specific block number (requires --mode-value)
Before,
}
impl Command {
/// Execute the command
pub fn execute<N: ProviderNodeTypes>(self, tool: &DbTool<N>) -> eyre::Result<()> {
match self.command {
Subcommands::Get { segment } => Self::get(tool, segment),
Subcommands::Set(args) => Self::set(tool, args),
}
}
fn get<N: ProviderNodeTypes>(
tool: &DbTool<N>,
segment: Option<SegmentArg>,
) -> eyre::Result<()> {
let provider = tool.provider_factory.provider()?;
match segment {
Some(seg) => {
let segment: PruneSegment = seg.into();
match provider.get_prune_checkpoint(segment)? {
Some(checkpoint) => print_checkpoint(segment, &checkpoint),
None => println!("No checkpoint found for {segment}"),
}
}
None => {
let mut checkpoints = provider.get_prune_checkpoints()?;
checkpoints.sort_by_key(|(seg, _)| *seg);
if checkpoints.is_empty() {
println!("No prune checkpoints found.");
} else {
println!(
"{:<25} {:>15} {:>15} {:>20}",
"Segment", "Block Number", "Tx Number", "Prune Mode"
);
println!("{}", "-".repeat(80));
for (segment, checkpoint) in &checkpoints {
println!(
"{:<25} {:>15} {:>15} {:>20}",
segment.to_string(),
fmt_opt(checkpoint.block_number),
fmt_opt(checkpoint.tx_number),
fmt_mode(&checkpoint.prune_mode),
);
}
}
}
}
Ok(())
}
fn set<N: ProviderNodeTypes>(tool: &DbTool<N>, args: SetArgs) -> eyre::Result<()> {
eyre::ensure!(
args.block_number.is_some() || args.tx_number.is_some(),
"at least one of --block-number or --tx-number must be provided"
);
let prune_mode = match args.mode {
PruneModeArg::Full => PruneMode::Full,
PruneModeArg::Distance => PruneMode::Distance(
args.mode_value
.ok_or_else(|| eyre::eyre!("--mode-value is required for distance mode"))?,
),
PruneModeArg::Before => PruneMode::Before(
args.mode_value
.ok_or_else(|| eyre::eyre!("--mode-value is required for before mode"))?,
),
};
let segment: PruneSegment = args.segment.into();
let checkpoint = PruneCheckpoint {
block_number: args.block_number,
tx_number: args.tx_number,
prune_mode,
};
let provider_rw = tool.provider_factory.database_provider_rw()?;
// Show previous value if any
if let Some(prev) = provider_rw.get_prune_checkpoint(segment)? {
println!("Previous checkpoint for {segment}:");
print_checkpoint(segment, &prev);
println!();
}
provider_rw.save_prune_checkpoint(segment, checkpoint)?;
provider_rw.commit()?;
println!("Updated checkpoint for {segment}:");
print_checkpoint(segment, &checkpoint);
Ok(())
}
}
fn print_checkpoint(segment: PruneSegment, checkpoint: &PruneCheckpoint) {
println!(" Segment: {segment}");
println!(" Block Number: {}", fmt_opt(checkpoint.block_number));
println!(" Tx Number: {}", fmt_opt(checkpoint.tx_number));
println!(" Prune Mode: {}", fmt_mode(&checkpoint.prune_mode));
}
fn fmt_opt(val: Option<u64>) -> String {
val.map_or("-".to_string(), |n| n.to_string())
}
fn fmt_mode(mode: &PruneMode) -> String {
match mode {
PruneMode::Full => "Full".to_string(),
PruneMode::Distance(d) => format!("Distance({d})"),
PruneMode::Before(b) => format!("Before({b})"),
}
}

View File

@@ -5,7 +5,6 @@ use reth_cli_util::parse_socket_address;
use reth_db_api::{
cursor::{DbCursorRO, DbCursorRW, DbDupCursorRO},
database::Database,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_db_common::DbTool;
@@ -21,13 +20,15 @@ use reth_node_metrics::{
};
use reth_provider::{providers::ProviderNodeTypes, ChainSpecProvider, StageCheckpointReader};
use reth_stages::StageId;
use reth_storage_api::StorageSettingsCache;
use reth_tasks::TaskExecutor;
use reth_trie::{
verify::{Output, Verifier},
Nibbles,
};
use reth_trie_common::{StorageTrieEntry, StoredNibbles, StoredNibblesSubKey};
use reth_trie_db::{DatabaseHashedCursorFactory, DatabaseTrieCursorFactory};
use reth_trie_db::{
DatabaseHashedCursorFactory, DatabaseTrieCursorFactory, StorageTrieEntryLike, TrieTableAdapter,
};
use std::{
net::SocketAddr,
time::{Duration, Instant},
@@ -116,9 +117,13 @@ fn verify_only<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()> {
let mut tx = db.tx()?;
tx.disable_long_read_transaction_safety();
reth_trie_db::with_adapter!(tool.provider_factory, |A| do_verify_only::<_, A>(&tx))
}
fn do_verify_only<TX: DbTx, A: TrieTableAdapter>(tx: &TX) -> eyre::Result<()> {
// Create the verifier
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(&tx);
let trie_cursor_factory = DatabaseTrieCursorFactory::new(&tx);
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx);
let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx);
let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?;
let metrics = RepairTrieMetrics::new();
@@ -209,17 +214,37 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
// Check that a pipeline sync isn't in progress.
verify_checkpoints(provider_rw.as_ref())?;
let inconsistent_nodes = reth_trie_db::with_adapter!(tool.provider_factory, |A| {
do_verify_and_repair::<_, A>(&mut provider_rw)?
});
if inconsistent_nodes == 0 {
info!("No inconsistencies found");
} else {
provider_rw.commit()?;
info!("Repaired {} inconsistencies and committed changes", inconsistent_nodes);
}
Ok(())
}
fn do_verify_and_repair<N: ProviderNodeTypes, A: TrieTableAdapter>(
provider_rw: &mut reth_provider::DatabaseProviderRW<N::DB, N>,
) -> eyre::Result<usize>
where
<N::DB as reth_db_api::database::Database>::TXMut: DbTxMut + DbTx,
{
// Create cursors for making modifications with
let tx = provider_rw.tx_mut();
tx.disable_long_read_transaction_safety();
let mut account_trie_cursor = tx.cursor_write::<tables::AccountsTrie>()?;
let mut storage_trie_cursor = tx.cursor_dup_write::<tables::StoragesTrie>()?;
let mut account_trie_cursor = tx.cursor_write::<A::AccountTrieTable>()?;
let mut storage_trie_cursor = tx.cursor_dup_write::<A::StorageTrieTable>()?;
// Create the cursor factories. These cannot accept the `&mut` tx above because they require it
// to be AsRef.
// Create the cursor factories. These cannot accept the `&mut` tx above because they
// require it to be AsRef.
let tx = provider_rw.tx_ref();
let hashed_cursor_factory = DatabaseHashedCursorFactory::new(tx);
let trie_cursor_factory = DatabaseTrieCursorFactory::new(tx);
let trie_cursor_factory = DatabaseTrieCursorFactory::<_, A>::new(tx);
// Create the verifier
let verifier = Verifier::new(&trie_cursor_factory, hashed_cursor_factory)?;
@@ -257,17 +282,17 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
match output {
Output::AccountExtra(path, _node) => {
// Extra account node in trie, remove it
let nibbles = StoredNibbles(path);
if account_trie_cursor.seek_exact(nibbles)?.is_some() {
let key: A::AccountKey = path.into();
if account_trie_cursor.seek_exact(key)?.is_some() {
account_trie_cursor.delete_current()?;
}
}
Output::StorageExtra(account, path, _node) => {
// Extra storage node in trie, remove it
let nibbles = StoredNibblesSubKey(path);
let subkey: A::StorageSubKey = path.into();
if storage_trie_cursor
.seek_by_key_subkey(account, nibbles.clone())?
.filter(|e| e.nibbles == nibbles)
.seek_by_key_subkey(account, subkey.clone())?
.filter(|e| *e.nibbles() == subkey)
.is_some()
{
storage_trie_cursor.delete_current()?;
@@ -276,19 +301,19 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
Output::AccountWrong { path, expected: node, .. } |
Output::AccountMissing(path, node) => {
// Wrong/missing account node value, upsert it
let nibbles = StoredNibbles(path);
account_trie_cursor.upsert(nibbles, &node)?;
let key: A::AccountKey = path.into();
account_trie_cursor.upsert(key, &node)?;
}
Output::StorageWrong { account, path, expected: node, .. } |
Output::StorageMissing(account, path, node) => {
// Wrong/missing storage node value, upsert it
// (We can't just use `upsert` method with a dup cursor, it's not properly
// supported)
let nibbles = StoredNibblesSubKey(path);
let entry = StorageTrieEntry { nibbles: nibbles.clone(), node };
let subkey: A::StorageSubKey = path.into();
let entry = A::StorageValue::new(subkey.clone(), node);
if storage_trie_cursor
.seek_by_key_subkey(account, nibbles.clone())?
.filter(|v| v.nibbles == nibbles)
.seek_by_key_subkey(account, subkey.clone())?
.filter(|v| *v.nibbles() == subkey)
.is_some()
{
storage_trie_cursor.delete_current()?;
@@ -304,14 +329,7 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
}
}
if inconsistent_nodes == 0 {
info!("No inconsistencies found");
} else {
provider_rw.commit()?;
info!("Repaired {} inconsistencies and committed changes", inconsistent_nodes);
}
Ok(())
Ok(inconsistent_nodes as usize)
}
/// Output progress information based on the last seen account path.

View File

@@ -39,50 +39,12 @@ enum Subcommands {
#[derive(Debug, Clone, Copy, Subcommand)]
#[clap(rename_all = "snake_case")]
pub enum SetCommand {
/// Store receipts in static files instead of the database
Receipts {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction senders in static files instead of the database
TransactionSenders {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account changesets in static files instead of the database
AccountChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage history in rocksdb instead of MDBX
StoragesHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction hash to number mapping in rocksdb instead of MDBX
TransactionHashNumbers {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account history in rocksdb instead of MDBX
AccountHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage changesets in static files instead of the database
StorageChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Use hashed state tables (HashedAccounts/HashedStorages) as canonical state
/// Enable or disable v2 storage layout
///
/// When enabled, execution writes directly to hashed tables, eliminating need for
/// separate hashing stages. State reads come from hashed tables.
///
/// WARNING: Changing this setting in either direction requires re-syncing the database.
/// Enabling on an existing plain-state database leaves hashed tables empty.
/// Disabling on an existing hashed-state database leaves plain tables empty.
UseHashedState {
/// When enabled, uses static files for receipts/senders/changesets and RocksDB for
/// history indices and transaction hashes. When disabled, uses v1/legacy layout (everything in
/// MDBX).
V2 {
#[clap(action(ArgAction::Set))]
value: bool,
},
@@ -125,87 +87,18 @@ impl Command {
println!("No storage settings found, creating new settings.");
}
let mut settings @ StorageSettings {
receipts_in_static_files: _,
transaction_senders_in_static_files: _,
storages_history_in_rocksdb: _,
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
storage_changesets_in_static_files: _,
use_hashed_state: _,
} = settings.unwrap_or_else(StorageSettings::v1);
let mut settings @ StorageSettings { storage_v2: _ } =
settings.unwrap_or_else(StorageSettings::v1);
// Update the setting based on the key
match cmd {
SetCommand::Receipts { value } => {
if settings.receipts_in_static_files == value {
println!("receipts_in_static_files is already set to {}", value);
SetCommand::V2 { value } => {
if settings.storage_v2 == value {
println!("storage_v2 is already set to {}", value);
return Ok(());
}
settings.receipts_in_static_files = value;
println!("Set receipts_in_static_files = {}", value);
}
SetCommand::TransactionSenders { value } => {
if settings.transaction_senders_in_static_files == value {
println!("transaction_senders_in_static_files is already set to {}", value);
return Ok(());
}
settings.transaction_senders_in_static_files = value;
println!("Set transaction_senders_in_static_files = {}", value);
}
SetCommand::AccountChangesets { value } => {
if settings.account_changesets_in_static_files == value {
println!("account_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.account_changesets_in_static_files = value;
println!("Set account_changesets_in_static_files = {}", value);
}
SetCommand::StoragesHistory { value } => {
if settings.storages_history_in_rocksdb == value {
println!("storages_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.storages_history_in_rocksdb = value;
println!("Set storages_history_in_rocksdb = {}", value);
}
SetCommand::TransactionHashNumbers { value } => {
if settings.transaction_hash_numbers_in_rocksdb == value {
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.transaction_hash_numbers_in_rocksdb = value;
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
}
SetCommand::AccountHistory { value } => {
if settings.account_history_in_rocksdb == value {
println!("account_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
SetCommand::StorageChangesets { value } => {
if settings.storage_changesets_in_static_files == value {
println!("storage_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.storage_changesets_in_static_files = value;
println!("Set storage_changesets_in_static_files = {}", value);
}
SetCommand::UseHashedState { value } => {
if settings.use_hashed_state == value {
println!("use_hashed_state is already set to {}", value);
return Ok(());
}
if settings.use_hashed_state && !value {
println!("WARNING: Disabling use_hashed_state on an existing hashed-state database requires a full resync.");
} else {
println!("WARNING: Enabling use_hashed_state on an existing plain-state database requires a full resync.");
}
settings.use_hashed_state = value;
println!("Set use_hashed_state = {}", value);
settings.storage_v2 = value;
println!("Set storage_v2 = {}", value);
}
}

View File

@@ -0,0 +1,297 @@
//! `reth db stage-checkpoints` command for viewing and setting stage checkpoint values.
use clap::{Args, Parser, Subcommand, ValueEnum};
use reth_db_common::DbTool;
use reth_provider::{
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
StageCheckpointWriter,
};
use reth_stages::StageId;
use crate::common::AccessRights;
/// `reth db stage-checkpoints` subcommand
#[derive(Debug, Parser)]
pub struct Command {
#[command(subcommand)]
command: Subcommands,
}
impl Command {
/// Returns database access rights required for the command.
pub fn access_rights(&self) -> AccessRights {
match &self.command {
Subcommands::Get { .. } => AccessRights::RO,
Subcommands::Set(_) => AccessRights::RW,
}
}
/// Execute the command
pub fn execute<N: ProviderNodeTypes>(self, tool: &DbTool<N>) -> eyre::Result<()> {
match self.command {
Subcommands::Get { stage } => Self::get(tool, stage),
Subcommands::Set(args) => Self::set(tool, args),
}
}
fn get<N: ProviderNodeTypes>(tool: &DbTool<N>, stage: Option<StageArg>) -> eyre::Result<()> {
let provider = tool.provider_factory.provider()?;
match stage {
Some(stage) => {
let stage_id = stage.into();
let checkpoint = provider.get_stage_checkpoint(stage_id)?;
println!("{stage_id}: {checkpoint:?}");
}
None => {
let mut checkpoints = provider.get_all_checkpoints()?;
checkpoints.sort_by(|a, b| a.0.cmp(&b.0));
for (stage, checkpoint) in checkpoints {
println!("{stage}: {checkpoint:?}");
}
}
}
Ok(())
}
fn set<N: ProviderNodeTypes>(tool: &DbTool<N>, args: SetArgs) -> eyre::Result<()> {
let stage_id: StageId = args.stage.into();
let provider_rw = tool.provider_factory.database_provider_rw()?;
let previous = provider_rw.get_stage_checkpoint(stage_id)?;
let mut checkpoint = previous.unwrap_or_default();
checkpoint.block_number = args.block_number;
if args.clear_stage_unit {
checkpoint.stage_checkpoint = None;
}
provider_rw.save_stage_checkpoint(stage_id, checkpoint)?;
provider_rw.commit()?;
println!("Updated checkpoint for {stage_id}: {checkpoint:?}");
Ok(())
}
}
#[derive(Debug, Subcommand)]
enum Subcommands {
/// Get stage checkpoint(s) from database.
Get {
/// Specific stage to query. If omitted, shows all stages.
#[arg(long, value_enum)]
stage: Option<StageArg>,
},
/// Set a stage checkpoint.
Set(SetArgs),
}
/// Arguments for the `set` subcommand.
#[derive(Debug, Args)]
pub struct SetArgs {
/// Stage to update.
#[arg(long, value_enum)]
stage: StageArg,
/// Block number to set as stage checkpoint.
#[arg(long)]
block_number: u64,
/// Clear stage-specific unit checkpoint payload.
#[arg(long)]
clear_stage_unit: bool,
}
/// CLI-friendly stage names.
#[derive(Debug, Clone, Copy, ValueEnum)]
#[clap(rename_all = "kebab-case")]
pub enum StageArg {
Era,
Headers,
Bodies,
SenderRecovery,
Execution,
PruneSenderRecovery,
MerkleUnwind,
AccountHashing,
StorageHashing,
MerkleExecute,
TransactionLookup,
IndexStorageHistory,
IndexAccountHistory,
Prune,
Finish,
}
impl From<StageArg> for StageId {
fn from(arg: StageArg) -> Self {
match arg {
StageArg::Era => Self::Era,
StageArg::Headers => Self::Headers,
StageArg::Bodies => Self::Bodies,
StageArg::SenderRecovery => Self::SenderRecovery,
StageArg::Execution => Self::Execution,
StageArg::PruneSenderRecovery => Self::PruneSenderRecovery,
StageArg::MerkleUnwind => Self::MerkleUnwind,
StageArg::AccountHashing => Self::AccountHashing,
StageArg::StorageHashing => Self::StorageHashing,
StageArg::MerkleExecute => Self::MerkleExecute,
StageArg::TransactionLookup => Self::TransactionLookup,
StageArg::IndexStorageHistory => Self::IndexStorageHistory,
StageArg::IndexAccountHistory => Self::IndexAccountHistory,
StageArg::Prune => Self::Prune,
StageArg::Finish => Self::Finish,
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use clap::Parser;
use reth_provider::{
test_utils::create_test_provider_factory, DBProvider, DatabaseProviderFactory,
StageCheckpointReader, StageCheckpointWriter,
};
use reth_stages::StageCheckpoint;
#[test]
fn parse_set_args() {
let command = Command::parse_from([
"stage-checkpoints",
"set",
"--stage",
"headers",
"--block-number",
"123",
]);
assert!(matches!(
command.command,
Subcommands::Set(SetArgs {
stage: StageArg::Headers,
block_number: 123,
clear_stage_unit: false,
})
));
}
#[test]
fn set_overwrites_block_number() {
let provider_factory = create_test_provider_factory();
let tool = DbTool::new(provider_factory.clone()).expect("db tool");
{
let provider_rw = provider_factory.database_provider_rw().expect("rw provider");
provider_rw
.save_stage_checkpoint(StageId::Headers, StageCheckpoint::new(10))
.expect("save checkpoint");
provider_rw.commit().expect("commit initial checkpoint");
}
let command = Command {
command: Subcommands::Set(SetArgs {
stage: StageArg::Headers,
block_number: 42,
clear_stage_unit: false,
}),
};
command.execute(&tool).expect("execute command");
let provider = provider_factory.provider().expect("provider");
let checkpoint = provider
.get_stage_checkpoint(StageId::Headers)
.expect("get stage checkpoint")
.expect("missing stage checkpoint");
assert_eq!(checkpoint.block_number, 42);
}
#[test]
fn set_preserves_stage_unit_checkpoint_unless_cleared() {
let provider_factory = create_test_provider_factory();
let tool = DbTool::new(provider_factory.clone()).expect("db tool");
{
let provider_rw = provider_factory.database_provider_rw().expect("rw provider");
let checkpoint = StageCheckpoint::new(10).with_block_range(&StageId::Execution, 5, 10);
provider_rw
.save_stage_checkpoint(StageId::Execution, checkpoint)
.expect("save checkpoint");
provider_rw.commit().expect("commit initial checkpoint");
}
Command {
command: Subcommands::Set(SetArgs {
stage: StageArg::Execution,
block_number: 11,
clear_stage_unit: false,
}),
}
.execute(&tool)
.expect("execute command");
let provider = provider_factory.provider().expect("provider");
let checkpoint = provider
.get_stage_checkpoint(StageId::Execution)
.expect("get stage checkpoint")
.expect("missing stage checkpoint");
assert!(checkpoint.stage_checkpoint.is_some());
Command {
command: Subcommands::Set(SetArgs {
stage: StageArg::Execution,
block_number: 12,
clear_stage_unit: true,
}),
}
.execute(&tool)
.expect("execute command");
let checkpoint = provider_factory
.provider()
.expect("provider")
.get_stage_checkpoint(StageId::Execution)
.expect("get stage checkpoint")
.expect("missing stage checkpoint");
assert!(checkpoint.stage_checkpoint.is_none());
}
#[test]
fn set_preserves_checkpoint_progress() {
let provider_factory = create_test_provider_factory();
let tool = DbTool::new(provider_factory.clone()).expect("db tool");
{
let provider_rw = provider_factory.database_provider_rw().expect("rw provider");
provider_rw
.save_stage_checkpoint(StageId::MerkleExecute, StageCheckpoint::new(10))
.expect("save checkpoint");
provider_rw
.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![1, 2, 3])
.expect("save progress");
provider_rw.commit().expect("commit initial checkpoint");
}
Command {
command: Subcommands::Set(SetArgs {
stage: StageArg::MerkleExecute,
block_number: 20,
clear_stage_unit: false,
}),
}
.execute(&tool)
.expect("execute command");
let provider = provider_factory.provider().expect("provider");
let progress = provider
.get_stage_checkpoint_progress(StageId::MerkleExecute)
.expect("get stage checkpoint progress");
assert_eq!(progress, Some(vec![1, 2, 3]));
}
}

View File

@@ -1,4 +1,4 @@
use alloy_primitives::{Address, BlockNumber, B256, U256};
use alloy_primitives::{keccak256, Address, BlockNumber, B256, U256};
use clap::Parser;
use parking_lot::Mutex;
use reth_db_api::{
@@ -63,39 +63,65 @@ impl Command {
address: Address,
limit: usize,
) -> eyre::Result<()> {
let use_hashed_state = tool.provider_factory.cached_storage_settings().use_hashed_state();
let entries = tool.provider_factory.db_ref().view(|tx| {
// Get account info
let account = tx.get::<tables::PlainAccountState>(address)?;
// Get storage entries
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
let walker = cursor.walk_dup(Some(address), None)?;
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
let (account, walker_entries) = if use_hashed_state {
let hashed_address = keccak256(address);
let account = tx.get::<tables::HashedAccounts>(hashed_address)?;
let mut cursor = tx.cursor_dup_read::<tables::HashedStorages>()?;
let walker = cursor.walk_dup(Some(hashed_address), None)?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
if entries.len() >= limit {
break;
(account, entries)
} else {
// Get account info
let account = tx.get::<tables::PlainAccountState>(address)?;
// Get storage entries
let mut cursor = tx.cursor_dup_read::<tables::PlainStorageState>()?;
let walker = cursor.walk_dup(Some(address), None)?;
let mut entries = Vec::new();
let mut last_log = Instant::now();
for (idx, entry) in walker.enumerate() {
let (_, storage_entry) = entry?;
if storage_entry.value != U256::ZERO {
entries.push((storage_entry.key, storage_entry.value));
}
if entries.len() >= limit {
break;
}
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
(account, entries)
};
if last_log.elapsed() >= LOG_INTERVAL {
info!(
target: "reth::cli",
address = %address,
slots_scanned = idx,
"Scanning storage slots"
);
last_log = Instant::now();
}
}
Ok::<_, eyre::Report>((account, entries))
Ok::<_, eyre::Report>((account, walker_entries))
})??;
let (account, storage_entries) = entries;
@@ -119,7 +145,7 @@ impl Command {
// Check storage settings to determine where history is stored
let storage_settings = tool.provider_factory.cached_storage_settings();
let history_in_rocksdb = storage_settings.storages_history_in_rocksdb;
let history_in_rocksdb = storage_settings.storage_v2;
// For historical queries, enumerate keys from history indices only
// (not PlainStorageState, which reflects current state)

Some files were not shown because too many files have changed in this diff Show More