Compare commits

...

236 Commits

Author SHA1 Message Date
yongkangc
6be57ce9e1 fix: address review nits 2026-01-22 12:26:42 +00:00
Georgios Konstantopoulos
b997bee71b fix: add backticks to remaining RocksDB in doc comments 2026-01-22 12:08:21 +00:00
Georgios Konstantopoulos
e3c7413747 fix: clippy warnings and add missing import for rocksdb provider 2026-01-22 12:08:19 +00:00
Georgios Konstantopoulos
0bf2335c8c fix(rocksdb): add assume_history_complete flag to fix test semantics
When testing RocksDB with identical data to MDBX, both backends should
return the same results. The MaybeInPlainState fallback (for hybrid
storage safety) was causing test failures because MDBX returns
NotYetWritten when querying before first history entry.

Added assume_history_complete flag to RocksTx that:
- When false (default): returns MaybeInPlainState for hybrid storage
- When true: returns NotYetWritten to match MDBX semantics for tests

This preserves the correct hybrid storage behavior in production while
allowing tests with identical data to verify semantic equivalence.
2026-01-22 12:08:16 +00:00
yongkangc
1f7a5acae1 fix(rocksdb): correct history index divergence causing nonce errors after unwind
Fixes RocksDB history indices diverging from MDBX after unwind operations,
causing 'max nonce mismatch' errors during block replay.

Root causes:
1. write_account_history/write_storage_history included all accounts/slots
   instead of only those with actual changes (account-info or storage value)
2. history_info in RocksTx returned NotYetWritten for missing entries instead
   of MaybeInPlainState, incorrectly treating pre-RocksDB accounts as new
3. History writes accumulated during save_blocks calls without deferred commit
4. Sentinel shards (highest_block_number=u64::MAX) not properly handled in
   invariant checks

Changes:
- Filter account history to only info-changed or destroyed accounts
- Filter storage history to only actually-changed slots
- Always return MaybeInPlainState from history_info for missing entries
- Add PendingHistoryWrites to accumulate history across save_blocks calls
- Commit pending history after MDBX (not before) via commit_pending_history
- Check sentinel shard contents when validating/pruning history
2026-01-22 12:08:14 +00:00
Matthias Seitz
5c3e45cd6b fix: handle incomplete receipts gracefully in receipt root task (#21285) 2026-01-22 10:52:56 +00:00
Emma Jamieson-Hoare
68fdba32d2 chore(release): prep v1.10.2 release (#21287)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-22 10:50:10 +00:00
Matthias Seitz
8f6a0a2992 ci: add on-demand workflow to check alloy breaking changes (#21267) 2026-01-22 10:47:38 +00:00
Matthias Seitz
ec9c7f8d3e perf(db): use ArrayVec for StoredNibbles key encoding (#21279) 2026-01-22 02:05:50 +00:00
Matthias Seitz
dbdaf068f0 fix(engine): clear execution cache when block validation fails (#21282) 2026-01-22 01:01:22 +00:00
Matthias Seitz
055bf63ee9 refactor: use Default::default() for Header in tests (#21277) 2026-01-21 22:50:10 +00:00
Georgios Konstantopoulos
2305c3ebeb feat(rpc): respect history expiry in block() and map to PrunedHistoryUnavailable (#21270) 2026-01-21 22:22:05 +00:00
joshieDo
eb55c3c3da feat(grafana): add RocksDB metrics dashboard (#21243)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 22:09:42 +00:00
Alexey Shekhirin
72e1467ba3 fix(prune): avoid panic in tx lookup (#21275) 2026-01-21 21:21:53 +00:00
Alexey Shekhirin
74edce0089 revert: feat(trie): add V2 account proof computation and refactor proof types (#21214) (#21274) 2026-01-21 21:07:13 +00:00
Georgios Konstantopoulos
8c645d5762 feat(reth-bench): accept short notation for --target-gas-limit (#21273) 2026-01-21 21:04:10 +00:00
Georgios Konstantopoulos
b7d2ee2566 feat(engine): add metric for execution cache unavailability due to concurrent use (#21265)
Co-authored-by: Tempo AI <ai@tempo.xyz>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-21 20:17:45 +00:00
Matthias Seitz
7609deddda perf(trie): parallelize merge_ancestors_into_overlay (#21202) 2026-01-21 20:08:03 +00:00
Matthias Seitz
ec50fd40b3 chore(chainspec): use ..Default::default() in create_chain_config (#21266) 2026-01-21 19:19:24 +00:00
YK
624ddc5779 feat(stages): add RocksDB support for IndexStorageHistoryStage (#21175) 2026-01-21 17:05:19 +00:00
Georgios Konstantopoulos
dd72cfe23e refactor: remove static_files.to_settings() and add edge feature to RocksDB flags (#21225) 2026-01-21 16:52:24 +00:00
joshieDo
ff8ac97e33 fix(stages): clear ETL collectors on HeaderStage error paths (#21258) 2026-01-21 16:27:30 +00:00
Alexey Shekhirin
0974485863 feat(reth-bench): add --target-gas-limit option to gas-limit-ramp (#21262) 2026-01-21 16:19:22 +00:00
かりんとう
274394e777 fix: fix payload file filter prefix in replay-payloads (#21255) 2026-01-21 16:11:03 +00:00
Emma Jamieson-Hoare
1954c91a60 chore: update CODEOWNERS (#21223)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-21 14:40:54 +00:00
Sergei Shulepov
9cf82c8403 fix: supply a real ptr to mdbx_dbi_flags_ex (#21230) 2026-01-21 14:23:26 +00:00
Brian Picciano
f85fcba872 feat(trie): add V2 account proof computation and refactor proof types (#21214)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 14:18:44 +00:00
joshieDo
ebaa4bda3a feat(rocksdb): add missing observability (#21253) 2026-01-21 14:14:34 +00:00
joshieDo
04d4c9a02f fix(rocksdb): flush all column families on drop and show SST/memtable sizes (#21251)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 12:44:08 +00:00
Arsenii Kulikov
3065a328f9 fix: clear overlay_cache in with_extended_hashed_state_overlay (#21233) 2026-01-21 12:08:24 +00:00
Sergei Shulepov
43a84f1231 refactor(engine): move execution logic from metrics to payload_validator (#21226) 2026-01-21 11:17:30 +00:00
Matthias Seitz
5a5c21cc1b feat(txpool): add IntoIterator for AllPoolTransactions (#21241) 2026-01-21 10:01:32 +00:00
Matthias Seitz
8a8a9126d6 feat(execution-types): add receipts_iter and logs_iter helpers to Chain (#21240) 2026-01-21 09:59:15 +00:00
Emilia Hane
6f73c2447d feat(trie): Add serde-bincode-compat feature to reth-trie (#21235) 2026-01-21 09:42:52 +00:00
Sergei Shulepov
2cae438642 fix: sigsegv handler (#21231)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-21 09:42:36 +00:00
Georgios Konstantopoulos
37b5db0d47 feat(cli): add RocksDB table stats to reth db stats command (#21221)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-21 08:45:17 +00:00
joshieDo
238433e146 fix(rocksdb): flush memtables before dropping (#21234) 2026-01-21 02:19:36 +00:00
Georgios Konstantopoulos
660964a0f5 feat(node): log storage settings after genesis init (#21229) 2026-01-21 00:58:23 +00:00
Matthias Seitz
22b465dd64 chore(trie): remove unnecessary clone in into_sorted_ref (#21232) 2026-01-20 22:57:08 +00:00
Georgios Konstantopoulos
3ff575b877 feat(engine): add --engine.disable-cache-metrics flag (#21228)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-20 22:03:12 +00:00
かりんとう
d12752dc8a feat(engine): add time_between_forkchoice_updated metric (#21227) 2026-01-20 21:06:11 +00:00
Georgios Konstantopoulos
869b5d0851 feat(edge): enable transaction_hash_numbers_in_rocksdb for edge builds (#21224)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 20:02:02 +00:00
Georgios Konstantopoulos
78de3d8f61 perf(db): use Cow::Borrowed in walk_dup to avoid allocation (#21220) 2026-01-20 19:31:50 +00:00
YK
bc79cc44c9 feat(cli): add --rocksdb.* flags for RocksDB table routing (#21191) 2026-01-20 19:29:05 +00:00
Georgios Konstantopoulos
ff8f434dcd feat(cli): add reth db checksum rocksdb command (#21217)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 19:10:34 +00:00
Arsenii Kulikov
9662dc5271 fix: properly save history indices in pipeline (#21222) 2026-01-20 18:20:28 +00:00
Alexey Shekhirin
3ba37082dc fix(reth-bench): replay-payloads prefix (#21219) 2026-01-20 18:36:35 +01:00
Ahsen Kamal
7934294988 perf(trie): dispatch storage proofs in lexicographical order (#21213)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
2026-01-20 17:09:20 +00:00
Georgios Konstantopoulos
7371bd3f29 chore(db-api): remove sharded_key_encode benchmark (#21215)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 17:01:12 +00:00
Georgios Konstantopoulos
80980b8e4d feat(pruning): add DefaultPruningValues for overridable pruning defaults (#21207)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-20 16:58:29 +00:00
Matthias Seitz
2e2cd67663 perf(chain-state): parallelize into_sorted with rayon (#21193) 2026-01-20 16:42:16 +00:00
Georgios Konstantopoulos
4f009728e2 feat(cli): add reth db checksum mdbx/static-file command (#21211)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:11:51 +00:00
Georgios Konstantopoulos
39d5ae73e8 feat(storage): add read-only mode for RocksDB provider (#21210)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:09:51 +00:00
Georgios Konstantopoulos
5ef200eaad perf(db): stack-allocate ShardedKey and StorageShardedKey encoding (#21200)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 15:58:43 +00:00
ethfanWilliam
d002dacc13 chore: remove deprecated and unused ExecuteOutput struct (#20887)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 15:06:26 +00:00
Alexey Shekhirin
bb39cba504 ci: partition bench codspeed job (#20332)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 14:29:48 +00:00
YK
bd144a4c42 feat(stages): add RocksDB support for IndexAccountHistoryStage (#21165)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 14:23:29 +00:00
tonis
a0845bab18 feat: Check CL/Reth capability compatibility (#20348)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 14:19:31 +00:00
Brian Picciano
346cc0da71 feat(trie): add AsyncAccountValueEncoder for V2 proof computation (#21197)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:50:29 +00:00
Matthias Seitz
ea3d4663ae perf(trie): use HashMap reserve heuristic in MultiProof::extend (#21199) 2026-01-20 13:34:41 +00:00
Hwangjae Lee
3667d3b5aa perf(trie): defer child RLP conversion in proof_v2 for async encoder support (#20873)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-20 13:33:08 +00:00
Brian Picciano
7cfb19c98e feat(trie): Add V2 reveal method and target types (#21196)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:25:54 +00:00
joshieDo
5a38871489 fix: set StaticFileArgs defaults for edge (#21208) 2026-01-20 12:39:36 +00:00
Brian Picciano
c825c8c187 chore(trie): Move hybrid check for trie input merges into common code (#21198) 2026-01-20 12:38:46 +00:00
Matthias Seitz
8f37cd08fc feat(engine-api): add EIP-7928 BAL stub methods (#21204) 2026-01-20 11:33:27 +00:00
Matthias Seitz
c9dad4765d chore: bump version to 1.10.1 (#21188) 2026-01-19 14:04:08 +00:00
Dan Cline
1d55abeef3 chore: rename extend_ref methods on sorted data structures (#21043) 2026-01-19 13:04:57 +00:00
Niven
f7460e219c fix(flashblocks): Add flashblock ws connection retry period (#20510) 2026-01-19 12:01:33 +00:00
Georgios Konstantopoulos
0c66315f20 chore(bench): add --disable-tx-gossip to benchmark node args (#21171) 2026-01-19 11:45:56 +00:00
MozirDmitriy
6a2010e595 refactor(stages): reuse history index cache buffers in collect_history_indices (#21017) 2026-01-19 11:39:52 +00:00
Georgios Konstantopoulos
c2435ff6f8 feat(download): resumable snapshot downloads with auto-retry (#21161) 2026-01-19 10:26:24 +00:00
DaniPopes
52ec8e9491 ci: update to tempoxyz (#21176) 2026-01-19 10:21:37 +00:00
Georgios Konstantopoulos
a901d80ee6 chore: apply spelling and typo fixes (#21182) 2026-01-19 10:21:25 +00:00
MoNyAvA
915164078f docs: document minimal storage mode in pruning FAQ (#21025)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-19 10:27:45 +01:00
github-actions[bot]
be3234d848 chore(deps): weekly cargo update (#21167)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-18 14:57:20 +00:00
Matthias Seitz
f624372334 feat(execution-types): add receipts_iter helper (#21162)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-17 19:20:28 +01:00
Matthias Seitz
40bc9d3860 revert: undo Chain crate, add LazyTrieData to trie-common (#21155) 2026-01-17 15:57:09 +00:00
Georgios Konstantopoulos
1ea574417f feat(engine): add new_payload_interval metric (start-to-start) (#21159) 2026-01-17 12:15:45 +00:00
Georgios Konstantopoulos
27e055f790 feat(engine): add time_between_new_payloads metric (#21158) 2026-01-17 10:20:22 +00:00
Georgios Konstantopoulos
d5dc0b27eb fix(storage-api): gate reth-chain dependency behind std feature
The reth-chain crate is inherently std-only (uses BTreeMap, Arc, etc.)
and was breaking the riscv32imac no_std builds by pulling in serde_core
which doesn't support no_std properly.

This makes reth-chain optional and only enables it when std feature is
active, gating the block_writer module that uses Chain behind std.
2026-01-17 08:32:10 +00:00
Georgios Konstantopoulos
c11c13000f perf(storage): batch trie updates across blocks in save_blocks (#21142)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-17 07:15:40 +00:00
Matthias Seitz
6bf43ab24a refactor: use ExecutionOutcome::single instead of tuple From (#21152) 2026-01-17 01:51:26 +00:00
Matthias Seitz
574bde0d6f chore(chain-state): reorganize deferred_trie.rs impl blocks (#21151) 2026-01-17 01:39:29 +00:00
Matthias Seitz
79b8ffb828 feat(primitives-traits): add try_recover_signers for parallel batch recovery (#21103) 2026-01-17 01:24:53 +00:00
Dan Cline
c617d25c36 perf: make Chain use DeferredTrieData (#21137)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-17 01:05:35 +00:00
Georgios Konstantopoulos
b96a30821f fix(engine): request head block download when not buffered after backfill (#21150) 2026-01-17 00:33:27 +00:00
Mablr
012fbf5110 fix(docs/cli): update help.rs to use nightly toolchain (#21149) 2026-01-16 23:35:26 +00:00
Arsenii Kulikov
d7a5d1f872 fix: properly record span fields (#21148) 2026-01-16 23:25:54 +00:00
Matthias Seitz
3a39251f79 fix: release mutex before dropping ancestors in wait_cloned (#21146) 2026-01-16 22:32:23 +00:00
Julian Meyer
f6dbf2d82d feat(db): implement extra dup methods (#20964)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 21:31:52 +00:00
Brian Picciano
13707faf1a feat(consensus): incremental receipt root computation in background task (#21131) 2026-01-16 19:53:59 +00:00
Arsenii Kulikov
6e6415690c perf: start saving cache sooner (#21130) 2026-01-16 18:55:18 +00:00
Matthias Seitz
b81e373d78 chore(deps): bump vergen and vergen-git2 to 9.1.0 (#21141) 2026-01-16 20:00:43 +01:00
Arun Dhyani
a164654145 fix(exex): prevent ExExManager deadlock when buffer clears after being full (#21135) 2026-01-16 18:42:23 +00:00
Matthias Seitz
905bb95f8b perf(engine): defer trie overlay computation with LazyOverlay (#21133) 2026-01-16 18:25:04 +00:00
YK
13c32625bc feat(storage): add EitherReader for routing history queries to MDBX or RocksDB (#21063)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-16 17:44:43 +00:00
YK
1be9fab5bf perf: Optimize multiproof sequencer add_proof (#21129) 2026-01-16 17:33:48 +00:00
Arsenii Kulikov
80eb0d0fb6 refactor: use BlockExecutionOutcome in ExecutedBlock (#21123) 2026-01-16 17:07:19 +00:00
Matthias Seitz
5e178f6ac6 chore(deps): update alloy-evm and alloy-op-evm to 0.26.3 (#21126) 2026-01-16 17:24:45 +01:00
Matthias Seitz
b4b64096c8 perf(cli): use available_parallelism as default for re-execute (#21010) 2026-01-16 16:08:30 +00:00
figtracer
e313de818b chore(provider): pre alloc tx hashes (#21114) 2026-01-16 15:40:47 +00:00
rakita
86c414081a feat: stagging revm v34.0.0 (#20627)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 14:56:27 +00:00
Brian Picciano
a74cb9cbc3 feat(trie): in-memory trie changesets (#20997) 2026-01-16 01:06:31 +00:00
YK
e25411c32b perf(trie): fix extend_sorted_vec O(n log n) → O(n+m) merge (#21098) 2026-01-16 00:17:22 +00:00
Matthias Seitz
ec3323bba0 refactor(chain-state): extract blocks_to_chain helper (#21110) 2026-01-15 23:27:11 +00:00
Dan Cline
26cd132631 fix(reth-bench): use requests hash (#21111) 2026-01-15 19:19:16 +00:00
DaniPopes
079f59c2be perf: reserve in extend_sorted_vec (#21109) 2026-01-15 19:10:20 +00:00
joshieDo
e9b079ad62 feat: add rocksdb to save_blocks (#21003)
Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com>
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-15 18:33:19 +00:00
Dan Cline
b1f107b171 feat(reth-bench): add generate-big-block command (#21082) 2026-01-15 15:30:04 +00:00
YK
7d0e7e72de perf(trie): add k-way merge batch optimization for merge_overlay_trie_input (#21080) 2026-01-15 15:22:15 +00:00
joshieDo
f012b3391e feat: parallelize save_blocks (#20993)
Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com>
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-15 14:58:06 +00:00
joshieDo
d225fc1d7f feat: add get/set db settings for rocksdb (#21095) 2026-01-15 14:48:05 +00:00
Dan Cline
d469b7f1d0 feat(rpc): add flag to skip invalid transactions in testing_buildBlockV1 (#21094)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-15 12:05:30 +00:00
YK
9bcd3712c8 test(storage): add parametrized MDBX/RocksDB history lookup equivalence tests (#20871) 2026-01-15 11:16:40 +00:00
Emma Jamieson-Hoare
b25f32a977 chore(release): set version v1.10.0 (#21091)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-15 10:50:35 +00:00
Emma Jamieson-Hoare
905de96944 chore: release 1.9.4 (#21048)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-15 09:41:54 +00:00
Sergei Shulepov
27fbd9a7de fix(db): change commit return type from Result<bool> to Result<()> (#21077)
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
2026-01-14 23:56:27 +00:00
DaniPopes
26a99ac5a3 perf: small improvement to extend_sorted_vec (#21032) 2026-01-14 23:46:58 +00:00
James Prestwich
1265a89c21 refactor: make use of dbi consistent across mdbx interface (#21079) 2026-01-14 23:42:42 +00:00
Matthias Seitz
b9ff5941eb feat(primitives): add SealedBlock::decode_sealed for efficient RLP decoding (#21030) 2026-01-14 22:49:55 +00:00
Sergei Shulepov
a75a0a5db7 feat(cli): support file:// URLs in reth download (#21026)
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
2026-01-14 22:30:42 +00:00
Matthias Seitz
0a4bac77d0 feat(primitives): add From<Sealed<B>> for SealedBlock<B> (#21078) 2026-01-14 22:19:09 +00:00
Kamil Szczygieł
1fbd5a95f8 feat: Support for sending logs through OTLP (#21039)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-14 21:29:00 +00:00
Arsenii Kulikov
1bc07fad8e perf: use binary search in ForwardInMemoryCursor (#21049) 2026-01-14 19:31:11 +00:00
Arsenii Kulikov
8cb506c4d3 perf: don't clone entire keys set (#21042) 2026-01-14 19:26:23 +00:00
ethfanWilliam
15f16a5a2e fix: propagate keccak-cache-global feature to reth-optimism-cli (#21051)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-14 19:22:22 +00:00
Brian Picciano
5cf1d2a0b0 fix(trie): Update branch masks when revealing blinded nodes (#20937) 2026-01-14 19:12:15 +00:00
Matthias Seitz
59fb25d892 feat(bench-compare): add --skip-wait-syncing flag (#21035) 2026-01-14 16:24:19 +01:00
Alexey Shekhirin
665a0a8553 feat(cli): parse URL path and display ETA in reth download (#21014) 2026-01-14 10:01:01 +00:00
DaniPopes
54735ce0f4 perf: use fixed-map for StaticFileSegment maps (#21001)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-14 00:52:54 +00:00
joshieDo
a73e73adef feat(storage): split static file commit into sync_all and finalize (#20984) 2026-01-13 16:27:55 +00:00
github-actions[bot]
4f3bd3eac1 chore(deps): weekly cargo update (#20924)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-13 14:42:54 +00:00
YK
ae41823be6 fix: propagate edge feature to reth-node-core for version output (#20998) 2026-01-13 14:35:24 +00:00
Matthias Seitz
1fa71f893c test: add testing_buildBlockV1 RPC method and Osaka test (#20990) 2026-01-13 15:18:52 +01:00
ANtutov
c6b17848dd fix(trie): remove redundant storage trie root calculation in witness (#20965) 2026-01-13 13:12:39 +00:00
Alexey Shekhirin
a5dd7d0106 feat(node): --minimal flag (#20960) 2026-01-13 12:54:26 +00:00
Emilia Hane
61354e6c21 chore(test): use reth_optimism_chainspec::BASE_SEPOLIA in tests (#20988) 2026-01-13 12:07:47 +00:00
DaniPopes
2444533a04 perf: use in-memory length for static files metrics (#20987) 2026-01-13 11:37:00 +00:00
kurahin
8fa01eb62e fix: use global default for rpc_proof_permits CLI flag (#20967) 2026-01-12 23:03:51 +00:00
DaniPopes
c5e00e4aeb perf(db): throttle metrics reporting (#20974)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-12 22:44:24 +00:00
joshieDo
98a35cc870 fix: propagate FEATURES to sub-makes (#20975) 2026-01-12 20:03:34 +00:00
YK
46d670eca5 fix(stages): use static files for unwind in SenderRecovery stage (#20972)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-12 19:22:49 +00:00
DaniPopes
25906b7b3e fix(libmdbx): use correct size for freelist u32 values (#20970) 2026-01-12 18:52:03 +00:00
Matthias Seitz
1b3d815cb8 fix(rpc): validate eth_feeHistory newest_block against chain head (#20969)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-12 18:48:46 +00:00
DaniPopes
23f3f8e820 feat: add tracing-tracy (#20958)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-12 18:37:37 +00:00
DaniPopes
2663942b50 chore(deps): bump metrics (#20968) 2026-01-12 18:13:38 +00:00
YK
369c629b9b perf(trie): reuse overlay in deferred trie overlay computation (#20774) 2026-01-12 15:04:26 +00:00
GarmashAlex
6fec4603cf refactor(trie): avoid building prefix set for v2 storage proofs (#20898) 2026-01-12 12:49:24 +00:00
DaniPopes
515fd597f3 perf(net): use alloy_primitives::Keccak256 (#20957) 2026-01-12 11:21:27 +00:00
Crypto Nomad
126a7c9570 perf(engine): save one clock read in sparse trie metrics (#20947) 2026-01-12 07:40:30 +00:00
Matthias Seitz
8aeee5018e perf(trie): save one clock read in elapsed time calculation (#20916) 2026-01-12 03:57:54 +00:00
Matthias Seitz
210309ca76 docs: fix typos and incorrect documentation (#20943) 2026-01-12 00:48:01 +01:00
Matthias Seitz
551918b0d8 refactor(engine): defer sparse trie setup to spawned task (#20942) 2026-01-11 23:30:14 +00:00
iPLAY888
89677e1bd9 docs(rpc): fix incorrect transport in with_ipc comment (#20939) 2026-01-11 23:04:32 +00:00
pepes
0e2b3afa3f chore: correct deprecation message for SealedBlockFor (#20929) 2026-01-11 15:08:25 +00:00
David Klank
5d551eab29 perf(payload): remove unnecessary parent_header clone (#20930) 2026-01-11 15:07:51 +00:00
David Klank
12c4c04f7d fix(optimism): add missing Holocene hardfork to DEV_HARDFORKS (#20931) 2026-01-11 15:03:35 +00:00
Matthias Seitz
392f8e6e13 refactor(engine): simplify is_done signature in MultiProofTask (#20906) 2026-01-11 09:46:20 +00:00
Crypto Nomad
1a94d1f091 docs: fix re-export source comments (#20913) 2026-01-10 15:36:03 +00:00
viktorking7
97ae89c7f0 docs: fix dead link (#20914) 2026-01-10 15:18:56 +00:00
Matthias Seitz
a4921119e4 perf(trie): defer consuming remaining storage proof receivers (#20915) 2026-01-10 15:17:20 +00:00
VolodymyrBg
0f3d3695f5 docs: document account_change_sets static files config (#20903) 2026-01-10 09:02:42 +00:00
phrwlk
54355dfc78 docs: fix Performant card link on landing page (#20904) 2026-01-10 08:54:58 +00:00
FT
44a6035fa3 fix: correct typos in error messages and logs (#20894) 2026-01-10 08:54:31 +00:00
Matthias Seitz
746baed2b1 feat(cli): add CliRunnerConfig for configurable graceful shutdown timeout (#20899)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-09 21:52:03 +00:00
Dan Cline
e86c5fba53 fix(stages): advance sender static file in sender recovery (#20897) 2026-01-09 20:23:17 +00:00
joshieDo
485fa3448d fix: call cancel_all_background_work on RocksDBProviderInner drop (#20895) 2026-01-09 19:53:31 +00:00
DaniPopes
0db3813941 fix(rbc): fail early if node exits while waiting for startup (#20892) 2026-01-09 17:58:04 +00:00
FT
52c2ae3362 docs: fix typos in documentation files (#20890)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-09 18:41:37 +01:00
YK
b1d75f2771 feat(bench-compare): add --wait-for-persistence flag support (#20891) 2026-01-09 16:47:46 +00:00
Matthias Seitz
ef80ee1687 chore: remove env clone (#20889) 2026-01-09 16:42:50 +00:00
radik878
8dacfb3d9c refactor(ecies): avoid duplicate keccak digest in MAC::update_body (#20854) 2026-01-09 15:35:51 +00:00
joshieDo
425a021e3b feat: add edge feature flag to reth (#20841) 2026-01-09 15:33:21 +00:00
Hwangjae Lee
08c0d30ea7 docs(reth): fix outdated comments and document missing features (#20849)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-09 15:32:17 +00:00
かりんとう
84e970e4c9 perf: remove redundant contains_key (#20820) 2026-01-09 15:22:06 +00:00
Fibonacci747
020f20db42 chore: correct StorageHistory prune map size constant name (#20828) 2026-01-09 15:20:02 +00:00
ANtutov
f53929e0c8 docs: clarify bodies downloader set_download_range semantics (#20821) 2026-01-09 15:18:37 +00:00
ethfanWilliam
4a8fbe15e3 chore: remove unused implementation (#20885) 2026-01-09 15:08:06 +00:00
yyhrnk
a59e9832e6 docs: document optional block param for trace_rawTransaction (#20812)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-09 15:04:29 +00:00
YK
07beb76cf7 feat(reth-bench-compare): add persistence-based flow optimization for reth-bench (#20869)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-09 14:58:21 +00:00
FT
3ddf0bd729 docs: correct typo in hive.yml workflow comment (#20884) 2026-01-09 14:50:05 +00:00
iPLAY888
c3d92ddfc2 docs(engine): update outdated EthBuiltPayload comment (#20883) 2026-01-09 14:45:11 +00:00
kurahin
c0628dfbff refactor(config): delegate PruneConfig::has_receipts_pruning (#20809) 2026-01-09 14:44:43 +00:00
Sabnock
a2aa1f18df feat(rpc): add debug_getBlockAccessList endpoint for EIP-7928 (#20824)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-09 13:29:37 +00:00
Arun Dhyani
d489f80f6b feat: Add TrieUpdatesSorted and HashedPostStateSorted in all ExEx notifications (#20333)
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-09 13:06:41 +00:00
Emilia Hane
bf272c9432 chore(consensus): Add trait object error variant to ConsensusError (#20875)
Co-authored-by: leeli <Leeliren@proton.me>
2026-01-09 13:01:22 +00:00
FT
ebb54d0dcc docs: typo in comment (#20879) 2026-01-09 13:00:22 +00:00
Matthias Seitz
1d7367c389 perf(engine): simplify get_prefetch_proof_targets (#20864)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-09 12:53:28 +00:00
refcell
824ae12d75 feat(exex): Make WAL Block Threshold Configurable (#20867) 2026-01-09 12:45:53 +00:00
Alexey Shekhirin
2db281e51d feat(reth-bench-compare): nP latency mean change percent (#20881) 2026-01-09 11:58:08 +00:00
Brian Picciano
8367ba473e feat(metrics): Add metrics for save_block steps and computed trie input sizes (#20878) 2026-01-09 11:40:35 +00:00
fig
f2abad5f5c perf(engine): destructure leaf to avoid clone() (#20863) 2026-01-09 11:19:49 +00:00
Matthias Seitz
4673d77c03 perf(trie): optimize ChunkedHashedPostState sorting (#20866) 2026-01-09 11:18:28 +00:00
Matthias Seitz
33bcd60348 feat(rpc): add persisted block subscription (#20877)
Co-authored-by: cakevm <cakevm@proton.me>
2026-01-09 10:37:46 +00:00
Matthias Seitz
8a9b5d90f4 feat(chain-state): add persisted block tracking (#20876)
Co-authored-by: cakevm <cakevm@proton.me>
2026-01-09 09:56:20 +00:00
joshieDo
c26cfa3dcb fix: pre-calculate transitions on append_blocks_with_state (#20850) 2026-01-09 09:26:46 +00:00
joshieDo
13e59651f1 fix: initialize transaction-senders sf during genesis (#20846) 2026-01-09 09:26:26 +00:00
Hwangjae Lee
0f4995d1ea chore(trie): fix typo in comment (#20870)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
2026-01-09 09:19:14 +00:00
Matthias Seitz
cff7e8be53 perf(engine): avoid unnecessary B256 copy in get_proof_targets (#20845)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-09 04:57:23 +00:00
YK
5433d7a4ac feat(storage): add RocksDB history lookup methods and owned batch type [2/3] (#20543) 2026-01-09 04:52:15 +00:00
fig
1866db4d50 chore(engine): remove unnecessary debug-level clone() (#20862) 2026-01-08 22:21:29 +00:00
Danno Ferrin
c9b92550b6 feat(network): add customizable announcement filtering policy to APIs (#20861)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-08 22:08:41 +00:00
Sebastian Stammler
8e81ebfc1f feat(optimism): Also require non-zero elasticity in payload attributes (#20858) 2026-01-08 21:32:46 +00:00
joshieDo
1363205b5d feat: allow TransactionHashNumbers to be written to rocksdb during live sync (#20853) 2026-01-08 20:02:49 +00:00
DaniPopes
ed201cae0e chore(rbc): improve compilation log message (#20855) 2026-01-08 19:30:04 +00:00
Matthias Seitz
a5b10f11ce perf(engine): handle EmptyProof inline during prefetch batching (#20848)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-08 19:12:14 +00:00
Brian Picciano
a06644944f feat(trie): Keep cached storage roots on proof workers (#20838) 2026-01-08 17:04:42 +00:00
Matthias Seitz
8eecad3d1d chore(engine): remove state update batching in multiproof (#20842)
Co-authored-by: Claude Opus 4.5 <noreply@anthropic.com>
2026-01-08 16:04:12 +00:00
Emilia Hane
412f39e223 chore(consensus): Remove associated type Consensus::Error (#20843)
Co-authored-by: Josh_dfG <126518346+JoshdfG@users.noreply.github.com>
2026-01-08 15:54:31 +00:00
Matthias Seitz
13106233e4 perf(engine): use crossbeam select for persistence events (#20813) 2026-01-08 15:47:50 +00:00
joshieDo
e63fef0e79 ci: rocksdb job to unit.yml (#20839) 2026-01-08 13:20:43 +00:00
Dan Cline
eed34254f5 feat: add StaticFileSegment::AccountChangeSets (#18882)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-08 12:05:05 +00:00
Emilia Hane
b38d37a1e1 feat(sdk): Add custom TrieType (#20804)
Co-authored-by: jagroot <4516586+itschaindev@users.noreply.github.com>
2026-01-08 11:53:27 +00:00
Maxim Evtush
7efaf4ca97 docs: mention optional EraStage in DefaultStages documentation (#20836) 2026-01-08 11:51:10 +00:00
Emilia Hane
ef708792a9 chore(storage): Add trait object error variant to DatabaseError (#20096) 2026-01-08 11:40:09 +00:00
Alexey Shekhirin
bcd74d021b feat(metrics): configurable jeprof pprof dumps directory (#20834) 2026-01-08 11:21:42 +00:00
bigbear
0f0a181fe2 fix(trie): account for all flag in PrefixSet::is_empty() (#20801) 2026-01-08 11:20:55 +00:00
Matthias Seitz
9678d6c76d chore: tighten iat timeout (#20835) 2026-01-08 11:09:03 +00:00
Brian Picciano
7ceca70353 feat(trie): Add flag to enable proof v2 for storage proof workers (#20617)
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-08 10:53:24 +00:00
Matthias Seitz
4412a501eb perf(chain-state): avoid clones in deferred trie computation (#20816) 2026-01-08 09:25:32 +00:00
YK
3ca5cf49b6 refactor(storage): extract shared find_changeset_block_from_index algorithm [1/3] (#20542) 2026-01-08 02:56:38 +00:00
Matthias Seitz
1d4603769f perf(trie): use sorted_unstable for proof target chunking (#20827) 2026-01-08 01:05:14 +00:00
Lorsmirq Benton
9bba8c7a98 docs(net): complete incomplete doc comment (#20793) 2026-01-07 21:16:00 +00:00
Alexey Shekhirin
6f0ef914b9 feat(metrics): jemalloc heap dump endpoint (#20811) 2026-01-07 19:36:08 +00:00
Alexey Shekhirin
d756e8310a chore(engine): more logs when cache is not available (#20817)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-01-07 19:35:27 +00:00
DaniPopes
74a7ba581c feat(rbc): don't wait in between FCUs when warming up (#20818) 2026-01-07 19:20:33 +00:00
Matthias Seitz
a8980bf7c1 chore: ignore RUSTSEC-2026-0002 (#20819) 2026-01-07 18:47:09 +00:00
Matthias Seitz
050d9f440f chore: ignore RUSTSEC-2025-0141 bincode advisory (#20815) 2026-01-07 19:10:30 +01:00
Brian Picciano
df33a8200f feat(reth-bench-compare): Do unwind first (#20808) 2026-01-07 16:49:07 +00:00
Matthias Seitz
d3dab613fc revert: "perf(engine): parellelize multiproof_targets_from_state (#206… (#20807) 2026-01-07 15:49:10 +00:00
Matthias Seitz
1b31739adf revert: "perf(engine): paralellize evm_state_to_hashed_post_state() (#… (#20806) 2026-01-07 15:47:15 +00:00
DaniPopes
6280abedd0 chore(reth-bench-compare): skip last unwind (#20805) 2026-01-07 15:44:36 +00:00
Mohan Somnath
4c064a4d20 docs: fix article and grammar errors in comments (#20794) 2026-01-07 15:00:13 +00:00
phrwlk
8d19a36492 docs: clarify pending pending_block build_block docs (#20800) 2026-01-07 14:09:54 +00:00
cui
78f2685ee9 perf: remove unnecessary code (#20719)
Co-authored-by: weixie.cui <weixie.cui@okg.com>
2026-01-07 12:12:17 +00:00
YK
fee7e997ff refactor(trie): replace TrieMasks with Option<BranchNodeMasks> (#20707) 2026-01-07 11:27:23 +00:00
534 changed files with 30404 additions and 7634 deletions

View File

@@ -12,7 +12,7 @@ workflows:
# Check that `A` activates the features of `B`.
"propagate-feature",
# These are the features to check:
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer,portable,keccak-cache-global",
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global",
# Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
"--left-side-feature-missing=ignore",
# Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.

43
.github/CODEOWNERS vendored
View File

@@ -1,45 +1,52 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
crates/era/ @mattsse
crates/era-downloader/ @mattsse
crates/era-utils/ @mattsse
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
crates/ethereum/ @mattsse @Rjected
crates/etl/ @joshieDo @shekhirin
crates/evm/ @rakita @mattsse @Rjected
crates/evm/ @mattsse @Rjected @klkvr
crates/exex/ @shekhirin
crates/fs-util/ @mattsse
crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected @fgimenez
crates/optimism/ @mattsse @Rjected
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/ress/ @shekhirin @Rjected
crates/revm/ @mattsse
crates/rpc/ @mattsse @Rjected
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/stateless/ @mattsse
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
crates/storage/db-api/ @joshieDo
crates/storage/db-common/ @Rjected
crates/storage/db/ @joshieDo @rakita
crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @fgimenez
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
bin/reth/ @mattsse @shekhirin @Rjected
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

View File

@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
cmd=(cargo codspeed build --profile profiling)
crates=(
-p reth-primitives
-p reth-trie
-p reth-trie-common
-p reth-trie-sparse
)
"${cmd[@]}" --features test-utils "${crates[@]}"

View File

@@ -17,6 +17,16 @@ name: bench
jobs:
codspeed:
runs-on: depot-ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
include:
- partition: 1
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
- partition: 2
crates: "-p reth-trie"
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
steps:
- uses: actions/checkout@v6
with:
@@ -32,10 +42,10 @@ jobs:
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: ./.github/scripts/codspeed-build.sh
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run --workspace
run: cargo codspeed run ${{ matrix.crates }}
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}

66
.github/workflows/check-alloy.yml vendored Normal file
View File

@@ -0,0 +1,66 @@
# Checks reth compilation against alloy branches to detect breaking changes.
# Run on-demand via workflow_dispatch.
name: Check Alloy Breaking Changes
on:
workflow_dispatch:
inputs:
alloy_branch:
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
required: false
type: string
alloy_evm_branch:
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
required: false
type: string
op_alloy_branch:
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
required: false
type: string
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Apply alloy patches
run: |
ARGS=""
if [ -n "${{ inputs.alloy_branch }}" ]; then
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
fi
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
fi
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
fi
if [ -z "$ARGS" ]; then
echo "No branches specified, nothing to patch"
exit 1
fi
./scripts/patch-alloy.sh $ARGS
echo "=== Final patch section ==="
tail -50 Cargo.toml
- name: Check workspace
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features

View File

@@ -15,6 +15,6 @@ permissions:
jobs:
update:
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
secrets:
token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -28,10 +28,14 @@ jobs:
build:
- name: 'Build and push the nightly reth image'
command: 'make PROFILE=maxperf docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
- name: 'Build and push the nightly op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
steps:

View File

@@ -15,11 +15,21 @@ concurrency:
cancel-in-progress: true
jobs:
prepare-reth:
prepare-reth-stable:
uses: ./.github/workflows/prepare-reth.yml
with:
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak"
artifact_name: "reth-stable"
prepare-reth-edge:
uses: ./.github/workflows/prepare-reth.yml
with:
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak edge"
artifact_name: "reth-edge"
prepare-hive:
if: github.repository == 'paradigmxyz/reth'
@@ -77,6 +87,7 @@ jobs:
strategy:
fail-fast: false
matrix:
storage: [stable, edge]
# ethereum/rpc to be deprecated:
# https://github.com/ethereum/hive/pull/1117
scenario:
@@ -86,7 +97,7 @@ jobs:
- sim: devp2p
limit: discv4
# started failing after https://github.com/ethereum/go-ethereum/pull/31843, no
# action on our side, remove from here when we get unxpected passes on these tests
# action on our side, remove from here when we get unexpected passes on these tests
# - sim: devp2p
# limit: eth
# include:
@@ -176,9 +187,10 @@ jobs:
- sim: ethereum/eels/consume-rlp
limit: .*tests/paris.*
needs:
- prepare-reth
- prepare-reth-stable
- prepare-reth-edge
- prepare-hive
name: run ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
runs-on:
group: Reth
permissions:
@@ -197,7 +209,7 @@ jobs:
- name: Download reth image
uses: actions/download-artifact@v7
with:
name: artifacts
name: reth-${{ matrix.storage }}
path: /tmp
- name: Load Docker images

View File

@@ -285,7 +285,7 @@ jobs:
- run: zepter run check
deny:
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
uses: tempoxyz/ci/.github/workflows/deny.yml@main
lint-success:
name: lint success

View File

@@ -21,6 +21,11 @@ on:
required: false
type: string
description: "Optional cargo package path"
artifact_name:
required: false
type: string
default: "artifacts"
description: "Name for the uploaded artifact"
jobs:
prepare-reth:
@@ -52,5 +57,5 @@ jobs:
id: upload
uses: actions/upload-artifact@v6
with:
name: artifacts
name: ${{ inputs.artifact_name }}
path: ./artifacts

View File

@@ -19,29 +19,22 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.type }} (${{ matrix.partition }}/${{ matrix.total_partitions }})
name: test / ${{ matrix.type }} / ${{ matrix.storage }}
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
strategy:
matrix:
type: [ethereum, optimism]
storage: [stable, edge]
include:
- type: ethereum
args: --features "asm-keccak ethereum" --locked
partition: 1
total_partitions: 2
- type: ethereum
args: --features "asm-keccak ethereum" --locked
partition: 2
total_partitions: 2
features: asm-keccak ethereum
exclude_args: ""
- type: optimism
args: --features "asm-keccak" --locked --exclude reth --exclude reth-bench --exclude "example-*" --exclude "reth-ethereum-*" --exclude "*-ethereum"
partition: 1
total_partitions: 2
- type: optimism
args: --features "asm-keccak" --locked --exclude reth --exclude reth-bench --exclude "example-*" --exclude "reth-ethereum-*" --exclude "*-ethereum"
partition: 2
total_partitions: 2
features: asm-keccak
exclude_args: --exclude reth --exclude reth-bench --exclude "example-*" --exclude "reth-ethereum-*" --exclude "*-ethereum"
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -59,9 +52,9 @@ jobs:
- name: Run tests
run: |
cargo nextest run \
${{ matrix.args }} --workspace \
--features "${{ matrix.features }} $EDGE_FEATURES" --locked \
${{ matrix.exclude_args }} --workspace \
--exclude ef-tests --no-tests=warn \
--partition hash:${{ matrix.partition }}/2 \
-E "!kind(test) and not binary(e2e_testsuite)"
state:

View File

@@ -18,7 +18,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
6. **Pipeline (`crates/stages/`)**: Staged sync architecture for blockchain synchronization
7. **Trie (`crates/trie/`)**: Merkle Patricia Trie implementation with parallel state root computation
8. **Node Builder (`crates/node/`)**: High-level node orchestration and configuration
9 **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated)
9. **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated)
### Key Design Principles
@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
unsafe impl GlobalAlloc for LimitedAllocator { ... }
// Binary search requires sorted input. Panics on unsorted slices.
fn find_index(items: &[Item], target: &Item) -> Option
fn find_index(items: &[Item], target: &Item) -> Option<usize>
// Timeout set to 5s to match EVM block processing limits
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);

View File

@@ -51,9 +51,7 @@ elsewhere.
<!-- - **Asking in the support Telegram:** The [Foundry Support Telegram][support-tg] is a fast and easy way to ask questions. -->
<!-- - **Opening a discussion:** This repository comes with a discussions board where you can also ask for help. Click the "Discussions" tab at the top. -->
If you have reviewed existing documentation and still have questions, or you are having problems, you can get help by *
*opening a discussion**. This repository comes with a discussions board where you can also ask for help. Click the "
Discussions" tab at the top.
If you have reviewed existing documentation and still have questions, or you are having problems, you can get help by **opening a discussion**. This repository comes with a discussions board where you can also ask for help. Click the "Discussions" tab at the top.
As Reth is still in heavy development, the documentation can be a bit scattered. The [Reth Docs][reth-docs] is our
current best-effort attempt at keeping up-to-date information.

1572
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.9.3"
version = "1.10.2"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
@@ -473,22 +473,22 @@ reth-ress-protocol = { path = "crates/ress/protocol" }
reth-ress-provider = { path = "crates/ress/provider" }
# revm
revm = { version = "33.1.0", default-features = false }
revm-bytecode = { version = "7.1.1", default-features = false }
revm-database = { version = "9.0.5", default-features = false }
revm-state = { version = "8.1.1", default-features = false }
revm-primitives = { version = "21.0.2", default-features = false }
revm-interpreter = { version = "31.1.0", default-features = false }
revm-database-interface = { version = "8.0.5", default-features = false }
op-revm = { version = "14.1.0", default-features = false }
revm-inspectors = "0.33.2"
revm = { version = "34.0.0", default-features = false }
revm-bytecode = { version = "8.0.0", default-features = false }
revm-database = { version = "10.0.0", default-features = false }
revm-state = { version = "9.0.0", default-features = false }
revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.0"
# eth
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.4.1"
alloy-dyn-abi = "1.4.3"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.1.0" }
alloy-evm = { version = "0.25.1", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
@@ -497,36 +497,36 @@ alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.2.1", default-features = false }
alloy-contract = { version = "1.2.1", default-features = false }
alloy-eips = { version = "1.2.1", default-features = false }
alloy-genesis = { version = "1.2.1", default-features = false }
alloy-json-rpc = { version = "1.2.1", default-features = false }
alloy-network = { version = "1.2.1", default-features = false }
alloy-network-primitives = { version = "1.2.1", default-features = false }
alloy-provider = { version = "1.2.1", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.2.1", default-features = false }
alloy-rpc-client = { version = "1.2.1", default-features = false }
alloy-rpc-types = { version = "1.2.1", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.2.1", default-features = false }
alloy-rpc-types-anvil = { version = "1.2.1", default-features = false }
alloy-rpc-types-beacon = { version = "1.2.1", default-features = false }
alloy-rpc-types-debug = { version = "1.2.1", default-features = false }
alloy-rpc-types-engine = { version = "1.2.1", default-features = false }
alloy-rpc-types-eth = { version = "1.2.1", default-features = false }
alloy-rpc-types-mev = { version = "1.2.1", default-features = false }
alloy-rpc-types-trace = { version = "1.2.1", default-features = false }
alloy-rpc-types-txpool = { version = "1.2.1", default-features = false }
alloy-serde = { version = "1.2.1", default-features = false }
alloy-signer = { version = "1.2.1", default-features = false }
alloy-signer-local = { version = "1.2.1", default-features = false }
alloy-transport = { version = "1.2.1" }
alloy-transport-http = { version = "1.2.1", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.2.1", default-features = false }
alloy-transport-ws = { version = "1.2.1", default-features = false }
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
# op
alloy-op-evm = { version = "0.25.0", default-features = false }
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -555,6 +555,7 @@ dirs-next = "2.0.0"
dyn-clone = "1.0.17"
eyre = "0.6"
fdlimit = "0.3.0"
fixed-map = { version = "0.9", default-features = false }
humantime = "2.1"
humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
@@ -596,9 +597,9 @@ chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-exporter-prometheus = { version = "0.16.0", default-features = false }
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.19.0" }
metrics-util = { default-features = false, version = "0.20.0" }
# proc-macros
proc-macro2 = "1.0"
@@ -664,6 +665,7 @@ opentelemetry_sdk = "0.31"
opentelemetry = "0.31"
opentelemetry-otlp = "0.31"
opentelemetry-semantic-conventions = "0.31"
opentelemetry-appender-tracing = "0.31"
tracing-opentelemetry = "0.32"
# misc-testing
@@ -684,6 +686,7 @@ ethereum_ssz = "0.9.0"
ethereum_ssz_derive = "0.9.0"
# allocators
jemalloc_pprof = { version = "0.8", default-features = false }
tikv-jemalloc-ctl = "0.6"
tikv-jemallocator = "0.6"
tracy-client = "0.18.0"
@@ -733,17 +736,18 @@ tracing-journald = "0.3"
tracing-logfmt = "0.3.3"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
triehash = "0.8"
typenum = "1.15.0"
vergen = "9.0.4"
vergen = "9.1.0"
visibility = "0.1.1"
walkdir = "2.3.3"
vergen-git2 = "1.0.5"
vergen-git2 = "9.1.0"
# networking
ipnet = "2.11"
# [patch.crates-io]
[patch.crates-io]
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
@@ -788,3 +792,37 @@ ipnet = "2.11"
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

View File

@@ -276,13 +276,18 @@ docker-build-push-latest: ## Build and push a cross-arch Docker image tagged wit
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call docker_build_push,nightly,nightly)
.PHONY: docker-build-push-nightly-edge-profiling
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Create a cross-arch Docker image with the given tags and push it
define docker_build_push
$(MAKE) build-x86_64-unknown-linux-gnu
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
$(MAKE) build-aarch64-unknown-linux-gnu
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
@@ -328,6 +333,11 @@ op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly,nightly)
.PHONY: op-docker-build-push-nightly-edge-profiling
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
@@ -347,11 +357,11 @@ op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker imag
# Create a cross-arch Docker image with the given tags and push it
define op_docker_build_push
$(MAKE) op-build-x86_64-unknown-linux-gnu
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
$(MAKE) op-build-aarch64-unknown-linux-gnu
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth

View File

@@ -44,7 +44,7 @@ More historical context below:
- We released 1.0 "production-ready" stable Reth in June 2024.
- Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf).
- Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon.
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024,the last beta release.
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024, the last beta release.
- We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4, 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives.
- We shipped iterative improvements until the last alpha release on February 28, 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21).
- We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) on June 20, 2023.

View File

@@ -25,7 +25,9 @@ reth-chainspec.workspace = true
# alloy
alloy-provider = { workspace = true, features = ["reqwest-rustls-tls"], default-features = false }
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
alloy-rpc-types-eth.workspace = true
alloy-transport-ws.workspace = true
alloy-primitives.workspace = true
# CLI and argument parsing
@@ -69,7 +71,11 @@ jemalloc = [
"reth-node-core/jemalloc",
]
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
tracy-allocator = ["reth-cli-util/tracy-allocator"]
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
tracy = [
"reth-node-core/tracy",
"reth-tracing/tracy",
]
min-error-logs = [
"tracing/release_max_level_error",

View File

@@ -0,0 +1,50 @@
# reth-bench-compare
Compare reth performance between two git references.
## Usage
```bash
reth-bench-compare \
--baseline-ref main \
--feature-ref my-feature \
--blocks 100 \
--wait-for-persistence
```
## Arguments
| Argument | Description | Default | Required |
|----------|-------------|---------|----------|
| `--baseline-ref <REF>` | Git reference for baseline | - | Yes |
| `--feature-ref <REF>` | Git reference to compare | - | Yes |
| `--blocks <N>` | Number of blocks to benchmark | `100` | No |
| `--chain <CHAIN>` | Chain to benchmark | `mainnet` | No |
| `--datadir <PATH>` | Data directory path | OS-specific | No |
| `--rpc-url <URL>` | RPC endpoint for block data | Chain default | No |
| `--output-dir <PATH>` | Output directory | `./reth-bench-compare` | No |
| `--wait-for-persistence` | Wait for block persistence | `false` | No |
| `--persistence-threshold <N>` | Wait after every N+1 blocks | `2` | No |
| `--wait-time <DURATION>` | Fixed delay (legacy) | - | No |
| `--warmup-blocks <N>` | Cache warmup blocks | Same as `--blocks` | No |
| `--draw` | Generate charts (needs Python/uv) | `false` | No |
| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
| `-vvvv` | Debug logging | Info | No |
| `--features <FEATURES>` | Rust features for both builds | `jemalloc,asm-keccak` | No |
| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |
| `--baseline-rustflags <FLAGS>` | RUSTFLAGS for baseline only | Inherits `--rustflags` | No |
| `--feature-rustflags <FLAGS>` | RUSTFLAGS for feature only | Inherits `--rustflags` | No |
| `--baseline-args <ARGS>` | Extra args for baseline node | - | No |
| `--feature-args <ARGS>` | Extra args for feature node | - | No |
| `--metrics-port <PORT>` | Metrics endpoint port | `5005` | No |
| `--sudo` | Run with elevated privileges | `false` | No |
## Output
Results in `./reth-bench-compare/results/<timestamp>/`:
- `comparison_report.json` - Metrics comparison
- `per_block_comparison.csv` - Per-block statistics
- `baseline/` and `feature/` - Individual run results
- `latency_comparison.png` - Chart (if `--draw` used)

View File

@@ -18,6 +18,8 @@ pub(crate) struct BenchmarkRunner {
rpc_url: String,
jwt_secret: String,
wait_time: Option<String>,
wait_for_persistence: bool,
persistence_threshold: Option<u64>,
warmup_blocks: u64,
}
@@ -28,6 +30,8 @@ impl BenchmarkRunner {
rpc_url: args.get_rpc_url(),
jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(),
wait_time: args.wait_time.clone(),
wait_for_persistence: args.wait_for_persistence,
persistence_threshold: args.persistence_threshold,
warmup_blocks: args.get_warmup_blocks(),
}
}
@@ -96,13 +100,9 @@ impl BenchmarkRunner {
&from_block.to_string(),
"--to",
&to_block.to_string(),
"--wait-time=0ms", // Warmup should avoid persistence waits.
]);
// Add wait-time argument if provided
if let Some(ref wait_time) = self.wait_time {
cmd.args(["--wait-time", wait_time]);
}
cmd.env("RUST_LOG_STYLE", "never")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
@@ -186,9 +186,16 @@ impl BenchmarkRunner {
&output_dir.to_string_lossy(),
]);
// Add wait-time argument if provided
// Configure wait mode: wait-time takes precedence over persistence-based flow
if let Some(ref wait_time) = self.wait_time {
cmd.args(["--wait-time", wait_time]);
} else if self.wait_for_persistence {
cmd.arg("--wait-for-persistence");
// Add persistence threshold if specified
if let Some(threshold) = self.persistence_threshold {
cmd.args(["--persistence-threshold", &threshold.to_string()]);
}
}
cmd.env("RUST_LOG_STYLE", "never")

View File

@@ -114,10 +114,29 @@ pub(crate) struct Args {
#[arg(long)]
pub profile: bool,
/// Wait time between engine API calls (passed to reth-bench)
#[arg(long, value_name = "DURATION")]
/// Optional fixed delay between engine API calls (passed to reth-bench).
///
/// When set, reth-bench uses wait-time mode and disables persistence-based flow.
/// This flag remains for compatibility with older scripts.
#[arg(long, value_name = "DURATION", hide = true)]
pub wait_time: Option<String>,
/// Wait for blocks to be persisted before sending the next batch (passed to reth-bench).
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
#[arg(long)]
pub wait_for_persistence: bool,
/// Engine persistence threshold (passed to reth-bench).
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's default persistence threshold (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(long, value_name = "PERSISTENCE_THRESHOLD")]
pub persistence_threshold: Option<u64>,
/// Number of blocks to run for cache warmup after clearing caches.
/// If not specified, defaults to the same as --blocks
#[arg(long, value_name = "N")]
@@ -128,6 +147,11 @@ pub(crate) struct Args {
#[arg(long)]
pub no_clear_cache: bool,
/// Skip waiting for the node to sync before starting benchmarks.
/// When enabled, assumes the node is already synced and skips the initial tip check.
#[arg(long)]
pub skip_wait_syncing: bool,
#[command(flatten)]
pub logs: LogArgs,
@@ -512,6 +536,7 @@ async fn run_compilation_phase(
Ok((baseline_commit, feature_commit))
}
#[allow(clippy::too_many_arguments)]
/// Run warmup phase to warm up caches before benchmarking
async fn run_warmup_phase(
git_manager: &GitManager,
@@ -521,9 +546,15 @@ async fn run_warmup_phase(
args: &Args,
is_optimism: bool,
baseline_commit: &str,
starting_tip: u64,
) -> Result<()> {
info!("=== Running warmup phase ===");
// Unwind to starting block minus warmup blocks, so we end up back at starting_tip
let warmup_blocks = args.get_warmup_blocks();
let unwind_target = starting_tip.saturating_sub(warmup_blocks);
node_manager.unwind_to_block(unwind_target).await?;
// Use baseline for warmup
let warmup_ref = &args.baseline_ref;
@@ -552,12 +583,13 @@ async fn run_warmup_phase(
node_manager.start_node(&binary_path, warmup_ref, "warmup", &additional_args).await?;
// Wait for node to be ready and get its current tip
let current_tip = node_manager.wait_for_node_ready_and_get_tip().await?;
let current_tip = if args.skip_wait_syncing {
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
} else {
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
};
info!("Warmup node is ready at tip: {}", current_tip);
// Store the tip we'll unwind back to
let original_tip = current_tip;
// Clear filesystem caches before warmup run only (unless disabled)
if args.no_clear_cache {
info!("Skipping filesystem cache clearing (--no-clear-cache flag set)");
@@ -568,12 +600,9 @@ async fn run_warmup_phase(
// Run warmup to warm up caches
benchmark_runner.run_warmup(current_tip).await?;
// Stop node before unwinding (node must be stopped to release database lock)
// Stop node after warmup
node_manager.stop_node(&mut node_process).await?;
// Unwind back to starting block after warmup
node_manager.unwind_to_block(original_tip).await?;
info!("Warmup phase completed");
Ok(())
}
@@ -595,6 +624,31 @@ async fn run_benchmark_workflow(
let (baseline_commit, feature_commit) =
run_compilation_phase(git_manager, compilation_manager, args, is_optimism).await?;
// Switch to baseline reference and get the starting tip
git_manager.switch_ref(&args.baseline_ref)?;
let binary_path =
compilation_manager.get_cached_binary_path_for_commit(&baseline_commit, is_optimism);
if !binary_path.exists() {
return Err(eyre!(
"Cached baseline binary not found at {:?}. Compilation phase should have created it.",
binary_path
));
}
// Start node briefly to get the current tip, then stop it
info!("=== Determining initial block height ===");
let additional_args = args.build_additional_args("baseline", args.baseline_args.as_ref());
let (mut node_process, _) = node_manager
.start_node(&binary_path, &args.baseline_ref, "baseline", &additional_args)
.await?;
let starting_tip = if args.skip_wait_syncing {
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
} else {
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
};
info!("Node starting tip: {}", starting_tip);
node_manager.stop_node(&mut node_process).await?;
// Run warmup phase before benchmarking (skip if warmup_blocks is 0)
if args.get_warmup_blocks() > 0 {
run_warmup_phase(
@@ -605,6 +659,7 @@ async fn run_benchmark_workflow(
args,
is_optimism,
&baseline_commit,
starting_tip,
)
.await?;
} else {
@@ -620,6 +675,10 @@ async fn run_benchmark_workflow(
let commit = commits[i];
info!("=== Processing {} reference: {} ===", ref_type, git_ref);
// Unwind to starting block minus benchmark blocks, so we end up back at starting_tip
let unwind_target = starting_tip.saturating_sub(args.blocks);
node_manager.unwind_to_block(unwind_target).await?;
// Switch to target reference
git_manager.switch_ref(git_ref)?;
@@ -653,17 +712,18 @@ async fn run_benchmark_workflow(
node_manager.start_node(&binary_path, git_ref, ref_type, &additional_args).await?;
// Wait for node to be ready and get its current tip (wherever it is)
let current_tip = node_manager.wait_for_node_ready_and_get_tip().await?;
let current_tip = if args.skip_wait_syncing {
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
} else {
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
};
info!("Node is ready at tip: {}", current_tip);
// Store the tip we'll unwind back to
let original_tip = current_tip;
// Calculate benchmark range
// Note: reth-bench has an off-by-one error where it consumes the first block
// of the range, so we add 1 to compensate and get exactly args.blocks blocks
let from_block = original_tip;
let to_block = original_tip + args.blocks;
let from_block = current_tip;
let to_block = current_tip + args.blocks;
// Run benchmark
let output_dir = comparison_generator.get_ref_output_dir(ref_type);
@@ -680,9 +740,6 @@ async fn run_benchmark_workflow(
// Stop node
node_manager.stop_node(&mut node_process).await?;
// Unwind back to original tip
node_manager.unwind_to_block(original_tip).await?;
// Store results for comparison
comparison_generator.add_ref_results(ref_type, &output_dir)?;

View File

@@ -99,6 +99,7 @@ pub(crate) struct RefInfo {
/// Summary of the comparison between references.
///
/// Percent deltas are `(feature - baseline) / baseline * 100`:
/// - `new_payload_latency_mean_change_percent`: percent changes of the per-block means.
/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective
/// per-block percentiles.
/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the
@@ -116,6 +117,7 @@ pub(crate) struct ComparisonSummary {
pub per_block_latency_change_median_percent: f64,
pub per_block_latency_change_std_dev_percent: f64,
pub new_payload_total_latency_change_percent: f64,
pub new_payload_latency_mean_change_percent: f64,
pub new_payload_latency_p50_change_percent: f64,
pub new_payload_latency_p90_change_percent: f64,
pub new_payload_latency_p99_change_percent: f64,
@@ -445,6 +447,10 @@ impl ComparisonGenerator {
per_block_latency_change_median_percent,
per_block_latency_change_std_dev_percent,
new_payload_total_latency_change_percent,
new_payload_latency_mean_change_percent: calc_percent_change(
baseline.mean_new_payload_latency_ms,
feature.mean_new_payload_latency_ms,
),
new_payload_latency_p50_change_percent: calc_percent_change(
baseline.median_new_payload_latency_ms,
feature.median_new_payload_latency_ms,
@@ -575,6 +581,10 @@ impl ComparisonGenerator {
" Total newPayload time change: {:+.2}%",
summary.new_payload_total_latency_change_percent
);
println!(
" NewPayload Latency mean: {:+.2}%",
summary.new_payload_latency_mean_change_percent
);
println!(
" NewPayload Latency p50: {:+.2}%",
summary.new_payload_latency_p50_change_percent

View File

@@ -121,8 +121,7 @@ impl CompilationManager {
cmd.env("RUSTFLAGS", rustflags);
info!("Using RUSTFLAGS: {rustflags}");
// Debug log the command
debug!("Executing cargo command: {:?}", cmd);
info!("Compiling {binary_name} with {cmd:?}");
let output = cmd.output().wrap_err("Failed to execute cargo build command")?;
@@ -231,8 +230,7 @@ impl CompilationManager {
let mut cmd = Command::new("cargo");
cmd.args(["install", "--locked", "samply"]);
// Debug log the command
debug!("Executing cargo command: {:?}", cmd);
info!("Installing samply with {cmd:?}");
let output = cmd.output().wrap_err("Failed to execute cargo install samply command")?;
@@ -307,8 +305,7 @@ impl CompilationManager {
let mut cmd = Command::new("make");
cmd.arg("install-reth-bench").current_dir(&self.repo_root);
// Debug log the command
debug!("Executing make command: {:?}", cmd);
info!("Compiling reth-bench with {cmd:?}");
let output = cmd.output().wrap_err("Failed to execute make install-reth-bench command")?;

View File

@@ -2,7 +2,9 @@
use crate::cli::Args;
use alloy_provider::{Provider, ProviderBuilder};
use alloy_rpc_client::RpcClient;
use alloy_rpc_types_eth::SyncStatus;
use alloy_transport_ws::WsConnect;
use eyre::{eyre, OptionExt, Result, WrapErr};
#[cfg(unix)]
use nix::sys::signal::{killpg, Signal};
@@ -18,6 +20,9 @@ use tokio::{
};
use tracing::{debug, info, warn};
/// Default websocket RPC port used by reth
const DEFAULT_WS_RPC_PORT: u16 = 8546;
/// Manages reth node lifecycle and operations
pub(crate) struct NodeManager {
datadir: Option<String>,
@@ -152,9 +157,13 @@ impl NodeManager {
metrics_arg,
"--http".to_string(),
"--http.api".to_string(),
"eth".to_string(),
"eth,reth".to_string(),
"--ws".to_string(),
"--ws.api".to_string(),
"eth,reth".to_string(),
"--disable-discovery".to_string(),
"--trusted-only".to_string(),
"--disable-tx-gossip".to_string(),
]);
// Add tracing arguments if OTLP endpoint is configured
@@ -359,8 +368,13 @@ impl NodeManager {
Ok((child, reth_command))
}
/// Wait for the node to be ready and return its current tip
pub(crate) async fn wait_for_node_ready_and_get_tip(&self) -> Result<u64> {
/// Wait for the node to be ready and return its current tip.
///
/// Fails early if the node process exits before becoming ready.
pub(crate) async fn wait_for_node_ready_and_get_tip(
&self,
child: &mut tokio::process::Child,
) -> Result<u64> {
info!("Waiting for node to be ready and synced...");
let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync
@@ -371,8 +385,23 @@ impl NodeManager {
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
let provider = ProviderBuilder::new().connect_http(url);
let start_time = tokio::time::Instant::now();
let mut iteration = 0;
timeout(max_wait, async {
loop {
iteration += 1;
debug!(
"Readiness check iteration {} (elapsed: {:?})",
iteration,
start_time.elapsed()
);
// Check if the node process has exited.
if let Some(status) = child.try_wait()? {
return Err(eyre!("Node process exited unexpectedly with {status}"));
}
// First check if RPC is up and node is not syncing
match provider.syncing().await {
Ok(sync_result) => {
@@ -381,21 +410,115 @@ impl NodeManager {
debug!("Node is still syncing {sync_info:?}, waiting...");
}
_ => {
debug!("HTTP RPC is up and node is not syncing, checking block number...");
// Node is not syncing, now get the tip
match provider.get_block_number().await {
Ok(tip) => {
info!("Node is ready and not syncing at block: {}", tip);
return Ok(tip);
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
// Verify WebSocket RPC is ready (public endpoint, no JWT required)
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
debug!("Attempting WebSocket connection to {} (public endpoint)", ws_url);
let ws_connect = WsConnect::new(&ws_url);
match RpcClient::connect_pubsub(ws_connect).await
{
Ok(_) => {
info!(
"Node is ready (HTTP and WebSocket) at block: {} (took {:?}, {} iterations)",
tip, start_time.elapsed(), iteration
);
return Ok(tip);
}
Err(e) => {
debug!(
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
iteration, e
);
debug!("WebSocket error details: {}", e);
}
}
}
Err(e) => {
debug!("Failed to get block number: {}", e);
debug!("Failed to get block number (iteration {}): {:?}", iteration, e);
}
}
}
}
}
Err(e) => {
debug!("Node RPC not ready yet or failed to check sync status: {}", e);
debug!("Node RPC not ready yet or failed to check sync status (iteration {}): {:?}", iteration, e);
}
}
debug!("Sleeping for {:?} before next check", check_interval);
sleep(check_interval).await;
}
})
.await
.wrap_err("Timed out waiting for node to be ready and synced")?
}
/// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
///
/// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
/// the node is still syncing.
pub(crate) async fn wait_for_rpc_and_get_tip(
&self,
child: &mut tokio::process::Child,
) -> Result<u64> {
info!("Waiting for node RPC to be ready (skipping sync wait)...");
let max_wait = Duration::from_secs(60);
let check_interval = Duration::from_secs(2);
let rpc_url = "http://localhost:8545";
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
let provider = ProviderBuilder::new().connect_http(url);
let start_time = tokio::time::Instant::now();
let mut iteration = 0;
timeout(max_wait, async {
loop {
iteration += 1;
debug!(
"RPC readiness check iteration {} (elapsed: {:?})",
iteration,
start_time.elapsed()
);
if let Some(status) = child.try_wait()? {
return Err(eyre!("Node process exited unexpectedly with {status}"));
}
match provider.get_block_number().await {
Ok(tip) => {
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
let ws_connect = WsConnect::new(&ws_url);
match RpcClient::connect_pubsub(ws_connect).await {
Ok(_) => {
info!(
"Node RPC is ready at block: {} (took {:?}, {} iterations)",
tip,
start_time.elapsed(),
iteration
);
return Ok(tip);
}
Err(e) => {
debug!(
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
iteration, e
);
}
}
}
Err(e) => {
debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
}
}
@@ -403,7 +526,7 @@ impl NodeManager {
}
})
.await
.wrap_err("Timed out waiting for node to be ready and synced")?
.wrap_err("Timed out waiting for node RPC to be ready")?
}
/// Stop the reth node gracefully

View File

@@ -16,20 +16,27 @@ workspace = true
# reth
reth-cli-runner.workspace = true
reth-cli-util.workspace = true
reth-engine-primitives.workspace = true
reth-ethereum-primitives.workspace = true
reth-fs-util.workspace = true
reth-node-api.workspace = true
reth-node-core.workspace = true
reth-primitives-traits.workspace = true
reth-rpc-api.workspace = true
reth-tracing.workspace = true
reth-chainspec.workspace = true
# alloy
alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives.workspace = true
alloy-provider = { workspace = true, features = ["engine-api", "reqwest-rustls-tls"], default-features = false }
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client.workspace = true
alloy-rpc-types-engine.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
alloy-rpc-types-engine = { workspace = true, features = ["kzg"] }
alloy-transport-http.workspace = true
alloy-transport-ipc.workspace = true
alloy-transport-ws.workspace = true
@@ -50,6 +57,9 @@ tracing.workspace = true
serde.workspace = true
serde_json.workspace = true
# url parsing
url.workspace = true
# async
async-trait.workspace = true
futures.workspace = true
@@ -80,7 +90,11 @@ jemalloc = [
"reth-node-core/jemalloc",
]
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
tracy-allocator = ["reth-cli-util/tracy-allocator"]
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
tracy = [
"reth-node-core/tracy",
"reth-tracing/tracy",
]
min-error-logs = [
"tracing/release_max_level_error",

View File

@@ -31,6 +31,14 @@ Otherwise, running `make maxperf` at the root of the repo should be sufficient f
`reth-bench` contains different commands to benchmark different patterns of engine API calls.
The `reth-bench new-payload-fcu` command is the most representative of ethereum mainnet live sync, alternating between sending `engine_newPayload` calls and `engine_forkchoiceUpdated` calls.
The `new-payload-fcu` command supports two optional waiting modes that can be used together or independently:
- `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms`)
- `--wait-for-persistence`: Waits for blocks to be persisted using the `reth_subscribePersistedBlock` subscription
When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
By default, the WebSocket URL for persistence subscriptions is derived from `--engine-rpc-url` (converting to ws:// on port 8546). Use `--ws-rpc-url` to override this.
Below is an overview of how to run a benchmark:
### Setup

View File

@@ -163,7 +163,7 @@ impl AuthenticatedTransport {
// shift the iat forward by one second so there is some buffer time
let mut shifted_claims = inner_and_claims.1;
shifted_claims.iat -= 1;
shifted_claims.iat -= 30;
// if the claims are out of date, reset the inner transport
if !shifted_claims.is_within_time_window() {

View File

@@ -0,0 +1,286 @@
//! Benchmarks empty block processing by ramping the block gas limit.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
};
use alloy_eips::BlockNumberOrTag;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
use clap::Parser;
use reqwest::Url;
use reth_chainspec::ChainSpec;
use reth_cli_runner::CliContext;
use reth_ethereum_primitives::TransactionSigned;
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
engine_rpc_url: String,
/// Path to the JWT secret for Engine API authentication.
#[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
jwt_secret: PathBuf,
/// Output directory for benchmark results and generated payloads.
#[arg(long, value_name = "OUTPUT")]
output: PathBuf,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
// Ensure output directory exists
if self.output.is_file() {
return Err(eyre::eyre!("Output path must be a directory"));
}
if !self.output.exists() {
std::fs::create_dir_all(&self.output)?;
info!("Created output directory: {:?}", self.output);
}
// Set up authenticated provider (used for both Engine API and eth_ methods)
let jwt = std::fs::read_to_string(&self.jwt_secret)?;
let jwt = JwtSecret::from_hex(jwt)?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
let client = ClientBuilder::default().connect_with(auth_transport).await?;
let provider = RootProvider::<AnyNetwork>::new(client);
// Get chain spec - required for fork detection
let chain_id = provider.get_chain_id().await?;
let chain_spec = ChainSpec::from_chain_id(chain_id)
.ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
// Fetch the current head block as parent
let parent_block = provider
.get_block_by_number(BlockNumberOrTag::Latest)
.full()
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
match mode {
RampMode::Blocks(blocks) => {
info!(
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
let new_payload_version = request.new_payload_version;
let (payload, sidecar) = build_payload(&provider, request).await?;
let mut block =
payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
let max_increase = max_gas_limit_increase(parent_header.gas_limit);
let gas_limit =
parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
block.header.gas_limit = gas_limit;
let block_hash = block.header.hash_slow();
// Regenerate the payload from the modified block, but keep the original sidecar
// which contains the actual execution requests data (not just the hash)
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
let (version, params) = payload_to_new_payload(
payload,
sidecar,
false,
block.header.withdrawals_root,
Some(new_payload_version),
)?;
// Save payload to file with version info for replay
let payload_path =
self.output.join(format!("payload_block_{}.json", block.header.number));
let file =
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
let payload_json = serde_json::to_string_pretty(&file)?;
std::fs::write(&payload_path, &payload_json)?;
info!(block_number = block.header.number, path = %payload_path.display(), "Saved payload");
call_new_payload(&provider, version, params).await?;
let forkchoice_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: block_hash,
finalized_block_hash: block_hash,
};
call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
parent_header = block.header;
parent_hash = block_hash;
blocks_processed += 1;
}
let final_gas_limit = parent_header.gas_limit;
info!(
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed,
final_gas_limit,
"Benchmark complete"
);
Ok(())
}
}
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -0,0 +1,617 @@
//! Command for generating large blocks by packing transactions from real blocks.
//!
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::authenticated_transport::AuthenticatedTransportConnect;
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ForkchoiceState, JwtSecret,
PayloadAttributes,
};
use alloy_transport::layers::RetryBackoffLayer;
use clap::Parser;
use eyre::Context;
use reqwest::Url;
use reth_cli_runner::CliContext;
use reth_rpc_api::TestingBuildBlockRequestV1;
use std::future::Future;
use tokio::sync::mpsc;
use tracing::{info, warn};
/// A single transaction with its gas used and raw encoded bytes.
#[derive(Debug, Clone)]
pub struct RawTransaction {
/// The actual gas used by the transaction (from receipt).
pub gas_used: u64,
/// The transaction type (e.g., 3 for EIP-4844 blob txs).
pub tx_type: u8,
/// The raw RLP-encoded transaction bytes.
pub raw: Bytes,
}
/// Abstraction over sources of transactions for big block generation.
///
/// Implementors provide transactions from different sources (RPC, database, files, etc.)
pub trait TransactionSource {
/// Fetch transactions from a specific block number.
///
/// Returns `Ok(None)` if the block doesn't exist.
/// Returns `Ok(Some((transactions, gas_used)))` with the block's transactions and total gas.
fn fetch_block_transactions(
&self,
block_number: u64,
) -> impl Future<Output = eyre::Result<Option<(Vec<RawTransaction>, u64)>>> + Send;
}
/// RPC-based transaction source that fetches from a remote node.
#[derive(Debug)]
pub struct RpcTransactionSource {
provider: RootProvider<AnyNetwork>,
}
impl RpcTransactionSource {
/// Create a new RPC transaction source.
pub const fn new(provider: RootProvider<AnyNetwork>) -> Self {
Self { provider }
}
/// Create from an RPC URL with retry backoff.
pub fn from_url(rpc_url: &str) -> eyre::Result<Self> {
let client = ClientBuilder::default()
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let provider = RootProvider::<AnyNetwork>::new(client);
Ok(Self { provider })
}
}
impl TransactionSource for RpcTransactionSource {
async fn fetch_block_transactions(
&self,
block_number: u64,
) -> eyre::Result<Option<(Vec<RawTransaction>, u64)>> {
// Fetch block and receipts in parallel
let (block, receipts) = tokio::try_join!(
self.provider.get_block_by_number(block_number.into()).full(),
self.provider.get_block_receipts(block_number.into())
)?;
let Some(block) = block else {
return Ok(None);
};
let Some(receipts) = receipts else {
return Err(eyre::eyre!("Receipts not found for block {}", block_number));
};
let block_gas_used = block.header.gas_used;
// Convert cumulative gas from receipts to per-tx gas_used
let mut prev_cumulative = 0u64;
let transactions: Vec<RawTransaction> = block
.transactions
.txns()
.zip(receipts.iter())
.map(|(tx, receipt)| {
let cumulative = receipt.inner.inner.inner.receipt.cumulative_gas_used;
let gas_used = cumulative - prev_cumulative;
prev_cumulative = cumulative;
let with_encoded = tx.inner.inner.clone().into_encoded();
RawTransaction {
gas_used,
tx_type: tx.inner.ty(),
raw: with_encoded.encoded_bytes().clone(),
}
})
.collect();
Ok(Some((transactions, block_gas_used)))
}
}
/// Collects transactions from a source up to a target gas usage.
#[derive(Debug)]
pub struct TransactionCollector<S> {
source: S,
target_gas: u64,
}
impl<S: TransactionSource> TransactionCollector<S> {
/// Create a new transaction collector.
pub const fn new(source: S, target_gas: u64) -> Self {
Self { source, target_gas }
}
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < self.target_gas {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(block = current_block, "Block not found, stopping");
break;
};
for tx in block_txs {
// Skip blob transactions (EIP-4844, type 3)
if tx.tx_type == 3 {
continue;
}
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
total_gas += tx.gas_used;
}
if total_gas >= self.target_gas {
break;
}
}
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = self.target_gas.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
}
info!(
total_txs = transactions.len(),
total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok((transactions, total_gas, current_block))
}
}
/// `reth bench generate-big-block` command
///
/// Generates a large block by fetching transactions from existing blocks and packing them
/// into a single block using the `testing_buildBlockV1` RPC endpoint.
#[derive(Debug, Parser)]
pub struct Command {
/// The RPC URL to use for fetching blocks (can be an external archive node).
#[arg(long, value_name = "RPC_URL")]
rpc_url: String,
/// The engine RPC URL (with JWT authentication).
#[arg(long, value_name = "ENGINE_RPC_URL", default_value = "http://localhost:8551")]
engine_rpc_url: String,
/// The RPC URL for `testing_buildBlockV1` calls (same node as engine, regular RPC port).
#[arg(long, value_name = "TESTING_RPC_URL", default_value = "http://localhost:8545")]
testing_rpc_url: String,
/// Path to the JWT secret file for engine API authentication.
#[arg(long, value_name = "JWT_SECRET")]
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
target_gas: u64,
/// Starting block number to fetch transactions from.
/// If not specified, starts from the engine's latest block.
#[arg(long, value_name = "FROM_BLOCK")]
from_block: Option<u64>,
/// Execute the payload (call newPayload + forkchoiceUpdated).
/// If false, only builds the payload and prints it.
#[arg(long, default_value = "false")]
execute: bool,
/// Number of payloads to generate. Each payload uses the previous as parent.
/// When count == 1, the payload is only generated and saved, not executed.
/// When count > 1, each payload is executed before building the next.
#[arg(long, default_value = "1")]
count: u64,
/// Number of transaction batches to prefetch in background when count > 1.
/// Higher values reduce latency but use more memory.
#[arg(long, default_value = "4")]
prefetch_buffer: usize,
/// Output directory for generated payloads. Each payload is saved as `payload_block_N.json`.
#[arg(long, value_name = "OUTPUT_DIR")]
output_dir: std::path::PathBuf,
}
/// A built payload ready for execution.
struct BuiltPayload {
block_number: u64,
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
}
impl Command {
/// Execute the `generate-big-block` command
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(target_gas = self.target_gas, count = self.count, "Generating big block(s)");
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
let jwt = JwtSecret::from_hex(jwt.trim())?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
// Set up testing RPC provider (for testing_buildBlockV1)
info!("Connecting to Testing RPC at {}", self.testing_rpc_url);
let testing_client = ClientBuilder::default()
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(self.testing_rpc_url.parse()?);
let testing_provider = RootProvider::<AnyNetwork>::new(testing_client);
// Get the parent block (latest canonical block)
info!(endpoint = "engine", method = "eth_getBlockByNumber", block = "latest", "RPC call");
let parent_block = auth_provider
.get_block_by_number(BlockNumberOrTag::Latest)
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let parent_hash = parent_block.header.hash;
let parent_number = parent_block.header.number;
let parent_timestamp = parent_block.header.timestamp;
info!(
parent_hash = %parent_hash,
parent_number = parent_number,
"Using initial parent block"
);
// Create output directory
std::fs::create_dir_all(&self.output_dir).wrap_err_with(|| {
format!("Failed to create output directory: {:?}", self.output_dir)
})?;
let start_block = self.from_block.unwrap_or(parent_number);
// Use pipelined execution when generating multiple payloads
if self.count > 1 {
self.execute_pipelined(
&auth_provider,
&testing_provider,
start_block,
parent_hash,
parent_timestamp,
)
.await?;
} else {
// Single payload - collect transactions and build
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential(
&auth_provider,
&testing_provider,
transactions,
parent_hash,
parent_timestamp,
)
.await?;
}
info!(count = self.count, output_dir = %self.output_dir.display(), "All payloads generated");
Ok(())
}
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
transactions: Vec<Bytes>,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let built = self
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
}
parent_hash = built.block_hash;
parent_timestamp = built.timestamp;
}
Ok(())
}
/// Pipelined execution - fetches transactions and builds payloads in background.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
start_block: u64,
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(error = %e, "Failed to create transaction source");
return;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
});
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
let is_last = i == self.count - 1;
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
);
self.build_payload(
testing_provider,
&transactions,
i,
parent_hash,
parent_timestamp,
)
.await?
};
self.save_payload(&current_payload)?;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_receiver);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
&self,
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
}
/// Static version for use in spawned tasks.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
let request = TestingBuildBlockRequestV1 {
parent_block_hash: parent_hash,
payload_attributes: PayloadAttributes {
timestamp: parent_timestamp + 12,
prev_randao: B256::ZERO,
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
},
transactions: transactions.to_vec(),
extra_data: None,
};
let total_tx_bytes: usize = transactions.iter().map(|tx| tx.len()).sum();
info!(
payload = index + 1,
tx_count = transactions.len(),
total_tx_bytes = total_tx_bytes,
parent_hash = %parent_hash,
"Sending to testing_buildBlockV1"
);
let envelope: ExecutionPayloadEnvelopeV5 =
testing_provider.client().request("testing_buildBlockV1", [request]).await?;
let v4_envelope = envelope.try_into_v4()?;
let inner = &v4_envelope.envelope_inner.execution_payload.payload_inner.payload_inner;
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
}
/// Save a payload to disk.
fn save_payload(&self, payload: &BuiltPayload) -> eyre::Result<()> {
let filename = format!("payload_block_{}.json", payload.block_number);
let filepath = self.output_dir.join(&filename);
let json = serde_json::to_string_pretty(&payload.envelope)?;
std::fs::write(&filepath, &json)
.wrap_err_with(|| format!("Failed to write payload to {:?}", filepath))?;
info!(block_number = payload.block_number, block_hash = %payload.block_hash, path = %filepath.display(), "Payload saved");
Ok(())
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload,
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
if !fcu_result.is_valid() {
return Err(eyre::eyre!("FCU rejected: {:?}", fcu_result));
}
Ok(())
}
}

View File

@@ -0,0 +1,196 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
PayloadAttributes, PayloadId, PraguePayloadFields,
};
use eyre::OptionExt;
use reth_chainspec::{ChainSpec, EthereumHardforks};
use reth_node_api::EngineApiMessageVersion;
use tracing::debug;
/// Prepared payload request data for triggering block building.
pub(crate) struct PayloadRequest {
/// The payload attributes for the new block.
pub(crate) attributes: PayloadAttributes,
/// The forkchoice state pointing to the parent block.
pub(crate) forkchoice_state: ForkchoiceState,
/// The engine API version for FCU calls.
pub(crate) fcu_version: EngineApiMessageVersion,
/// The getPayload version to use (1-5).
pub(crate) get_payload_version: u8,
/// The newPayload version to use.
pub(crate) new_payload_version: EngineApiMessageVersion,
}
/// Prepare payload attributes and forkchoice state for a new block.
pub(crate) fn prepare_payload_request(
chain_spec: &ChainSpec,
timestamp: u64,
parent_hash: B256,
) -> PayloadRequest {
let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
// FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
let fcu_version = if cancun_active {
EngineApiMessageVersion::V3
} else if shanghai_active {
EngineApiMessageVersion::V2
} else {
EngineApiMessageVersion::V1
};
// getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
// newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
let (get_payload_version, new_payload_version) = if osaka_active {
(5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
} else if prague_active {
(4, EngineApiMessageVersion::V4)
} else if cancun_active {
(3, EngineApiMessageVersion::V3)
} else if shanghai_active {
(2, EngineApiMessageVersion::V2)
} else {
(1, EngineApiMessageVersion::V1)
};
PayloadRequest {
attributes: PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: Address::ZERO,
withdrawals: shanghai_active.then(Vec::new),
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
},
forkchoice_state: ForkchoiceState {
head_block_hash: parent_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
},
fcu_version,
get_payload_version,
new_payload_version,
}
}
/// Trigger payload building via FCU and retrieve the built payload.
///
/// This sends a forkchoiceUpdated with payload attributes to start building,
/// then calls getPayload to retrieve the result.
pub(crate) async fn build_payload(
provider: &RootProvider<AnyNetwork>,
request: PayloadRequest,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
let fcu_result = call_forkchoice_updated(
provider,
request.fcu_version,
request.forkchoice_state,
Some(request.attributes.clone()),
)
.await?;
let payload_id =
fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
get_payload_with_sidecar(
provider,
request.get_payload_version,
payload_id,
request.attributes.parent_beacon_block_root,
)
.await
}
/// Convert an RPC block to a consensus header and block hash.
pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
let block_hash = block.header.hash;
let header = block.header.inner.clone().into_header_with_defaults();
(header, block_hash)
}
/// Compute versioned hashes from KZG commitments.
fn versioned_hashes_from_commitments(
commitments: &[alloy_primitives::FixedBytes<48>],
) -> Vec<B256> {
commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
}
/// Fetch an execution payload using the appropriate engine API version.
pub(crate) async fn get_payload_with_sidecar(
provider: &RootProvider<AnyNetwork>,
version: u8,
payload_id: PayloadId,
parent_beacon_block_root: Option<B256>,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
debug!(get_payload_version = ?version, ?payload_id, "Sending getPayload");
match version {
1 => {
let payload = provider.get_payload_v1(payload_id).await?;
Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
}
2 => {
let envelope = provider.get_payload_v2(payload_id).await?;
let payload = match envelope.execution_payload {
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
};
Ok((payload, ExecutionPayloadSidecar::none()))
}
3 => {
let envelope = provider.get_payload_v3(payload_id).await?;
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V3")?,
versioned_hashes,
};
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v3(cancun_fields),
))
}
4 => {
let envelope = provider.get_payload_v4(payload_id).await?;
let versioned_hashes = versioned_hashes_from_commitments(
&envelope.envelope_inner.blobs_bundle.commitments,
);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V4")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.envelope_inner.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
5 => {
// V5 (Osaka) - use raw request since alloy doesn't have get_payload_v5 yet
let envelope = provider.get_payload_v5(payload_id).await?;
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V5")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
_ => panic!("This tool does not support getPayload versions past v5"),
}
}

View File

@@ -6,9 +6,16 @@ use reth_node_core::args::LogArgs;
use reth_tracing::FileWorkerGuard;
mod context;
mod gas_limit_ramp;
mod generate_big_block;
pub(crate) mod helpers;
pub use generate_big_block::{
RawTransaction, RpcTransactionSource, TransactionCollector, TransactionSource,
};
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod replay_payloads;
mod send_payload;
/// `reth bench` command
@@ -27,6 +34,9 @@ pub enum Subcommands {
/// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
NewPayloadFcu(new_payload_fcu::Command),
/// Benchmark which builds empty blocks with a ramped gas limit.
GasLimitRamp(gas_limit_ramp::Command),
/// Benchmark which only calls subsequent `newPayload` calls.
NewPayloadOnly(new_payload_only::Command),
@@ -41,6 +51,29 @@ pub enum Subcommands {
/// `cast block latest --full --json | reth-bench send-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex)`
SendPayload(send_payload::Command),
/// Generate a large block by packing transactions from existing blocks.
///
/// This command fetches transactions from real blocks and packs them into a single
/// block using the `testing_buildBlockV1` RPC endpoint.
///
/// Example:
///
/// `reth-bench generate-big-block --rpc-url http://localhost:8545 --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex --target-gas
/// 30000000`
GenerateBigBlock(generate_big_block::Command),
/// Replay pre-generated payloads from a directory.
///
/// This command reads payload files from a previous `generate-big-block` run and replays
/// them in sequence using `newPayload` followed by `forkchoiceUpdated`.
///
/// Example:
///
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
}
impl BenchmarkCommand {
@@ -51,8 +84,11 @@ impl BenchmarkCommand {
match self.command {
Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
}
}

View File

@@ -1,26 +1,42 @@
//! Runs the `reth bench` command, calling first newPayload for each block, then calling
//! forkchoiceUpdated.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
bench::{
context::BenchContext,
output::{
CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow, COMBINED_OUTPUT_SUFFIX,
GAS_OUTPUT_SUFFIX,
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
};
use alloy_provider::Provider;
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use csv::Writer;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use url::Url;
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -30,8 +46,31 @@ pub struct Command {
rpc_url: String,
/// How long to wait after a forkchoice update before sending the next payload.
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, default_value = "250ms", verbatim_doc_comment)]
wait_time: Duration,
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(
long = "persistence-threshold",
value_name = "PERSISTENCE_THRESHOLD",
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
verbatim_doc_comment
)]
persistence_threshold: u64,
/// The size of the block buffer (channel capacity) for prefetching blocks from the RPC
/// endpoint.
@@ -50,12 +89,39 @@ pub struct Command {
impl Command {
/// Execute `benchmark new-payload-fcu` command
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
// Log mode configuration
if let Some(duration) = self.wait_time {
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options (duration takes precedence)
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let sub = self.setup_persistence_subscription().await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
PERSISTENCE_CHECKPOINT_TIMEOUT,
))
}
(None, false) => None,
};
let BenchContext {
benchmark_mode,
block_provider,
auth_provider,
mut next_block,
is_optimism,
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let buffer_size = self.rpc_block_buffer_size;
@@ -110,7 +176,6 @@ impl Command {
}
});
// put results in a summary vec so they can be printed at the end
let mut results = Vec::new();
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
@@ -121,14 +186,13 @@ impl Command {
total_wait_time += wait_start.elapsed();
result
} {
// just put gas used here
let gas_used = block.header.gas_used;
let gas_limit = block.header.gas_limit;
let block_number = block.header.number;
let transaction_count = block.transactions.len() as u64;
debug!(target: "reth-bench", ?block_number, "Sending payload",);
debug!(target: "reth-bench", ?block_number, "Sending payload");
// construct fcu to call
let forkchoice_state = ForkchoiceState {
head_block_hash: head,
safe_block_hash: safe,
@@ -143,28 +207,26 @@ impl Command {
call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?;
// calculate the total duration and the fcu latency, record
let total_latency = start.elapsed();
let fcu_latency = total_latency - new_payload_result.latency;
let combined_result = CombinedResult {
block_number,
gas_limit,
transaction_count,
new_payload_result,
fcu_latency,
total_latency,
};
// current duration since the start of the benchmark minus the time
// waiting for blocks
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
// We want to measure engine throughput, not RPC fetch latency.
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
// convert gas used to gigagas, then compute gigagas per second
info!(%combined_result);
// wait before sending the next payload
tokio::time::sleep(self.wait_time).await;
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
// record the current result
let gas_row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
@@ -175,34 +237,19 @@ impl Command {
return Err(error);
}
let (gas_output_results, combined_results): (_, Vec<CombinedResult>) =
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
// write the csv output to files
if let Some(path) = self.benchmark.output {
// first write the combined results to a file
let output_path = path.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(output_path)?;
for result in combined_results {
writer.serialize(result)?;
}
writer.flush()?;
// now write the gas output to a file
let output_path = path.join(GAS_OUTPUT_SUFFIX);
info!("Writing total gas output to file: {:?}", output_path);
let mut writer = Writer::from_path(output_path)?;
for row in &gas_output_results {
writer.serialize(row)?;
}
writer.flush()?;
info!("Finished writing benchmark output files to {:?}.", path);
if let Some(ref path) = self.benchmark.output {
write_benchmark_results(path, &gas_output_results, combined_results)?;
}
// accumulate the results and calculate the overall Ggas/s
let gas_output = TotalGasOutput::new(gas_output_results)?;
info!(
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
@@ -213,4 +260,278 @@ impl Command {
Ok(())
}
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `--ws-rpc-url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %self.benchmark.engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
let ws_url = self.derive_ws_rpc_url()?;
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -49,6 +49,7 @@ impl Command {
auth_provider,
mut next_block,
is_optimism,
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let buffer_size = self.rpc_block_buffer_size;
@@ -96,11 +97,7 @@ impl Command {
let transaction_count = block.transactions.len() as u64;
let gas_used = block.header.gas_used;
debug!(
target: "reth-bench",
number=?block.header.number,
"Sending payload to engine",
);
debug!(number=?block.header.number, "Sending payload to engine");
let (version, params) = block_to_new_payload(block, is_optimism)?;

View File

@@ -1,10 +1,13 @@
//! Contains various benchmark output formats, either for logging or for
//! serialization to / from files.
use alloy_primitives::B256;
use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Serialize};
use std::time::Duration;
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{path::Path, time::Duration};
use tracing::info;
/// This is the suffix for gas output csv files.
pub(crate) const GAS_OUTPUT_SUFFIX: &str = "total_gas.csv";
@@ -15,6 +18,17 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
/// This is the suffix for new payload output csv files.
pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
/// Serialized format for gas ramp payloads on disk.
#[derive(Debug, Serialize, Deserialize)]
pub(crate) struct GasRampPayloadFile {
/// Engine API version (1-5).
pub(crate) version: u8,
/// The block hash for FCU.
pub(crate) block_hash: B256,
/// The params to pass to newPayload.
pub(crate) params: serde_json::Value,
}
/// This represents the results of a single `newPayload` call in the benchmark, containing the gas
/// used and the `newPayload` latency.
#[derive(Debug)]
@@ -67,6 +81,8 @@ impl Serialize for NewPayloadResult {
pub(crate) struct CombinedResult {
/// The block number of the block being processed.
pub(crate) block_number: u64,
/// The gas limit of the block.
pub(crate) gas_limit: u64,
/// The number of transactions in the block.
pub(crate) transaction_count: u64,
/// The `newPayload` result.
@@ -88,7 +104,7 @@ impl std::fmt::Display for CombinedResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
write!(
f,
"Payload {} processed at {:.4} Ggas/s, used {} total gas. Combined gas per second: {:.4} Ggas/s. fcu latency: {:?}, newPayload latency: {:?}",
"Block {} processed at {:.4} Ggas/s, used {} total gas. Combined: {:.4} Ggas/s. fcu: {:?}, newPayload: {:?}",
self.block_number,
self.new_payload_result.gas_per_second() / GIGAGAS as f64,
self.new_payload_result.gas_used,
@@ -110,10 +126,11 @@ impl Serialize for CombinedResult {
let fcu_latency = self.fcu_latency.as_micros();
let new_payload_latency = self.new_payload_result.latency.as_micros();
let total_latency = self.total_latency.as_micros();
let mut state = serializer.serialize_struct("CombinedResult", 6)?;
let mut state = serializer.serialize_struct("CombinedResult", 7)?;
// flatten the new payload result because this is meant for CSV writing
state.serialize_field("block_number", &self.block_number)?;
state.serialize_field("gas_limit", &self.gas_limit)?;
state.serialize_field("transaction_count", &self.transaction_count)?;
state.serialize_field("gas_used", &self.new_payload_result.gas_used)?;
state.serialize_field("new_payload_latency", &new_payload_latency)?;
@@ -167,6 +184,36 @@ impl TotalGasOutput {
}
}
/// Write benchmark results to CSV files.
///
/// Writes two files to the output directory:
/// - `combined_latency.csv`: Per-block latency results
/// - `total_gas.csv`: Per-block gas usage over time
pub(crate) fn write_benchmark_results(
output_dir: &Path,
gas_results: &[TotalGasRow],
combined_results: Vec<CombinedResult>,
) -> eyre::Result<()> {
let output_path = output_dir.join(COMBINED_OUTPUT_SUFFIX);
info!("Writing engine api call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;
for result in combined_results {
writer.serialize(result)?;
}
writer.flush()?;
let output_path = output_dir.join(GAS_OUTPUT_SUFFIX);
info!("Writing total gas output to file: {:?}", output_path);
let mut writer = Writer::from_path(&output_path)?;
for row in gas_results {
writer.serialize(row)?;
}
writer.flush()?;
info!("Finished writing benchmark output files to {:?}.", output_dir);
Ok(())
}
/// This serializes the `time` field of the [`TotalGasRow`] to microseconds.
///
/// This is essentially just for the csv writer, which would have headers

View File

@@ -0,0 +1,332 @@
//! Command for replaying pre-generated payloads from disk.
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::output::GasRampPayloadFile,
valid_payload::{call_forkchoice_updated, call_new_payload},
};
use alloy_primitives::B256;
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
use clap::Parser;
use eyre::Context;
use reqwest::Url;
use reth_cli_runner::CliContext;
use reth_node_api::EngineApiMessageVersion;
use std::path::PathBuf;
use tracing::{debug, info};
/// `reth bench replay-payloads` command
///
/// Replays pre-generated payloads from a directory by calling `newPayload` followed by
/// `forkchoiceUpdated` for each payload in sequence.
#[derive(Debug, Parser)]
pub struct Command {
/// The engine RPC URL (with JWT authentication).
#[arg(long, value_name = "ENGINE_RPC_URL", default_value = "http://localhost:8551")]
engine_rpc_url: String,
/// Path to the JWT secret file for engine API authentication.
#[arg(long, value_name = "JWT_SECRET")]
jwt_secret: PathBuf,
/// Directory containing payload files (`payload_block_N.json`).
#[arg(long, value_name = "PAYLOAD_DIR")]
payload_dir: PathBuf,
/// Optional limit on the number of payloads to replay.
/// If not specified, replays all payloads in the directory.
#[arg(long, value_name = "COUNT")]
count: Option<usize>,
/// Skip the first N payloads.
#[arg(long, value_name = "SKIP", default_value = "0")]
skip: usize,
/// Optional directory containing gas ramp payloads to replay first.
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
}
/// A loaded payload ready for execution.
struct LoadedPayload {
/// The index (from filename).
index: u64,
/// The payload envelope.
envelope: ExecutionPayloadEnvelopeV4,
/// The block hash.
block_hash: B256,
}
/// A gas ramp payload loaded from disk.
struct GasRampPayload {
/// Block number from filename.
block_number: u64,
/// Engine API version for newPayload.
version: EngineApiMessageVersion,
/// The file contents.
file: GasRampPayloadFile,
}
impl Command {
/// Execute the `replay-payloads` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(payload_dir = %self.payload_dir.display(), "Replaying payloads");
// Set up authenticated engine provider
let jwt =
std::fs::read_to_string(&self.jwt_secret).wrap_err("Failed to read JWT secret file")?;
let jwt = JwtSecret::from_hex(jwt.trim())?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
// Get parent block (latest canonical block) - we need this for the first FCU
let parent_block = auth_provider
.get_block_by_number(alloy_eips::BlockNumberOrTag::Latest)
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let initial_parent_hash = parent_block.header.hash;
let initial_parent_number = parent_block.header.number;
info!(
parent_hash = %initial_parent_hash,
parent_number = initial_parent_number,
"Using initial parent block"
);
// Load all payloads upfront to avoid I/O delays between phases
let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
if payloads.is_empty() {
return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
}
info!(count = payloads.len(), "Loaded gas ramp payloads from disk");
payloads
} else {
Vec::new()
};
let payloads = self.load_payloads()?;
if payloads.is_empty() {
return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
}
info!(count = payloads.len(), "Loaded main payloads from disk");
let mut parent_hash = initial_parent_hash;
// Replay gas ramp payloads first
for (i, payload) in gas_ramp_payloads.iter().enumerate() {
info!(
gas_ramp_payload = i + 1,
total = gas_ramp_payloads.len(),
block_number = payload.block_number,
block_hash = %payload.file.block_hash,
"Executing gas ramp payload (newPayload + FCU)"
);
call_new_payload(&auth_provider, payload.version, payload.file.params.clone()).await?;
let fcu_state = ForkchoiceState {
head_block_hash: payload.file.block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
parent_hash = payload.file.block_hash;
}
if !gas_ramp_payloads.is_empty() {
info!(count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
for (i, payload) in payloads.iter().enumerate() {
info!(
payload = i + 1,
total = payloads.len(),
index = payload.index,
block_hash = %payload.block_hash,
"Executing payload (newPayload + FCU)"
);
self.execute_payload_v4(&auth_provider, &payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
parent_hash = payload.block_hash;
}
info!(count = payloads.len(), "All payloads replayed successfully");
Ok(())
}
/// Load and parse all payload files from the directory.
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
let mut payloads = Vec::new();
// Read directory entries
let entries: Vec<_> = std::fs::read_dir(&self.payload_dir)
.wrap_err_with(|| format!("Failed to read directory {:?}", self.payload_dir))?
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
// Parse filenames to get indices and sort
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
.into_iter()
.filter_map(|e| {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract index from "payload_NNN.json"
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let index: u64 = index_str.parse().ok()?;
Some((index, e.path()))
})
.collect();
indexed_paths.sort_by_key(|(idx, _)| *idx);
// Apply skip and count
let indexed_paths: Vec<_> = indexed_paths.into_iter().skip(self.skip).collect();
let indexed_paths: Vec<_> = match self.count {
Some(count) => indexed_paths.into_iter().take(count).collect(),
None => indexed_paths,
};
// Load each payload
for (index, path) in indexed_paths {
let content = std::fs::read_to_string(&path)
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
info!(
index = index,
block_hash = %block_hash,
path = %path.display(),
"Loaded payload"
);
payloads.push(LoadedPayload { index, envelope, block_hash });
}
Ok(payloads)
}
/// Load and parse gas ramp payload files from a directory.
fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
let mut payloads = Vec::new();
let entries: Vec<_> = std::fs::read_dir(dir)
.wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
// Parse filenames to get block numbers and sort
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
.into_iter()
.filter_map(|e| {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract block number from "payload_block_NNN.json"
let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let block_number: u64 = block_str.parse().ok()?;
Some((block_number, e.path()))
})
.collect();
indexed_paths.sort_by_key(|(num, _)| *num);
for (block_number, path) in indexed_paths {
let content = std::fs::read_to_string(&path)
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
let file: GasRampPayloadFile = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let version = match file.version {
1 => EngineApiMessageVersion::V1,
2 => EngineApiMessageVersion::V2,
3 => EngineApiMessageVersion::V3,
4 => EngineApiMessageVersion::V4,
5 => EngineApiMessageVersion::V5,
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
};
info!(
block_number,
block_hash = %file.block_hash,
path = %path.display(),
"Loaded gas ramp payload"
);
payloads.push(GasRampPayload { block_number, version, file });
}
Ok(payloads)
}
async fn execute_payload_v4(
&self,
provider: &RootProvider<AnyNetwork>,
envelope: &ExecutionPayloadEnvelopeV4,
parent_hash: B256,
) -> eyre::Result<()> {
let block_hash =
envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
debug!(
method = "engine_newPayloadV4",
block_hash = %block_hash,
"Sending newPayload"
);
let status = provider
.new_payload_v4(
envelope.envelope_inner.execution_payload.clone(),
vec![],
B256::ZERO,
envelope.execution_requests.to_vec(),
)
.await?;
info!(?status, "newPayloadV4 response");
if !status.is_valid() {
return Err(eyre::eyre!("Payload rejected: {:?}", status));
}
let fcu_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
debug!(method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_result = provider.fork_choice_updated_v3(fcu_state, None).await?;
info!(?fcu_result, "forkchoiceUpdatedV3 response");
Ok(())
}
}

View File

@@ -3,15 +3,16 @@
//! before sending additional calls.
use alloy_eips::eip7685::Requests;
use alloy_primitives::B256;
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
use alloy_rpc_types_engine::{
ExecutionPayload, ExecutionPayloadInputV2, ForkchoiceState, ForkchoiceUpdated,
PayloadAttributes, PayloadStatus,
ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ForkchoiceState,
ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
};
use alloy_transport::TransportResult;
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
use reth_node_api::EngineApiMessageVersion;
use tracing::error;
use tracing::{debug, error};
/// An extension trait for providers that implement the engine API, to wait for a VALID response.
#[async_trait::async_trait]
@@ -52,6 +53,13 @@ where
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
method = "engine_forkchoiceUpdatedV1",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v1(fork_choice_state, payload_attributes.clone()).await?;
@@ -82,6 +90,13 @@ where
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
method = "engine_forkchoiceUpdatedV2",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v2(fork_choice_state, payload_attributes.clone()).await?;
@@ -112,6 +127,13 @@ where
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
method = "engine_forkchoiceUpdatedV3",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v3(fork_choice_state, payload_attributes.clone()).await?;
@@ -148,33 +170,47 @@ pub(crate) fn block_to_new_payload(
// Convert to execution payload
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
}
pub(crate) fn payload_to_new_payload(
payload: ExecutionPayload,
sidecar: ExecutionPayloadSidecar,
is_optimism: bool,
withdrawals_root: Option<B256>,
target_version: Option<EngineApiMessageVersion>,
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
let (version, params) = match payload {
ExecutionPayload::V3(payload) => {
let cancun = sidecar.cancun().unwrap();
if let Some(prague) = sidecar.prague() {
// Use target version if provided (for Osaka), otherwise default to V4
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
if is_optimism {
let withdrawals_root = withdrawals_root.ok_or_else(|| {
eyre::eyre!("Missing withdrawals root for Optimism payload")
})?;
(
EngineApiMessageVersion::V4,
version,
serde_json::to_value((
OpExecutionPayloadV4 {
payload_inner: payload,
withdrawals_root: block.withdrawals_root.unwrap(),
},
OpExecutionPayloadV4 { payload_inner: payload, withdrawals_root },
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
Requests::default(),
))?,
)
} else {
// Extract actual Requests from RequestsOrHash
let requests = prague.requests.requests_hash();
(
EngineApiMessageVersion::V4,
version,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
prague.requests.requests_hash(),
requests,
))?,
)
}
@@ -217,6 +253,8 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
) -> TransportResult<()> {
let method = version.method_name();
debug!(method, "Sending newPayload");
let mut status: PayloadStatus = provider.client().request(method, &params).await?;
while !status.is_valid() {
@@ -237,12 +275,15 @@ pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
/// `EngineApiMessageVersion`, using the provided forkchoice state and payload attributes for the
/// actual engine api message call.
///
/// Note: For Prague (V4), we still use forkchoiceUpdatedV3 as there is no V4.
pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
provider: P,
message_version: EngineApiMessageVersion,
forkchoice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await

View File

@@ -81,12 +81,16 @@ backon.workspace = true
tempfile.workspace = true
[features]
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
default = ["jemalloc", "otlp", "otlp-logs", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
otlp = [
"reth-ethereum-cli/otlp",
"reth-node-core/otlp",
]
otlp-logs = [
"reth-ethereum-cli/otlp-logs",
"reth-node-core/otlp-logs",
]
js-tracer = [
"reth-node-builder/js-tracer",
"reth-node-ethereum/js-tracer",
@@ -116,6 +120,11 @@ jemalloc-prof = [
"reth-cli-util/jemalloc",
"reth-cli-util/jemalloc-prof",
"reth-ethereum-cli/jemalloc-prof",
"reth-node-metrics/jemalloc-prof",
]
jemalloc-symbols = [
"jemalloc-prof",
"reth-ethereum-cli/jemalloc-symbols",
]
jemalloc-unprefixed = [
"reth-cli-util/jemalloc-unprefixed",
@@ -126,6 +135,11 @@ jemalloc-unprefixed = [
tracy-allocator = [
"reth-cli-util/tracy-allocator",
"reth-ethereum-cli/tracy-allocator",
"tracy",
]
tracy = [
"reth-ethereum-cli/tracy",
"reth-node-core/tracy",
]
# Because jemalloc is default and preferred over snmalloc when both features are
@@ -166,6 +180,8 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
edge = ["reth-ethereum-cli/edge", "reth-node-core/edge"]
[[bin]]
name = "reth"
path = "src/main.rs"

View File

@@ -2,22 +2,46 @@
//!
//! ## Feature Flags
//!
//! ### Default Features
//!
//! - `jemalloc`: Uses [jemallocator](https://github.com/tikv/jemallocator) as the global allocator.
//! This is **not recommended on Windows**. See [here](https://rust-lang.github.io/rfcs/1974-global-allocators.html#jemalloc)
//! for more info.
//! - `otlp`: Enables [OpenTelemetry](https://opentelemetry.io/) metrics export to a configured OTLP
//! collector endpoint.
//! - `js-tracer`: Enables the `JavaScript` tracer for the `debug_trace` endpoints, allowing custom
//! `JavaScript`-based transaction tracing.
//! - `keccak-cache-global`: Enables global caching for Keccak256 hashes to improve performance.
//! - `asm-keccak`: Replaces the default, pure-Rust implementation of Keccak256 with one implemented
//! in assembly; see [the `keccak-asm` crate](https://github.com/DaniPopes/keccak-asm) for more
//! details and supported targets.
//!
//! ### Allocator Features
//!
//! - `jemalloc-prof`: Enables [jemallocator's](https://github.com/tikv/jemallocator) heap profiling
//! and leak detection functionality. See [jemalloc's opt.prof](https://jemalloc.net/jemalloc.3.html#opt.prof)
//! documentation for usage details. This is **not recommended on Windows**. See [here](https://rust-lang.github.io/rfcs/1974-global-allocators.html#jemalloc)
//! for more info.
//! - `asm-keccak`: replaces the default, pure-Rust implementation of Keccak256 with one implemented
//! in assembly; see [the `keccak-asm` crate](https://github.com/DaniPopes/keccak-asm) for more
//! details and supported targets
//! documentation for usage details. This is **not recommended on Windows**.
//! - `jemalloc-symbols`: Enables jemalloc symbols for profiling. Includes `jemalloc-prof`.
//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
//! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
//! integration for memory profiling.
//! - `snmalloc`: Uses [snmalloc](https://github.com/snmalloc/snmalloc) as the global allocator. Use
//! `--no-default-features` when enabling this, as jemalloc takes precedence.
//! - `snmalloc-native`: Uses snmalloc with native CPU optimizations. Use `--no-default-features`
//! when enabling this.
//!
//! ### Log Level Features
//!
//! - `min-error-logs`: Disables all logs below `error` level.
//! - `min-warn-logs`: Disables all logs below `warn` level.
//! - `min-info-logs`: Disables all logs below `info` level. This can speed up the node, since fewer
//! calls to the logging component are made.
//! - `min-debug-logs`: Disables all logs below `debug` level.
//! - `min-trace-logs`: Disables all logs below `trace` level.
//!
//! ### Development Features
//!
//! - `dev`: Enables development mode features, including test vector generation commands.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
@@ -170,7 +194,7 @@ pub mod rpc {
pub use reth_rpc::eth::*;
}
/// Re-exported from `reth_rpc::rpc`.
/// Re-exported from `reth_rpc_server_types::result`.
pub mod result {
pub use reth_rpc_server_types::result::*;
}

View File

@@ -3,6 +3,10 @@
#[global_allocator]
static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
#[cfg(all(feature = "jemalloc-prof", unix))]
#[unsafe(export_name = "_rjem_malloc_conf")]
static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0";
use clap::Parser;
use reth::{args::RessArgs, cli::Cli, ress::install_ress_subprotocol};
use reth_ethereum_cli::chainspec::EthereumChainSpecParser;

View File

@@ -41,6 +41,7 @@ derive_more.workspace = true
metrics.workspace = true
parking_lot.workspace = true
pin-project.workspace = true
rayon = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
# optional deps for test-utils
@@ -84,6 +85,7 @@ test-utils = [
"reth-trie/test-utils",
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"

View File

@@ -33,6 +33,7 @@ where
) -> Self {
let (finalized_block, _) = watch::channel(finalized);
let (safe_block, _) = watch::channel(safe);
let (persisted_block, _) = watch::channel(None);
Self {
inner: Arc::new(ChainInfoInner {
@@ -42,6 +43,7 @@ where
canonical_head: RwLock::new(head),
safe_block,
finalized_block,
persisted_block,
}),
}
}
@@ -97,6 +99,11 @@ where
self.inner.finalized_block.borrow().as_ref().map(SealedHeader::num_hash)
}
/// Returns the `BlockNumHash` of the persisted block.
pub fn get_persisted_num_hash(&self) -> Option<BlockNumHash> {
*self.inner.persisted_block.borrow()
}
/// Sets the canonical head of the chain.
pub fn set_canonical_head(&self, header: SealedHeader<N::BlockHeader>) {
let number = header.number();
@@ -130,6 +137,18 @@ where
});
}
/// Sets the persisted block of the chain.
pub fn set_persisted(&self, num_hash: BlockNumHash) {
self.inner.persisted_block.send_if_modified(|current| {
if current.map(|b| b.hash) != Some(num_hash.hash) {
let _ = current.replace(num_hash);
return true
}
false
});
}
/// Subscribe to the finalized block.
pub fn subscribe_finalized_block(
&self,
@@ -141,6 +160,11 @@ where
pub fn subscribe_safe_block(&self) -> watch::Receiver<Option<SealedHeader<N::BlockHeader>>> {
self.inner.safe_block.subscribe()
}
/// Subscribe to the persisted block.
pub fn subscribe_persisted_block(&self) -> watch::Receiver<Option<BlockNumHash>> {
self.inner.persisted_block.subscribe()
}
}
/// Container type for all chain info fields
@@ -159,11 +183,14 @@ struct ChainInfoInner<N: NodePrimitives = reth_ethereum_primitives::EthPrimitive
safe_block: watch::Sender<Option<SealedHeader<N::BlockHeader>>>,
/// The block that the beacon node considers finalized.
finalized_block: watch::Sender<Option<SealedHeader<N::BlockHeader>>>,
/// The last block that was persisted to disk.
persisted_block: watch::Sender<Option<BlockNumHash>>,
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::B256;
use reth_ethereum_primitives::EthPrimitives;
use reth_testing_utils::{generators, generators::random_header};
@@ -338,4 +365,28 @@ mod tests {
// Assert that the BlockNumHash returned matches the safe header
assert_eq!(tracker.get_safe_num_hash(), Some(safe_header.num_hash()));
}
#[test]
fn test_set_persisted() {
let mut rng = generators::rng();
let header = random_header(&mut rng, 10, None);
let tracker: ChainInfoTracker<EthPrimitives> = ChainInfoTracker::new(header, None, None);
// Initial state: persisted block should be None
assert!(tracker.get_persisted_num_hash().is_none());
// Set a persisted block
let num_hash1 = BlockNumHash::new(10, B256::random());
tracker.set_persisted(num_hash1);
assert_eq!(tracker.get_persisted_num_hash(), Some(num_hash1));
// Setting the same block again should not change anything
tracker.set_persisted(num_hash1);
assert_eq!(tracker.get_persisted_num_hash(), Some(num_hash1));
// Set a different block
let num_hash2 = BlockNumHash::new(20, B256::random());
tracker.set_persisted(num_hash2);
assert_eq!(tracker.get_persisted_num_hash(), Some(num_hash2));
}
}

View File

@@ -37,12 +37,19 @@ pub struct ComputedTrieData {
/// Trie input bundled with its anchor hash.
///
/// This is used to store the trie input and anchor hash for a block together.
/// The `trie_input` contains the **cumulative** overlay of all in-memory ancestor blocks,
/// not just this block's changes. Child blocks reuse the parent's overlay in O(1) by
/// cloning the Arc-wrapped data.
///
/// The `anchor_hash` is metadata indicating which persisted base state this overlay
/// sits on top of. It is CRITICAL for overlay reuse decisions: an overlay built on top
/// of Anchor A cannot be reused for a block anchored to Anchor B, as it would result
/// in an incorrect state.
#[derive(Clone, Debug)]
pub struct AnchoredTrieInput {
/// The persisted ancestor hash this trie input is anchored to.
pub anchor_hash: B256,
/// Trie input constructed from in-memory overlays.
/// Cumulative trie input overlay from all in-memory ancestors.
pub trie_input: Arc<TrieInputSorted>,
}
@@ -62,7 +69,8 @@ static DEFERRED_TRIE_METRICS: LazyLock<DeferredTrieMetrics> =
/// Internal state for deferred trie data.
enum DeferredState {
/// Data is not yet available; raw inputs stored for fallback computation.
Pending(PendingInputs),
/// Wrapped in `Option` to allow taking ownership during computation.
Pending(Option<PendingInputs>),
/// Data has been computed and is ready.
Ready(ComputedTrieData),
}
@@ -112,12 +120,12 @@ impl DeferredTrieData {
ancestors: Vec<Self>,
) -> Self {
Self {
state: Arc::new(Mutex::new(DeferredState::Pending(PendingInputs {
state: Arc::new(Mutex::new(DeferredState::Pending(Some(PendingInputs {
hashed_state,
trie_updates,
anchor_hash,
ancestors,
}))),
})))),
}
}
@@ -138,8 +146,9 @@ impl DeferredTrieData {
///
/// # Process
/// 1. Sort the current block's hashed state and trie updates
/// 2. Merge ancestor overlays (oldest -> newest, so later state takes precedence)
/// 3. Extend the merged overlay with this block's sorted data
/// 2. Reuse parent's cached overlay if available (O(1) - the common case)
/// 3. Otherwise, rebuild overlay from ancestors (rare fallback)
/// 4. Extend the overlay with this block's sorted data
///
/// Used by both the async background task and the synchronous fallback path.
///
@@ -147,49 +156,163 @@ impl DeferredTrieData {
/// * `hashed_state` - Unsorted hashed post-state (account/storage changes) from execution
/// * `trie_updates` - Unsorted trie node updates from state root computation
/// * `anchor_hash` - The persisted ancestor hash this trie input is anchored to
/// * `ancestors` - Deferred trie data from ancestor blocks for merging
/// * `ancestors` - Deferred trie data from ancestor blocks for merging (oldest -> newest)
pub fn sort_and_build_trie_input(
hashed_state: &HashedPostState,
trie_updates: &TrieUpdates,
hashed_state: Arc<HashedPostState>,
trie_updates: Arc<TrieUpdates>,
anchor_hash: B256,
ancestors: &[Self],
) -> ComputedTrieData {
// Sort the current block's hashed state and trie updates
let sorted_hashed_state = Arc::new(hashed_state.clone_into_sorted());
let sorted_trie_updates = Arc::new(trie_updates.clone_into_sorted());
#[cfg(feature = "rayon")]
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|| match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
|| match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
// Merge trie data from ancestors (oldest -> newest so later state takes precedence)
let mut overlay = TrieInputSorted::default();
for ancestor in ancestors {
let ancestor_data = ancestor.wait_cloned();
{
let state_mut = Arc::make_mut(&mut overlay.state);
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
}
{
let nodes_mut = Arc::make_mut(&mut overlay.nodes);
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
}
}
#[cfg(not(feature = "rayon"))]
let (sorted_hashed_state, sorted_trie_updates) = (
match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
// Extend overlay with current block's sorted data
{
let state_mut = Arc::make_mut(&mut overlay.state);
state_mut.extend_ref(sorted_hashed_state.as_ref());
}
{
let nodes_mut = Arc::make_mut(&mut overlay.nodes);
nodes_mut.extend_ref(sorted_trie_updates.as_ref());
}
// Reuse parent's overlay if available and anchors match.
// We can only reuse the parent's overlay if it was built on top of the same
// persisted anchor. If the anchor has changed (e.g., due to persistence),
// the parent's overlay is relative to an old state and cannot be used.
let overlay = if let Some(parent) = ancestors.last() {
let parent_data = parent.wait_cloned();
match &parent_data.anchored_trie_input {
// Case 1: Parent has cached overlay AND anchors match.
Some(AnchoredTrieInput { anchor_hash: parent_anchor, trie_input })
if *parent_anchor == anchor_hash =>
{
// O(1): Reuse parent's overlay, extend with current block's data.
let mut overlay = TrieInputSorted::new(
Arc::clone(&trie_input.nodes),
Arc::clone(&trie_input.state),
Default::default(), // prefix_sets are per-block, not cumulative
);
// Only trigger COW clone if there's actually data to add.
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
}
overlay
}
// Case 2: Parent exists but anchor mismatch or no cached overlay.
// We must rebuild from the ancestors list (which only contains unpersisted blocks).
_ => Self::merge_ancestors_into_overlay(
ancestors,
&sorted_hashed_state,
&sorted_trie_updates,
),
}
} else {
// Case 3: No in-memory ancestors (first block after persisted anchor).
// Build overlay with just this block's data.
Self::merge_ancestors_into_overlay(&[], &sorted_hashed_state, &sorted_trie_updates)
};
ComputedTrieData::with_trie_input(
sorted_hashed_state,
sorted_trie_updates,
Arc::new(sorted_hashed_state),
Arc::new(sorted_trie_updates),
anchor_hash,
Arc::new(overlay),
)
}
/// Merge all ancestors and current block's data into a single overlay.
///
/// This is a rare fallback path, only used when no ancestor has a cached
/// `anchored_trie_input` (e.g., blocks created via alternative constructors).
/// In normal operation, the parent always has a cached overlay and this
/// function is never called.
///
/// When the `rayon` feature is enabled, uses parallel collection and merge:
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
/// 2. Merges hashed state and trie updates in parallel with each other
/// 3. Uses tree reduction within each merge for O(log n) depth
#[cfg(feature = "rayon")]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
// Early exit: no ancestors means just wrap current block's data
if ancestors.is_empty() {
return TrieInputSorted::new(
Arc::new(sorted_trie_updates.clone()),
Arc::new(sorted_hashed_state.clone()),
Default::default(),
);
}
// Collect ancestor data, unzipping states and updates into Arc slices
let (states, updates): (Vec<_>, Vec<_>) = ancestors
.iter()
.map(|a| {
let data = a.wait_cloned();
(data.hashed_state, data.trie_updates)
})
.unzip();
// Merge state and nodes in parallel with each other using tree reduction
let (state, nodes) = rayon::join(
|| {
let mut merged = HashedPostStateSorted::merge_parallel(&states);
merged.extend_ref_and_sort(sorted_hashed_state);
merged
},
|| {
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
merged.extend_ref_and_sort(sorted_trie_updates);
merged
},
);
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
}
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
#[cfg(not(feature = "rayon"))]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
let mut overlay = TrieInputSorted::default();
let state_mut = Arc::make_mut(&mut overlay.state);
let nodes_mut = Arc::make_mut(&mut overlay.nodes);
for ancestor in ancestors {
let ancestor_data = ancestor.wait_cloned();
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
}
// Extend with current block's sorted data last (takes precedence)
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
overlay
}
/// Returns trie data, computing synchronously if the async task hasn't completed.
///
/// - If the async task has completed (`Ready`), returns the cached result.
@@ -204,7 +327,7 @@ impl DeferredTrieData {
#[instrument(level = "debug", target = "engine::tree::deferred_trie", skip_all)]
pub fn wait_cloned(&self) -> ComputedTrieData {
let mut state = self.state.lock();
match &*state {
match &mut *state {
// If the deferred trie data is ready, return the cached result.
DeferredState::Ready(bundle) => {
DEFERRED_TRIE_METRICS.deferred_trie_async_ready.increment(1);
@@ -212,15 +335,23 @@ impl DeferredTrieData {
}
// If the deferred trie data is pending, compute the trie data synchronously and return
// the result. This is the fallback path if the async task hasn't completed.
DeferredState::Pending(inputs) => {
DeferredState::Pending(maybe_inputs) => {
DEFERRED_TRIE_METRICS.deferred_trie_sync_fallback.increment(1);
let inputs = maybe_inputs.take().expect("inputs must be present in Pending state");
let computed = Self::sort_and_build_trie_input(
&inputs.hashed_state,
&inputs.trie_updates,
inputs.hashed_state,
inputs.trie_updates,
inputs.anchor_hash,
&inputs.ancestors,
);
*state = DeferredState::Ready(computed.clone());
// Release lock before inputs (and its ancestors) drop to avoid holding it
// while their potential last Arc refs drop (which could trigger recursive locking)
drop(state);
computed
}
}
@@ -441,4 +572,365 @@ mod tests {
let (_, account) = &overlay_state[0];
assert_eq!(account.unwrap().nonce, 2);
}
/// Helper to create a ready block with anchored trie input containing specific state.
fn ready_block_with_state(
anchor_hash: B256,
accounts: Vec<(B256, Option<Account>)>,
) -> DeferredTrieData {
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
let trie_updates = Arc::default();
let mut overlay = TrieInputSorted::default();
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
DeferredTrieData::ready(ComputedTrieData {
hashed_state,
trie_updates,
anchored_trie_input: Some(AnchoredTrieInput {
anchor_hash,
trie_input: Arc::new(overlay),
}),
})
}
/// Verifies that first block after anchor (no ancestors) creates empty base overlay.
#[test]
fn first_block_after_anchor_creates_empty_base() {
let anchor = B256::with_last_byte(1);
let key = B256::with_last_byte(42);
let account = Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None };
// First block after anchor - no ancestors
let first_block = DeferredTrieData::pending(
Arc::new(HashedPostState::default().with_accounts([(key, Some(account))])),
Arc::new(TrieUpdates::default()),
anchor,
vec![], // No ancestors
);
let result = first_block.wait_cloned();
// Should have overlay with just this block's data
let overlay = result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.anchor_hash, anchor);
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
let (found_key, found_account) = &overlay.trie_input.state.accounts[0];
assert_eq!(*found_key, key);
assert_eq!(found_account.unwrap().nonce, 1);
}
/// Verifies that parent's overlay is reused regardless of anchor.
#[test]
fn reuses_parent_overlay() {
let anchor = B256::with_last_byte(1);
let key = B256::with_last_byte(42);
let account = Account { nonce: 100, balance: U256::ZERO, bytecode_hash: None };
// Create parent with anchored trie input
let parent = ready_block_with_state(anchor, vec![(key, Some(account))]);
// Create child - should reuse parent's overlay
let child = DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
anchor,
vec![parent],
);
let result = child.wait_cloned();
// Verify parent's account is in the overlay
let overlay = result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.anchor_hash, anchor);
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
let (found_key, found_account) = &overlay.trie_input.state.accounts[0];
assert_eq!(*found_key, key);
assert_eq!(found_account.unwrap().nonce, 100);
}
/// Verifies that parent's overlay is NOT reused when anchor changes (after persist).
/// The overlay data is dependent on the anchor, so it must be rebuilt from the
/// remaining ancestors.
#[test]
fn rebuilds_overlay_when_anchor_changes() {
let old_anchor = B256::with_last_byte(1);
let new_anchor = B256::with_last_byte(2);
let key = B256::with_last_byte(42);
let account = Account { nonce: 50, balance: U256::ZERO, bytecode_hash: None };
// Create parent with OLD anchor
let parent = ready_block_with_state(old_anchor, vec![(key, Some(account))]);
// Create child with NEW anchor (simulates after persist)
// Should NOT reuse parent's overlay because anchor changed
let child = DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
new_anchor,
vec![parent],
);
let result = child.wait_cloned();
// Verify result uses new anchor
let overlay = result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.anchor_hash, new_anchor);
// Crucially, since we provided `parent` in ancestors but it has a different anchor,
// the code falls back to `merge_ancestors_into_overlay`.
// `merge_ancestors_into_overlay` reads `parent.hashed_state` (which has the account).
// So the account IS present, but it was obtained via REBUILD, not REUSE.
// We can check `DEFERRED_TRIE_METRICS` if we want to be sure, but functionally:
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
let (found_key, found_account) = &overlay.trie_input.state.accounts[0];
assert_eq!(*found_key, key);
assert_eq!(found_account.unwrap().nonce, 50);
}
/// Verifies that parent without `anchored_trie_input` triggers rebuild path.
#[test]
fn rebuilds_when_parent_has_no_anchored_input() {
let anchor = B256::with_last_byte(1);
let key = B256::with_last_byte(42);
let account = Account { nonce: 25, balance: U256::ZERO, bytecode_hash: None };
// Create parent WITHOUT anchored trie input (e.g., from without_trie_input constructor)
let parent_state =
HashedPostStateSorted::new(vec![(key, Some(account))], B256Map::default());
let parent = DeferredTrieData::ready(ComputedTrieData {
hashed_state: Arc::new(parent_state),
trie_updates: Arc::default(),
anchored_trie_input: None, // No anchored input
});
// Create child - should rebuild from parent's hashed_state
let child = DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
anchor,
vec![parent],
);
let result = child.wait_cloned();
// Verify overlay is built and contains parent's data
let overlay = result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.anchor_hash, anchor);
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
}
/// Verifies that a chain of blocks with matching anchors builds correct cumulative overlay.
#[test]
fn chain_of_blocks_builds_cumulative_overlay() {
let anchor = B256::with_last_byte(1);
let key1 = B256::with_last_byte(1);
let key2 = B256::with_last_byte(2);
let key3 = B256::with_last_byte(3);
// Block 1: sets account at key1
let block1 = ready_block_with_state(
anchor,
vec![(key1, Some(Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None }))],
);
// Block 2: adds account at key2, ancestor is block1
let block2_hashed = HashedPostState::default().with_accounts([(
key2,
Some(Account { nonce: 2, balance: U256::ZERO, bytecode_hash: None }),
)]);
let block2 = DeferredTrieData::pending(
Arc::new(block2_hashed),
Arc::new(TrieUpdates::default()),
anchor,
vec![block1.clone()],
);
// Compute block2's trie data
let block2_computed = block2.wait_cloned();
let block2_ready = DeferredTrieData::ready(block2_computed);
// Block 3: adds account at key3, ancestor is block2 (which includes block1)
let block3_hashed = HashedPostState::default().with_accounts([(
key3,
Some(Account { nonce: 3, balance: U256::ZERO, bytecode_hash: None }),
)]);
let block3 = DeferredTrieData::pending(
Arc::new(block3_hashed),
Arc::new(TrieUpdates::default()),
anchor,
vec![block1, block2_ready],
);
let result = block3.wait_cloned();
// Verify all three accounts are in the cumulative overlay
let overlay = result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.trie_input.state.accounts.len(), 3);
// Accounts should be sorted by key (B256 ordering)
let accounts = &overlay.trie_input.state.accounts;
assert!(accounts.iter().any(|(k, a)| *k == key1 && a.unwrap().nonce == 1));
assert!(accounts.iter().any(|(k, a)| *k == key2 && a.unwrap().nonce == 2));
assert!(accounts.iter().any(|(k, a)| *k == key3 && a.unwrap().nonce == 3));
}
/// Verifies that child block's state overwrites parent's state for the same key.
#[test]
fn child_state_overwrites_parent() {
let anchor = B256::with_last_byte(1);
let key = B256::with_last_byte(42);
// Parent sets nonce to 10
let parent = ready_block_with_state(
anchor,
vec![(key, Some(Account { nonce: 10, balance: U256::ZERO, bytecode_hash: None }))],
);
// Child overwrites nonce to 99
let child_hashed = HashedPostState::default().with_accounts([(
key,
Some(Account { nonce: 99, balance: U256::ZERO, bytecode_hash: None }),
)]);
let child = DeferredTrieData::pending(
Arc::new(child_hashed),
Arc::new(TrieUpdates::default()),
anchor,
vec![parent],
);
let result = child.wait_cloned();
// Verify child's value wins (extend_ref uses later value)
let overlay = result.anchored_trie_input.as_ref().unwrap();
// Note: extend_ref may result in duplicate keys; check the last occurrence
let accounts = &overlay.trie_input.state.accounts;
let last_account = accounts.iter().rfind(|(k, _)| *k == key).unwrap();
assert_eq!(last_account.1.unwrap().nonce, 99);
}
/// Stress test: verify O(N) behavior by building a chain of many blocks.
/// This test ensures the fix doesn't regress - previously this would be O(N²).
#[test]
fn long_chain_builds_in_linear_time() {
let anchor = B256::with_last_byte(1);
let num_blocks = 50; // Enough to notice O(N²) vs O(N) difference
let mut ancestors: Vec<DeferredTrieData> = Vec::new();
let start = Instant::now();
for i in 0..num_blocks {
let key = B256::with_last_byte(i as u8);
let account = Account { nonce: i as u64, balance: U256::ZERO, bytecode_hash: None };
let hashed = HashedPostState::default().with_accounts([(key, Some(account))]);
let block = DeferredTrieData::pending(
Arc::new(hashed),
Arc::new(TrieUpdates::default()),
anchor,
ancestors.clone(),
);
// Compute and add to ancestors for next iteration
let computed = block.wait_cloned();
ancestors.push(DeferredTrieData::ready(computed));
}
let elapsed = start.elapsed();
// With O(N) fix, 50 blocks should complete quickly (< 1 second)
// With O(N²), this would take significantly longer
assert!(
elapsed < Duration::from_secs(2),
"Chain of {num_blocks} blocks took {:?}, possible O(N²) regression",
elapsed
);
// Verify final overlay has all accounts
let final_result = ancestors.last().unwrap().wait_cloned();
let overlay = final_result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.trie_input.state.accounts.len(), num_blocks);
}
/// Verifies that a multi-ancestor overlay is rebuilt when anchor changes.
/// This simulates the "persist prefix then keep building" scenario where:
/// 1. A chain of blocks is built with anchor A
/// 2. Some blocks are persisted, changing anchor to B
/// 3. New blocks must rebuild the overlay from the remaining ancestors
#[test]
fn multi_ancestor_overlay_rebuilt_after_anchor_change() {
let old_anchor = B256::with_last_byte(1);
let new_anchor = B256::with_last_byte(2);
let key1 = B256::with_last_byte(1);
let key2 = B256::with_last_byte(2);
let key3 = B256::with_last_byte(3);
let key4 = B256::with_last_byte(4);
// Build a chain of 3 blocks with old_anchor
let block1 = ready_block_with_state(
old_anchor,
vec![(key1, Some(Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None }))],
);
let block2_hashed = HashedPostState::default().with_accounts([(
key2,
Some(Account { nonce: 2, balance: U256::ZERO, bytecode_hash: None }),
)]);
let block2 = DeferredTrieData::pending(
Arc::new(block2_hashed),
Arc::new(TrieUpdates::default()),
old_anchor,
vec![block1.clone()],
);
let block2_ready = DeferredTrieData::ready(block2.wait_cloned());
let block3_hashed = HashedPostState::default().with_accounts([(
key3,
Some(Account { nonce: 3, balance: U256::ZERO, bytecode_hash: None }),
)]);
let block3 = DeferredTrieData::pending(
Arc::new(block3_hashed),
Arc::new(TrieUpdates::default()),
old_anchor,
vec![block1.clone(), block2_ready.clone()],
);
let block3_ready = DeferredTrieData::ready(block3.wait_cloned());
// Verify block3's overlay has all 3 accounts with old_anchor
let block3_overlay = block3_ready.wait_cloned().anchored_trie_input.unwrap();
assert_eq!(block3_overlay.anchor_hash, old_anchor);
assert_eq!(block3_overlay.trie_input.state.accounts.len(), 3);
// Now simulate persist: create block4 with NEW anchor but same ancestors.
// To verify correct rebuilding, we must provide ALL unpersisted ancestors.
// If we only provided block3, the rebuild would only see block3's state.
// We pass block1, block2, block3 to simulate that they are all still in memory
// but the anchor check forces a rebuild (e.g. artificial anchor change).
let block4_hashed = HashedPostState::default().with_accounts([(
key4,
Some(Account { nonce: 4, balance: U256::ZERO, bytecode_hash: None }),
)]);
let block4 = DeferredTrieData::pending(
Arc::new(block4_hashed),
Arc::new(TrieUpdates::default()),
new_anchor, // Different anchor - simulates post-persist
vec![block1, block2_ready, block3_ready],
);
let result = block4.wait_cloned();
// Verify:
// 1. New anchor is used in result
assert_eq!(result.anchor_hash(), Some(new_anchor));
// 2. All 4 accounts are in the overlay (rebuilt from ancestors + extended)
let overlay = result.anchored_trie_input.as_ref().unwrap();
assert_eq!(overlay.trie_input.state.accounts.len(), 4);
// 3. All accounts have correct values
let accounts = &overlay.trie_input.state.accounts;
assert!(accounts.iter().any(|(k, a)| *k == key1 && a.unwrap().nonce == 1));
assert!(accounts.iter().any(|(k, a)| *k == key2 && a.unwrap().nonce == 2));
assert!(accounts.iter().any(|(k, a)| *k == key3 && a.unwrap().nonce == 3));
assert!(accounts.iter().any(|(k, a)| *k == key4 && a.unwrap().nonce == 4));
}
}

View File

@@ -10,15 +10,15 @@ use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256};
use parking_lot::RwLock;
use reth_chainspec::ChainInfo;
use reth_ethereum_primitives::EthPrimitives;
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_metrics::{metrics::Gauge, Metrics};
use reth_primitives_traits::{
BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
SignedTransaction,
};
use reth_storage_api::StateProviderBox;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::{collections::BTreeMap, ops::Deref, sync::Arc, time::Instant};
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, TrieInputSorted};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tokio::sync::{broadcast, watch};
/// Size of the broadcast channel used to notify canonical state events.
@@ -317,6 +317,7 @@ impl<N: NodePrimitives> CanonicalInMemoryState<N> {
/// This will update the links between blocks and remove all blocks that are [..
/// `persisted_height`].
pub fn remove_persisted_blocks(&self, persisted_num_hash: BlockNumHash) {
self.set_persisted(persisted_num_hash);
// if the persisted hash is not in the canonical in memory state, do nothing, because it
// means canonical blocks were not actually persisted.
//
@@ -444,6 +445,11 @@ impl<N: NodePrimitives> CanonicalInMemoryState<N> {
self.inner.chain_info_tracker.set_finalized(header);
}
/// Persisted block setter.
pub fn set_persisted(&self, num_hash: BlockNumHash) {
self.inner.chain_info_tracker.set_persisted(num_hash);
}
/// Canonical head getter.
pub fn get_canonical_head(&self) -> SealedHeader<N::BlockHeader> {
self.inner.chain_info_tracker.get_canonical_head()
@@ -459,6 +465,11 @@ impl<N: NodePrimitives> CanonicalInMemoryState<N> {
self.inner.chain_info_tracker.get_safe_header()
}
/// Persisted block `BlockNumHash` getter.
pub fn get_persisted_num_hash(&self) -> Option<BlockNumHash> {
self.inner.chain_info_tracker.get_persisted_num_hash()
}
/// Returns the `SealedHeader` corresponding to the pending state.
pub fn pending_sealed_header(&self) -> Option<SealedHeader<N::BlockHeader>> {
self.pending_state().map(|h| h.block_ref().recovered_block().clone_sealed_header())
@@ -511,6 +522,11 @@ impl<N: NodePrimitives> CanonicalInMemoryState<N> {
self.inner.chain_info_tracker.subscribe_finalized_block()
}
/// Subscribe to new persisted block events.
pub fn subscribe_persisted_block(&self) -> watch::Receiver<Option<BlockNumHash>> {
self.inner.chain_info_tracker.subscribe_persisted_block()
}
/// Attempts to send a new [`CanonStateNotification`] to all active Receiver handles.
pub fn notify_canon_state(&self, event: CanonStateNotification<N>) {
self.inner.canon_state_notification_sender.send(event).ok();
@@ -632,7 +648,7 @@ impl<N: NodePrimitives> BlockState<N> {
}
/// Returns the `Receipts` of executed block that determines the state.
pub fn receipts(&self) -> &Vec<Vec<N::Receipt>> {
pub fn receipts(&self) -> &Vec<N::Receipt> {
&self.block.execution_outcome().receipts
}
@@ -643,15 +659,7 @@ impl<N: NodePrimitives> BlockState<N> {
///
/// This clones the vector of receipts. To avoid it, use [`Self::executed_block_receipts_ref`].
pub fn executed_block_receipts(&self) -> Vec<N::Receipt> {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().cloned().unwrap_or_default()
self.receipts().clone()
}
/// Returns a slice of `Receipt` of executed block that determines the state.
@@ -659,15 +667,7 @@ impl<N: NodePrimitives> BlockState<N> {
/// has only one element corresponding to the executed block associated to
/// the state.
pub fn executed_block_receipts_ref(&self) -> &[N::Receipt] {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().map(|receipts| receipts.deref()).unwrap_or_default()
self.receipts()
}
/// Returns an iterator over __parent__ `BlockStates`.
@@ -751,7 +751,7 @@ pub struct ExecutedBlock<N: NodePrimitives = EthPrimitives> {
/// Recovered Block
pub recovered_block: Arc<RecoveredBlock<N::Block>>,
/// Block's execution outcome.
pub execution_output: Arc<ExecutionOutcome<N::Receipt>>,
pub execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
/// Deferred trie data produced by execution.
///
/// This allows deferring the computation of the trie data which can be expensive.
@@ -763,7 +763,15 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
fn default() -> Self {
Self {
recovered_block: Default::default(),
execution_output: Default::default(),
execution_output: Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: Default::default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: Default::default(),
}),
trie_data: DeferredTrieData::ready(ComputedTrieData::default()),
}
}
@@ -784,7 +792,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// payload builders). This is the safe default path.
pub fn new(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: ComputedTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data: DeferredTrieData::ready(trie_data) }
@@ -806,7 +814,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Use [`Self::new()`] instead when trie data is already computed and available immediately.
pub const fn with_deferred_trie_data(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: DeferredTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data }
@@ -826,7 +834,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Returns a reference to the block's execution outcome
#[inline]
pub fn execution_outcome(&self) -> &ExecutionOutcome<N::Receipt> {
pub fn execution_outcome(&self) -> &BlockExecutionOutput<N::Receipt> {
&self.execution_output
}
@@ -926,31 +934,39 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
pub fn to_chain_notification(&self) -> CanonStateNotification<N> {
match self {
Self::Commit { new } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
);
chain
}));
CanonStateNotification::Commit { new }
CanonStateNotification::Commit { new: Arc::new(Self::blocks_to_chain(new)) }
}
Self::Reorg { new, old } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
Self::Reorg { new, old } => CanonStateNotification::Reorg {
new: Arc::new(Self::blocks_to_chain(new)),
old: Arc::new(Self::blocks_to_chain(old)),
},
}
}
/// Converts a slice of executed blocks into a [`Chain`].
fn blocks_to_chain(blocks: &[ExecutedBlock<N>]) -> Chain<N> {
match blocks {
[] => Chain::default(),
[first, rest @ ..] => {
let mut chain = Chain::from_block(
first.recovered_block().clone(),
ExecutionOutcome::from((
first.execution_outcome().clone(),
first.block_number(),
)),
LazyTrieData::ready(first.hashed_state(), first.trie_updates()),
);
for exec in rest {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
ExecutionOutcome::from((
exec.execution_outcome().clone(),
exec.block_number(),
)),
LazyTrieData::ready(exec.hashed_state(), exec.trie_updates()),
);
chain
}));
let old = Arc::new(old.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
);
chain
}));
CanonStateNotification::Reorg { new, old }
}
chain
}
}
}
@@ -1244,7 +1260,7 @@ mod tests {
let state = BlockState::new(block);
assert_eq!(state.receipts(), &receipts);
assert_eq!(state.receipts(), receipts.first().unwrap());
}
#[test]
@@ -1521,22 +1537,31 @@ mod tests {
let block2a =
test_block_builder.get_executed_block_with_number(2, block1.recovered_block.hash());
let sample_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
..Default::default()
};
// Test commit notification
let chain_commit = NewCanonicalChain::Commit { new: vec![block0.clone(), block1.clone()] };
// Build expected trie data map
let mut expected_trie_data = BTreeMap::new();
expected_trie_data
.insert(0, LazyTrieData::ready(block0.hashed_state(), block0.trie_updates()));
expected_trie_data
.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
// Build expected execution outcome (first_block matches first block number)
let commit_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 0,
..Default::default()
};
assert_eq!(
chain_commit.to_chain_notification(),
CanonStateNotification::Commit {
new: Arc::new(Chain::new(
vec![block0.recovered_block().clone(), block1.recovered_block().clone()],
sample_execution_outcome.clone(),
None
commit_execution_outcome,
expected_trie_data,
))
}
);
@@ -1547,18 +1572,39 @@ mod tests {
old: vec![block1.clone(), block2.clone()],
};
// Build expected trie data for old chain
let mut old_trie_data = BTreeMap::new();
old_trie_data.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
old_trie_data.insert(2, LazyTrieData::ready(block2.hashed_state(), block2.trie_updates()));
// Build expected trie data for new chain
let mut new_trie_data = BTreeMap::new();
new_trie_data
.insert(1, LazyTrieData::ready(block1a.hashed_state(), block1a.trie_updates()));
new_trie_data
.insert(2, LazyTrieData::ready(block2a.hashed_state(), block2a.trie_updates()));
// Build expected execution outcome for reorg chains (first_block matches first block
// number)
let reorg_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 1,
..Default::default()
};
assert_eq!(
chain_reorg.to_chain_notification(),
CanonStateNotification::Reorg {
old: Arc::new(Chain::new(
vec![block1.recovered_block().clone(), block2.recovered_block().clone()],
sample_execution_outcome.clone(),
None
reorg_execution_outcome.clone(),
old_trie_data,
)),
new: Arc::new(Chain::new(
vec![block1a.recovered_block().clone(), block2a.recovered_block().clone()],
sample_execution_outcome,
None
reorg_execution_outcome,
new_trie_data,
))
}
);

View File

@@ -0,0 +1,186 @@
//! Lazy overlay computation for trie input.
//!
//! This module provides [`LazyOverlay`], a type that computes the [`TrieInputSorted`]
//! lazily on first access. This allows execution to start before the trie overlay
//! is fully computed.
use crate::DeferredTrieData;
use alloy_primitives::B256;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::sync::{Arc, OnceLock};
use tracing::{debug, trace};
/// Inputs captured for lazy overlay computation.
#[derive(Clone)]
struct LazyOverlayInputs {
/// The persisted ancestor hash (anchor) this overlay should be built on.
anchor_hash: B256,
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
blocks: Vec<DeferredTrieData>,
}
/// Lazily computed trie overlay.
///
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
/// but for overlay computation.
///
/// # Fast Path vs Slow Path
///
/// - **Fast path**: If the tip block's cached `anchored_trie_input` is ready and its `anchor_hash`
/// matches our expected anchor, we can reuse it directly (O(1)).
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
#[derive(Clone)]
pub struct LazyOverlay {
/// Computed result, cached after first access.
inner: Arc<OnceLock<TrieInputSorted>>,
/// Inputs for lazy computation.
inputs: LazyOverlayInputs,
}
impl std::fmt::Debug for LazyOverlay {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LazyOverlay")
.field("anchor_hash", &self.inputs.anchor_hash)
.field("num_blocks", &self.inputs.blocks.len())
.field("computed", &self.inner.get().is_some())
.finish()
}
}
impl LazyOverlay {
/// Create a new lazy overlay with the given anchor hash and block handles.
///
/// # Arguments
///
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
}
/// Returns the anchor hash this overlay is built on.
pub const fn anchor_hash(&self) -> B256 {
self.inputs.anchor_hash
}
/// Returns the number of in-memory blocks this overlay covers.
pub const fn num_blocks(&self) -> usize {
self.inputs.blocks.len()
}
/// Returns true if the overlay has already been computed.
pub fn is_computed(&self) -> bool {
self.inner.get().is_some()
}
/// Returns the computed trie input, computing it if necessary.
///
/// The first call triggers computation (which may block waiting for deferred data).
/// Subsequent calls return the cached result immediately.
pub fn get(&self) -> &TrieInputSorted {
self.inner.get_or_init(|| self.compute())
}
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
let input = self.get();
(Arc::clone(&input.nodes), Arc::clone(&input.state))
}
/// Compute the trie input overlay.
fn compute(&self) -> TrieInputSorted {
let anchor_hash = self.inputs.anchor_hash;
let blocks = &self.inputs.blocks;
if blocks.is_empty() {
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
return TrieInputSorted::default();
}
// Fast path: Check if tip block's overlay is ready and anchor matches.
// The tip block (first in list) has the cumulative overlay from all ancestors.
if let Some(tip) = blocks.first() {
let data = tip.wait_cloned();
if let Some(anchored) = &data.anchored_trie_input {
if anchored.anchor_hash == anchor_hash {
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
return (*anchored.trie_input).clone();
}
debug!(
target: "chain_state::lazy_overlay",
computed_anchor = %anchored.anchor_hash,
%anchor_hash,
"Anchor mismatch, falling back to merge"
);
}
}
// Slow path: Merge all blocks' trie data into a new overlay.
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
Self::merge_blocks(blocks)
}
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
///
/// Blocks are ordered newest to oldest.
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
if blocks.is_empty() {
return TrieInputSorted::default();
}
let state =
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
let nodes =
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
}
}
#[cfg(test)]
mod tests {
use super::*;
use reth_trie::{updates::TrieUpdates, HashedPostState};
fn empty_deferred(anchor: B256) -> DeferredTrieData {
DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
anchor,
Vec::new(),
)
}
#[test]
fn empty_blocks_returns_default() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
let result = overlay.get();
assert!(result.state.is_empty());
assert!(result.nodes.is_empty());
}
#[test]
fn single_block_uses_data_directly() {
let anchor = B256::random();
let deferred = empty_deferred(anchor);
let overlay = LazyOverlay::new(anchor, vec![deferred]);
assert!(!overlay.is_computed());
let _ = overlay.get();
assert!(overlay.is_computed());
}
#[test]
fn cached_after_first_access() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
// First access computes
let _ = overlay.get();
assert!(overlay.is_computed());
// Second access uses cache
let _ = overlay.get();
assert!(overlay.is_computed());
}
}

View File

@@ -14,6 +14,9 @@ pub use in_memory::*;
mod deferred_trie;
pub use deferred_trie::*;
mod lazy_overlay;
pub use lazy_overlay::*;
mod noop;
mod chain_info;
@@ -23,7 +26,8 @@ mod notifications;
pub use notifications::{
CanonStateNotification, CanonStateNotificationSender, CanonStateNotificationStream,
CanonStateNotifications, CanonStateSubscriptions, ForkChoiceNotifications, ForkChoiceStream,
ForkChoiceSubscriptions,
ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions,
WatchValueStream,
};
mod memory_overlay;

View File

@@ -2,7 +2,7 @@
use crate::{
CanonStateNotifications, CanonStateSubscriptions, ForkChoiceNotifications,
ForkChoiceSubscriptions,
ForkChoiceSubscriptions, PersistedBlockNotifications, PersistedBlockSubscriptions,
};
use reth_primitives_traits::NodePrimitives;
use reth_storage_api::noop::NoopProvider;
@@ -27,3 +27,10 @@ impl<C: Send + Sync, N: NodePrimitives> ForkChoiceSubscriptions for NoopProvider
ForkChoiceNotifications(rx)
}
}
impl<C: Send + Sync, N: NodePrimitives> PersistedBlockSubscriptions for NoopProvider<C, N> {
fn subscribe_persisted_block(&self) -> PersistedBlockNotifications {
let (_, rx) = watch::channel(None);
PersistedBlockNotifications(rx)
}
}

View File

@@ -1,6 +1,6 @@
//! Canonical chain state notification trait and types.
use alloy_eips::eip2718::Encodable2718;
use alloy_eips::{eip2718::Encodable2718, BlockNumHash};
use derive_more::{Deref, DerefMut};
use reth_execution_types::{BlockReceipts, Chain};
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedHeader};
@@ -205,22 +205,22 @@ pub trait ForkChoiceSubscriptions: Send + Sync {
}
}
/// A stream for fork choice watch channels (pending, safe or finalized watchers)
/// A stream that yields values from a `watch::Receiver<Option<T>>`, filtering out `None` values.
#[derive(Debug)]
#[pin_project::pin_project]
pub struct ForkChoiceStream<T> {
pub struct WatchValueStream<T> {
#[pin]
st: WatchStream<Option<T>>,
}
impl<T: Clone + Sync + Send + 'static> ForkChoiceStream<T> {
/// Creates a new `ForkChoiceStream`
impl<T: Clone + Sync + Send + 'static> WatchValueStream<T> {
/// Creates a new [`WatchValueStream`]
pub fn new(rx: watch::Receiver<Option<T>>) -> Self {
Self { st: WatchStream::from_changes(rx) }
}
}
impl<T: Clone + Sync + Send + 'static> Stream for ForkChoiceStream<T> {
impl<T: Clone + Sync + Send + 'static> Stream for WatchValueStream<T> {
type Item = T;
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
@@ -234,6 +234,24 @@ impl<T: Clone + Sync + Send + 'static> Stream for ForkChoiceStream<T> {
}
}
/// Alias for [`WatchValueStream`] for fork choice watch channels.
pub type ForkChoiceStream<T> = WatchValueStream<T>;
/// Wrapper around a watch receiver that receives persisted block notifications.
#[derive(Debug, Deref, DerefMut)]
pub struct PersistedBlockNotifications(pub watch::Receiver<Option<BlockNumHash>>);
/// A trait that allows subscribing to persisted block events.
pub trait PersistedBlockSubscriptions: Send + Sync {
/// Get notified when a new block is persisted to disk.
fn subscribe_persisted_block(&self) -> PersistedBlockNotifications;
/// Convenience method to get a stream of the persisted blocks.
fn persisted_block_stream(&self) -> WatchValueStream<BlockNumHash> {
WatchValueStream::new(self.subscribe_persisted_block().0)
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -242,6 +260,7 @@ mod tests {
use reth_ethereum_primitives::{Receipt, TransactionSigned, TxType};
use reth_execution_types::ExecutionOutcome;
use reth_primitives_traits::SealedBlock;
use std::collections::BTreeMap;
#[test]
fn test_commit_notification() {
@@ -260,7 +279,7 @@ mod tests {
let chain: Arc<Chain> = Arc::new(Chain::new(
vec![block1.clone(), block2.clone()],
ExecutionOutcome::default(),
None,
BTreeMap::new(),
));
// Create a commit notification
@@ -295,12 +314,15 @@ mod tests {
block3.set_block_number(3);
block3.set_hash(block3_hash);
let old_chain: Arc<Chain> =
Arc::new(Chain::new(vec![block1.clone()], ExecutionOutcome::default(), None));
let old_chain: Arc<Chain> = Arc::new(Chain::new(
vec![block1.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
));
let new_chain = Arc::new(Chain::new(
vec![block2.clone(), block3.clone()],
ExecutionOutcome::default(),
None,
BTreeMap::new(),
));
// Create a reorg notification
@@ -362,8 +384,11 @@ mod tests {
let execution_outcome = ExecutionOutcome { receipts, ..Default::default() };
// Create a new chain segment with `block1` and `block2` and the execution outcome.
let new_chain: Arc<Chain> =
Arc::new(Chain::new(vec![block1.clone(), block2.clone()], execution_outcome, None));
let new_chain: Arc<Chain> = Arc::new(Chain::new(
vec![block1.clone(), block2.clone()],
execution_outcome,
BTreeMap::new(),
));
// Create a commit notification containing the new chain segment.
let notification = CanonStateNotification::Commit { new: new_chain };
@@ -421,7 +446,7 @@ mod tests {
// Create an old chain segment to be reverted, containing `old_block1`.
let old_chain: Arc<Chain> =
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, None));
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, BTreeMap::new()));
// Define block2 for the new chain segment, which will be committed.
let mut body = BlockBody::<TransactionSigned>::default();
@@ -449,7 +474,8 @@ mod tests {
ExecutionOutcome { receipts: new_receipts, ..Default::default() };
// Create a new chain segment to be committed, containing `new_block1`.
let new_chain = Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, None));
let new_chain =
Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, BTreeMap::new()));
// Create a reorg notification with both reverted (old) and committed (new) chain segments.
let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain };

View File

@@ -3,10 +3,7 @@ use crate::{
CanonStateSubscriptions, ComputedTrieData,
};
use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH};
use alloy_eips::{
eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE},
eip7685::Requests,
};
use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE};
use alloy_primitives::{Address, BlockNumber, B256, U256};
use alloy_signer::SignerSync;
use alloy_signer_local::PrivateKeySigner;
@@ -16,7 +13,7 @@ use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS};
use reth_ethereum_primitives::{
Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned,
};
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_primitives_traits::{
proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root},
Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader,
@@ -201,7 +198,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
fn get_executed_block(
&mut self,
block_number: BlockNumber,
receipts: Vec<Vec<Receipt>>,
mut receipts: Vec<Vec<Receipt>>,
parent_hash: B256,
) -> ExecutedBlock {
let block = self.generate_random_block(block_number, parent_hash);
@@ -209,12 +206,15 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
let trie_data = ComputedTrieData::default();
ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(ExecutionOutcome::new(
BundleState::default(),
receipts,
block_number,
vec![Requests::default()],
)),
Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: receipts.pop().unwrap_or_default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: BundleState::default(),
}),
trie_data,
)
}

View File

@@ -278,6 +278,7 @@ pub fn create_chain_config(
// Check if DAO fork is supported (it has an activation block)
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
#[allow(clippy::needless_update)]
ChainConfig {
chain_id: chain.map(|c| c.id()).unwrap_or(0),
homestead_block: block_num(EthereumHardfork::Homestead),
@@ -313,6 +314,7 @@ pub fn create_chain_config(
extra_fields: Default::default(),
deposit_contract_address,
blob_schedule,
..Default::default()
}
}
@@ -460,6 +462,18 @@ impl ChainSpec {
pub fn builder() -> ChainSpecBuilder {
ChainSpecBuilder::default()
}
/// Map a chain ID to a known chain spec, if available.
pub fn from_chain_id(chain_id: u64) -> Option<Arc<Self>> {
match NamedChain::try_from(chain_id).ok()? {
NamedChain::Mainnet => Some(MAINNET.clone()),
NamedChain::Sepolia => Some(SEPOLIA.clone()),
NamedChain::Holesky => Some(HOLESKY.clone()),
NamedChain::Hoodi => Some(HOODI.clone()),
NamedChain::Dev => Some(DEV.clone()),
_ => None,
}
}
}
impl<H: BlockHeader> ChainSpec<H> {

View File

@@ -83,6 +83,7 @@ backon.workspace = true
secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] }
tokio-stream.workspace = true
reqwest.workspace = true
url.workspace = true
metrics.workspace = true
# io
@@ -129,3 +130,5 @@ arbitrary = [
"reth-primitives-traits/arbitrary",
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -27,7 +27,7 @@ use reth_provider::{
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
ProviderFactory, StaticFileProviderFactory,
ProviderFactory, StaticFileProviderFactory, StorageSettings,
};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
/// All static files related arguments
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
@@ -107,13 +122,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let (db, sfp) = match access {
AccessRights::RW => (
Arc::new(init_db(db_path, self.db.database_args())?),
StaticFileProviderBuilder::read_write(sf_path)?
StaticFileProviderBuilder::read_write(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?,
),
AccessRights::RO | AccessRights::RoInconsistent => {
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
let provider = StaticFileProviderBuilder::read_only(sf_path)?
let provider = StaticFileProviderBuilder::read_only(sf_path)
.with_genesis_block_number(genesis_block_number)
.build()?;
provider.watch_directory();
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
// TransactionDB only support read-write mode
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
if access.is_read_write() {
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
}
Ok(Environment { config, provider_factory, data_dir })

View File

@@ -1,145 +0,0 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use reth_chainspec::EthereumHardforks;
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
self.table.view(&ChecksumViewer {
tool,
start_key: self.start_key,
end_key: self.end_key,
limit: self.limit,
})?;
Ok(())
}
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(100_000) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,288 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use itertools::Itertools;
use reth_chainspec::EthereumHardforks;
use reth_db::{static_file::iter_static_files, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[cfg(all(unix, feature = "edge"))]
mod rocksdb;
/// Interval for logging progress during checksum computation.
const PROGRESS_LOG_INTERVAL: usize = 100_000;
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
#[command(subcommand)]
subcommand: Subcommand,
}
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Calculates the checksum of a database table
Mdbx {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a static file segment
StaticFile {
/// The static file segment
#[arg(value_enum)]
segment: StaticFileSegment,
/// The block number to start from (inclusive).
#[arg(long)]
start_block: Option<u64>,
/// The block number to end at (inclusive).
#[arg(long)]
end_block: Option<u64>,
/// The maximum number of rows to checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a RocksDB table
#[cfg(all(unix, feature = "edge"))]
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: rocksdb::RocksDbTable,
/// The maximum number of records to checksum.
#[arg(long)]
limit: Option<usize>,
},
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
match self.subcommand {
Subcommand::Mdbx { table, start_key, end_key, limit } => {
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
}
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
checksum_static_file(tool, segment, start_block, end_block, limit)?;
}
#[cfg(all(unix, feature = "edge"))]
Subcommand::Rocksdb { table, limit } => {
rocksdb::checksum_rocksdb(tool, table, limit)?;
}
}
Ok(())
}
}
/// Creates a new hasher with the standard seed used for checksum computation.
fn checksum_hasher() -> impl Hasher {
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,
limit: Option<usize>,
) -> eyre::Result<()> {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
warn!("Error checking consistency of static files: {err}");
}
let static_files = iter_static_files(static_file_provider.directory())?;
let ranges = static_files
.get(segment)
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
segment,
start_block,
end_block,
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
let jar_provider = static_file_provider
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
.ok_or_else(|| {
eyre::eyre!(
"Failed to get segment provider for segment {} at range {}",
segment,
block_range
)
})?;
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!(
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
segment, checksum, total, elapsed
);
Ok(())
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,106 @@
//! RocksDB checksum implementation.
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
use crate::common::CliNodeTypes;
use clap::ValueEnum;
use reth_chainspec::EthereumHardforks;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl RocksDbTable {
/// Returns the table name as a string
const fn name(&self) -> &'static str {
match self {
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
Self::AccountsHistory => tables::AccountsHistory::NAME,
Self::StoragesHistory => tables::StoragesHistory::NAME,
}
}
}
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
let start_time = Instant::now();
let limit = limit.unwrap_or(usize::MAX);
info!(
"Computing checksum for RocksDB table `{}`, limit={:?}",
table.name(),
if limit == usize::MAX { None } else { Some(limit) }
);
let (checksum, total) = match table {
RocksDbTable::TransactionHashNumbers => {
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
}
RocksDbTable::AccountsHistory => {
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
}
RocksDbTable::StoragesHistory => {
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
}
};
let elapsed = start_time.elapsed();
info!(
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
table.name(),
checksum,
total,
elapsed
);
Ok(())
}
/// Computes checksum for a specific RocksDB table by iterating over rows.
fn checksum_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
limit: usize,
) -> eyre::Result<(u64, usize)> {
let iter = rocksdb.raw_iter::<T>()?;
let mut hasher = checksum_hasher();
let mut total = 0usize;
for entry in iter {
let (key_bytes, value_bytes) = entry?;
hasher.write(&key_bytes);
hasher.write(&value_bytes);
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break;
}
}
Ok((hasher.finish(), total))
}

View File

@@ -29,7 +29,7 @@ impl Command {
let static_file_provider = tool.provider_factory.static_file_provider();
let static_files = iter_static_files(static_file_provider.directory())?;
if let Some(segment_static_files) = static_files.get(&segment) {
if let Some(segment_static_files) = static_files.get(segment) {
for (block_range, _) in segment_static_files {
static_file_provider.delete_jar(segment, block_range.start())?;
}

View File

@@ -2,8 +2,8 @@ use alloy_primitives::{hex, BlockHash};
use clap::Parser;
use reth_db::{
static_file::{
ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask, ReceiptMask, TransactionMask,
TransactionSenderMask,
AccountChangesetMask, ColumnSelectorOne, ColumnSelectorTwo, HeaderWithHashMask,
ReceiptMask, TransactionMask, TransactionSenderMask,
},
RawDupSort,
};
@@ -19,7 +19,7 @@ use reth_db_common::DbTool;
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, StaticFileProviderFactory};
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use tracing::error;
@@ -64,6 +64,10 @@ enum Subcommand {
#[arg(value_parser = maybe_json_value_parser)]
key: String,
/// The subkey to get content for, for example address in changeset
#[arg(value_parser = maybe_json_value_parser)]
subkey: Option<String>,
/// Output bytes instead of human-readable decoded value
#[arg(long)]
raw: bool,
@@ -77,33 +81,77 @@ impl Command {
Subcommand::Mdbx { table, key, subkey, end_key, end_subkey, raw } => {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::StaticFile { segment, key, raw } => {
let (key, mask): (u64, _) = match segment {
Subcommand::StaticFile { segment, key, subkey, raw } => {
let (key, subkey, mask): (u64, _, _) = match segment {
StaticFileSegment::Headers => (
table_key::<tables::Headers>(&key)?,
None,
<HeaderWithHashMask<HeaderTy<N>>>::MASK,
),
StaticFileSegment::Transactions => {
(table_key::<tables::Transactions>(&key)?, <TransactionMask<TxTy<N>>>::MASK)
}
StaticFileSegment::Receipts => {
(table_key::<tables::Receipts>(&key)?, <ReceiptMask<ReceiptTy<N>>>::MASK)
}
StaticFileSegment::Transactions => (
table_key::<tables::Transactions>(&key)?,
None,
<TransactionMask<TxTy<N>>>::MASK,
),
StaticFileSegment::Receipts => (
table_key::<tables::Receipts>(&key)?,
None,
<ReceiptMask<ReceiptTy<N>>>::MASK,
),
StaticFileSegment::TransactionSenders => (
table_key::<tables::TransactionSenders>(&key)?,
<TransactionSenderMask>::MASK,
None,
TransactionSenderMask::MASK,
),
StaticFileSegment::AccountChangeSets => {
let subkey =
table_subkey::<tables::AccountChangeSets>(subkey.as_deref()).ok();
(
table_key::<tables::AccountChangeSets>(&key)?,
subkey,
AccountChangesetMask::MASK,
)
}
};
let content = tool
.provider_factory
.static_file_provider()
.get_segment_provider(segment, key)?
.cursor()?
.get(key.into(), mask)
.map(|result| {
result.map(|vec| vec.iter().map(|slice| slice.to_vec()).collect::<Vec<_>>())
})?;
// handle account changesets differently if a subkey is provided.
if let StaticFileSegment::AccountChangeSets = segment {
let Some(subkey) = subkey else {
// get all changesets for the block
let changesets = tool
.provider_factory
.static_file_provider()
.account_block_changeset(key)?;
println!("{}", serde_json::to_string_pretty(&changesets)?);
return Ok(())
};
let account = tool
.provider_factory
.static_file_provider()
.get_account_before_block(key, subkey)?;
if let Some(account) = account {
println!("{}", serde_json::to_string_pretty(&account)?);
} else {
error!(target: "reth::cli", "No content for the given table key.");
}
return Ok(())
}
let content = tool.provider_factory.static_file_provider().find_static_file(
segment,
|provider| {
let mut cursor = provider.cursor()?;
cursor.get(key.into(), mask).map(|result| {
result.map(|vec| {
vec.iter().map(|slice| slice.to_vec()).collect::<Vec<_>>()
})
})
},
)?;
match content {
Some(content) => {
@@ -139,6 +187,9 @@ impl Command {
)?;
println!("{}", serde_json::to_string_pretty(&sender)?);
}
StaticFileSegment::AccountChangeSets => {
unreachable!("account changeset static files are special cased before this match")
}
}
}
}

View File

@@ -100,7 +100,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
tx.disable_long_read_transaction_safety();
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
let total_entries = stats.entries();
let final_entry_idx = total_entries.saturating_sub(1);
if self.args.skip > final_entry_idx {

View File

@@ -39,7 +39,7 @@ pub enum Subcommands {
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Calculates the content checksum of a table
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),
@@ -162,7 +162,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
let access_rights =
if command.dry_run { AccessRights::RO } else { AccessRights::RW };
db_exec!(self.env, tool, N, access_rights, {
command.execute(&tool, ctx.task_executor.clone())?;
command.execute(&tool, ctx.task_executor.clone(), &data_dir)?;
});
}
Subcommands::StaticFileHeader(command) => {

View File

@@ -9,7 +9,10 @@ use reth_db_api::{
transaction::{DbTx, DbTxMut},
};
use reth_db_common::DbTool;
use reth_node_core::version::version_metadata;
use reth_node_core::{
dirs::{ChainPath, DataDirPath},
version::version_metadata,
};
use reth_node_metrics::{
chain::ChainSpecInfo,
hooks::Hooks,
@@ -53,11 +56,13 @@ impl Command {
self,
tool: &DbTool<N>,
task_executor: TaskExecutor,
data_dir: &ChainPath<DataDirPath>,
) -> eyre::Result<()> {
// Set up metrics server if requested
let _metrics_handle = if let Some(listen_addr) = self.metrics {
let chain_name = tool.provider_factory.chain_spec().chain().to_string();
let executor = task_executor.clone();
let pprof_dump_dir = data_dir.pprof_dumps();
let handle = task_executor.spawn_critical("metrics server", async move {
let config = MetricServerConfig::new(
@@ -73,6 +78,7 @@ impl Command {
ChainSpecInfo { name: chain_name },
executor,
Hooks::builder().build(),
pprof_dump_dir,
);
// Spawn the metrics server

View File

@@ -40,12 +40,32 @@ enum Subcommands {
#[clap(rename_all = "snake_case")]
pub enum SetCommand {
/// Store receipts in static files instead of the database
ReceiptsInStaticFiles {
Receipts {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction senders in static files instead of the database
TransactionSendersInStaticFiles {
TransactionSenders {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account changesets in static files instead of the database
AccountChangesets {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store storage history in rocksdb instead of MDBX
StoragesHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store transaction hash to number mapping in rocksdb instead of MDBX
TransactionHashNumbers {
#[clap(action(ArgAction::Set))]
value: bool,
},
/// Store account history in rocksdb instead of MDBX
AccountHistory {
#[clap(action(ArgAction::Set))]
value: bool,
},
@@ -94,11 +114,12 @@ impl Command {
storages_history_in_rocksdb: _,
transaction_hash_numbers_in_rocksdb: _,
account_history_in_rocksdb: _,
account_changesets_in_static_files: _,
} = settings.unwrap_or_else(StorageSettings::legacy);
// Update the setting based on the key
match cmd {
SetCommand::ReceiptsInStaticFiles { value } => {
SetCommand::Receipts { value } => {
if settings.receipts_in_static_files == value {
println!("receipts_in_static_files is already set to {}", value);
return Ok(());
@@ -106,7 +127,7 @@ impl Command {
settings.receipts_in_static_files = value;
println!("Set receipts_in_static_files = {}", value);
}
SetCommand::TransactionSendersInStaticFiles { value } => {
SetCommand::TransactionSenders { value } => {
if settings.transaction_senders_in_static_files == value {
println!("transaction_senders_in_static_files is already set to {}", value);
return Ok(());
@@ -114,6 +135,38 @@ impl Command {
settings.transaction_senders_in_static_files = value;
println!("Set transaction_senders_in_static_files = {}", value);
}
SetCommand::AccountChangesets { value } => {
if settings.account_changesets_in_static_files == value {
println!("account_changesets_in_static_files is already set to {}", value);
return Ok(());
}
settings.account_changesets_in_static_files = value;
println!("Set account_changesets_in_static_files = {}", value);
}
SetCommand::StoragesHistory { value } => {
if settings.storages_history_in_rocksdb == value {
println!("storages_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.storages_history_in_rocksdb = value;
println!("Set storages_history_in_rocksdb = {}", value);
}
SetCommand::TransactionHashNumbers { value } => {
if settings.transaction_hash_numbers_in_rocksdb == value {
println!("transaction_hash_numbers_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.transaction_hash_numbers_in_rocksdb = value;
println!("Set transaction_hash_numbers_in_rocksdb = {}", value);
}
SetCommand::AccountHistory { value } => {
if settings.account_history_in_rocksdb == value {
println!("account_history_in_rocksdb is already set to {}", value);
return Ok(());
}
settings.account_history_in_rocksdb = value;
println!("Set account_history_in_rocksdb = {}", value);
}
}
// Write updated settings

View File

@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
use reth_provider::{
providers::{ProviderNodeTypes, StaticFileProvider},
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
@@ -61,6 +64,11 @@ impl Command {
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
println!("\n");
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
println!("{rocksdb_stats_table}");
Ok(())
}
@@ -88,7 +96,7 @@ impl Command {
let stats = tx
.inner
.db_stat(&table_db)
.db_stat(table_db.dbi())
.wrap_err(format!("Could not find table: {db_table}"))?;
// Defaults to 16KB right now but we should
@@ -129,7 +137,8 @@ impl Command {
table.add_row(row);
let freelist = tx.inner.env().freelist()?;
let pagesize = tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;
let pagesize =
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
let freelist_size = freelist * pagesize;
let mut row = Row::new();
@@ -147,6 +156,60 @@ impl Command {
Ok(table)
}
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"RocksDB Table Name",
"# Entries",
"SST Size",
"Memtable Size",
"Total Size",
"Pending Compaction",
]);
let stats = tool.provider_factory.rocksdb_provider().table_stats();
let mut total_sst: u64 = 0;
let mut total_memtable: u64 = 0;
let mut total_size: u64 = 0;
let mut total_pending: u64 = 0;
for stat in &stats {
total_sst += stat.sst_size_bytes;
total_memtable += stat.memtable_size_bytes;
total_size += stat.estimated_size_bytes;
total_pending += stat.pending_compaction_bytes;
let mut row = Row::new();
row.add_cell(Cell::new(&stat.name))
.add_cell(Cell::new(stat.estimated_num_keys))
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
table.add_row(row);
}
if !stats.is_empty() {
let max_widths = table.column_max_content_widths();
let mut separator = Row::new();
for width in max_widths {
separator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(separator);
let mut row = Row::new();
row.add_cell(Cell::new("RocksDB Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_sst as f64)))
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
}
table
}
fn static_files_stats_table<N: NodePrimitives>(
&self,
data_dir: ChainPath<DataDirPath>,

View File

@@ -2,20 +2,22 @@ use crate::common::EnvironmentArgs;
use clap::Parser;
use eyre::Result;
use lz4::Decoder;
use reqwest::Client;
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_fs_util as fs;
use std::{
borrow::Cow,
io::{self, Read, Write},
path::Path,
fs::OpenOptions,
io::{self, BufWriter, Read, Write},
path::{Path, PathBuf},
sync::{Arc, OnceLock},
time::{Duration, Instant},
};
use tar::Archive;
use tokio::task;
use tracing::info;
use url::Url;
use zstd::stream::read::Decoder as ZstdDecoder;
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
@@ -85,6 +87,9 @@ impl DownloadDefaults {
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
);
help.push_str(self.default_base_url.as_ref());
help.push_str(
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
);
help
}
@@ -170,12 +175,14 @@ struct DownloadProgress {
downloaded: u64,
total_size: u64,
last_displayed: Instant,
started_at: Instant,
}
impl DownloadProgress {
/// Creates new progress tracker with given total size
fn new(total_size: u64) -> Self {
Self { downloaded: 0, total_size, last_displayed: Instant::now() }
let now = Instant::now();
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
}
/// Converts bytes to human readable format (B, KB, MB, GB)
@@ -191,6 +198,18 @@ impl DownloadProgress {
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
}
/// Format duration as human readable string
fn format_duration(duration: Duration) -> String {
let secs = duration.as_secs();
if secs < 60 {
format!("{secs}s")
} else if secs < 3600 {
format!("{}m {}s", secs / 60, secs % 60)
} else {
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
}
}
/// Updates progress bar
fn update(&mut self, chunk_size: u64) -> Result<()> {
self.downloaded += chunk_size;
@@ -201,8 +220,24 @@ impl DownloadProgress {
let formatted_total = Self::format_size(self.total_size);
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
// Calculate ETA based on current speed
let elapsed = self.started_at.elapsed();
let eta = if self.downloaded > 0 {
let remaining = self.total_size.saturating_sub(self.downloaded);
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
if speed > 0.0 {
Duration::from_secs_f64(remaining as f64 / speed)
} else {
Duration::ZERO
}
} else {
Duration::ZERO
};
let eta_str = Self::format_duration(eta);
// Pad with spaces to clear any previous longer line
print!(
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total})",
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
);
io::stdout().flush()?;
self.last_displayed = Instant::now();
@@ -246,29 +281,30 @@ enum CompressionFormat {
impl CompressionFormat {
/// Detect compression format from file extension
fn from_url(url: &str) -> Result<Self> {
if url.ends_with(EXTENSION_TAR_LZ4) {
let path =
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
if path.ends_with(EXTENSION_TAR_LZ4) {
Ok(Self::Lz4)
} else if url.ends_with(EXTENSION_TAR_ZSTD) {
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
Ok(Self::Zstd)
} else {
Err(eyre::eyre!("Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}", url))
Err(eyre::eyre!(
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
path
))
}
}
}
/// Downloads and extracts a snapshot, blocking until finished.
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
let client = reqwest::blocking::Client::builder().build()?;
let response = client.get(url).send()?.error_for_status()?;
let total_size = response.content_length().ok_or_else(|| {
eyre::eyre!(
"Server did not provide Content-Length header. This is required for snapshot downloads"
)
})?;
let progress_reader = ProgressReader::new(response, total_size);
let format = CompressionFormat::from_url(url)?;
/// Extracts a compressed tar archive to the target directory with progress tracking.
fn extract_archive<R: Read>(
reader: R,
total_size: u64,
format: CompressionFormat,
target_dir: &Path,
) -> Result<()> {
let progress_reader = ProgressReader::new(reader, total_size);
match format {
CompressionFormat::Lz4 => {
@@ -285,6 +321,185 @@ fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
Ok(())
}
/// Extracts a snapshot from a local file.
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let file = std::fs::File::open(path)?;
let total_size = file.metadata()?.len();
extract_archive(file, total_size, format, target_dir)
}
const MAX_DOWNLOAD_RETRIES: u32 = 10;
const RETRY_BACKOFF_SECS: u64 = 5;
/// Wrapper that tracks download progress while writing data.
/// Used with [`io::copy`] to display progress during downloads.
struct ProgressWriter<W> {
inner: W,
progress: DownloadProgress,
}
impl<W: Write> Write for ProgressWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
let _ = self.progress.update(n as u64);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Downloads a file with resume support using HTTP Range requests.
/// Automatically retries on failure, resuming from where it left off.
/// Returns the path to the downloaded file and its total size.
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
let file_name = Url::parse(url)
.ok()
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
.unwrap_or_else(|| "snapshot.tar".to_string());
let final_path = target_dir.join(&file_name);
let part_path = target_dir.join(format!("{file_name}.part"));
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
let mut total_size: Option<u64> = None;
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
if let Some(total) = total_size &&
existing_size >= total
{
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, total));
}
if attempt > 1 {
info!(target: "reth::cli",
"Retry attempt {}/{} - resuming from {} bytes",
attempt, MAX_DOWNLOAD_RETRIES, existing_size
);
}
let mut request = client.get(url);
if existing_size > 0 {
request = request.header(RANGE, format!("bytes={existing_size}-"));
if attempt == 1 {
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
}
}
let response = match request.send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(e) => {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
let size = if is_partial {
response
.headers()
.get("Content-Range")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.split('/').next_back())
.and_then(|v| v.parse().ok())
} else {
response.content_length()
};
if total_size.is_none() {
total_size = size;
}
let current_total = total_size.ok_or_else(|| {
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
})?;
let file = if is_partial && existing_size > 0 {
OpenOptions::new()
.append(true)
.open(&part_path)
.map_err(|e| fs::FsPathError::open(e, &part_path))?
} else {
fs::create_file(&part_path)?
};
let start_offset = if is_partial { existing_size } else { 0 };
let mut progress = DownloadProgress::new(current_total);
progress.downloaded = start_offset;
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
let mut reader = response;
let copy_result = io::copy(&mut reader, &mut writer);
let flush_result = writer.inner.flush();
println!();
if let Err(e) = copy_result.and(flush_result) {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, current_total));
}
Err(last_error
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
info!(target: "reth::cli", "Extracting snapshot...");
let file = fs::open(&downloaded_path)?;
extract_archive(file, total_size, format, target_dir)?;
fs::remove_file(&downloaded_path)?;
info!(target: "reth::cli", "Removed downloaded archive");
Ok(())
}
/// Downloads and extracts a snapshot, blocking until finished.
///
/// Supports both `file://` URLs for local files and HTTP(S) URLs for remote downloads.
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
let format = CompressionFormat::from_url(url)?;
if let Ok(parsed_url) = Url::parse(url) &&
parsed_url.scheme() == "file"
{
let file_path = parsed_url
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
extract_from_file(&file_path, format, target_dir)
} else {
download_and_extract(url, format, target_dir)
}
}
async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
let target_dir = target_dir.to_path_buf();
let url = url.to_string();
@@ -343,6 +558,7 @@ mod tests {
assert!(help.contains("Available snapshot sources:"));
assert!(help.contains("merkle.io"));
assert!(help.contains("publicnode.com"));
assert!(help.contains("file://"));
}
#[test]
@@ -367,4 +583,25 @@ mod tests {
assert_eq!(defaults.available_snapshots.len(), 4); // 2 defaults + 2 added
assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string()));
}
#[test]
fn test_compression_format_detection() {
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
}
}

View File

@@ -69,9 +69,7 @@ pub async fn import_blocks_from_file<N>(
provider_factory: ProviderFactory<N>,
config: &Config,
executor: impl ConfigureEvm<Primitives = N::Primitives> + 'static,
consensus: Arc<
impl FullConsensus<N::Primitives, Error = reth_consensus::ConsensusError> + 'static,
>,
consensus: Arc<impl FullConsensus<N::Primitives> + 'static>,
) -> eyre::Result<ImportResult>
where
N: ProviderNodeTypes,
@@ -198,7 +196,7 @@ pub fn build_import_pipeline_impl<N, C, E>(
) -> eyre::Result<(Pipeline<N>, impl futures::Stream<Item = NodeEvent<N::Primitives>> + use<N, C, E>)>
where
N: ProviderNodeTypes,
C: FullConsensus<N::Primitives, Error = reth_consensus::ConsensusError> + 'static,
C: FullConsensus<N::Primitives> + 'static,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
{
if !file_client.has_canonical_blocks() {

View File

@@ -99,6 +99,7 @@ where
/// * Headers: It will push an empty block.
/// * Transactions: It will not push any tx, only increments the end block range.
/// * Receipts: It will not push any receipt, only increments the end block range.
/// * TransactionSenders: If the segment exists, increments the end block range.
fn append_dummy_chain<N, F>(
sf_provider: &StaticFileProvider<N>,
target_height: BlockNumber,
@@ -110,8 +111,15 @@ where
{
let (tx, rx) = std::sync::mpsc::channel();
// Spawn jobs for incrementing the block end range of transactions and receipts
for segment in [StaticFileSegment::Transactions, StaticFileSegment::Receipts] {
// Spawn jobs for incrementing the block end range of transactions, receipts, and senders.
for segment in [
StaticFileSegment::Transactions,
StaticFileSegment::Receipts,
StaticFileSegment::TransactionSenders,
] {
if sf_provider.get_highest_static_file_block(segment).is_none() {
continue
}
let tx_clone = tx.clone();
let provider = sf_provider.clone();
std::thread::spawn(move || {
@@ -151,9 +159,15 @@ where
// If, for any reason, rayon crashes this verifies if all segments are at the same
// target_height.
for segment in
[StaticFileSegment::Headers, StaticFileSegment::Receipts, StaticFileSegment::Transactions]
{
for segment in [
StaticFileSegment::Headers,
StaticFileSegment::Receipts,
StaticFileSegment::Transactions,
StaticFileSegment::TransactionSenders,
] {
if sf_provider.get_highest_static_file_block(segment).is_none() {
continue
}
assert_eq!(
sf_provider.latest_writer(segment)?.user_header().block_end(),
Some(target_height),

View File

@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -166,12 +171,16 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -187,6 +196,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,

View File

@@ -42,9 +42,9 @@ pub struct Command<C: ChainSpecParser> {
#[arg(long)]
to: Option<u64>,
/// Number of tasks to run in parallel
#[arg(long, default_value = "10")]
num_tasks: u64,
/// Number of tasks to run in parallel. Defaults to the number of available CPUs.
#[arg(long)]
num_tasks: Option<u64>,
/// Continues with execution when an invalid block is encountered and collects these blocks.
#[arg(long)]
@@ -84,12 +84,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
}
};
let num_tasks = self.num_tasks.unwrap_or_else(|| {
std::thread::available_parallelism().map(|n| n.get() as u64).unwrap_or(10)
});
let total_blocks = max_block - min_block;
let total_gas = calculate_gas_used_from_headers(
&provider_factory.static_file_provider(),
min_block..=max_block,
)?;
let blocks_per_task = total_blocks / self.num_tasks;
let blocks_per_task = total_blocks / num_tasks;
let db_at = {
let provider_factory = provider_factory.clone();
@@ -107,10 +111,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let _guard = cancellation.drop_guard();
let mut tasks = JoinSet::new();
for i in 0..self.num_tasks {
for i in 0..num_tasks {
let start_block = min_block + i * blocks_per_task;
let end_block =
if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task };
if i == num_tasks - 1 { max_block } else { start_block + blocks_per_task };
// Spawn thread executing blocks
let provider_factory = provider_factory.clone();
@@ -148,7 +152,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
};
if let Err(err) = consensus
.validate_block_post_execution(&block, &result)
.validate_block_post_execution(&block, &result, None)
.wrap_err_with(|| {
format!("Failed to validate block {} {}", block.number(), block.hash())
})

View File

@@ -15,7 +15,7 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, TrieWriter,
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -87,6 +87,9 @@ impl<C: ChainSpecParser> Command<C> {
.unwrap_or_default();
writer.prune_transaction_senders(to_delete, 0)?;
}
StaticFileSegment::AccountChangeSets => {
writer.prune_account_changesets(highest_block)?;
}
}
}
}
@@ -164,10 +167,6 @@ impl<C: ChainSpecParser> Command<C> {
None,
)?;
}
StageEnum::MerkleChangeSets => {
provider_rw.clear_trie_changesets()?;
reset_stage_checkpoint(tx, StageId::MerkleChangeSets)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;

View File

@@ -1,5 +1,5 @@
use super::setup;
use reth_consensus::{noop::NoopConsensus, ConsensusError, FullConsensus};
use reth_consensus::{noop::NoopConsensus, FullConsensus};
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, database::Database, table::TableImporter, tables, transaction::DbTx,
@@ -28,7 +28,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
C: FullConsensus<E::Primitives, Error = ConsensusError> + 'static,
C: FullConsensus<E::Primitives> + 'static,
{
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
@@ -169,7 +169,7 @@ fn dry_run<N, E, C>(
where
N: ProviderNodeTypes,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
C: FullConsensus<E::Primitives, Error = ConsensusError> + 'static,
C: FullConsensus<E::Primitives> + 'static,
{
info!(target: "reth::cli", "Executing stage. [dry-run]");

View File

@@ -4,7 +4,7 @@ use super::setup;
use alloy_primitives::{Address, BlockNumber};
use eyre::Result;
use reth_config::config::EtlConfig;
use reth_consensus::{ConsensusError, FullConsensus};
use reth_consensus::FullConsensus;
use reth_db::DatabaseEnv;
use reth_db_api::{database::Database, models::BlockNumberAddress, table::TableImporter, tables};
use reth_db_common::DbTool;
@@ -31,7 +31,7 @@ pub(crate) async fn dump_merkle_stage<N>(
output_datadir: ChainPath<DataDirPath>,
should_run: bool,
evm_config: impl ConfigureEvm<Primitives = N::Primitives>,
consensus: impl FullConsensus<N::Primitives, Error = ConsensusError> + 'static,
consensus: impl FullConsensus<N::Primitives> + 'static,
) -> Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
@@ -79,7 +79,7 @@ fn unwind_and_copy<N: ProviderNodeTypes>(
tip_block_number: u64,
output_db: &DatabaseEnv,
evm_config: impl ConfigureEvm<Primitives = N::Primitives>,
consensus: impl FullConsensus<N::Primitives, Error = ConsensusError> + 'static,
consensus: impl FullConsensus<N::Primitives> + 'static,
) -> eyre::Result<()> {
let (from, to) = range;
let provider = db_tool.provider_factory.database_provider_rw()?;

View File

@@ -11,7 +11,6 @@ use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_cli_util::get_secret_key;
use reth_config::config::{HashingConfig, SenderRecoveryConfig, TransactionLookupConfig};
use reth_db_api::database_metrics::DatabaseMetrics;
use reth_downloaders::{
bodies::bodies::BodiesDownloaderBuilder,
headers::reverse_headers::ReverseHeadersDownloaderBuilder,
@@ -19,19 +18,19 @@ use reth_downloaders::{
use reth_exex::ExExManagerHandle;
use reth_network::BlockDownloaderProvider;
use reth_network_p2p::HeadersClient;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{
args::{NetworkArgs, StageEnum},
version::version_metadata,
};
use reth_node_metrics::{
chain::ChainSpecInfo,
hooks::Hooks,
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
use reth_provider::{
ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
StageCheckpointWriter, StaticFileProviderFactory,
StageCheckpointWriter,
};
use reth_stages::{
stages::{
@@ -139,20 +138,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
Hooks::builder()
.with_hook({
let db = provider_factory.db_ref().clone();
move || db.report_metrics()
})
.with_hook({
let sfp = provider_factory.static_file_provider();
move || {
if let Err(error) = sfp.report_metrics() {
error!(%error, "Failed to report metrics from static file provider");
}
}
})
.build(),
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
MetricServer::new(config).serve().await?;

View File

@@ -18,8 +18,8 @@ use tracing::{debug, error, trace};
///
/// Provides utilities for running a cli command to completion.
#[derive(Debug)]
#[non_exhaustive]
pub struct CliRunner {
config: CliRunnerConfig,
tokio_runtime: tokio::runtime::Runtime,
}
@@ -29,12 +29,18 @@ impl CliRunner {
///
/// The default tokio runtime is multi-threaded, with both I/O and time drivers enabled.
pub fn try_default_runtime() -> Result<Self, std::io::Error> {
Ok(Self { tokio_runtime: tokio_runtime()? })
Ok(Self { config: CliRunnerConfig::default(), tokio_runtime: tokio_runtime()? })
}
/// Create a new [`CliRunner`] from a provided tokio [`Runtime`](tokio::runtime::Runtime).
pub const fn from_runtime(tokio_runtime: tokio::runtime::Runtime) -> Self {
Self { tokio_runtime }
Self { config: CliRunnerConfig::new(), tokio_runtime }
}
/// Sets the [`CliRunnerConfig`] for this runner.
pub const fn with_config(mut self, config: CliRunnerConfig) -> Self {
self.config = config;
self
}
/// Executes an async block on the runtime and blocks until completion.
@@ -74,7 +80,7 @@ impl CliRunner {
// after the command has finished or exit signal was received we shutdown the task
// manager which fires the shutdown signal to all tasks spawned via the task
// executor and awaiting on tasks spawned with graceful shutdown
task_manager.graceful_shutdown_with_timeout(Duration::from_secs(5));
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
}
// `drop(tokio_runtime)` would block the current thread until its pools
@@ -128,7 +134,7 @@ impl CliRunner {
error!(target: "reth::cli", "shutting down due to error");
} else {
debug!(target: "reth::cli", "shutting down gracefully");
task_manager.graceful_shutdown_with_timeout(Duration::from_secs(5));
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
}
// Shutdown the runtime on a separate thread
@@ -211,6 +217,38 @@ pub struct CliContext {
pub task_executor: TaskExecutor,
}
/// Default timeout for graceful shutdown of tasks.
const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
/// Configuration for [`CliRunner`].
#[derive(Debug, Clone)]
pub struct CliRunnerConfig {
/// Timeout for graceful shutdown of tasks.
///
/// After the command completes, this is the maximum time to wait for spawned tasks
/// to finish before forcefully terminating them.
pub graceful_shutdown_timeout: Duration,
}
impl Default for CliRunnerConfig {
fn default() -> Self {
Self::new()
}
}
impl CliRunnerConfig {
/// Creates a new config with default values.
pub const fn new() -> Self {
Self { graceful_shutdown_timeout: DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT }
}
/// Sets the graceful shutdown timeout.
pub const fn with_graceful_shutdown_timeout(mut self, timeout: Duration) -> Self {
self.graceful_shutdown_timeout = timeout;
self
}
}
/// Creates a new default tokio multi-thread [Runtime](tokio::runtime::Runtime) with all features
/// enabled
pub fn tokio_runtime() -> Result<tokio::runtime::Runtime, std::io::Error> {

View File

@@ -26,7 +26,8 @@ rand_08.workspace = true
thiserror.workspace = true
serde.workspace = true
tracy-client = { workspace = true, optional = true, features = ["demangle"] }
tracy-client = { workspace = true, optional = true }
reth-tracing = { workspace = true, optional = true }
[dev-dependencies]
rand.workspace = true
@@ -46,7 +47,7 @@ jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"]
# Wraps the selected allocator in the tracy profiling allocator
tracy-allocator = ["dep:tracy-client"]
tracy-allocator = ["dep:tracy-client", "dep:reth-tracing"]
snmalloc = ["dep:snmalloc-rs"]

View File

@@ -25,7 +25,6 @@ cfg_if::cfg_if! {
cfg_if::cfg_if! {
if #[cfg(feature = "tracy-allocator")] {
type AllocatorWrapper = tracy_client::ProfiledAllocator<AllocatorInner>;
tracy_client::register_demangler!();
const fn new_allocator_wrapper() -> AllocatorWrapper {
AllocatorWrapper::new(AllocatorInner {}, 100)
}

View File

@@ -8,6 +8,9 @@
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
#![cfg_attr(docsrs, feature(doc_cfg))]
#[cfg(feature = "tracy-allocator")]
use reth_tracing as _;
pub mod allocator;
pub mod cancellation;

View File

@@ -121,7 +121,16 @@ pub fn install() {
unsafe {
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
let mut alt_stack: libc::stack_t = mem::zeroed();
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
// generous here and just use a size of a page.
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
let page_sz = if raw_page_sz == -1 {
// Fallback alignment in case sysconf fails.
4096_usize
} else {
raw_page_sz as usize
};
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
alt_stack.ss_size = alt_stack_size;
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());

View File

@@ -2,9 +2,8 @@
use reth_network_types::{PeersConfig, SessionsConfig};
use reth_prune_types::PruneModes;
use reth_stages_types::ExecutionStageThresholds;
use reth_static_file_types::StaticFileSegment;
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
use std::{
collections::HashMap,
path::{Path, PathBuf},
time::Duration,
};
@@ -437,6 +436,8 @@ pub struct BlocksPerFileConfig {
pub receipts: Option<u64>,
/// Number of blocks per file for the transaction senders segment.
pub transaction_senders: Option<u64>,
/// Number of blocks per file for the account changesets segment.
pub account_change_sets: Option<u64>,
}
impl StaticFilesConfig {
@@ -444,8 +445,13 @@ impl StaticFilesConfig {
///
/// Returns an error if any blocks per file value is zero.
pub fn validate(&self) -> eyre::Result<()> {
let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
self.blocks_per_file;
let BlocksPerFileConfig {
headers,
transactions,
receipts,
transaction_senders,
account_change_sets,
} = self.blocks_per_file;
eyre::ensure!(headers != Some(0), "Headers segment blocks per file must be greater than 0");
eyre::ensure!(
transactions != Some(0),
@@ -459,15 +465,24 @@ impl StaticFilesConfig {
transaction_senders != Some(0),
"Transaction senders segment blocks per file must be greater than 0"
);
eyre::ensure!(
account_change_sets != Some(0),
"Account changesets segment blocks per file must be greater than 0"
);
Ok(())
}
/// Converts the blocks per file configuration into a [`HashMap`] per segment.
pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
let BlocksPerFileConfig { headers, transactions, receipts, transaction_senders } =
self.blocks_per_file;
/// Converts the blocks per file configuration into a [`StaticFileMap`].
pub fn as_blocks_per_file_map(&self) -> StaticFileMap<u64> {
let BlocksPerFileConfig {
headers,
transactions,
receipts,
transaction_senders,
account_change_sets,
} = self.blocks_per_file;
let mut map = HashMap::new();
let mut map = StaticFileMap::default();
// Iterating over all possible segments allows us to do an exhaustive match here,
// to not forget to configure new segments in the future.
for segment in StaticFileSegment::iter() {
@@ -476,6 +491,7 @@ impl StaticFilesConfig {
StaticFileSegment::Transactions => transactions,
StaticFileSegment::Receipts => receipts,
StaticFileSegment::TransactionSenders => transaction_senders,
StaticFileSegment::AccountChangeSets => account_change_sets,
};
if let Some(blocks_per_file) = blocks_per_file {
@@ -527,14 +543,13 @@ impl PruneConfig {
/// Returns whether there is any kind of receipt pruning configuration.
pub fn has_receipts_pruning(&self) -> bool {
self.segments.receipts.is_some() || !self.segments.receipts_log_filter.is_empty()
self.segments.has_receipts_pruning()
}
/// Merges values from `other` into `self`.
/// - `Option<PruneMode>` fields: set from `other` only if `self` is `None`.
/// - `block_interval`: set from `other` only if `self.block_interval ==
/// DEFAULT_BLOCK_INTERVAL`.
/// - `merkle_changesets`: always set from `other`.
/// - `receipts_log_filter`: set from `other` only if `self` is empty and `other` is non-empty.
pub fn merge(&mut self, other: Self) {
let Self {
@@ -547,7 +562,6 @@ impl PruneConfig {
account_history,
storage_history,
bodies_history,
merkle_changesets,
receipts_log_filter,
},
} = other;
@@ -564,8 +578,6 @@ impl PruneConfig {
self.segments.account_history = self.segments.account_history.or(account_history);
self.segments.storage_history = self.segments.storage_history.or(storage_history);
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
// Merkle changesets is not optional; always take the value from `other`
self.segments.merkle_changesets = merkle_changesets;
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
self.segments.receipts_log_filter = receipts_log_filter;
@@ -1062,18 +1074,6 @@ transaction_lookup = 'full'
receipts = { distance = 16384 }
#";
let _conf: Config = toml::from_str(s).unwrap();
let s = r"#
[prune]
block_interval = 5
[prune.segments]
sender_recovery = { distance = 16384 }
transaction_lookup = 'full'
receipts = 'full'
#";
let err = toml::from_str::<Config>(s).unwrap_err().to_string();
assert!(err.contains("invalid value: string \"full\""), "{}", err);
}
#[test]
@@ -1087,7 +1087,6 @@ receipts = 'full'
account_history: None,
storage_history: Some(PruneMode::Before(5000)),
bodies_history: None,
merkle_changesets: PruneMode::Before(0),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Full,
@@ -1104,7 +1103,6 @@ receipts = 'full'
account_history: Some(PruneMode::Distance(2000)),
storage_history: Some(PruneMode::Distance(3000)),
bodies_history: None,
merkle_changesets: PruneMode::Distance(10000),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
(Address::random(), PruneMode::Distance(1000)),
(Address::random(), PruneMode::Before(2000)),
@@ -1123,7 +1121,6 @@ receipts = 'full'
assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000));
assert_eq!(config1.segments.receipts_log_filter, original_filter);
}

View File

@@ -500,13 +500,11 @@ mod tests {
let expected_blob_gas_used = 10 * DATA_GAS_PER_BLOB;
// validate blob, it should fail blob gas used validation
assert_eq!(
validate_block_pre_execution(&block, &chain_spec),
Err(ConsensusError::BlobGasUsedDiff(GotExpected {
got: 1,
expected: expected_blob_gas_used
}))
);
assert!(matches!(
validate_block_pre_execution(&block, &chain_spec).unwrap_err(),
ConsensusError::BlobGasUsedDiff(diff)
if diff.got == 1 && diff.expected == expected_blob_gas_used
));
}
#[test]
@@ -517,10 +515,10 @@ mod tests {
// Test exceeding default - should fail
let header_33 = Header { extra_data: Bytes::from(vec![0; 33]), ..Default::default() };
assert_eq!(
validate_header_extra_data(&header_33, 32),
Err(ConsensusError::ExtraDataExceedsMax { len: 33 })
);
assert!(matches!(
validate_header_extra_data(&header_33, 32).unwrap_err(),
ConsensusError::ExtraDataExceedsMax { len } if len == 33
));
// Test with custom larger limit - should pass
assert!(validate_header_extra_data(&header_33, 64).is_ok());

View File

@@ -11,9 +11,16 @@
extern crate alloc;
use alloc::{boxed::Box, fmt::Debug, string::String, vec::Vec};
use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
use alloy_consensus::Header;
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
use core::error::Error;
/// Pre-computed receipt root and logs bloom.
///
/// When provided to [`FullConsensus::validate_block_post_execution`], this allows skipping
/// the receipt root computation and using the pre-computed values instead.
pub type ReceiptRootBloom = (B256, Bloom);
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
@@ -38,26 +45,27 @@ pub trait FullConsensus<N: NodePrimitives>: Consensus<N::Block> {
///
/// See the Yellow Paper sections 4.3.2 "Holistic Validity".
///
/// If `receipt_root_bloom` is provided, the implementation should use the pre-computed
/// receipt root and logs bloom instead of computing them from the receipts.
///
/// Note: validating blocks does not include other validations of the Consensus
fn validate_block_post_execution(
&self,
block: &RecoveredBlock<N::Block>,
result: &BlockExecutionResult<N::Receipt>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError>;
}
/// Consensus is a protocol that chooses canonical chain.
#[auto_impl::auto_impl(&, Arc)]
pub trait Consensus<B: Block>: HeaderValidator<B::Header> {
/// The error type related to consensus.
type Error;
/// Ensures that body field values match the header.
fn validate_body_against_header(
&self,
body: &B::Body,
header: &SealedHeader<B::Header>,
) -> Result<(), Self::Error>;
) -> Result<(), ConsensusError>;
/// Validate a block disregarding world state, i.e. things that can be checked before sender
/// recovery and execution.
@@ -69,7 +77,7 @@ pub trait Consensus<B: Block>: HeaderValidator<B::Header> {
/// **This should not be called for the genesis block**.
///
/// Note: validating blocks does not include other validations of the Consensus
fn validate_block_pre_execution(&self, block: &SealedBlock<B>) -> Result<(), Self::Error>;
fn validate_block_pre_execution(&self, block: &SealedBlock<B>) -> Result<(), ConsensusError>;
}
/// `HeaderValidator` is a protocol that validates headers and their relationships.
@@ -125,7 +133,7 @@ pub trait HeaderValidator<H = Header>: Debug + Send + Sync {
}
/// Consensus Errors
#[derive(Debug, PartialEq, Eq, Clone, thiserror::Error)]
#[derive(Debug, Clone, thiserror::Error)]
pub enum ConsensusError {
/// Error when the gas used in the header exceeds the gas limit.
#[error("block used gas ({gas_used}) is greater than gas limit ({gas_limit})")]
@@ -410,6 +418,9 @@ pub enum ConsensusError {
/// Other, likely an injected L2 error.
#[error("{0}")]
Other(String),
/// Other unspecified error.
#[error(transparent)]
Custom(#[from] Arc<dyn Error + Send + Sync>),
}
impl ConsensusError {
@@ -447,3 +458,34 @@ pub struct TxGasLimitTooHighErr {
/// The maximum allowed gas limit
pub max_allowed: u64,
}
#[cfg(test)]
mod tests {
use super::*;
#[derive(thiserror::Error, Debug)]
#[error("Custom L2 consensus error")]
struct CustomL2Error;
#[test]
fn test_custom_error_conversion() {
// Test conversion from custom error to ConsensusError
let custom_err = CustomL2Error;
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
let consensus_err: ConsensusError = arc_err.into();
// Verify it's the Custom variant
assert!(matches!(consensus_err, ConsensusError::Custom(_)));
}
#[test]
fn test_custom_error_display() {
let custom_err = CustomL2Error;
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
let consensus_err: ConsensusError = arc_err.into();
// Verify the error message is preserved through transparent attribute
let error_message = format!("{}", consensus_err);
assert_eq!(error_message, "Custom L2 consensus error");
}
}

View File

@@ -18,7 +18,7 @@
//!
//! **Not for production use** - provides no security guarantees or consensus validation.
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use alloc::sync::Arc;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
@@ -55,19 +55,17 @@ impl<H> HeaderValidator<H> for NoopConsensus {
}
impl<B: Block> Consensus<B> for NoopConsensus {
type Error = ConsensusError;
/// Validates body against header (no-op implementation).
fn validate_body_against_header(
&self,
_body: &B::Body,
_header: &SealedHeader<B::Header>,
) -> Result<(), Self::Error> {
) -> Result<(), ConsensusError> {
Ok(())
}
/// Validates block before execution (no-op implementation).
fn validate_block_pre_execution(&self, _block: &SealedBlock<B>) -> Result<(), Self::Error> {
fn validate_block_pre_execution(&self, _block: &SealedBlock<B>) -> Result<(), ConsensusError> {
Ok(())
}
}
@@ -78,6 +76,7 @@ impl<N: NodePrimitives> FullConsensus<N> for NoopConsensus {
&self,
_block: &RecoveredBlock<N::Block>,
_result: &BlockExecutionResult<N::Receipt>,
_receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
Ok(())
}

View File

@@ -1,4 +1,4 @@
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use core::sync::atomic::{AtomicBool, Ordering};
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
@@ -51,6 +51,7 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
&self,
_block: &RecoveredBlock<N::Block>,
_result: &BlockExecutionResult<N::Receipt>,
_receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
if self.fail_validation() {
Err(ConsensusError::BaseFeeMissing)
@@ -61,13 +62,11 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
}
impl<B: Block> Consensus<B> for TestConsensus {
type Error = ConsensusError;
fn validate_body_against_header(
&self,
_body: &B::Body,
_header: &SealedHeader<B::Header>,
) -> Result<(), Self::Error> {
) -> Result<(), ConsensusError> {
if self.fail_body_against_header() {
Err(ConsensusError::BaseFeeMissing)
} else {
@@ -75,7 +74,7 @@ impl<B: Block> Consensus<B> for TestConsensus {
}
}
fn validate_block_pre_execution(&self, _block: &SealedBlock<B>) -> Result<(), Self::Error> {
fn validate_block_pre_execution(&self, _block: &SealedBlock<B>) -> Result<(), ConsensusError> {
if self.fail_validation() {
Err(ConsensusError::BaseFeeMissing)
} else {

View File

@@ -2,11 +2,11 @@ use crate::{network::NetworkTestContext, payload::PayloadTestContext, rpc::RpcTe
use alloy_consensus::{transaction::TxHashRef, BlockHeader};
use alloy_eips::BlockId;
use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B256};
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState};
use alloy_rpc_types_eth::BlockNumberOrTag;
use eyre::Ok;
use futures_util::Future;
use jsonrpsee::http_client::HttpClient;
use jsonrpsee::{core::client::ClientT, http_client::HttpClient};
use reth_chainspec::EthereumHardforks;
use reth_network_api::test_utils::PeersHandleProvider;
use reth_node_api::{
@@ -20,6 +20,7 @@ use reth_provider::{
BlockReader, BlockReaderIdExt, CanonStateNotificationStream, CanonStateSubscriptions,
HeaderProvider, StageCheckpointReader,
};
use reth_rpc_api::TestingBuildBlockRequestV1;
use reth_rpc_builder::auth::AuthServerHandle;
use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt};
use reth_stages_types::StageId;
@@ -319,4 +320,20 @@ where
Ok(crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle))
}
/// Calls the `testing_buildBlockV1` RPC on this node.
///
/// This endpoint builds a block using the provided parent, payload attributes, and
/// transactions. Requires the `Testing` RPC module to be enabled.
pub async fn testing_build_block_v1(
&self,
request: TestingBuildBlockRequestV1,
) -> eyre::Result<ExecutionPayloadEnvelopeV5> {
let client =
self.rpc_client().ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?;
let res: ExecutionPayloadEnvelopeV5 =
client.request("testing_buildBlockV1", [request]).await?;
eyre::Ok(res)
}
}

View File

@@ -113,7 +113,6 @@ pub async fn setup_engine_with_chain_import(
let rocksdb_dir_path = datadir.join("rocksdb");
// Initialize the database using init_db (same as CLI import command)
// Use the same database arguments as the node will use
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args)?;
let db = Arc::new(db_env);
@@ -317,7 +316,8 @@ mod tests {
// Import the chain
{
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
let db = Arc::new(db_env);
let provider_factory: ProviderFactory<
@@ -475,7 +475,8 @@ mod tests {
let datadir = temp_dir.path().join("datadir");
std::fs::create_dir_all(&datadir).unwrap();
let db_path = datadir.join("db");
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
let db = Arc::new(reth_db::test_utils::TempDatabase::new(db_env, db_path));
// Create static files path

View File

@@ -448,12 +448,14 @@ mod tests {
nonce: account.nonce,
code_hash: account.bytecode_hash.unwrap_or_default(),
code: None,
account_id: None,
}),
original_info: (i == 0).then(|| AccountInfo {
balance: account.balance.checked_div(U256::from(2)).unwrap_or(U256::ZERO),
nonce: 0,
code_hash: account.bytecode_hash.unwrap_or_default(),
code: None,
account_id: None,
}),
storage,
status: AccountStatus::default(),

View File

@@ -135,6 +135,10 @@ pub struct TreeConfig {
storage_worker_count: usize,
/// Number of account proof worker threads.
account_worker_count: usize,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl Default for TreeConfig {
@@ -163,6 +167,8 @@ impl Default for TreeConfig {
allow_unwind_canonical_header: false,
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
enable_proof_v2: false,
disable_cache_metrics: false,
}
}
}
@@ -194,6 +200,8 @@ impl TreeConfig {
allow_unwind_canonical_header: bool,
storage_worker_count: usize,
account_worker_count: usize,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
persistence_threshold,
@@ -219,6 +227,8 @@ impl TreeConfig {
allow_unwind_canonical_header,
storage_worker_count,
account_worker_count,
enable_proof_v2,
disable_cache_metrics,
}
}
@@ -500,4 +510,26 @@ impl TreeConfig {
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
self
}
/// Return whether V2 storage proofs are enabled.
pub const fn enable_proof_v2(&self) -> bool {
self.enable_proof_v2
}
/// Setter for whether to enable V2 storage proofs.
pub const fn with_enable_proof_v2(mut self, enable_proof_v2: bool) -> Self {
self.enable_proof_v2 = enable_proof_v2;
self
}
/// Returns whether cache metrics recording is disabled.
pub const fn disable_cache_metrics(&self) -> bool {
self.disable_cache_metrics
}
/// Setter for whether to disable cache metrics recording.
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
self.disable_cache_metrics = disable_cache_metrics;
self
}
}

View File

@@ -25,6 +25,7 @@ reth-tasks.workspace = true
reth-node-types.workspace = true
reth-chainspec.workspace = true
reth-engine-primitives.workspace = true
reth-trie-db.workspace = true
# async
futures.workspace = true
@@ -40,6 +41,8 @@ reth-evm-ethereum.workspace = true
reth-exex-types.workspace = true
reth-primitives-traits.workspace = true
reth-node-ethereum.workspace = true
reth-trie-db.workspace = true
alloy-eips.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio-stream.workspace = true

View File

@@ -1,7 +1,7 @@
use futures::{Stream, StreamExt};
use pin_project::pin_project;
use reth_chainspec::EthChainSpec;
use reth_consensus::{ConsensusError, FullConsensus};
use reth_consensus::FullConsensus;
use reth_engine_primitives::{BeaconEngineMessage, ConsensusEngineEvent};
use reth_engine_tree::{
backfill::PipelineSync,
@@ -26,6 +26,7 @@ use reth_provider::{
use reth_prune::PrunerWithFactory;
use reth_stages_api::{MetricEventsSender, Pipeline};
use reth_tasks::TaskSpawner;
use reth_trie_db::ChangesetCache;
use std::{
pin::Pin,
sync::Arc,
@@ -70,7 +71,7 @@ where
/// Constructor for `EngineService`.
#[expect(clippy::too_many_arguments)]
pub fn new<V, C>(
consensus: Arc<dyn FullConsensus<N::Primitives, Error = ConsensusError>>,
consensus: Arc<dyn FullConsensus<N::Primitives>>,
chain_spec: Arc<N::ChainSpec>,
client: Client,
incoming_requests: EngineMessageStream<N::Payload>,
@@ -84,6 +85,7 @@ where
tree_config: TreeConfig,
sync_metrics_tx: MetricEventsSender,
evm_config: C,
changeset_cache: ChangesetCache,
) -> Self
where
V: EngineValidator<N::Payload>,
@@ -109,6 +111,7 @@ where
tree_config,
engine_kind,
evm_config,
changeset_cache,
);
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
@@ -156,6 +159,7 @@ mod tests {
};
use reth_prune::Pruner;
use reth_tasks::TokioTaskExecutor;
use reth_trie_db::ChangesetCache;
use std::sync::Arc;
use tokio::sync::{mpsc::unbounded_channel, watch};
use tokio_stream::wrappers::UnboundedReceiverStream;
@@ -188,6 +192,8 @@ mod tests {
let pruner = Pruner::new_with_factory(provider_factory.clone(), vec![], 0, 0, None, rx);
let evm_config = EthEvmConfig::new(chain_spec.clone());
let changeset_cache = ChangesetCache::new();
let engine_validator = BasicEngineValidator::new(
blockchain_db.clone(),
consensus.clone(),
@@ -195,6 +201,7 @@ mod tests {
engine_payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache.clone(),
);
let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel();
@@ -214,6 +221,7 @@ mod tests {
TreeConfig::default(),
sync_metrics_tx,
evm_config,
changeset_cache,
);
}
}

View File

@@ -12,7 +12,7 @@ workspace = true
[dependencies]
# reth
reth-chain-state.workspace = true
reth-chain-state = { workspace = true, features = ["rayon"] }
reth-chainspec = { workspace = true, optional = true }
reth-consensus.workspace = true
reth-db.workspace = true
@@ -29,12 +29,13 @@ reth-provider.workspace = true
reth-prune.workspace = true
reth-revm.workspace = true
reth-stages-api.workspace = true
reth-storage-errors.workspace = true
reth-tasks.workspace = true
reth-trie-parallel.workspace = true
reth-trie-sparse = { workspace = true, features = ["std", "metrics"] }
reth-trie-sparse-parallel = { workspace = true, features = ["std"] }
reth-trie.workspace = true
reth-trie-common.workspace = true
reth-trie-db.workspace = true
# alloy
alloy-evm.workspace = true
@@ -95,7 +96,7 @@ reth-tracing.workspace = true
reth-node-ethereum.workspace = true
reth-e2e-test-utils.workspace = true
# alloy
# revm
revm-state.workspace = true
assert_matches.workspace = true
@@ -134,6 +135,8 @@ test-utils = [
"reth-static-file",
"reth-tracing",
"reth-trie/test-utils",
"reth-trie-common/test-utils",
"reth-trie-db/test-utils",
"reth-trie-sparse/test-utils",
"reth-prune-types?/test-utils",
"reth-trie-parallel/test-utils",

View File

@@ -26,7 +26,9 @@ fn create_bench_state(num_accounts: usize) -> EvmState {
nonce: 10,
code_hash: B256::from_slice(&rng.random::<[u8; 32]>()),
code: Default::default(),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::empty(),
transaction_id: 0,

View File

@@ -62,6 +62,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
storage: HashMap::default(),
status: AccountStatus::SelfDestructed,
transaction_id: 0,
original_info: Box::new(AccountInfo::default()),
}
} else {
RevmAccount {
@@ -70,6 +71,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
nonce: rng.random::<u64>(),
code_hash: KECCAK_EMPTY,
code: Some(Default::default()),
account_id: None,
},
storage: (0..rng.random_range(0..=params.storage_slots_per_account))
.map(|_| {
@@ -84,6 +86,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
})
.collect(),
status: AccountStatus::Touched,
original_info: Box::new(AccountInfo::default()),
transaction_id: 0,
}
};
@@ -239,7 +242,10 @@ fn bench_state_root(c: &mut Criterion) {
std::convert::identity,
),
StateProviderBuilder::new(provider.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(provider),
OverlayStateProviderFactory::new(
provider,
reth_trie_db::ChangesetCache::new(),
),
&TreeConfig::default(),
None,
);

View File

@@ -4,7 +4,7 @@ use crate::{engine::DownloadRequest, metrics::BlockDownloaderMetrics};
use alloy_consensus::BlockHeader;
use alloy_primitives::B256;
use futures::FutureExt;
use reth_consensus::{Consensus, ConsensusError};
use reth_consensus::Consensus;
use reth_network_p2p::{
full_block::{FetchFullBlockFuture, FetchFullBlockRangeFuture, FullBlockClient},
BlockClient,
@@ -81,7 +81,7 @@ where
B: Block,
{
/// Create a new instance
pub fn new(client: Client, consensus: Arc<dyn Consensus<B, Error = ConsensusError>>) -> Self {
pub fn new(client: Client, consensus: Arc<dyn Consensus<B>>) -> Self {
Self {
full_block_client: FullBlockClient::new(client, consensus),
inflight_full_block_requests: Vec::new(),

View File

@@ -6,6 +6,7 @@ use crate::{
download::{BlockDownloader, DownloadAction, DownloadOutcome},
};
use alloy_primitives::B256;
use crossbeam_channel::Sender;
use futures::{Stream, StreamExt};
use reth_chain_state::ExecutedBlock;
use reth_engine_primitives::{BeaconEngineMessage, ConsensusEngineEvent};
@@ -15,7 +16,6 @@ use reth_primitives_traits::{Block, NodePrimitives, SealedBlock};
use std::{
collections::HashSet,
fmt::Display,
sync::mpsc::Sender,
task::{ready, Context, Poll},
};
use tokio::sync::mpsc::UnboundedReceiver;

View File

@@ -1,12 +1,13 @@
use crate::metrics::PersistenceMetrics;
use alloy_eips::BlockNumHash;
use crossbeam_channel::Sender as CrossbeamSender;
use reth_chain_state::ExecutedBlock;
use reth_errors::ProviderError;
use reth_ethereum_primitives::EthPrimitives;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::ProviderNodeTypes, BlockExecutionWriter, BlockHashReader, ChainStateBlockWriter,
DBProvider, DatabaseProviderFactory, ProviderFactory,
DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksMode,
};
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
use reth_stages_api::{MetricEvent, MetricEventsSender};
@@ -15,7 +16,6 @@ use std::{
time::Instant,
};
use thiserror::Error;
use tokio::sync::oneshot;
use tracing::{debug, error};
/// Writes parts of reth's in memory tree state to the database and static files.
@@ -151,7 +151,7 @@ where
if last_block.is_some() {
let provider_rw = self.provider.database_provider_rw()?;
provider_rw.save_blocks(blocks)?;
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
provider_rw.commit()?;
}
@@ -159,6 +159,7 @@ where
self.metrics.save_blocks_block_count.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
Ok(last_block)
}
}
@@ -183,13 +184,13 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
///
/// First, header, transaction, and receipt-related data should be written to static files.
/// Then the execution history-related data will be written to the database.
SaveBlocks(Vec<ExecutedBlock<N>>, oneshot::Sender<Option<BlockNumHash>>),
SaveBlocks(Vec<ExecutedBlock<N>>, CrossbeamSender<Option<BlockNumHash>>),
/// Removes block data above the given block number from the database.
///
/// This will first update checkpoints from the database, then remove actual block data from
/// static files.
RemoveBlocksAbove(u64, oneshot::Sender<Option<BlockNumHash>>),
RemoveBlocksAbove(u64, CrossbeamSender<Option<BlockNumHash>>),
/// Update the persisted finalized block on disk
SaveFinalizedBlock(u64),
@@ -261,7 +262,7 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
pub fn save_blocks(
&self,
blocks: Vec<ExecutedBlock<T>>,
tx: oneshot::Sender<Option<BlockNumHash>>,
tx: CrossbeamSender<Option<BlockNumHash>>,
) -> Result<(), SendError<PersistenceAction<T>>> {
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
}
@@ -290,7 +291,7 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
pub fn remove_blocks_above(
&self,
block_num: u64,
tx: oneshot::Sender<Option<BlockNumHash>>,
tx: CrossbeamSender<Option<BlockNumHash>>,
) -> Result<(), SendError<PersistenceAction<T>>> {
self.send_action(PersistenceAction::RemoveBlocksAbove(block_num, tx))
}
@@ -319,22 +320,22 @@ mod tests {
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx)
}
#[tokio::test]
async fn test_save_blocks_empty() {
#[test]
fn test_save_blocks_empty() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let blocks = vec![];
let (tx, rx) = oneshot::channel();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
let hash = rx.await.unwrap();
let hash = rx.recv().unwrap();
assert_eq!(hash, None);
}
#[tokio::test]
async fn test_save_blocks_single_block() {
#[test]
fn test_save_blocks_single_block() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let block_number = 0;
@@ -344,37 +345,35 @@ mod tests {
let block_hash = executed.recovered_block().hash();
let blocks = vec![executed];
let (tx, rx) = oneshot::channel();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } =
tokio::time::timeout(std::time::Duration::from_secs(10), rx)
.await
.expect("test timed out")
.expect("channel closed unexpectedly")
.expect("no hash returned");
let BlockNumHash { hash: actual_hash, number: _ } = rx
.recv_timeout(std::time::Duration::from_secs(10))
.expect("test timed out")
.expect("no hash returned");
assert_eq!(block_hash, actual_hash);
}
#[tokio::test]
async fn test_save_blocks_multiple_blocks() {
#[test]
fn test_save_blocks_multiple_blocks() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let mut test_block_builder = TestBlockBuilder::eth();
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = oneshot::channel();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.await.unwrap().unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);
}
#[tokio::test]
async fn test_save_blocks_multiple_calls() {
#[test]
fn test_save_blocks_multiple_calls() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
@@ -383,11 +382,11 @@ mod tests {
for range in ranges {
let blocks = test_block_builder.get_executed_blocks(range).collect::<Vec<_>>();
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = oneshot::channel();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.await.unwrap().unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);
}
}

View File

@@ -606,12 +606,21 @@ pub(crate) struct SavedCache {
/// A guard to track in-flight usage of this cache.
/// The cache is considered available if the strong count is 1.
usage_guard: Arc<()>,
/// Whether to skip cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl SavedCache {
/// Creates a new instance with the internals
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
}
/// Sets whether to disable cache metrics recording.
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
self.disable_cache_metrics = disable;
self
}
/// Returns the hash for this cache
@@ -619,9 +628,9 @@ impl SavedCache {
self.hash
}
/// Splits the cache into its caches and metrics, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
(self.caches, self.metrics)
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
(self.caches, self.metrics, self.disable_cache_metrics)
}
/// Returns true if the cache is available for use (no other tasks are currently using it).
@@ -629,6 +638,11 @@ impl SavedCache {
Arc::strong_count(&self.usage_guard) == 1
}
/// Returns the current strong count of the usage guard.
pub(crate) fn usage_count(&self) -> usize {
Arc::strong_count(&self.usage_guard)
}
/// Returns the [`ExecutionCache`] belonging to the tracked hash.
pub(crate) const fn cache(&self) -> &ExecutionCache {
&self.caches
@@ -640,7 +654,13 @@ impl SavedCache {
}
/// Updates the metrics for the [`ExecutionCache`].
///
/// Note: This can be expensive with large cached state as it iterates over
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
pub(crate) fn update_metrics(&self) {
if self.disable_cache_metrics {
return;
}
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);

View File

@@ -6,15 +6,13 @@ use reth_errors::{BlockExecutionError, BlockValidationError, ProviderError};
use reth_evm::execute::InternalBlockExecutionError;
use reth_payload_primitives::NewPayloadError;
use reth_primitives_traits::{Block, BlockBody, SealedBlock};
use tokio::sync::oneshot::error::TryRecvError;
/// This is an error that can come from advancing persistence. Either this can be a
/// [`TryRecvError`], or this can be a [`ProviderError`]
/// This is an error that can come from advancing persistence.
#[derive(Debug, thiserror::Error)]
pub enum AdvancePersistenceError {
/// An error that can be from failing to receive a value from persistence
#[error(transparent)]
RecvError(#[from] TryRecvError),
/// The persistence channel was closed unexpectedly
#[error("persistence channel closed")]
ChannelClosed,
/// A provider error
#[error(transparent)]
Provider(#[from] ProviderError),

View File

@@ -1,25 +1,15 @@
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
use alloy_consensus::transaction::TxHashRef;
use alloy_evm::{
block::{BlockExecutor, ExecutableTx},
Evm,
};
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
use core::borrow::BorrowMut;
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
use reth_errors::{BlockExecutionError, ProviderError};
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
use reth_errors::ProviderError;
use reth_evm::metrics::ExecutorMetrics;
use reth_execution_types::BlockExecutionOutput;
use reth_metrics::{
metrics::{Counter, Gauge, Histogram},
Metrics,
};
use reth_primitives_traits::SignedTransaction;
use reth_trie::updates::TrieUpdates;
use revm::database::{states::bundle_state::BundleRetention, State};
use revm_primitives::Address;
use std::time::Instant;
use tracing::{debug_span, trace};
use std::time::{Duration, Instant};
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
@@ -35,101 +25,24 @@ pub(crate) struct EngineApiMetrics {
}
impl EngineApiMetrics {
/// Helper function for metered execution
fn metered<F, R>(&self, f: F) -> R
where
F: FnOnce() -> (u64, R),
{
// Execute the block and record the elapsed time.
let execute_start = Instant::now();
let (gas_used, output) = f();
let execution_duration = execute_start.elapsed().as_secs_f64();
// Update gas metrics.
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_duration);
self.executor.execution_duration.set(execution_duration);
output
}
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
/// execution.
/// Records metrics for block execution.
///
/// This method updates metrics for execution time, gas usage, and the number
/// of accounts, storage slots and bytecodes loaded and updated.
pub(crate) fn execute_metered<E, DB>(
/// of accounts, storage slots and bytecodes updated.
pub(crate) fn record_block_execution<R>(
&self,
executor: E,
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
transaction_count: usize,
state_hook: Box<dyn OnStateHook>,
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
where
DB: alloy_evm::Database,
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
{
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
// they are globally registered so that the data recorded in the hook will
// be accessible.
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
output: &BlockExecutionOutput<R>,
execution_duration: Duration,
) {
let execution_secs = execution_duration.as_secs_f64();
let gas_used = output.result.gas_used;
let mut senders = Vec::with_capacity(transaction_count);
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
let f = || {
let start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.entered()
.in_scope(|| executor.apply_pre_execution_changes())?;
self.executor.pre_execution_histogram.record(start.elapsed());
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
loop {
let start = Instant::now();
let Some(tx) = transactions.next() else { break };
self.executor.transaction_wait_histogram.record(start.elapsed());
let tx = tx?;
senders.push(*tx.signer());
let span =
debug_span!(target: "engine::tree", "execute tx", tx_hash=?tx.tx().tx_hash());
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.executor.transaction_execution_histogram.record(start.elapsed());
// record the tx gas used
enter.record("gas_used", gas_used);
}
drop(exec_span);
let start = Instant::now();
let result = debug_span!(target: "engine::tree", "finish")
.entered()
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result));
self.executor.post_execution_histogram.record(start.elapsed());
result
};
// Use metered to execute and track timing/gas metrics
let (mut db, result) = self.metered(|| {
let res = f();
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
(gas_used, res)
})?;
// merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.entered()
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
// Update gas metrics
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_secs);
self.executor.execution_duration.set(execution_secs);
// Update the metrics for the number of accounts, storage slots and bytecodes updated
let accounts = output.state.state.len();
@@ -140,8 +53,31 @@ impl EngineApiMetrics {
self.executor.accounts_updated_histogram.record(accounts as f64);
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
}
Ok((output, senders))
/// Returns a reference to the executor metrics for use in state hooks.
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
&self.executor
}
/// Records the duration of block pre-execution changes (e.g., beacon root update).
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
self.executor.pre_execution_histogram.record(elapsed);
}
/// Records the duration of block post-execution changes (e.g., finalization).
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
self.executor.post_execution_histogram.record(elapsed);
}
/// Records the time spent waiting for the next transaction from the iterator.
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
self.executor.transaction_wait_histogram.record(elapsed);
}
/// Records the duration of a single transaction execution.
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
self.executor.transaction_execution_histogram.record(elapsed);
}
}
@@ -197,6 +133,12 @@ pub(crate) struct EngineMetrics {
#[derive(Metrics)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ForkchoiceUpdatedMetrics {
/// Finish time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of forkchoice updated messages received.
pub(crate) forkchoice_updated_messages: Counter,
/// The total count of forkchoice updated messages with payload received.
@@ -219,18 +161,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
pub(crate) forkchoice_updated_last: Gauge,
/// Time diff between new payload call response and the next forkchoice updated call request.
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
/// time).
pub(crate) time_between_forkchoice_updated: Histogram,
/// Time from previous forkchoice updated start to current forkchoice updated start (total
/// interval).
pub(crate) forkchoice_updated_interval: Histogram,
}
impl ForkchoiceUpdatedMetrics {
/// Increment the forkchoiceUpdated counter based on the given result
pub(crate) fn update_response_metrics(
&self,
&mut self,
start: Instant,
latest_new_payload_at: &mut Option<Instant>,
has_attrs: bool,
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
) {
let elapsed = start.elapsed();
let finish = Instant::now();
let elapsed = finish - start;
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_forkchoice_updated.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.forkchoice_updated_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.forkchoice_status() {
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
@@ -257,7 +216,10 @@ impl ForkchoiceUpdatedMetrics {
pub(crate) struct NewPayloadStatusMetrics {
/// Finish time of the latest new payload call.
#[metric(skip)]
pub(crate) latest_at: Option<Instant>,
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest new payload call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of new payload messages received.
pub(crate) new_payload_messages: Counter,
/// The total count of new payload messages that we responded to with
@@ -285,6 +247,10 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) new_payload_latency: Histogram,
/// Latency for the last new payload call.
pub(crate) new_payload_last: Gauge,
/// Time from previous payload finish to current payload start (idle time).
pub(crate) time_between_new_payloads: Histogram,
/// Time from previous payload start to current payload start (total interval).
pub(crate) new_payload_interval: Histogram,
}
impl NewPayloadStatusMetrics {
@@ -298,7 +264,14 @@ impl NewPayloadStatusMetrics {
let finish = Instant::now();
let elapsed = finish - start;
self.latest_at = Some(finish);
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_new_payloads.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.new_payload_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.status {
PayloadStatusEnum::Valid => {
@@ -321,7 +294,7 @@ impl NewPayloadStatusMetrics {
}
/// Metrics for non-execution related block validation.
#[derive(Metrics)]
#[derive(Metrics, Clone)]
#[metrics(scope = "sync.block_validation")]
pub(crate) struct BlockValidationMetrics {
/// Total number of storage tries updated in the state root calculation
@@ -334,10 +307,6 @@ pub(crate) struct BlockValidationMetrics {
pub(crate) state_root_histogram: Histogram,
/// Histogram of deferred trie computation duration.
pub(crate) deferred_trie_compute_duration: Histogram,
/// Histogram of time spent waiting for deferred trie data to become available.
pub(crate) deferred_trie_wait_duration: Histogram,
/// Trie input computation duration
pub(crate) trie_input_duration: Histogram,
/// Payload conversion and validation latency
pub(crate) payload_validation_duration: Gauge,
/// Histogram of payload validation latency
@@ -348,6 +317,14 @@ pub(crate) struct BlockValidationMetrics {
pub(crate) post_execution_validation_duration: Histogram,
/// Total duration of the new payload call
pub(crate) total_duration: Histogram,
/// Size of `HashedPostStateSorted` (`total_len`)
pub(crate) hashed_post_state_size: Histogram,
/// Size of `TrieUpdatesSorted` (`total_len`)
pub(crate) trie_updates_sorted_size: Histogram,
/// Size of `AnchoredTrieInput` overlay `TrieUpdatesSorted` (`total_len`)
pub(crate) anchored_overlay_trie_updates_size: Histogram,
/// Size of `AnchoredTrieInput` overlay `HashedPostStateSorted` (`total_len`)
pub(crate) anchored_overlay_hashed_state_size: Histogram,
}
impl BlockValidationMetrics {
@@ -379,133 +356,10 @@ pub(crate) struct BlockBufferMetrics {
mod tests {
use super::*;
use alloy_eips::eip7685::Requests;
use alloy_evm::block::StateChangeSource;
use alloy_primitives::{B256, U256};
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm_ethereum::EthEvm;
use reth_ethereum_primitives::Receipt;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::RecoveredBlock;
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
database_interface::EmptyDB,
inspector::NoOpInspector,
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
Context, MainBuilder, MainContext,
};
use revm_primitives::Bytes;
use std::sync::mpsc;
/// A simple mock executor for testing that doesn't require complex EVM setup
struct MockExecutor {
state: EvmState,
hook: Option<Box<dyn OnStateHook>>,
}
impl MockExecutor {
fn new(state: EvmState) -> Self {
Self { state, hook: None }
}
}
// Mock Evm type for testing
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
impl BlockExecutor for MockExecutor {
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Evm = MockEvm;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
}
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
// Call hook with our mock state for each transaction
if let Some(hook) = self.hook.as_mut() {
hook.on_state(StateChangeSource::Transaction(0), &self.state);
}
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 1000, // Mock gas used
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(1000)
}
fn finish(
self,
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
let Self { hook, state, .. } = self;
// Call hook with our mock state
if let Some(mut hook) = hook {
hook.on_state(StateChangeSource::Transaction(0), &state);
}
// Create a mock EVM
let db = State::builder()
.with_database(EmptyDB::default())
.with_bundle_update()
.without_state_clear()
.build();
let evm = EthEvm::new(
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
false,
);
// Return successful result like the original tests
Ok((
evm,
BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 1000,
blob_gas_used: 0,
},
))
}
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
self.hook = hook;
}
fn evm(&self) -> &Self::Evm {
panic!("Mock executor evm() not implemented")
}
fn evm_mut(&mut self) -> &mut Self::Evm {
panic!("Mock executor evm_mut() not implemented")
}
}
struct ChannelStateHook {
output: i32,
sender: mpsc::Sender<i32>,
}
impl OnStateHook for ChannelStateHook {
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
let _ = self.sender.send(self.output);
}
}
use reth_revm::db::BundleState;
fn setup_test_recorder() -> Snapshotter {
let recorder = DebuggingRecorder::new();
@@ -515,37 +369,7 @@ mod tests {
}
#[test]
fn test_executor_metrics_hook_called() {
let metrics = EngineApiMetrics::default();
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, rx) = mpsc::channel();
let expected_output = 42;
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
let state = EvmState::default();
let executor = MockExecutor::new(state);
// This will fail to create the EVM but should still call the hook
let _result = metrics.execute_metered::<_, EmptyDB>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
);
// Check if hook was called (it might not be if finish() fails early)
match rx.try_recv() {
Ok(actual_output) => assert_eq!(actual_output, expected_output),
Err(_) => {
// Hook wasn't called, which is expected if the mock fails early
// The test still validates that the code compiles and runs
}
}
}
#[test]
fn test_executor_metrics_hook_metrics_recorded() {
fn test_record_block_execution_metrics() {
let snapshotter = setup_test_recorder();
let metrics = EngineApiMetrics::default();
@@ -554,42 +378,17 @@ mod tests {
metrics.executor.gas_per_second.set(0.0);
metrics.executor.gas_used_histogram.record(0.0);
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, _rx) = mpsc::channel();
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
// Create a state with some data
let state = {
let mut state = EvmState::default();
let storage =
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
state.insert(
Default::default(),
Account {
info: AccountInfo {
balance: U256::from(100),
nonce: 10,
code_hash: B256::random(),
code: Default::default(),
},
storage,
status: AccountStatus::default(),
transaction_id: 0,
},
);
state
let output = BlockExecutionOutput::<Receipt> {
state: BundleState::default(),
result: BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 21000,
blob_gas_used: 0,
},
};
let executor = MockExecutor::new(state);
// Execute (will fail but should still update some metrics)
let _result = metrics.execute_metered::<_, EmptyDB>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
);
metrics.record_block_execution(&output, Duration::from_millis(100));
let snapshot = snapshotter.snapshot().into_vec();

View File

@@ -30,25 +30,21 @@ use reth_payload_primitives::{
};
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
use reth_provider::{
BlockReader, DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StateProviderBox,
StateProviderFactory, StateReader, TransactionVariant, TrieReader,
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
use reth_trie_db::ChangesetCache;
use revm::state::EvmState;
use state::TreeState;
use std::{
fmt::Debug,
ops,
sync::{
mpsc::{Receiver, RecvError, RecvTimeoutError, Sender},
Arc,
},
time::Instant,
};
use std::{fmt::Debug, ops, sync::Arc, time::Instant};
use crossbeam_channel::{Receiver, Sender};
use tokio::sync::{
mpsc::{unbounded_channel, UnboundedReceiver, UnboundedSender},
oneshot::{self, error::TryRecvError},
oneshot,
};
use tracing::*;
@@ -240,7 +236,7 @@ where
C: ConfigureEvm<Primitives = N> + 'static,
{
provider: P,
consensus: Arc<dyn FullConsensus<N, Error = ConsensusError>>,
consensus: Arc<dyn FullConsensus<N>>,
payload_validator: V,
/// Keeps track of internals such as executed and buffered blocks.
state: EngineApiTreeState<N>,
@@ -277,6 +273,8 @@ where
engine_kind: EngineApiKind,
/// The EVM configuration.
evm_config: C,
/// Changeset cache for in-memory trie changesets
changeset_cache: ChangesetCache,
}
impl<N, P: Debug, T: PayloadTypes + Debug, V: Debug, C> std::fmt::Debug
@@ -301,6 +299,7 @@ where
.field("metrics", &self.metrics)
.field("engine_kind", &self.engine_kind)
.field("evm_config", &self.evm_config)
.field("changeset_cache", &self.changeset_cache)
.finish()
}
}
@@ -313,11 +312,12 @@ where
+ StateProviderFactory
+ StateReader<Receipt = N::Receipt>
+ HashedPostStateProvider
+ TrieReader
+ Clone
+ 'static,
<P as DatabaseProviderFactory>::Provider:
BlockReader<Block = N::Block, Header = N::BlockHeader>,
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ BlockNumReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
V: EngineValidator<T>,
@@ -326,7 +326,7 @@ where
#[expect(clippy::too_many_arguments)]
pub fn new(
provider: P,
consensus: Arc<dyn FullConsensus<N, Error = ConsensusError>>,
consensus: Arc<dyn FullConsensus<N>>,
payload_validator: V,
outgoing: UnboundedSender<EngineApiEvent<N>>,
state: EngineApiTreeState<N>,
@@ -337,8 +337,9 @@ where
config: TreeConfig,
engine_kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
) -> Self {
let (incoming_tx, incoming) = std::sync::mpsc::channel();
let (incoming_tx, incoming) = crossbeam_channel::unbounded();
Self {
provider,
@@ -357,6 +358,7 @@ where
incoming_tx,
engine_kind,
evm_config,
changeset_cache,
}
}
@@ -368,7 +370,7 @@ where
#[expect(clippy::complexity)]
pub fn spawn_new(
provider: P,
consensus: Arc<dyn FullConsensus<N, Error = ConsensusError>>,
consensus: Arc<dyn FullConsensus<N>>,
payload_validator: V,
persistence: PersistenceHandle<N>,
payload_builder: PayloadBuilderHandle<T>,
@@ -376,6 +378,7 @@ where
config: TreeConfig,
kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
) -> (Sender<FromEngine<EngineApiRequest<T, N>, N::Block>>, UnboundedReceiver<EngineApiEvent<N>>)
{
let best_block_number = provider.best_block_number().unwrap_or(0);
@@ -407,6 +410,7 @@ where
config,
kind,
evm_config,
changeset_cache,
);
let incoming = task.incoming_tx.clone();
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
@@ -423,8 +427,8 @@ where
/// This will block the current thread and process incoming messages.
pub fn run(mut self) {
loop {
match self.try_recv_engine_message() {
Ok(Some(msg)) => {
match self.wait_for_event() {
LoopEvent::EngineMessage(msg) => {
debug!(target: "engine::tree", %msg, "received new engine message");
match self.on_engine_message(msg) {
Ok(ops::ControlFlow::Break(())) => return,
@@ -435,15 +439,22 @@ where
}
}
}
Ok(None) => {
debug!(target: "engine::tree", "received no engine message for some time, while waiting for persistence task to complete");
LoopEvent::PersistenceComplete { result, start_time } => {
if let Err(err) = self.on_persistence_complete(result, start_time) {
error!(target: "engine::tree", %err, "Persistence complete handling failed");
return
}
}
Err(_err) => {
error!(target: "engine::tree", "Engine channel disconnected");
LoopEvent::Disconnected => {
error!(target: "engine::tree", "Channel disconnected");
return
}
}
// Always check if we need to trigger new persistence after any event:
// - After engine messages: new blocks may have been inserted that exceed the
// persistence threshold
// - After persistence completion: we can now persist more blocks if needed
if let Err(err) = self.advance_persistence() {
error!(target: "engine::tree", %err, "Advancing persistence failed");
return
@@ -451,6 +462,47 @@ where
}
}
/// Blocks until the next event is ready: either an incoming engine message or a persistence
/// completion (if one is in progress).
///
/// Uses biased selection to prioritize persistence completion to update in-memory state and
/// unblock further writes.
fn wait_for_event(&mut self) -> LoopEvent<T, N> {
// Take ownership of persistence rx if present
let maybe_persistence = self.persistence_state.rx.take();
if let Some((persistence_rx, start_time, action)) = maybe_persistence {
// Biased select prioritizes persistence completion to update in memory state and
// unblock further writes
crossbeam_channel::select_biased! {
recv(persistence_rx) -> result => {
// Don't put it back - consumed (oneshot-like behavior)
match result {
Ok(value) => LoopEvent::PersistenceComplete {
result: value,
start_time,
},
Err(_) => LoopEvent::Disconnected,
}
},
recv(self.incoming) -> msg => {
// Put the persistence rx back - we didn't consume it
self.persistence_state.rx = Some((persistence_rx, start_time, action));
match msg {
Ok(m) => LoopEvent::EngineMessage(m),
Err(_) => LoopEvent::Disconnected,
}
},
}
} else {
// No persistence in progress - just wait on incoming
match self.incoming.recv() {
Ok(m) => LoopEvent::EngineMessage(m),
Err(_) => LoopEvent::Disconnected,
}
}
}
/// Invoked when previously requested blocks were downloaded.
///
/// If the block count exceeds the configured batch size we're allowed to execute at once, this
@@ -1191,39 +1243,13 @@ where
.with_event(TreeEvent::Download(DownloadRequest::single_block(target))))
}
/// Attempts to receive the next engine request.
///
/// If there's currently no persistence action in progress, this will block until a new request
/// is received. If there's a persistence action in progress, this will try to receive the
/// next request with a timeout to not block indefinitely and return `Ok(None)` if no request is
/// received in time.
///
/// Returns an error if the engine channel is disconnected.
#[expect(clippy::type_complexity)]
fn try_recv_engine_message(
&self,
) -> Result<Option<FromEngine<EngineApiRequest<T, N>, N::Block>>, RecvError> {
if self.persistence_state.in_progress() {
// try to receive the next request with a timeout to not block indefinitely
match self.incoming.recv_timeout(std::time::Duration::from_millis(500)) {
Ok(msg) => Ok(Some(msg)),
Err(err) => match err {
RecvTimeoutError::Timeout => Ok(None),
RecvTimeoutError::Disconnected => Err(RecvError),
},
}
} else {
self.incoming.recv().map(Some)
}
}
/// Helper method to remove blocks and set the persistence state. This ensures we keep track of
/// the current persistence action while we're removing blocks.
fn remove_blocks(&mut self, new_tip_num: u64) {
debug!(target: "engine::tree", ?new_tip_num, last_persisted_block_number=?self.persistence_state.last_persisted_block.number, "Removing blocks using persistence task");
if new_tip_num < self.persistence_state.last_persisted_block.number {
debug!(target: "engine::tree", ?new_tip_num, "Starting remove blocks job");
let (tx, rx) = oneshot::channel();
let (tx, rx) = crossbeam_channel::bounded(1);
let _ = self.persistence.remove_blocks_above(new_tip_num, tx);
self.persistence_state.start_remove(new_tip_num, rx);
}
@@ -1245,35 +1271,17 @@ where
.expect("Checked non-empty persisting blocks");
debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(), "Persisting blocks");
let (tx, rx) = oneshot::channel();
let (tx, rx) = crossbeam_channel::bounded(1);
let _ = self.persistence.save_blocks(blocks_to_persist, tx);
self.persistence_state.start_save(highest_num_hash, rx);
}
/// Attempts to advance the persistence state.
/// Triggers new persistence actions if no persistence task is currently in progress.
///
/// If we're currently awaiting a response this will try to receive the response (non-blocking)
/// or send a new persistence action if necessary.
/// This checks if we need to remove blocks (disk reorg) or save new blocks to disk.
/// Persistence completion is handled separately via the `wait_for_event` method.
fn advance_persistence(&mut self) -> Result<(), AdvancePersistenceError> {
if self.persistence_state.in_progress() {
let (mut rx, start_time, current_action) = self
.persistence_state
.rx
.take()
.expect("if a persistence task is in progress Receiver must be Some");
// Check if persistence has complete
match rx.try_recv() {
Ok(last_persisted_hash_num) => {
self.on_persistence_complete(last_persisted_hash_num, start_time)?;
}
Err(TryRecvError::Closed) => return Err(TryRecvError::Closed.into()),
Err(TryRecvError::Empty) => {
self.persistence_state.rx = Some((rx, start_time, current_action))
}
}
}
if !self.persistence_state.in_progress() {
if let Some(new_tip_num) = self.find_disk_reorg()? {
self.remove_blocks(new_tip_num)
@@ -1306,7 +1314,7 @@ where
loop {
// Wait for any in-progress persistence to complete (blocking)
if let Some((rx, start_time, _action)) = self.persistence_state.rx.take() {
let result = rx.blocking_recv().map_err(|_| TryRecvError::Closed)?;
let result = rx.recv().map_err(|_| AdvancePersistenceError::ChannelClosed)?;
self.on_persistence_complete(result, start_time)?;
}
@@ -1322,6 +1330,31 @@ where
}
}
/// Tries to poll for a completed persistence task (non-blocking).
///
/// Returns `true` if a persistence task was completed, `false` otherwise.
#[cfg(test)]
pub fn try_poll_persistence(&mut self) -> Result<bool, AdvancePersistenceError> {
let Some((rx, start_time, action)) = self.persistence_state.rx.take() else {
return Ok(false);
};
match rx.try_recv() {
Ok(result) => {
self.on_persistence_complete(result, start_time)?;
Ok(true)
}
Err(crossbeam_channel::TryRecvError::Empty) => {
// Not ready yet, put it back
self.persistence_state.rx = Some((rx, start_time, action));
Ok(false)
}
Err(crossbeam_channel::TryRecvError::Disconnected) => {
Err(AdvancePersistenceError::ChannelClosed)
}
}
}
/// Handles a completed persistence task.
fn on_persistence_complete(
&mut self,
@@ -1342,6 +1375,21 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
let min_threshold = last_persisted_block_number.saturating_sub(64);
let eviction_threshold = finalized.number.min(min_threshold);
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = finalized.number,
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
}
self.on_new_persisted_block()?;
Ok(())
}
@@ -1430,7 +1478,7 @@ where
self.metrics.engine.forkchoice_updated.update_response_metrics(
start,
&mut self.metrics.engine.new_payload.latest_at,
&mut self.metrics.engine.new_payload.latest_finish_at,
has_attrs,
&output,
);
@@ -1628,6 +1676,18 @@ where
)));
return Ok(());
}
} else {
// We don't have the head block or any of its ancestors buffered. Request
// a download for the head block which will then trigger further sync.
debug!(
target: "engine::tree",
head_hash = %sync_target_state.head_block_hash,
"Backfill complete but head block not buffered, requesting download"
);
self.emit_event(EngineApiEvent::Download(DownloadRequest::single_block(
sync_target_state.head_block_hash,
)));
return Ok(());
}
// try to close the gap by executing buffered blocks that are child blocks of the new head
@@ -1795,6 +1855,7 @@ where
/// or the database. If the required historical data (such as trie change sets) has been
/// pruned for a given block, this operation will return an error. On archive nodes, it
/// can retrieve any block.
#[instrument(level = "debug", target = "engine::tree", skip(self))]
fn canonical_block_by_hash(&self, hash: B256) -> ProviderResult<Option<ExecutedBlock<N>>> {
trace!(target: "engine::tree", ?hash, "Fetching executed block by hash");
// check memory first
@@ -1807,12 +1868,23 @@ where
.sealed_block_with_senders(hash.into(), TransactionVariant::WithHash)?
.ok_or_else(|| ProviderError::HeaderNotFound(hash.into()))?
.split_sealed();
let execution_output = self
let mut execution_output = self
.provider
.get_state(block.header().number())?
.ok_or_else(|| ProviderError::StateForNumberNotFound(block.header().number()))?;
let hashed_state = self.provider.hashed_post_state(execution_output.state());
let trie_updates = self.provider.get_block_trie_updates(block.number())?;
debug!(
target: "engine::tree",
number = ?block.number(),
"computing block trie updates",
);
let db_provider = self.provider.database_provider_ro()?;
let trie_updates = reth_trie_db::compute_block_trie_updates(
&self.changeset_cache,
&db_provider,
block.number(),
)?;
let sorted_hashed_state = Arc::new(hashed_state.into_sorted());
let sorted_trie_updates = Arc::new(trie_updates);
@@ -1820,9 +1892,19 @@ where
let trie_data =
ComputedTrieData::without_trie_input(sorted_hashed_state, sorted_trie_updates);
let execution_output = Arc::new(BlockExecutionOutput {
state: execution_output.bundle,
result: BlockExecutionResult {
receipts: execution_output.receipts.pop().unwrap_or_default(),
requests: execution_output.requests.pop().unwrap_or_default(),
gas_used: block.gas_used(),
blob_gas_used: block.blob_gas_used().unwrap_or_default(),
},
});
Ok(Some(ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(execution_output),
execution_output,
trie_data,
)))
}
@@ -2848,6 +2930,26 @@ where
}
}
/// Events received in the main engine loop.
#[derive(Debug)]
enum LoopEvent<T, N>
where
N: NodePrimitives,
T: PayloadTypes,
{
/// An engine API message was received.
EngineMessage(FromEngine<EngineApiRequest<T, N>, N::Block>),
/// A persistence task completed.
PersistenceComplete {
/// The result of the persistence operation.
result: Option<BlockNumHash>,
/// When the persistence operation started.
start_time: Instant,
},
/// A channel was disconnected.
Disconnected,
}
/// Block inclusion can be valid, accepted, or invalid. Invalid blocks are returned as an error
/// variant.
///

View File

@@ -101,7 +101,7 @@ impl<'a> Iterator for BALSlotIter<'a> {
return None;
}
return Some((address, slot));
return Some((address, StorageKey::from(slot)));
}
// Move to next account
@@ -177,13 +177,11 @@ where
let mut storage_map = HashedStorage::new(false);
for slot_changes in &account_changes.storage_changes {
let hashed_slot = keccak256(slot_changes.slot);
let hashed_slot = keccak256(slot_changes.slot.to_be_bytes::<32>());
// Get the last change for this slot
if let Some(last_change) = slot_changes.changes.last() {
storage_map
.storage
.insert(hashed_slot, U256::from_be_bytes(last_change.new_value.0));
storage_map.storage.insert(hashed_slot, last_change.new_value);
}
}
@@ -237,8 +235,8 @@ mod tests {
let provider = StateProviderTest::default();
let address = Address::random();
let slot = StorageKey::random();
let value = B256::random();
let slot = U256::random();
let value = U256::random();
let slot_changes = SlotChanges { slot, changes: vec![StorageChange::new(0, value)] };
@@ -258,10 +256,10 @@ mod tests {
assert!(result.storages.contains_key(&hashed_address));
let storage = result.storages.get(&hashed_address).unwrap();
let hashed_slot = keccak256(slot);
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
let stored_value = storage.storage.get(&hashed_slot).unwrap();
assert_eq!(*stored_value, U256::from_be_bytes(value.0));
assert_eq!(*stored_value, value);
}
#[test]
@@ -392,15 +390,15 @@ mod tests {
let provider = StateProviderTest::default();
let address = Address::random();
let slot = StorageKey::random();
let slot = U256::random();
// Multiple changes to the same slot - should take the last one
let slot_changes = SlotChanges {
slot,
changes: vec![
StorageChange::new(0, B256::from(U256::from(100).to_be_bytes::<32>())),
StorageChange::new(1, B256::from(U256::from(200).to_be_bytes::<32>())),
StorageChange::new(2, B256::from(U256::from(300).to_be_bytes::<32>())),
StorageChange::new(0, U256::from(100)),
StorageChange::new(1, U256::from(200)),
StorageChange::new(2, U256::from(300)),
],
};
@@ -418,7 +416,7 @@ mod tests {
let hashed_address = keccak256(address);
let storage = result.storages.get(&hashed_address).unwrap();
let hashed_slot = keccak256(slot);
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
let stored_value = storage.storage.get(&hashed_slot).unwrap();
@@ -438,15 +436,15 @@ mod tests {
address: addr1,
storage_changes: vec![
SlotChanges {
slot: StorageKey::from(U256::from(100)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(100),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
SlotChanges {
slot: StorageKey::from(U256::from(101)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(101),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
],
storage_reads: vec![StorageKey::from(U256::from(102))],
storage_reads: vec![U256::from(102)],
balance_changes: vec![],
nonce_changes: vec![],
code_changes: vec![],
@@ -456,10 +454,10 @@ mod tests {
let account2 = AccountChanges {
address: addr2,
storage_changes: vec![SlotChanges {
slot: StorageKey::from(U256::from(200)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(200),
changes: vec![StorageChange::new(0, U256::ZERO)],
}],
storage_reads: vec![StorageKey::from(U256::from(201))],
storage_reads: vec![U256::from(201)],
balance_changes: vec![],
nonce_changes: vec![],
code_changes: vec![],
@@ -470,15 +468,15 @@ mod tests {
address: addr3,
storage_changes: vec![
SlotChanges {
slot: StorageKey::from(U256::from(300)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(300),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
SlotChanges {
slot: StorageKey::from(U256::from(301)),
changes: vec![StorageChange::new(0, B256::ZERO)],
slot: U256::from(301),
changes: vec![StorageChange::new(0, U256::ZERO)],
},
],
storage_reads: vec![StorageKey::from(U256::from(302))],
storage_reads: vec![U256::from(302)],
balance_changes: vec![],
nonce_changes: vec![],
code_changes: vec![],

View File

@@ -1,7 +1,7 @@
//! Configured sparse trie enum for switching between serial and parallel implementations.
use alloy_primitives::B256;
use reth_trie::{Nibbles, ProofTrieNode, TrieMasks, TrieNode};
use reth_trie::{BranchNodeMasks, Nibbles, ProofTrieNode, TrieNode};
use reth_trie_sparse::{
errors::SparseTrieResult, provider::TrieNodeProvider, LeafLookup, LeafLookupError,
SerialSparseTrie, SparseTrieInterface, SparseTrieUpdates,
@@ -44,7 +44,7 @@ impl SparseTrieInterface for ConfiguredSparseTrie {
fn with_root(
self,
root: TrieNode,
masks: TrieMasks,
masks: Option<BranchNodeMasks>,
retain_updates: bool,
) -> SparseTrieResult<Self> {
match self {
@@ -75,7 +75,7 @@ impl SparseTrieInterface for ConfiguredSparseTrie {
&mut self,
path: Nibbles,
node: TrieNode,
masks: TrieMasks,
masks: Option<BranchNodeMasks>,
) -> SparseTrieResult<()> {
match self {
Self::Serial(trie) => trie.reveal_node(path, node, masks),

View File

@@ -19,6 +19,7 @@ use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
@@ -28,10 +29,11 @@ use reth_evm::{
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
};
use reth_execution_types::ExecutionOutcome;
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockReader, DatabaseProviderROFactory, StateProvider, StateProviderFactory, StateReader,
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
StateProviderFactory, StateReader,
};
use reth_revm::{db::BundleState, state::EvmState};
use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory};
@@ -61,6 +63,7 @@ mod configured_sparse_trie;
pub mod executor;
pub mod multiproof;
pub mod prewarm;
pub mod receipt_root_task;
pub mod sparse_trie;
use configured_sparse_trie::ConfiguredSparseTrie;
@@ -138,6 +141,8 @@ where
disable_parallel_sparse_trie: bool,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
impl<N, Evm> PayloadProcessor<Evm>
@@ -170,6 +175,7 @@ where
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
}
@@ -274,24 +280,23 @@ where
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
let storage_worker_count = config.storage_worker_count();
let account_worker_count = config.account_worker_count();
let v2_proofs_enabled = config.enable_proof_v2();
let proof_handle = ProofWorkerHandle::new(
self.executor.handle().clone(),
task_ctx,
storage_worker_count,
account_worker_count,
v2_proofs_enabled,
);
let multi_proof_task = MultiProofTask::new(
proof_handle.clone(),
to_sparse_trie,
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
to_multi_proof,
to_multi_proof.clone(),
from_multi_proof,
);
// wire the multiproof task to the prewarm task
let to_multi_proof = Some(multi_proof_task.state_root_message_sender());
// spawn multi-proof task
let parent_span = span.clone();
let saved_cache = prewarm_handle.saved_cache.clone();
@@ -300,7 +305,7 @@ where
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics) = saved_cache.split();
let (cache, metrics, _) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
@@ -316,7 +321,7 @@ where
self.spawn_sparse_trie_task(sparse_trie_rx, proof_handle, state_root_tx);
PayloadHandle {
to_multi_proof,
to_multi_proof: Some(to_multi_proof),
prewarm_handle,
state_root: Some(state_root_rx),
transactions: execution_rx,
@@ -477,6 +482,7 @@ where
debug!("creating new execution cache on cache miss");
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
@@ -492,38 +498,40 @@ where
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
{
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration if
// there's none to reuse.
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
let default_trie = SparseTrie::blind_from(if self.disable_parallel_sparse_trie {
ConfiguredSparseTrie::Serial(Default::default())
} else {
ConfiguredSparseTrie::Parallel(Box::new(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
))
});
ClearedSparseStateTrie::from_state_trie(
SparseStateTrie::new()
.with_accounts_trie(default_trie.clone())
.with_default_storage_trie(default_trie)
.with_updates(true),
)
});
let task =
SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
self.trie_metrics.clone(),
sparse_state_trie,
);
let disable_parallel_sparse_trie = self.disable_parallel_sparse_trie;
let trie_metrics = self.trie_metrics.clone();
let span = Span::current();
self.executor.spawn_blocking(move || {
let _enter = span.entered();
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration
// if there's none to reuse.
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
let default_trie = SparseTrie::blind_from(if disable_parallel_sparse_trie {
ConfiguredSparseTrie::Serial(Default::default())
} else {
ConfiguredSparseTrie::Parallel(Box::new(
ParallelSparseTrie::default()
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
))
});
ClearedSparseStateTrie::from_state_trie(
SparseStateTrie::new()
.with_accounts_trie(default_trie.clone())
.with_default_storage_trie(default_trie)
.with_updates(true),
)
});
let task = SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
sparse_trie_rx,
proof_worker_handle,
trie_metrics,
sparse_state_trie,
);
let (result, trie) = task.run();
// Send state root computation result
let _ = state_root_tx.send(result);
@@ -556,6 +564,7 @@ where
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
) {
let disable_cache_metrics = self.disable_cache_metrics;
self.execution_cache.update_with_guard(|cached| {
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
debug!(
@@ -569,7 +578,8 @@ where
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
existing.split()
let (c, m, _) = existing.split();
(c, m)
}
None => (
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
@@ -578,7 +588,8 @@ where
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
@@ -664,13 +675,15 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
/// Terminates the entire caching task.
///
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` allows sharing with the main execution
/// path without cloning the expensive `BundleState`.
///
/// Returns a sender for the channel that should be notified on block validation success.
pub(super) fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
) {
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
self.prewarm_handle.terminate_caching(execution_outcome)
}
@@ -706,15 +719,21 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
/// Terminates the entire pre-warming task.
///
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
#[must_use = "sender must be used and notified on block validation success"]
pub(super) fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
) {
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
if let Some(tx) = self.to_prewarm_task.take() {
let event = PrewarmTaskEvent::Terminate { execution_outcome };
let (valid_block_tx, valid_block_rx) = mpsc::channel();
let event = PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx };
let _ = tx.send(event);
Some(valid_block_tx)
} else {
None
}
}
}
@@ -723,7 +742,10 @@ impl<R> Drop for CacheTaskHandle<R> {
fn drop(&mut self) {
// Ensure we always terminate on drop - send None without needing Send + Sync bounds
if let Some(tx) = self.to_prewarm_task.take() {
let _ = tx.send(PrewarmTaskEvent::Terminate { execution_outcome: None });
let _ = tx.send(PrewarmTaskEvent::Terminate {
execution_outcome: None,
valid_block_rx: mpsc::channel().1,
});
}
}
}
@@ -757,6 +779,8 @@ impl<R> Drop for CacheTaskHandle<R> {
struct ExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl ExecutionCache {
@@ -775,12 +799,38 @@ impl ExecutionCache {
warn!(blocked_for=?elapsed, "Blocked waiting for execution cache mutex");
}
cache
.as_ref()
if let Some(c) = cache.as_ref() {
let cached_hash = c.executed_block_hash();
// Check that the cache hash matches the parent hash of the current block. It won't
// match in case it's a fork block.
let hash_matches = cached_hash == parent_hash;
// Check `is_available()` to ensure no other tasks (e.g., prewarming) currently hold
// a reference to this cache. We can only reuse it when we have exclusive access.
.filter(|c| c.executed_block_hash() == parent_hash && c.is_available())
.cloned()
let available = c.is_available();
let usage_count = c.usage_count();
debug!(
target: "engine::caching",
%cached_hash,
%parent_hash,
hash_matches,
available,
usage_count,
"Existing cache found"
);
if hash_matches && available {
return Some(c.clone());
}
if hash_matches && !available {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
debug!(target: "engine::caching", %parent_hash, "No cache found");
}
None
}
/// Clears the tracked cache
@@ -811,6 +861,15 @@ impl ExecutionCache {
}
}
/// Metrics for execution cache operations.
#[derive(Metrics, Clone)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ExecutionCacheMetrics {
/// Counter for when the execution cache was unavailable because other threads
/// (e.g., prewarming) are still using it.
pub(crate) execution_cache_in_use: Counter,
}
/// EVM context required to execute a block.
#[derive(Debug, Clone)]
pub struct ExecutionEnv<Evm: ConfigureEvm> {
@@ -863,6 +922,7 @@ mod tests {
use reth_revm::db::BundleState;
use reth_testing_utils::generators;
use reth_trie::{test_utils::state_root, HashedPostState};
use reth_trie_db::ChangesetCache;
use revm_primitives::{Address, HashMap, B256, KECCAK_EMPTY, U256};
use revm_state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot};
use std::sync::Arc;
@@ -1035,7 +1095,9 @@ mod tests {
nonce: rng.random::<u64>(),
code_hash: KECCAK_EMPTY,
code: Some(Default::default()),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::Touched,
transaction_id: 0,
@@ -1118,7 +1180,7 @@ mod tests {
std::convert::identity,
),
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(provider_factory),
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
&TreeConfig::default(),
None, // No BAL for test
);

File diff suppressed because it is too large Load Diff

Some files were not shown because too many files have changed in this diff Show More