Compare commits

..

104 Commits

Author SHA1 Message Date
yongkangc
082daa7fc1 fix(rocksdb): filter account/storage history to match MDBX semantics 2026-01-22 12:58:59 +00:00
Matthias Seitz
5c3e45cd6b fix: handle incomplete receipts gracefully in receipt root task (#21285) 2026-01-22 10:52:56 +00:00
Emma Jamieson-Hoare
68fdba32d2 chore(release): prep v1.10.2 release (#21287)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
2026-01-22 10:50:10 +00:00
Matthias Seitz
8f6a0a2992 ci: add on-demand workflow to check alloy breaking changes (#21267) 2026-01-22 10:47:38 +00:00
Matthias Seitz
ec9c7f8d3e perf(db): use ArrayVec for StoredNibbles key encoding (#21279) 2026-01-22 02:05:50 +00:00
Matthias Seitz
dbdaf068f0 fix(engine): clear execution cache when block validation fails (#21282) 2026-01-22 01:01:22 +00:00
Matthias Seitz
055bf63ee9 refactor: use Default::default() for Header in tests (#21277) 2026-01-21 22:50:10 +00:00
Georgios Konstantopoulos
2305c3ebeb feat(rpc): respect history expiry in block() and map to PrunedHistoryUnavailable (#21270) 2026-01-21 22:22:05 +00:00
joshieDo
eb55c3c3da feat(grafana): add RocksDB metrics dashboard (#21243)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 22:09:42 +00:00
Alexey Shekhirin
72e1467ba3 fix(prune): avoid panic in tx lookup (#21275) 2026-01-21 21:21:53 +00:00
Alexey Shekhirin
74edce0089 revert: feat(trie): add V2 account proof computation and refactor proof types (#21214) (#21274) 2026-01-21 21:07:13 +00:00
Georgios Konstantopoulos
8c645d5762 feat(reth-bench): accept short notation for --target-gas-limit (#21273) 2026-01-21 21:04:10 +00:00
Georgios Konstantopoulos
b7d2ee2566 feat(engine): add metric for execution cache unavailability due to concurrent use (#21265)
Co-authored-by: Tempo AI <ai@tempo.xyz>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-21 20:17:45 +00:00
Matthias Seitz
7609deddda perf(trie): parallelize merge_ancestors_into_overlay (#21202) 2026-01-21 20:08:03 +00:00
Matthias Seitz
ec50fd40b3 chore(chainspec): use ..Default::default() in create_chain_config (#21266) 2026-01-21 19:19:24 +00:00
YK
624ddc5779 feat(stages): add RocksDB support for IndexStorageHistoryStage (#21175) 2026-01-21 17:05:19 +00:00
Georgios Konstantopoulos
dd72cfe23e refactor: remove static_files.to_settings() and add edge feature to RocksDB flags (#21225) 2026-01-21 16:52:24 +00:00
joshieDo
ff8ac97e33 fix(stages): clear ETL collectors on HeaderStage error paths (#21258) 2026-01-21 16:27:30 +00:00
Alexey Shekhirin
0974485863 feat(reth-bench): add --target-gas-limit option to gas-limit-ramp (#21262) 2026-01-21 16:19:22 +00:00
かりんとう
274394e777 fix: fix payload file filter prefix in replay-payloads (#21255) 2026-01-21 16:11:03 +00:00
Emma Jamieson-Hoare
1954c91a60 chore: update CODEOWNERS (#21223)
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmai.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-21 14:40:54 +00:00
Sergei Shulepov
9cf82c8403 fix: supply a real ptr to mdbx_dbi_flags_ex (#21230) 2026-01-21 14:23:26 +00:00
Brian Picciano
f85fcba872 feat(trie): add V2 account proof computation and refactor proof types (#21214)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 14:18:44 +00:00
joshieDo
ebaa4bda3a feat(rocksdb): add missing observability (#21253) 2026-01-21 14:14:34 +00:00
joshieDo
04d4c9a02f fix(rocksdb): flush all column families on drop and show SST/memtable sizes (#21251)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-21 12:44:08 +00:00
Arsenii Kulikov
3065a328f9 fix: clear overlay_cache in with_extended_hashed_state_overlay (#21233) 2026-01-21 12:08:24 +00:00
Sergei Shulepov
43a84f1231 refactor(engine): move execution logic from metrics to payload_validator (#21226) 2026-01-21 11:17:30 +00:00
Matthias Seitz
5a5c21cc1b feat(txpool): add IntoIterator for AllPoolTransactions (#21241) 2026-01-21 10:01:32 +00:00
Matthias Seitz
8a8a9126d6 feat(execution-types): add receipts_iter and logs_iter helpers to Chain (#21240) 2026-01-21 09:59:15 +00:00
Emilia Hane
6f73c2447d feat(trie): Add serde-bincode-compat feature to reth-trie (#21235) 2026-01-21 09:42:52 +00:00
Sergei Shulepov
2cae438642 fix: sigsegv handler (#21231)
Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
2026-01-21 09:42:36 +00:00
Georgios Konstantopoulos
37b5db0d47 feat(cli): add RocksDB table stats to reth db stats command (#21221)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-21 08:45:17 +00:00
joshieDo
238433e146 fix(rocksdb): flush memtables before dropping (#21234) 2026-01-21 02:19:36 +00:00
Georgios Konstantopoulos
660964a0f5 feat(node): log storage settings after genesis init (#21229) 2026-01-21 00:58:23 +00:00
Matthias Seitz
22b465dd64 chore(trie): remove unnecessary clone in into_sorted_ref (#21232) 2026-01-20 22:57:08 +00:00
Georgios Konstantopoulos
3ff575b877 feat(engine): add --engine.disable-cache-metrics flag (#21228)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
2026-01-20 22:03:12 +00:00
かりんとう
d12752dc8a feat(engine): add time_between_forkchoice_updated metric (#21227) 2026-01-20 21:06:11 +00:00
Georgios Konstantopoulos
869b5d0851 feat(edge): enable transaction_hash_numbers_in_rocksdb for edge builds (#21224)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 20:02:02 +00:00
Georgios Konstantopoulos
78de3d8f61 perf(db): use Cow::Borrowed in walk_dup to avoid allocation (#21220) 2026-01-20 19:31:50 +00:00
YK
bc79cc44c9 feat(cli): add --rocksdb.* flags for RocksDB table routing (#21191) 2026-01-20 19:29:05 +00:00
Georgios Konstantopoulos
ff8f434dcd feat(cli): add reth db checksum rocksdb command (#21217)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 19:10:34 +00:00
Arsenii Kulikov
9662dc5271 fix: properly save history indices in pipeline (#21222) 2026-01-20 18:20:28 +00:00
Alexey Shekhirin
3ba37082dc fix(reth-bench): replay-payloads prefix (#21219) 2026-01-20 18:36:35 +01:00
Ahsen Kamal
7934294988 perf(trie): dispatch storage proofs in lexicographical order (#21213)
Signed-off-by: Ahsen Kamal <itsahsenkamal@gmail.com>
2026-01-20 17:09:20 +00:00
Georgios Konstantopoulos
7371bd3f29 chore(db-api): remove sharded_key_encode benchmark (#21215)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 17:01:12 +00:00
Georgios Konstantopoulos
80980b8e4d feat(pruning): add DefaultPruningValues for overridable pruning defaults (#21207)
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-01-20 16:58:29 +00:00
Matthias Seitz
2e2cd67663 perf(chain-state): parallelize into_sorted with rayon (#21193) 2026-01-20 16:42:16 +00:00
Georgios Konstantopoulos
4f009728e2 feat(cli): add reth db checksum mdbx/static-file command (#21211)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:11:51 +00:00
Georgios Konstantopoulos
39d5ae73e8 feat(storage): add read-only mode for RocksDB provider (#21210)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 16:09:51 +00:00
Georgios Konstantopoulos
5ef200eaad perf(db): stack-allocate ShardedKey and StorageShardedKey encoding (#21200)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 15:58:43 +00:00
ethfanWilliam
d002dacc13 chore: remove deprecated and unused ExecuteOutput struct (#20887)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 15:06:26 +00:00
Alexey Shekhirin
bb39cba504 ci: partition bench codspeed job (#20332)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-20 14:29:48 +00:00
YK
bd144a4c42 feat(stages): add RocksDB support for IndexAccountHistoryStage (#21165)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-20 14:23:29 +00:00
tonis
a0845bab18 feat: Check CL/Reth capability compatibility (#20348)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 14:19:31 +00:00
Brian Picciano
346cc0da71 feat(trie): add AsyncAccountValueEncoder for V2 proof computation (#21197)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:50:29 +00:00
Matthias Seitz
ea3d4663ae perf(trie): use HashMap reserve heuristic in MultiProof::extend (#21199) 2026-01-20 13:34:41 +00:00
Hwangjae Lee
3667d3b5aa perf(trie): defer child RLP conversion in proof_v2 for async encoder support (#20873)
Signed-off-by: Hwangjae Lee <meetrick@gmail.com>
Co-authored-by: Brian Picciano <me@mediocregopher.com>
2026-01-20 13:33:08 +00:00
Brian Picciano
7cfb19c98e feat(trie): Add V2 reveal method and target types (#21196)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-20 13:25:54 +00:00
joshieDo
5a38871489 fix: set StaticFileArgs defaults for edge (#21208) 2026-01-20 12:39:36 +00:00
Brian Picciano
c825c8c187 chore(trie): Move hybrid check for trie input merges into common code (#21198) 2026-01-20 12:38:46 +00:00
Matthias Seitz
8f37cd08fc feat(engine-api): add EIP-7928 BAL stub methods (#21204) 2026-01-20 11:33:27 +00:00
Matthias Seitz
c9dad4765d chore: bump version to 1.10.1 (#21188) 2026-01-19 14:04:08 +00:00
Dan Cline
1d55abeef3 chore: rename extend_ref methods on sorted data structures (#21043) 2026-01-19 13:04:57 +00:00
Niven
f7460e219c fix(flashblocks): Add flashblock ws connection retry period (#20510) 2026-01-19 12:01:33 +00:00
Georgios Konstantopoulos
0c66315f20 chore(bench): add --disable-tx-gossip to benchmark node args (#21171) 2026-01-19 11:45:56 +00:00
MozirDmitriy
6a2010e595 refactor(stages): reuse history index cache buffers in collect_history_indices (#21017) 2026-01-19 11:39:52 +00:00
Georgios Konstantopoulos
c2435ff6f8 feat(download): resumable snapshot downloads with auto-retry (#21161) 2026-01-19 10:26:24 +00:00
DaniPopes
52ec8e9491 ci: update to tempoxyz (#21176) 2026-01-19 10:21:37 +00:00
Georgios Konstantopoulos
a901d80ee6 chore: apply spelling and typo fixes (#21182) 2026-01-19 10:21:25 +00:00
MoNyAvA
915164078f docs: document minimal storage mode in pruning FAQ (#21025)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-19 10:27:45 +01:00
github-actions[bot]
be3234d848 chore(deps): weekly cargo update (#21167)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-01-18 14:57:20 +00:00
Matthias Seitz
f624372334 feat(execution-types): add receipts_iter helper (#21162)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-17 19:20:28 +01:00
Matthias Seitz
40bc9d3860 revert: undo Chain crate, add LazyTrieData to trie-common (#21155) 2026-01-17 15:57:09 +00:00
Georgios Konstantopoulos
1ea574417f feat(engine): add new_payload_interval metric (start-to-start) (#21159) 2026-01-17 12:15:45 +00:00
Georgios Konstantopoulos
27e055f790 feat(engine): add time_between_new_payloads metric (#21158) 2026-01-17 10:20:22 +00:00
Georgios Konstantopoulos
d5dc0b27eb fix(storage-api): gate reth-chain dependency behind std feature
The reth-chain crate is inherently std-only (uses BTreeMap, Arc, etc.)
and was breaking the riscv32imac no_std builds by pulling in serde_core
which doesn't support no_std properly.

This makes reth-chain optional and only enables it when std feature is
active, gating the block_writer module that uses Chain behind std.
2026-01-17 08:32:10 +00:00
Georgios Konstantopoulos
c11c13000f perf(storage): batch trie updates across blocks in save_blocks (#21142)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: YK <chiayongkang@hotmail.com>
2026-01-17 07:15:40 +00:00
Matthias Seitz
6bf43ab24a refactor: use ExecutionOutcome::single instead of tuple From (#21152) 2026-01-17 01:51:26 +00:00
Matthias Seitz
574bde0d6f chore(chain-state): reorganize deferred_trie.rs impl blocks (#21151) 2026-01-17 01:39:29 +00:00
Matthias Seitz
79b8ffb828 feat(primitives-traits): add try_recover_signers for parallel batch recovery (#21103) 2026-01-17 01:24:53 +00:00
Dan Cline
c617d25c36 perf: make Chain use DeferredTrieData (#21137)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-17 01:05:35 +00:00
Georgios Konstantopoulos
b96a30821f fix(engine): request head block download when not buffered after backfill (#21150) 2026-01-17 00:33:27 +00:00
Mablr
012fbf5110 fix(docs/cli): update help.rs to use nightly toolchain (#21149) 2026-01-16 23:35:26 +00:00
Arsenii Kulikov
d7a5d1f872 fix: properly record span fields (#21148) 2026-01-16 23:25:54 +00:00
Matthias Seitz
3a39251f79 fix: release mutex before dropping ancestors in wait_cloned (#21146) 2026-01-16 22:32:23 +00:00
Julian Meyer
f6dbf2d82d feat(db): implement extra dup methods (#20964)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 21:31:52 +00:00
Brian Picciano
13707faf1a feat(consensus): incremental receipt root computation in background task (#21131) 2026-01-16 19:53:59 +00:00
Arsenii Kulikov
6e6415690c perf: start saving cache sooner (#21130) 2026-01-16 18:55:18 +00:00
Matthias Seitz
b81e373d78 chore(deps): bump vergen and vergen-git2 to 9.1.0 (#21141) 2026-01-16 20:00:43 +01:00
Arun Dhyani
a164654145 fix(exex): prevent ExExManager deadlock when buffer clears after being full (#21135) 2026-01-16 18:42:23 +00:00
Matthias Seitz
905bb95f8b perf(engine): defer trie overlay computation with LazyOverlay (#21133) 2026-01-16 18:25:04 +00:00
YK
13c32625bc feat(storage): add EitherReader for routing history queries to MDBX or RocksDB (#21063)
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-01-16 17:44:43 +00:00
YK
1be9fab5bf perf: Optimize multiproof sequencer add_proof (#21129) 2026-01-16 17:33:48 +00:00
Arsenii Kulikov
80eb0d0fb6 refactor: use BlockExecutionOutcome in ExecutedBlock (#21123) 2026-01-16 17:07:19 +00:00
Matthias Seitz
5e178f6ac6 chore(deps): update alloy-evm and alloy-op-evm to 0.26.3 (#21126) 2026-01-16 17:24:45 +01:00
Matthias Seitz
b4b64096c8 perf(cli): use available_parallelism as default for re-execute (#21010) 2026-01-16 16:08:30 +00:00
figtracer
e313de818b chore(provider): pre alloc tx hashes (#21114) 2026-01-16 15:40:47 +00:00
rakita
86c414081a feat: stagging revm v34.0.0 (#20627)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-01-16 14:56:27 +00:00
Brian Picciano
a74cb9cbc3 feat(trie): in-memory trie changesets (#20997) 2026-01-16 01:06:31 +00:00
YK
e25411c32b perf(trie): fix extend_sorted_vec O(n log n) → O(n+m) merge (#21098) 2026-01-16 00:17:22 +00:00
Matthias Seitz
ec3323bba0 refactor(chain-state): extract blocks_to_chain helper (#21110) 2026-01-15 23:27:11 +00:00
Dan Cline
26cd132631 fix(reth-bench): use requests hash (#21111) 2026-01-15 19:19:16 +00:00
DaniPopes
079f59c2be perf: reserve in extend_sorted_vec (#21109) 2026-01-15 19:10:20 +00:00
joshieDo
e9b079ad62 feat: add rocksdb to save_blocks (#21003)
Co-authored-by: Sergei Shulepov <s.pepyakin@gmail.com>
Co-authored-by: Sergei Shulepov <pep@tempo.xyz>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-01-15 18:33:19 +00:00
325 changed files with 13558 additions and 5751 deletions

43
.github/CODEOWNERS vendored
View File

@@ -1,45 +1,52 @@
* @gakonst
crates/blockchain-tree-api/ @rakita @mattsse @Rjected
crates/blockchain-tree/ @rakita @mattsse @Rjected
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez
crates/engine/ @mattsse @Rjected @fgimenez @mediocregopher @yongkangc
crates/era/ @mattsse @RomanHodulak
crates/engine/ @mattsse @Rjected @mediocregopher @yongkangc
crates/era/ @mattsse
crates/era-downloader/ @mattsse
crates/era-utils/ @mattsse
crates/errors/ @mattsse
crates/ethereum-forks/ @mattsse @Rjected
crates/ethereum/ @mattsse @Rjected
crates/etl/ @joshieDo @shekhirin
crates/evm/ @rakita @mattsse @Rjected
crates/evm/ @mattsse @Rjected @klkvr
crates/exex/ @shekhirin
crates/fs-util/ @mattsse
crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected @fgimenez
crates/optimism/ @mattsse @Rjected
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @RomanHodulak @mattsse @klkvr
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
crates/prune/ @shekhirin @joshieDo
crates/ress @shekhirin @Rjected
crates/revm/ @mattsse @rakita
crates/rpc/ @mattsse @Rjected @RomanHodulak
crates/ress/ @shekhirin @Rjected
crates/revm/ @mattsse
crates/rpc/ @mattsse @Rjected
crates/stages/ @shekhirin @mediocregopher
crates/static-file/ @joshieDo @shekhirin
crates/stateless/ @mattsse
crates/storage/codecs/ @joshieDo
crates/storage/db-api/ @joshieDo @rakita
crates/storage/db-api/ @joshieDo
crates/storage/db-common/ @Rjected
crates/storage/db/ @joshieDo @rakita
crates/storage/errors/ @rakita
crates/storage/libmdbx-rs/ @rakita @shekhirin
crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @rakita @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @fgimenez
crates/tokio-util/ @mattsse
crates/tracing/ @mattsse @shekhirin
crates/tracing-otlp/ @mattsse @Rjected
crates/transaction-pool/ @mattsse @yongkangc
crates/trie/ @Rjected @shekhirin @mediocregopher
crates/trie/ @Rjected @shekhirin @mediocregopher @yongkangc
bin/reth/ @mattsse @shekhirin @Rjected
bin/reth-bench/ @mattsse @Rjected @shekhirin @yongkangc
bin/reth-bench-compare/ @mediocregopher @shekhirin @yongkangc
etc/ @Rjected @shekhirin
.github/ @gakonst @DaniPopes

View File

@@ -11,17 +11,14 @@ go build .
# Run each hive command in the background for each simulator and wait
echo "Building images"
./hive -client reth --sim "ethereum/eels" \
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v2.0.0/fixtures_bal.tar.gz \
--sim.buildarg branch=eips/amsterdam/eip-7928 \
--sim.timelimit 1s || true &
# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &
./hive -client reth --sim "smoke/genesis" -sim.timelimit 1s || true &
./hive -client reth --sim "smoke/network" -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/sync" -sim.timelimit 1s || true &
wait
# Run docker save in parallel, wait and exit on error
@@ -40,7 +37,7 @@ for pid in "${saving_pids[@]}"; do
wait "$pid" || exit
done
# Make sure we don't rebuild images on the CI jobs
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/assets/hive/no_sim_build.diff
go build .
mv ./hive ../hive_assets/

View File

@@ -24,4 +24,4 @@ done
wait
docker image ls -a
docker image ls -a

View File

@@ -1,14 +0,0 @@
#!/usr/bin/env bash
set -eo pipefail
# TODO: Benchmarks run WAY too slow due to excessive amount of iterations.
cmd=(cargo codspeed build --profile profiling)
crates=(
-p reth-primitives
-p reth-trie
-p reth-trie-common
-p reth-trie-sparse
)
"${cmd[@]}" --features test-utils "${crates[@]}"

View File

@@ -3,9 +3,9 @@
on:
pull_request:
# TODO: Disabled temporarily for https://github.com/CodSpeedHQ/runner/issues/55
# merge_group :
# merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always
@@ -17,6 +17,16 @@ name: bench
jobs:
codspeed:
runs-on: depot-ubuntu-latest
strategy:
matrix:
partition: [1, 2]
total_partitions: [2]
include:
- partition: 1
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
- partition: 2
crates: "-p reth-trie"
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
steps:
- uses: actions/checkout@v6
with:
@@ -32,10 +42,10 @@ jobs:
with:
tool: cargo-codspeed
- name: Build the benchmark target(s)
run: ./.github/scripts/codspeed-build.sh
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
- name: Run the benchmarks
uses: CodSpeedHQ/action@v4
with:
run: cargo codspeed run --workspace
run: cargo codspeed run ${{ matrix.crates }}
mode: instrumentation
token: ${{ secrets.CODSPEED_TOKEN }}

View File

@@ -74,4 +74,4 @@ jobs:
steps:
- name: Deploy to GitHub Pages
id: deployment
uses: actions/deploy-pages@v4
uses: actions/deploy-pages@v4

66
.github/workflows/check-alloy.yml vendored Normal file
View File

@@ -0,0 +1,66 @@
# Checks reth compilation against alloy branches to detect breaking changes.
# Run on-demand via workflow_dispatch.
name: Check Alloy Breaking Changes
on:
workflow_dispatch:
inputs:
alloy_branch:
description: 'Branch/rev for alloy-rs/alloy (leave empty to skip)'
required: false
type: string
alloy_evm_branch:
description: 'Branch/rev for alloy-rs/evm (alloy-evm, alloy-op-evm) (leave empty to skip)'
required: false
type: string
op_alloy_branch:
description: 'Branch/rev for alloy-rs/op-alloy (leave empty to skip)'
required: false
type: string
env:
CARGO_TERM_COLOR: always
jobs:
check:
name: Check compilation with patched alloy
runs-on: depot-ubuntu-latest-16
timeout-minutes: 60
steps:
- uses: actions/checkout@v4
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Apply alloy patches
run: |
ARGS=""
if [ -n "${{ inputs.alloy_branch }}" ]; then
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
fi
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
fi
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
fi
if [ -z "$ARGS" ]; then
echo "No branches specified, nothing to patch"
exit 1
fi
./scripts/patch-alloy.sh $ARGS
echo "=== Final patch section ==="
tail -50 Cargo.toml
- name: Check workspace
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features

View File

@@ -9,7 +9,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -15,6 +15,6 @@ permissions:
jobs:
update:
uses: ithacaxyz/ci/.github/workflows/cargo-update-pr.yml@main
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
secrets:
token: ${{ secrets.GITHUB_TOKEN }}

View File

@@ -6,7 +6,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -6,9 +6,7 @@ on:
workflow_dispatch:
schedule:
- cron: "0 */6 * * *"
pull_request:
branches:
- main
env:
CARGO_TERM_COLOR: always
@@ -44,7 +42,6 @@ jobs:
uses: actions/checkout@v6
with:
repository: ethereum/hive
ref: master
path: hivetests
- name: Get hive commit hash
@@ -130,29 +127,27 @@ jobs:
# eth_ rpc methods
- sim: ethereum/rpc-compat
include:
# - eth_blockNumber
- eth_blockNumber
- eth_call
# - eth_chainId
# - eth_createAccessList
# - eth_estimateGas
# - eth_feeHistory
# - eth_getBalance
# - eth_getBlockBy
# - eth_getBlockTransactionCountBy
# - eth_getCode
# - eth_getProof
# - eth_getStorage
# - eth_getTransactionBy
# - eth_getTransactionCount
# - eth_getTransactionReceipt
# - eth_sendRawTransaction
# - eth_syncing
# # debug_ rpc methods
# - debug_
- eth_chainId
- eth_createAccessList
- eth_estimateGas
- eth_feeHistory
- eth_getBalance
- eth_getBlockBy
- eth_getBlockTransactionCountBy
- eth_getCode
- eth_getProof
- eth_getStorage
- eth_getTransactionBy
- eth_getTransactionCount
- eth_getTransactionReceipt
- eth_sendRawTransaction
- eth_syncing
# debug_ rpc methods
- debug_
# consume-engine
- sim: ethereum/eels/consume-engine
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-engine
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-engine
@@ -169,10 +164,10 @@ jobs:
limit: .*tests/homestead.*
- sim: ethereum/eels/consume-engine
limit: .*tests/frontier.*
- sim: ethereum/eels/consume-engine
limit: .*tests/paris.*
# consume-rlp
- sim: ethereum/eels/consume-rlp
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-rlp
@@ -189,6 +184,8 @@ jobs:
limit: .*tests/homestead.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/frontier.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/paris.*
needs:
- prepare-reth-stable
- prepare-reth-edge

View File

@@ -6,7 +6,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
schedule:
# Run once a day at 3:00 UTC
- cron: "0 3 * * *"

View File

@@ -4,7 +4,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always
@@ -285,7 +285,7 @@ jobs:
- run: zepter run check
deny:
uses: ithacaxyz/ci/.github/workflows/deny.yml@main
uses: tempoxyz/ci/.github/workflows/deny.yml@main
lint-success:
name: lint success

View File

@@ -6,7 +6,7 @@ on:
pull_request:
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -4,10 +4,9 @@ name: unit
on:
pull_request:
branches: ["**"]
merge_group:
push:
branches: ["**"]
branches: [main]
env:
CARGO_TERM_COLOR: always

View File

@@ -4,9 +4,9 @@ name: windows
on:
push:
branches: ["**"]
branches: [main]
pull_request:
branches: ["**"]
branches: [main]
merge_group:
env:

View File

@@ -249,7 +249,7 @@ Write comments that remain valuable after the PR is merged. Future readers won't
unsafe impl GlobalAlloc for LimitedAllocator { ... }
// Binary search requires sorted input. Panics on unsorted slices.
fn find_index(items: &[Item], target: &Item) -> Option
fn find_index(items: &[Item], target: &Item) -> Option<usize>
// Timeout set to 5s to match EVM block processing limits
const TRACER_TIMEOUT: Duration = Duration::from_secs(5);

894
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.10.0"
version = "1.10.2"
edition = "2024"
rust-version = "1.88"
license = "MIT OR Apache-2.0"
@@ -376,11 +376,11 @@ reth-era-utils = { path = "crates/era-utils" }
reth-errors = { path = "crates/errors" }
reth-eth-wire = { path = "crates/net/eth-wire" }
reth-eth-wire-types = { path = "crates/net/eth-wire-types" }
reth-ethereum-payload-builder = { path = "crates/ethereum/payload" }
reth-ethereum-cli = { path = "crates/ethereum/cli", default-features = false }
reth-ethereum-consensus = { path = "crates/ethereum/consensus", default-features = false }
reth-ethereum-engine-primitives = { path = "crates/ethereum/engine-primitives", default-features = false }
reth-ethereum-forks = { path = "crates/ethereum/hardforks", default-features = false }
reth-ethereum-payload-builder = { path = "crates/ethereum/payload" }
reth-ethereum-primitives = { path = "crates/ethereum/primitives", default-features = false }
reth-ethereum = { path = "crates/ethereum/reth" }
reth-etl = { path = "crates/etl" }
@@ -472,7 +472,7 @@ reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-feat
reth-ress-protocol = { path = "crates/ress/protocol" }
reth-ress-provider = { path = "crates/ress/provider" }
# revm v103 (revm v34.0.0 with BAL EIP-7928 support)
# revm
revm = { version = "34.0.0", default-features = false }
revm-bytecode = { version = "8.0.0", default-features = false }
revm-database = { version = "10.0.0", default-features = false }
@@ -480,21 +480,16 @@ revm-state = { version = "9.0.0", default-features = false }
revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
revm-context = { version = "13.0.0", default-features = false }
revm-context-interface = { version = "14.0.0", default-features = false }
revm-inspector = { version = "15.0.0", default-features = false }
revm-handler = { version = "15.0.0", default-features = false }
revm-precompile = { version = "32.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.33.2"
revm-inspectors = "0.34.0"
# eth
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-evm = { version = "0.25.2", default-features = false }
alloy-dyn-abi = "1.4.1"
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-dyn-abi = "1.4.3"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
alloy-sol-types = { version = "1.5.0", default-features = false }
@@ -531,7 +526,7 @@ alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
# op
alloy-op-evm = { version = "0.25.2", default-features = false }
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -744,54 +739,50 @@ tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
triehash = "0.8"
typenum = "1.15.0"
vergen = "9.0.4"
vergen = "9.1.0"
visibility = "0.1.1"
walkdir = "2.3.3"
vergen-git2 = "1.0.5"
vergen-git2 = "9.1.0"
# networking
ipnet = "2.11"
[patch.crates-io]
alloy-consensus = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-contract = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-eips = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-genesis = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-json-rpc = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-network = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-network-primitives = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-provider = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-pubsub = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-client = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-admin = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-anvil = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-beacon = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-debug = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-engine = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-eth = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-mev = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-trace = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-rpc-types-txpool = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-serde = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-signer = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-signer-local = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport-http = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport-ipc = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
alloy-transport-ws = { git = "https://github.com/Soubhik-10/alloy", branch = "bal-devnet-1" }
# alloy-hardforks = { git = "https://github.com/Rimeeeeee/hardforks", branch = "amsterdam" }
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-serde = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-op-hardforks = { git = "https://github.com/Rimeeeeee/hardforks", branch = "amsterdam" }
# op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-jsonrpsee = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
#
# alloy-evm with BAL support for revm v34/v103
alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "staging-revm" }
alloy-op-evm = { git = "https://github.com/alloy-rs/evm", branch = "staging-revm" }
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", branch = "staging-revm" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "1207e33" }
#
# jsonrpsee = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-core = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
@@ -801,3 +792,37 @@ revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", bran
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

View File

@@ -163,6 +163,7 @@ impl NodeManager {
"eth,reth".to_string(),
"--disable-discovery".to_string(),
"--trusted-only".to_string(),
"--disable-tx-gossip".to_string(),
]);
// Add tracing arguments if OTLP endpoint is configured

View File

@@ -22,12 +22,42 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate.
#[arg(long, value_name = "BLOCKS")]
blocks: u64,
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
@@ -42,12 +72,37 @@ pub struct Command {
output: PathBuf,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
if self.blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
// Ensure output directory exists
if self.output.is_file() {
@@ -84,14 +139,31 @@ impl Command {
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
let end_block = start_block + self.blocks - 1;
info!(canonical_parent, start_block, end_block, "Starting gas limit ramp benchmark");
match mode {
RampMode::Blocks(blocks) => {
info!(
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
let mut next_block_number = start_block;
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
while next_block_number <= end_block {
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
@@ -140,13 +212,13 @@ impl Command {
parent_header = block.header;
parent_hash = block_hash;
next_block_number += 1;
blocks_processed += 1;
}
let final_gas_limit = parent_header.gas_limit;
info!(
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed = self.blocks,
blocks_processed,
final_gas_limit,
"Benchmark complete"
);
@@ -158,3 +230,57 @@ impl Command {
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -541,7 +541,6 @@ impl Command {
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
},
transactions: transactions.to_vec(),
extra_data: None,

View File

@@ -69,7 +69,6 @@ pub(crate) fn prepare_payload_request(
suggested_fee_recipient: Address::ZERO,
withdrawals: shanghai_active.then(Vec::new),
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
slot_number: None,
},
forkchoice_state: ForkchoiceState {
head_block_hash: parent_hash,

View File

@@ -180,7 +180,7 @@ impl Command {
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_")
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
@@ -191,7 +191,7 @@ impl Command {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract index from "payload_NNN.json"
let index_str = name_str.strip_prefix("payload_")?.strip_suffix(".json")?;
let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let index: u64 = index_str.parse().ok()?;
Some((index, e.path()))
})

View File

@@ -181,45 +181,6 @@ pub(crate) fn payload_to_new_payload(
target_version: Option<EngineApiMessageVersion>,
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
let (version, params) = match payload {
ExecutionPayload::V4(payload) => {
let cancun = sidecar.cancun().unwrap();
if let Some(prague) = sidecar.prague() {
if is_optimism {
(
EngineApiMessageVersion::V4,
serde_json::to_value((
OpExecutionPayloadV4 {
payload_inner: payload.payload_inner,
withdrawals_root: withdrawals_root.unwrap(),
},
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
Requests::default(),
))?,
)
} else {
(
EngineApiMessageVersion::V4,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
prague.requests.requests_hash(),
))?,
)
}
} else {
(
EngineApiMessageVersion::V3,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
))?,
)
}
}
ExecutionPayload::V3(payload) => {
let cancun = sidecar.cancun().unwrap();
@@ -242,11 +203,7 @@ pub(crate) fn payload_to_new_payload(
)
} else {
// Extract actual Requests from RequestsOrHash
let requests = prague
.requests
.requests()
.cloned()
.ok_or_else(|| eyre::eyre!("Prague sidecar has hash, not requests"))?;
let requests = prague.requests.requests_hash();
(
version,
serde_json::to_value((
@@ -328,10 +285,7 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 |
EngineApiMessageVersion::V6 => {
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V2 => {

View File

@@ -41,6 +41,7 @@ derive_more.workspace = true
metrics.workspace = true
parking_lot.workspace = true
pin-project.workspace = true
rayon = { workspace = true, optional = true }
serde = { workspace = true, optional = true }
# optional deps for test-utils
@@ -84,6 +85,7 @@ test-utils = [
"reth-trie/test-utils",
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"

View File

@@ -163,14 +163,29 @@ impl DeferredTrieData {
anchor_hash: B256,
ancestors: &[Self],
) -> ComputedTrieData {
let sorted_hashed_state = match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
let sorted_trie_updates = match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
};
#[cfg(feature = "rayon")]
let (sorted_hashed_state, sorted_trie_updates) = rayon::join(
|| match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
|| match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
#[cfg(not(feature = "rayon"))]
let (sorted_hashed_state, sorted_trie_updates) = (
match Arc::try_unwrap(hashed_state) {
Ok(state) => state.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
match Arc::try_unwrap(trie_updates) {
Ok(updates) => updates.into_sorted(),
Err(arc) => arc.clone_into_sorted(),
},
);
// Reuse parent's overlay if available and anchors match.
// We can only reuse the parent's overlay if it was built on top of the same
@@ -192,10 +207,10 @@ impl DeferredTrieData {
);
// Only trigger COW clone if there's actually data to add.
if !sorted_hashed_state.is_empty() {
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(&sorted_hashed_state);
}
if !sorted_trie_updates.is_empty() {
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
Arc::make_mut(&mut overlay.nodes).extend_ref_and_sort(&sorted_trie_updates);
}
overlay
}
@@ -228,8 +243,53 @@ impl DeferredTrieData {
/// In normal operation, the parent always has a cached overlay and this
/// function is never called.
///
/// Iterates ancestors oldest -> newest, then extends with current block's data,
/// so later state takes precedence.
/// When the `rayon` feature is enabled, uses parallel collection and merge:
/// 1. Collects ancestor data in parallel (each `wait_cloned()` may compute)
/// 2. Merges hashed state and trie updates in parallel with each other
/// 3. Uses tree reduction within each merge for O(log n) depth
#[cfg(feature = "rayon")]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
sorted_trie_updates: &TrieUpdatesSorted,
) -> TrieInputSorted {
// Early exit: no ancestors means just wrap current block's data
if ancestors.is_empty() {
return TrieInputSorted::new(
Arc::new(sorted_trie_updates.clone()),
Arc::new(sorted_hashed_state.clone()),
Default::default(),
);
}
// Collect ancestor data, unzipping states and updates into Arc slices
let (states, updates): (Vec<_>, Vec<_>) = ancestors
.iter()
.map(|a| {
let data = a.wait_cloned();
(data.hashed_state, data.trie_updates)
})
.unzip();
// Merge state and nodes in parallel with each other using tree reduction
let (state, nodes) = rayon::join(
|| {
let mut merged = HashedPostStateSorted::merge_parallel(&states);
merged.extend_ref_and_sort(sorted_hashed_state);
merged
},
|| {
let mut merged = TrieUpdatesSorted::merge_parallel(&updates);
merged.extend_ref_and_sort(sorted_trie_updates);
merged
},
);
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
}
/// Merge all ancestors and current block's data into a single overlay (sequential fallback).
#[cfg(not(feature = "rayon"))]
fn merge_ancestors_into_overlay(
ancestors: &[Self],
sorted_hashed_state: &HashedPostStateSorted,
@@ -242,13 +302,13 @@ impl DeferredTrieData {
for ancestor in ancestors {
let ancestor_data = ancestor.wait_cloned();
state_mut.extend_ref(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
state_mut.extend_ref_and_sort(ancestor_data.hashed_state.as_ref());
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
}
// Extend with current block's sorted data last (takes precedence)
state_mut.extend_ref(sorted_hashed_state);
nodes_mut.extend_ref(sorted_trie_updates);
state_mut.extend_ref_and_sort(sorted_hashed_state);
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
overlay
}
@@ -287,6 +347,11 @@ impl DeferredTrieData {
&inputs.ancestors,
);
*state = DeferredState::Ready(computed.clone());
// Release lock before inputs (and its ancestors) drop to avoid holding it
// while their potential last Arc refs drop (which could trigger recursive locking)
drop(state);
computed
}
}
@@ -516,7 +581,7 @@ mod tests {
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
let trie_updates = Arc::default();
let mut overlay = TrieInputSorted::default();
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
Arc::make_mut(&mut overlay.state).extend_ref_and_sort(hashed_state.as_ref());
DeferredTrieData::ready(ComputedTrieData {
hashed_state,

View File

@@ -10,15 +10,15 @@ use alloy_primitives::{map::HashMap, BlockNumber, TxHash, B256};
use parking_lot::RwLock;
use reth_chainspec::ChainInfo;
use reth_ethereum_primitives::EthPrimitives;
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_metrics::{metrics::Gauge, Metrics};
use reth_primitives_traits::{
BlockBody as _, IndexedTx, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
SignedTransaction,
};
use reth_storage_api::StateProviderBox;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::{collections::BTreeMap, ops::Deref, sync::Arc, time::Instant};
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, LazyTrieData, TrieInputSorted};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
use tokio::sync::{broadcast, watch};
/// Size of the broadcast channel used to notify canonical state events.
@@ -648,7 +648,7 @@ impl<N: NodePrimitives> BlockState<N> {
}
/// Returns the `Receipts` of executed block that determines the state.
pub fn receipts(&self) -> &Vec<Vec<N::Receipt>> {
pub fn receipts(&self) -> &Vec<N::Receipt> {
&self.block.execution_outcome().receipts
}
@@ -659,15 +659,7 @@ impl<N: NodePrimitives> BlockState<N> {
///
/// This clones the vector of receipts. To avoid it, use [`Self::executed_block_receipts_ref`].
pub fn executed_block_receipts(&self) -> Vec<N::Receipt> {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().cloned().unwrap_or_default()
self.receipts().clone()
}
/// Returns a slice of `Receipt` of executed block that determines the state.
@@ -675,15 +667,7 @@ impl<N: NodePrimitives> BlockState<N> {
/// has only one element corresponding to the executed block associated to
/// the state.
pub fn executed_block_receipts_ref(&self) -> &[N::Receipt] {
let receipts = self.receipts();
debug_assert!(
receipts.len() <= 1,
"Expected at most one block's worth of receipts, found {}",
receipts.len()
);
receipts.first().map(|receipts| receipts.deref()).unwrap_or_default()
self.receipts()
}
/// Returns an iterator over __parent__ `BlockStates`.
@@ -767,7 +751,7 @@ pub struct ExecutedBlock<N: NodePrimitives = EthPrimitives> {
/// Recovered Block
pub recovered_block: Arc<RecoveredBlock<N::Block>>,
/// Block's execution outcome.
pub execution_output: Arc<ExecutionOutcome<N::Receipt>>,
pub execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
/// Deferred trie data produced by execution.
///
/// This allows deferring the computation of the trie data which can be expensive.
@@ -779,7 +763,15 @@ impl<N: NodePrimitives> Default for ExecutedBlock<N> {
fn default() -> Self {
Self {
recovered_block: Default::default(),
execution_output: Default::default(),
execution_output: Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: Default::default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: Default::default(),
}),
trie_data: DeferredTrieData::ready(ComputedTrieData::default()),
}
}
@@ -800,7 +792,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// payload builders). This is the safe default path.
pub fn new(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: ComputedTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data: DeferredTrieData::ready(trie_data) }
@@ -822,7 +814,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Use [`Self::new()`] instead when trie data is already computed and available immediately.
pub const fn with_deferred_trie_data(
recovered_block: Arc<RecoveredBlock<N::Block>>,
execution_output: Arc<ExecutionOutcome<N::Receipt>>,
execution_output: Arc<BlockExecutionOutput<N::Receipt>>,
trie_data: DeferredTrieData,
) -> Self {
Self { recovered_block, execution_output, trie_data }
@@ -842,7 +834,7 @@ impl<N: NodePrimitives> ExecutedBlock<N> {
/// Returns a reference to the block's execution outcome
#[inline]
pub fn execution_outcome(&self) -> &ExecutionOutcome<N::Receipt> {
pub fn execution_outcome(&self) -> &BlockExecutionOutput<N::Receipt> {
&self.execution_output
}
@@ -942,37 +934,39 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
pub fn to_chain_notification(&self) -> CanonStateNotification<N> {
match self {
Self::Commit { new } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
);
chain
}));
CanonStateNotification::Commit { new }
CanonStateNotification::Commit { new: Arc::new(Self::blocks_to_chain(new)) }
}
Self::Reorg { new, old } => {
let new = Arc::new(new.iter().fold(Chain::default(), |mut chain, exec| {
Self::Reorg { new, old } => CanonStateNotification::Reorg {
new: Arc::new(Self::blocks_to_chain(new)),
old: Arc::new(Self::blocks_to_chain(old)),
},
}
}
/// Converts a slice of executed blocks into a [`Chain`].
fn blocks_to_chain(blocks: &[ExecutedBlock<N>]) -> Chain<N> {
match blocks {
[] => Chain::default(),
[first, rest @ ..] => {
let mut chain = Chain::from_block(
first.recovered_block().clone(),
ExecutionOutcome::from((
first.execution_outcome().clone(),
first.block_number(),
)),
LazyTrieData::ready(first.hashed_state(), first.trie_updates()),
);
for exec in rest {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
ExecutionOutcome::from((
exec.execution_outcome().clone(),
exec.block_number(),
)),
LazyTrieData::ready(exec.hashed_state(), exec.trie_updates()),
);
chain
}));
let old = Arc::new(old.iter().fold(Chain::default(), |mut chain, exec| {
chain.append_block(
exec.recovered_block().clone(),
exec.execution_outcome().clone(),
exec.trie_updates(),
exec.hashed_state(),
);
chain
}));
CanonStateNotification::Reorg { new, old }
}
chain
}
}
}
@@ -1266,7 +1260,7 @@ mod tests {
let state = BlockState::new(block);
assert_eq!(state.receipts(), &receipts);
assert_eq!(state.receipts(), receipts.first().unwrap());
}
#[test]
@@ -1543,33 +1537,31 @@ mod tests {
let block2a =
test_block_builder.get_executed_block_with_number(2, block1.recovered_block.hash());
let sample_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
..Default::default()
};
// Test commit notification
let chain_commit = NewCanonicalChain::Commit { new: vec![block0.clone(), block1.clone()] };
// Build expected trie updates map
let mut expected_trie_updates = BTreeMap::new();
expected_trie_updates.insert(0, block0.trie_updates());
expected_trie_updates.insert(1, block1.trie_updates());
// Build expected trie data map
let mut expected_trie_data = BTreeMap::new();
expected_trie_data
.insert(0, LazyTrieData::ready(block0.hashed_state(), block0.trie_updates()));
expected_trie_data
.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
// Build expected hashed state map
let mut expected_hashed_state = BTreeMap::new();
expected_hashed_state.insert(0, block0.hashed_state());
expected_hashed_state.insert(1, block1.hashed_state());
// Build expected execution outcome (first_block matches first block number)
let commit_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 0,
..Default::default()
};
assert_eq!(
chain_commit.to_chain_notification(),
CanonStateNotification::Commit {
new: Arc::new(Chain::new(
vec![block0.recovered_block().clone(), block1.recovered_block().clone()],
sample_execution_outcome.clone(),
expected_trie_updates,
expected_hashed_state
commit_execution_outcome,
expected_trie_data,
))
}
);
@@ -1580,40 +1572,39 @@ mod tests {
old: vec![block1.clone(), block2.clone()],
};
// Build expected trie updates for old chain
let mut old_trie_updates = BTreeMap::new();
old_trie_updates.insert(1, block1.trie_updates());
old_trie_updates.insert(2, block2.trie_updates());
// Build expected trie data for old chain
let mut old_trie_data = BTreeMap::new();
old_trie_data.insert(1, LazyTrieData::ready(block1.hashed_state(), block1.trie_updates()));
old_trie_data.insert(2, LazyTrieData::ready(block2.hashed_state(), block2.trie_updates()));
// Build expected trie updates for new chain
let mut new_trie_updates = BTreeMap::new();
new_trie_updates.insert(1, block1a.trie_updates());
new_trie_updates.insert(2, block2a.trie_updates());
// Build expected trie data for new chain
let mut new_trie_data = BTreeMap::new();
new_trie_data
.insert(1, LazyTrieData::ready(block1a.hashed_state(), block1a.trie_updates()));
new_trie_data
.insert(2, LazyTrieData::ready(block2a.hashed_state(), block2a.trie_updates()));
// Build expected hashed state for old chain
let mut old_hashed_state = BTreeMap::new();
old_hashed_state.insert(1, block1.hashed_state());
old_hashed_state.insert(2, block2.hashed_state());
// Build expected hashed state for new chain
let mut new_hashed_state = BTreeMap::new();
new_hashed_state.insert(1, block1a.hashed_state());
new_hashed_state.insert(2, block2a.hashed_state());
// Build expected execution outcome for reorg chains (first_block matches first block
// number)
let reorg_execution_outcome = ExecutionOutcome {
receipts: vec![vec![], vec![]],
requests: vec![Requests::default(), Requests::default()],
first_block: 1,
..Default::default()
};
assert_eq!(
chain_reorg.to_chain_notification(),
CanonStateNotification::Reorg {
old: Arc::new(Chain::new(
vec![block1.recovered_block().clone(), block2.recovered_block().clone()],
sample_execution_outcome.clone(),
old_trie_updates,
old_hashed_state
reorg_execution_outcome.clone(),
old_trie_data,
)),
new: Arc::new(Chain::new(
vec![block1a.recovered_block().clone(), block2a.recovered_block().clone()],
sample_execution_outcome,
new_trie_updates,
new_hashed_state
reorg_execution_outcome,
new_trie_data,
))
}
);

View File

@@ -0,0 +1,186 @@
//! Lazy overlay computation for trie input.
//!
//! This module provides [`LazyOverlay`], a type that computes the [`TrieInputSorted`]
//! lazily on first access. This allows execution to start before the trie overlay
//! is fully computed.
use crate::DeferredTrieData;
use alloy_primitives::B256;
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
use std::sync::{Arc, OnceLock};
use tracing::{debug, trace};
/// Inputs captured for lazy overlay computation.
#[derive(Clone)]
struct LazyOverlayInputs {
/// The persisted ancestor hash (anchor) this overlay should be built on.
anchor_hash: B256,
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
blocks: Vec<DeferredTrieData>,
}
/// Lazily computed trie overlay.
///
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
/// but for overlay computation.
///
/// # Fast Path vs Slow Path
///
/// - **Fast path**: If the tip block's cached `anchored_trie_input` is ready and its `anchor_hash`
/// matches our expected anchor, we can reuse it directly (O(1)).
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
#[derive(Clone)]
pub struct LazyOverlay {
/// Computed result, cached after first access.
inner: Arc<OnceLock<TrieInputSorted>>,
/// Inputs for lazy computation.
inputs: LazyOverlayInputs,
}
impl std::fmt::Debug for LazyOverlay {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_struct("LazyOverlay")
.field("anchor_hash", &self.inputs.anchor_hash)
.field("num_blocks", &self.inputs.blocks.len())
.field("computed", &self.inner.get().is_some())
.finish()
}
}
impl LazyOverlay {
/// Create a new lazy overlay with the given anchor hash and block handles.
///
/// # Arguments
///
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
}
/// Returns the anchor hash this overlay is built on.
pub const fn anchor_hash(&self) -> B256 {
self.inputs.anchor_hash
}
/// Returns the number of in-memory blocks this overlay covers.
pub const fn num_blocks(&self) -> usize {
self.inputs.blocks.len()
}
/// Returns true if the overlay has already been computed.
pub fn is_computed(&self) -> bool {
self.inner.get().is_some()
}
/// Returns the computed trie input, computing it if necessary.
///
/// The first call triggers computation (which may block waiting for deferred data).
/// Subsequent calls return the cached result immediately.
pub fn get(&self) -> &TrieInputSorted {
self.inner.get_or_init(|| self.compute())
}
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
let input = self.get();
(Arc::clone(&input.nodes), Arc::clone(&input.state))
}
/// Compute the trie input overlay.
fn compute(&self) -> TrieInputSorted {
let anchor_hash = self.inputs.anchor_hash;
let blocks = &self.inputs.blocks;
if blocks.is_empty() {
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
return TrieInputSorted::default();
}
// Fast path: Check if tip block's overlay is ready and anchor matches.
// The tip block (first in list) has the cumulative overlay from all ancestors.
if let Some(tip) = blocks.first() {
let data = tip.wait_cloned();
if let Some(anchored) = &data.anchored_trie_input {
if anchored.anchor_hash == anchor_hash {
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
return (*anchored.trie_input).clone();
}
debug!(
target: "chain_state::lazy_overlay",
computed_anchor = %anchored.anchor_hash,
%anchor_hash,
"Anchor mismatch, falling back to merge"
);
}
}
// Slow path: Merge all blocks' trie data into a new overlay.
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
Self::merge_blocks(blocks)
}
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
///
/// Blocks are ordered newest to oldest.
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
if blocks.is_empty() {
return TrieInputSorted::default();
}
let state =
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
let nodes =
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
}
}
#[cfg(test)]
mod tests {
use super::*;
use reth_trie::{updates::TrieUpdates, HashedPostState};
fn empty_deferred(anchor: B256) -> DeferredTrieData {
DeferredTrieData::pending(
Arc::new(HashedPostState::default()),
Arc::new(TrieUpdates::default()),
anchor,
Vec::new(),
)
}
#[test]
fn empty_blocks_returns_default() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
let result = overlay.get();
assert!(result.state.is_empty());
assert!(result.nodes.is_empty());
}
#[test]
fn single_block_uses_data_directly() {
let anchor = B256::random();
let deferred = empty_deferred(anchor);
let overlay = LazyOverlay::new(anchor, vec![deferred]);
assert!(!overlay.is_computed());
let _ = overlay.get();
assert!(overlay.is_computed());
}
#[test]
fn cached_after_first_access() {
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
// First access computes
let _ = overlay.get();
assert!(overlay.is_computed());
// Second access uses cache
let _ = overlay.get();
assert!(overlay.is_computed());
}
}

View File

@@ -14,6 +14,9 @@ pub use in_memory::*;
mod deferred_trie;
pub use deferred_trie::*;
mod lazy_overlay;
pub use lazy_overlay::*;
mod noop;
mod chain_info;

View File

@@ -280,7 +280,6 @@ mod tests {
vec![block1.clone(), block2.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
// Create a commit notification
@@ -319,13 +318,11 @@ mod tests {
vec![block1.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
let new_chain = Arc::new(Chain::new(
vec![block2.clone(), block3.clone()],
ExecutionOutcome::default(),
BTreeMap::new(),
BTreeMap::new(),
));
// Create a reorg notification
@@ -391,7 +388,6 @@ mod tests {
vec![block1.clone(), block2.clone()],
execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
// Create a commit notification containing the new chain segment.
@@ -449,12 +445,8 @@ mod tests {
ExecutionOutcome { receipts: old_receipts, ..Default::default() };
// Create an old chain segment to be reverted, containing `old_block1`.
let old_chain: Arc<Chain> = Arc::new(Chain::new(
vec![old_block1.clone()],
old_execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
let old_chain: Arc<Chain> =
Arc::new(Chain::new(vec![old_block1.clone()], old_execution_outcome, BTreeMap::new()));
// Define block2 for the new chain segment, which will be committed.
let mut body = BlockBody::<TransactionSigned>::default();
@@ -482,12 +474,8 @@ mod tests {
ExecutionOutcome { receipts: new_receipts, ..Default::default() };
// Create a new chain segment to be committed, containing `new_block1`.
let new_chain = Arc::new(Chain::new(
vec![new_block1.clone()],
new_execution_outcome,
BTreeMap::new(),
BTreeMap::new(),
));
let new_chain =
Arc::new(Chain::new(vec![new_block1.clone()], new_execution_outcome, BTreeMap::new()));
// Create a reorg notification with both reverted (old) and committed (new) chain segments.
let notification = CanonStateNotification::Reorg { old: old_chain, new: new_chain };

View File

@@ -3,10 +3,7 @@ use crate::{
CanonStateSubscriptions, ComputedTrieData,
};
use alloy_consensus::{Header, SignableTransaction, TxEip1559, TxReceipt, EMPTY_ROOT_HASH};
use alloy_eips::{
eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE},
eip7685::Requests,
};
use alloy_eips::eip1559::{ETHEREUM_BLOCK_GAS_LIMIT_30M, INITIAL_BASE_FEE};
use alloy_primitives::{Address, BlockNumber, B256, U256};
use alloy_signer::SignerSync;
use alloy_signer_local::PrivateKeySigner;
@@ -16,7 +13,7 @@ use reth_chainspec::{ChainSpec, EthereumHardfork, MIN_TRANSACTION_GAS};
use reth_ethereum_primitives::{
Block, BlockBody, EthPrimitives, Receipt, Transaction, TransactionSigned,
};
use reth_execution_types::{Chain, ExecutionOutcome};
use reth_execution_types::{BlockExecutionOutput, BlockExecutionResult, Chain, ExecutionOutcome};
use reth_primitives_traits::{
proofs::{calculate_receipt_root, calculate_transaction_root, calculate_withdrawals_root},
Account, NodePrimitives, Recovered, RecoveredBlock, SealedBlock, SealedHeader,
@@ -173,7 +170,6 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
transactions: transactions.into_iter().map(|tx| tx.into_inner()).collect(),
ommers: Vec::new(),
withdrawals: Some(vec![].into()),
block_access_list: None,
},
)
}
@@ -202,7 +198,7 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
fn get_executed_block(
&mut self,
block_number: BlockNumber,
receipts: Vec<Vec<Receipt>>,
mut receipts: Vec<Vec<Receipt>>,
parent_hash: B256,
) -> ExecutedBlock {
let block = self.generate_random_block(block_number, parent_hash);
@@ -210,12 +206,15 @@ impl<N: NodePrimitives> TestBlockBuilder<N> {
let trie_data = ComputedTrieData::default();
ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(ExecutionOutcome::new(
BundleState::default(),
receipts,
block_number,
vec![Requests::default()],
)),
Arc::new(BlockExecutionOutput {
result: BlockExecutionResult {
receipts: receipts.pop().unwrap_or_default(),
requests: Default::default(),
gas_used: 0,
blob_gas_used: 0,
},
state: BundleState::default(),
}),
trie_data,
)
}

View File

@@ -45,10 +45,6 @@ use reth_network_peers::{
};
use reth_primitives_traits::{sync::LazyLock, BlockHeader, SealedHeader};
/// The hash of an empty block access list.
const EMPTY_BLOCK_ACCESS_LIST_HASH: B256 =
b256!("0x1dcc4de8dec75d7aab85b567b6ccd41ad312451b948a7413f0a142fd40d49347");
/// Helper method building a [`Header`] given [`Genesis`] and [`ChainHardforks`].
pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Header {
// If London is activated at genesis, we set the initial base fee as per EIP-1559.
@@ -83,12 +79,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
.active_at_timestamp(genesis.timestamp)
.then_some(EMPTY_REQUESTS_HASH);
// If Amsterdam is activated at genesis we set block access list hash empty hash.
let block_access_list_hash = hardforks
.fork(EthereumHardfork::Amsterdam)
.active_at_timestamp(genesis.timestamp)
.then_some(EMPTY_BLOCK_ACCESS_LIST_HASH);
Header {
number: genesis.number.unwrap_or_default(),
parent_hash: genesis.parent_hash.unwrap_or_default(),
@@ -106,7 +96,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
blob_gas_used,
excess_blob_gas,
requests_hash,
block_access_list_hash,
..Default::default()
}
}
@@ -289,6 +278,7 @@ pub fn create_chain_config(
// Check if DAO fork is supported (it has an activation block)
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
#[allow(clippy::needless_update)]
ChainConfig {
chain_id: chain.map(|c| c.id()).unwrap_or(0),
homestead_block: block_num(EthereumHardfork::Homestead),
@@ -311,7 +301,6 @@ pub fn create_chain_config(
cancun_time: timestamp(EthereumHardfork::Cancun),
prague_time: timestamp(EthereumHardfork::Prague),
osaka_time: timestamp(EthereumHardfork::Osaka),
amsterdam_time: timestamp(EthereumHardfork::Amsterdam),
bpo1_time: timestamp(EthereumHardfork::Bpo1),
bpo2_time: timestamp(EthereumHardfork::Bpo2),
bpo3_time: timestamp(EthereumHardfork::Bpo3),
@@ -325,6 +314,7 @@ pub fn create_chain_config(
extra_fields: Default::default(),
deposit_contract_address,
blob_schedule,
..Default::default()
}
}
@@ -913,7 +903,6 @@ impl From<Genesis> for ChainSpec {
(EthereumHardfork::Bpo3.boxed(), genesis.config.bpo3_time),
(EthereumHardfork::Bpo4.boxed(), genesis.config.bpo4_time),
(EthereumHardfork::Bpo5.boxed(), genesis.config.bpo5_time),
(EthereumHardfork::Amsterdam.boxed(), genesis.config.amsterdam_time),
];
let mut time_hardforks = time_hardfork_opts
@@ -1220,19 +1209,6 @@ impl ChainSpecBuilder {
self
}
/// Enable Amsterdam at genesis.
pub fn amsterdam_activated(mut self) -> Self {
self = self.osaka_activated();
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(0));
self
}
/// Enable Amsterdam at the given timestamp.
pub fn with_amsterdam_at(mut self, timestamp: u64) -> Self {
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(timestamp));
self
}
/// Build the resulting [`ChainSpec`].
///
/// # Panics

View File

@@ -131,4 +131,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]

View File

@@ -19,7 +19,7 @@ use reth_node_builder::{
Node, NodeComponents, NodeComponentsBuilder, NodeTypes, NodeTypesWithDBAdapter,
};
use reth_node_core::{
args::{DatabaseArgs, DatadirArgs, StaticFilesArgs},
args::{DatabaseArgs, DatadirArgs, RocksDbArgs, StaticFilesArgs},
dirs::{ChainPath, DataDirPath},
};
use reth_provider::{
@@ -27,7 +27,7 @@ use reth_provider::{
BlockchainProvider, NodeTypesForProvider, RocksDBProvider, StaticFileProvider,
StaticFileProviderBuilder,
},
ProviderFactory, StaticFileProviderFactory,
ProviderFactory, StaticFileProviderFactory, StorageSettings,
};
use reth_stages::{sets::DefaultStages, Pipeline, PipelineTarget};
use reth_static_file::StaticFileProducer;
@@ -66,9 +66,24 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
/// All static files related arguments
#[command(flatten)]
pub static_files: StaticFilesArgs,
/// All `RocksDB` related arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
}
impl<C: ChainSpecParser> EnvironmentArgs<C> {
/// Returns the effective storage settings derived from static-file and `RocksDB` CLI args.
pub fn storage_settings(&self) -> StorageSettings {
StorageSettings::base()
.with_receipts_in_static_files(self.static_files.receipts)
.with_transaction_senders_in_static_files(self.static_files.transaction_senders)
.with_account_changesets_in_static_files(self.static_files.account_changesets)
.with_transaction_hash_numbers_in_rocksdb(self.rocksdb.all || self.rocksdb.tx_hash)
.with_storages_history_in_rocksdb(self.rocksdb.all || self.rocksdb.storages_history)
.with_account_history_in_rocksdb(self.rocksdb.all || self.rocksdb.account_history)
}
/// Initializes environment according to [`AccessRights`] and returns an instance of
/// [`Environment`].
pub fn init<N: CliNodeTypes>(&self, access: AccessRights) -> eyre::Result<Environment<N>>
@@ -121,17 +136,17 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
})
}
};
// TransactionDB only support read-write mode
let rocksdb_provider = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?;
let provider_factory =
self.create_provider_factory(&config, db, sfp, rocksdb_provider, access)?;
if access.is_read_write() {
debug!(target: "reth::cli", chain=%self.chain.chain(), genesis=?self.chain.genesis_hash(), "Initializing genesis");
init_genesis_with_settings(&provider_factory, self.static_files.to_settings())?;
init_genesis_with_settings(&provider_factory, self.storage_settings())?;
}
Ok(Environment { config, provider_factory, data_dir })

View File

@@ -1,145 +0,0 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use reth_chainspec::EthereumHardforks;
use reth_db::DatabaseEnv;
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider};
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
self.table.view(&ChecksumViewer {
tool,
start_key: self.start_key,
end_key: self.end_key,
limit: self.limit,
})?;
Ok(())
}
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(100_000) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,288 @@
use crate::{
common::CliNodeTypes,
db::get::{maybe_json_value_parser, table_key},
};
use alloy_primitives::map::foldhash::fast::FixedState;
use clap::Parser;
use itertools::Itertools;
use reth_chainspec::EthereumHardforks;
use reth_db::{static_file::iter_static_files, DatabaseEnv};
use reth_db_api::{
cursor::DbCursorRO, table::Table, transaction::DbTx, RawKey, RawTable, RawValue, TableViewer,
Tables,
};
use reth_db_common::DbTool;
use reth_node_builder::{NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
#[cfg(all(unix, feature = "edge"))]
mod rocksdb;
/// Interval for logging progress during checksum computation.
const PROGRESS_LOG_INTERVAL: usize = 100_000;
#[derive(Parser, Debug)]
/// The arguments for the `reth db checksum` command
pub struct Command {
#[command(subcommand)]
subcommand: Subcommand,
}
#[derive(clap::Subcommand, Debug)]
enum Subcommand {
/// Calculates the checksum of a database table
Mdbx {
/// The table name
table: Tables,
/// The start of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
start_key: Option<String>,
/// The end of the range to checksum.
#[arg(long, value_parser = maybe_json_value_parser)]
end_key: Option<String>,
/// The maximum number of records that are queried and used to compute the
/// checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a static file segment
StaticFile {
/// The static file segment
#[arg(value_enum)]
segment: StaticFileSegment,
/// The block number to start from (inclusive).
#[arg(long)]
start_block: Option<u64>,
/// The block number to end at (inclusive).
#[arg(long)]
end_block: Option<u64>,
/// The maximum number of rows to checksum.
#[arg(long)]
limit: Option<usize>,
},
/// Calculates the checksum of a RocksDB table
#[cfg(all(unix, feature = "edge"))]
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: rocksdb::RocksDbTable,
/// The maximum number of records to checksum.
#[arg(long)]
limit: Option<usize>,
},
}
impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
match self.subcommand {
Subcommand::Mdbx { table, start_key, end_key, limit } => {
table.view(&ChecksumViewer { tool, start_key, end_key, limit })?;
}
Subcommand::StaticFile { segment, start_block, end_block, limit } => {
checksum_static_file(tool, segment, start_block, end_block, limit)?;
}
#[cfg(all(unix, feature = "edge"))]
Subcommand::Rocksdb { table, limit } => {
rocksdb::checksum_rocksdb(tool, table, limit)?;
}
}
Ok(())
}
}
/// Creates a new hasher with the standard seed used for checksum computation.
fn checksum_hasher() -> impl Hasher {
FixedState::with_seed(u64::from_be_bytes(*b"RETHRETH")).build_hasher()
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,
limit: Option<usize>,
) -> eyre::Result<()> {
let static_file_provider = tool.provider_factory.static_file_provider();
if let Err(err) = static_file_provider.check_consistency(&tool.provider_factory.provider()?) {
warn!("Error checking consistency of static files: {err}");
}
let static_files = iter_static_files(static_file_provider.directory())?;
let ranges = static_files
.get(segment)
.ok_or_else(|| eyre::eyre!("No static files found for segment: {}", segment))?;
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0usize;
let limit = limit.unwrap_or(usize::MAX);
let start_block = start_block.unwrap_or(0);
let end_block = end_block.unwrap_or(u64::MAX);
info!(
"Computing checksum for {} static files, start_block={}, end_block={}, limit={:?}",
segment,
start_block,
end_block,
if limit == usize::MAX { None } else { Some(limit) }
);
'outer: for (block_range, _header) in ranges.iter().sorted_by_key(|(range, _)| range.start()) {
if block_range.end() < start_block || block_range.start() > end_block {
continue;
}
let fixed_block_range = static_file_provider.find_fixed_range(segment, block_range.start());
let jar_provider = static_file_provider
.get_segment_provider_for_range(segment, || Some(fixed_block_range), None)?
.ok_or_else(|| {
eyre::eyre!(
"Failed to get segment provider for segment {} at range {}",
segment,
block_range
)
})?;
let mut cursor = jar_provider.cursor()?;
while let Ok(Some(row)) = cursor.next_row() {
for col_data in row.iter() {
hasher.write(col_data);
}
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break 'outer;
}
}
// Explicitly drop provider before removing from cache to avoid deadlock
drop(jar_provider);
static_file_provider.remove_cached_provider(segment, fixed_block_range.end());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!(
"Checksum for static file segment `{}`: {:#x} ({} entries, elapsed: {:?})",
segment, checksum, total, elapsed
);
Ok(())
}
pub(crate) struct ChecksumViewer<'a, N: NodeTypesWithDB> {
tool: &'a DbTool<N>,
start_key: Option<String>,
end_key: Option<String>,
limit: Option<usize>,
}
impl<N: NodeTypesWithDB> ChecksumViewer<'_, N> {
pub(crate) const fn new(tool: &'_ DbTool<N>) -> ChecksumViewer<'_, N> {
ChecksumViewer { tool, start_key: None, end_key: None, limit: None }
}
}
impl<N: ProviderNodeTypes> TableViewer<(u64, Duration)> for ChecksumViewer<'_, N> {
type Error = eyre::Report;
fn view<T: Table>(&self) -> Result<(u64, Duration), Self::Error> {
let provider =
self.tool.provider_factory.provider()?.disable_long_read_transaction_safety();
let tx = provider.tx_ref();
info!(
"Start computing checksum, start={:?}, end={:?}, limit={:?}",
self.start_key, self.end_key, self.limit
);
let mut cursor = tx.cursor_read::<RawTable<T>>()?;
let walker = match (self.start_key.as_deref(), self.end_key.as_deref()) {
(Some(start), Some(end)) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(start_key..=end_key)?
}
(None, Some(end)) => {
let end_key = table_key::<T>(end).map(RawKey::new)?;
cursor.walk_range(..=end_key)?
}
(Some(start), None) => {
let start_key = table_key::<T>(start).map(RawKey::new)?;
cursor.walk_range(start_key..)?
}
(None, None) => cursor.walk_range(..)?,
};
let start_time = Instant::now();
let mut hasher = checksum_hasher();
let mut total = 0;
let limit = self.limit.unwrap_or(usize::MAX);
let mut enumerate_start_key = None;
let mut enumerate_end_key = None;
for (index, entry) in walker.enumerate() {
let (k, v): (RawKey<T::Key>, RawValue<T::Value>) = entry?;
if index.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {index} entries.");
}
hasher.write(k.raw_key());
hasher.write(v.raw_value());
if enumerate_start_key.is_none() {
enumerate_start_key = Some(k.clone());
}
enumerate_end_key = Some(k);
total = index + 1;
if total >= limit {
break
}
}
info!("Hashed {total} entries.");
if let (Some(s), Some(e)) = (enumerate_start_key, enumerate_end_key) {
info!("start-key: {}", serde_json::to_string(&s.key()?).unwrap_or_default());
info!("end-key: {}", serde_json::to_string(&e.key()?).unwrap_or_default());
}
let checksum = hasher.finish();
let elapsed = start_time.elapsed();
info!("Checksum for table `{}`: {:#x} (elapsed: {:?})", T::NAME, checksum, elapsed);
Ok((checksum, elapsed))
}
}

View File

@@ -0,0 +1,106 @@
//! RocksDB checksum implementation.
use super::{checksum_hasher, PROGRESS_LOG_INTERVAL};
use crate::common::CliNodeTypes;
use clap::ValueEnum;
use reth_chainspec::EthereumHardforks;
use reth_db::{tables, DatabaseEnv};
use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
#[derive(Debug, Clone, Copy, ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl RocksDbTable {
/// Returns the table name as a string
const fn name(&self) -> &'static str {
match self {
Self::TransactionHashNumbers => tables::TransactionHashNumbers::NAME,
Self::AccountsHistory => tables::AccountsHistory::NAME,
Self::StoragesHistory => tables::StoragesHistory::NAME,
}
}
}
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
let start_time = Instant::now();
let limit = limit.unwrap_or(usize::MAX);
info!(
"Computing checksum for RocksDB table `{}`, limit={:?}",
table.name(),
if limit == usize::MAX { None } else { Some(limit) }
);
let (checksum, total) = match table {
RocksDbTable::TransactionHashNumbers => {
checksum_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, limit)?
}
RocksDbTable::AccountsHistory => {
checksum_rocksdb_table::<tables::AccountsHistory>(&rocksdb, limit)?
}
RocksDbTable::StoragesHistory => {
checksum_rocksdb_table::<tables::StoragesHistory>(&rocksdb, limit)?
}
};
let elapsed = start_time.elapsed();
info!(
"Checksum for RocksDB table `{}`: {:#x} ({} entries, elapsed: {:?})",
table.name(),
checksum,
total,
elapsed
);
Ok(())
}
/// Computes checksum for a specific RocksDB table by iterating over rows.
fn checksum_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
limit: usize,
) -> eyre::Result<(u64, usize)> {
let iter = rocksdb.raw_iter::<T>()?;
let mut hasher = checksum_hasher();
let mut total = 0usize;
for entry in iter {
let (key_bytes, value_bytes) = entry?;
hasher.write(&key_bytes);
hasher.write(&value_bytes);
total += 1;
if total.is_multiple_of(PROGRESS_LOG_INTERVAL) {
info!("Hashed {total} entries.");
}
if total >= limit {
break;
}
}
Ok((hasher.finish(), total))
}

View File

@@ -39,7 +39,7 @@ pub enum Subcommands {
Stats(stats::Command),
/// Lists the contents of a table
List(list::Command),
/// Calculates the content checksum of a table
/// Calculates the content checksum of a table or static file segment
Checksum(checksum::Command),
/// Create a diff between two database tables or two entire databases.
Diff(diff::Command),

View File

@@ -11,7 +11,10 @@ use reth_db_common::DbTool;
use reth_fs_util as fs;
use reth_node_builder::{NodePrimitives, NodeTypesWithDB, NodeTypesWithDBAdapter};
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::providers::{ProviderNodeTypes, StaticFileProvider};
use reth_provider::{
providers::{ProviderNodeTypes, StaticFileProvider},
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
@@ -61,6 +64,11 @@ impl Command {
let db_stats_table = self.db_stats_table(tool)?;
println!("{db_stats_table}");
println!("\n");
let rocksdb_stats_table = self.rocksdb_stats_table(tool);
println!("{rocksdb_stats_table}");
Ok(())
}
@@ -148,6 +156,60 @@ impl Command {
Ok(table)
}
fn rocksdb_stats_table<N: NodeTypesWithDB>(&self, tool: &DbTool<N>) -> ComfyTable {
let mut table = ComfyTable::new();
table.load_preset(comfy_table::presets::ASCII_MARKDOWN);
table.set_header([
"RocksDB Table Name",
"# Entries",
"SST Size",
"Memtable Size",
"Total Size",
"Pending Compaction",
]);
let stats = tool.provider_factory.rocksdb_provider().table_stats();
let mut total_sst: u64 = 0;
let mut total_memtable: u64 = 0;
let mut total_size: u64 = 0;
let mut total_pending: u64 = 0;
for stat in &stats {
total_sst += stat.sst_size_bytes;
total_memtable += stat.memtable_size_bytes;
total_size += stat.estimated_size_bytes;
total_pending += stat.pending_compaction_bytes;
let mut row = Row::new();
row.add_cell(Cell::new(&stat.name))
.add_cell(Cell::new(stat.estimated_num_keys))
.add_cell(Cell::new(human_bytes(stat.sst_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.memtable_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.estimated_size_bytes as f64)))
.add_cell(Cell::new(human_bytes(stat.pending_compaction_bytes as f64)));
table.add_row(row);
}
if !stats.is_empty() {
let max_widths = table.column_max_content_widths();
let mut separator = Row::new();
for width in max_widths {
separator.add_cell(Cell::new("-".repeat(width as usize)));
}
table.add_row(separator);
let mut row = Row::new();
row.add_cell(Cell::new("RocksDB Total"))
.add_cell(Cell::new(""))
.add_cell(Cell::new(human_bytes(total_sst as f64)))
.add_cell(Cell::new(human_bytes(total_memtable as f64)))
.add_cell(Cell::new(human_bytes(total_size as f64)))
.add_cell(Cell::new(human_bytes(total_pending as f64)));
table.add_row(row);
}
table
}
fn static_files_stats_table<N: NodePrimitives>(
&self,
data_dir: ChainPath<DataDirPath>,

View File

@@ -2,14 +2,15 @@ use crate::common::EnvironmentArgs;
use clap::Parser;
use eyre::Result;
use lz4::Decoder;
use reqwest::Client;
use reqwest::{blocking::Client as BlockingClient, header::RANGE, Client, StatusCode};
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_fs_util as fs;
use std::{
borrow::Cow,
io::{self, Read, Write},
path::Path,
fs::OpenOptions,
io::{self, BufWriter, Read, Write},
path::{Path, PathBuf},
sync::{Arc, OnceLock},
time::{Duration, Instant},
};
@@ -327,18 +328,158 @@ fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path)
extract_archive(file, total_size, format, target_dir)
}
/// Fetches the snapshot from a remote URL, uncompressing it in a streaming fashion.
const MAX_DOWNLOAD_RETRIES: u32 = 10;
const RETRY_BACKOFF_SECS: u64 = 5;
/// Wrapper that tracks download progress while writing data.
/// Used with [`io::copy`] to display progress during downloads.
struct ProgressWriter<W> {
inner: W,
progress: DownloadProgress,
}
impl<W: Write> Write for ProgressWriter<W> {
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
let n = self.inner.write(buf)?;
let _ = self.progress.update(n as u64);
Ok(n)
}
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Downloads a file with resume support using HTTP Range requests.
/// Automatically retries on failure, resuming from where it left off.
/// Returns the path to the downloaded file and its total size.
fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
let file_name = Url::parse(url)
.ok()
.and_then(|u| u.path_segments()?.next_back().map(|s| s.to_string()))
.unwrap_or_else(|| "snapshot.tar".to_string());
let final_path = target_dir.join(&file_name);
let part_path = target_dir.join(format!("{file_name}.part"));
let client = BlockingClient::builder().timeout(Duration::from_secs(30)).build()?;
let mut total_size: Option<u64> = None;
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
if let Some(total) = total_size &&
existing_size >= total
{
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, total));
}
if attempt > 1 {
info!(target: "reth::cli",
"Retry attempt {}/{} - resuming from {} bytes",
attempt, MAX_DOWNLOAD_RETRIES, existing_size
);
}
let mut request = client.get(url);
if existing_size > 0 {
request = request.header(RANGE, format!("bytes={existing_size}-"));
if attempt == 1 {
info!(target: "reth::cli", "Resuming download from {} bytes", existing_size);
}
}
let response = match request.send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(e) => {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download failed, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
let is_partial = response.status() == StatusCode::PARTIAL_CONTENT;
let size = if is_partial {
response
.headers()
.get("Content-Range")
.and_then(|v| v.to_str().ok())
.and_then(|v| v.split('/').next_back())
.and_then(|v| v.parse().ok())
} else {
response.content_length()
};
if total_size.is_none() {
total_size = size;
}
let current_total = total_size.ok_or_else(|| {
eyre::eyre!("Server did not provide Content-Length or Content-Range header")
})?;
let file = if is_partial && existing_size > 0 {
OpenOptions::new()
.append(true)
.open(&part_path)
.map_err(|e| fs::FsPathError::open(e, &part_path))?
} else {
fs::create_file(&part_path)?
};
let start_offset = if is_partial { existing_size } else { 0 };
let mut progress = DownloadProgress::new(current_total);
progress.downloaded = start_offset;
let mut writer = ProgressWriter { inner: BufWriter::new(file), progress };
let mut reader = response;
let copy_result = io::copy(&mut reader, &mut writer);
let flush_result = writer.inner.flush();
println!();
if let Err(e) = copy_result.and(flush_result) {
last_error = Some(e.into());
if attempt < MAX_DOWNLOAD_RETRIES {
info!(target: "reth::cli",
"Download interrupted, retrying in {} seconds...", RETRY_BACKOFF_SECS
);
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
fs::rename(&part_path, &final_path)?;
info!(target: "reth::cli", "Download complete: {}", final_path.display());
return Ok((final_path, current_total));
}
Err(last_error
.unwrap_or_else(|| eyre::eyre!("Download failed after {} attempts", MAX_DOWNLOAD_RETRIES)))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let client = reqwest::blocking::Client::builder().build()?;
let response = client.get(url).send()?.error_for_status()?;
let (downloaded_path, total_size) = resumable_download(url, target_dir)?;
let total_size = response.content_length().ok_or_else(|| {
eyre::eyre!(
"Server did not provide Content-Length header. This is required for snapshot downloads"
)
})?;
info!(target: "reth::cli", "Extracting snapshot...");
let file = fs::open(&downloaded_path)?;
extract_archive(file, total_size, format, target_dir)?;
extract_archive(response, total_size, format, target_dir)
fs::remove_file(&downloaded_path)?;
info!(target: "reth::cli", "Removed downloaded archive");
Ok(())
}
/// Downloads and extracts a snapshot, blocking until finished.

View File

@@ -10,7 +10,8 @@ use reth_node_builder::NodeBuilder;
use reth_node_core::{
args::{
DatabaseArgs, DatadirArgs, DebugArgs, DevArgs, EngineArgs, EraArgs, MetricArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RpcServerArgs, StaticFilesArgs, TxPoolArgs,
NetworkArgs, PayloadBuilderArgs, PruningArgs, RocksDbArgs, RpcServerArgs, StaticFilesArgs,
TxPoolArgs,
},
node_config::NodeConfig,
version,
@@ -102,6 +103,10 @@ pub struct NodeCommand<C: ChainSpecParser, Ext: clap::Args + fmt::Debug = NoArgs
#[command(flatten)]
pub pruning: PruningArgs,
/// All `RocksDB` table routing arguments
#[command(flatten)]
pub rocksdb: RocksDbArgs,
/// Engine cli arguments
#[command(flatten, next_help_heading = "Engine")]
pub engine: EngineArgs,
@@ -166,12 +171,16 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,
ext,
} = self;
// Validate RocksDB arguments
rocksdb.validate()?;
// set up node config
let mut node_config = NodeConfig {
datadir,
@@ -187,6 +196,7 @@ where
db,
dev,
pruning,
rocksdb,
engine,
era,
static_files,

View File

@@ -42,9 +42,9 @@ pub struct Command<C: ChainSpecParser> {
#[arg(long)]
to: Option<u64>,
/// Number of tasks to run in parallel
#[arg(long, default_value = "10")]
num_tasks: u64,
/// Number of tasks to run in parallel. Defaults to the number of available CPUs.
#[arg(long)]
num_tasks: Option<u64>,
/// Continues with execution when an invalid block is encountered and collects these blocks.
#[arg(long)]
@@ -84,12 +84,16 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
}
};
let num_tasks = self.num_tasks.unwrap_or_else(|| {
std::thread::available_parallelism().map(|n| n.get() as u64).unwrap_or(10)
});
let total_blocks = max_block - min_block;
let total_gas = calculate_gas_used_from_headers(
&provider_factory.static_file_provider(),
min_block..=max_block,
)?;
let blocks_per_task = total_blocks / self.num_tasks;
let blocks_per_task = total_blocks / num_tasks;
let db_at = {
let provider_factory = provider_factory.clone();
@@ -107,10 +111,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let _guard = cancellation.drop_guard();
let mut tasks = JoinSet::new();
for i in 0..self.num_tasks {
for i in 0..num_tasks {
let start_block = min_block + i * blocks_per_task;
let end_block =
if i == self.num_tasks - 1 { max_block } else { start_block + blocks_per_task };
if i == num_tasks - 1 { max_block } else { start_block + blocks_per_task };
// Spawn thread executing blocks
let provider_factory = provider_factory.clone();
@@ -148,7 +152,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
};
if let Err(err) = consensus
.validate_block_post_execution(&block, &result)
.validate_block_post_execution(&block, &result, None)
.wrap_err_with(|| {
format!("Failed to validate block {} {}", block.number(), block.hash())
})

View File

@@ -15,7 +15,7 @@ use reth_db_common::{
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_core::args::StageEnum;
use reth_provider::{
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter, TrieWriter,
DBProvider, DatabaseProviderFactory, StaticFileProviderFactory, StaticFileWriter,
};
use reth_prune::PruneSegment;
use reth_stages::StageId;
@@ -113,7 +113,6 @@ impl<C: ChainSpecParser> Command<C> {
tx.clear::<tables::TransactionBlocks>()?;
tx.clear::<tables::BlockOmmers<HeaderTy<N>>>()?;
tx.clear::<tables::BlockWithdrawals>()?;
tx.clear::<tables::BlockAccessLists>()?;
reset_stage_checkpoint(tx, StageId::Bodies)?;
insert_genesis_header(&provider_rw, &self.env.chain)?;
@@ -168,10 +167,6 @@ impl<C: ChainSpecParser> Command<C> {
None,
)?;
}
StageEnum::MerkleChangeSets => {
provider_rw.clear_trie_changesets()?;
reset_stage_checkpoint(tx, StageId::MerkleChangeSets)?;
}
StageEnum::AccountHistory | StageEnum::StorageHistory => {
tx.clear::<tables::AccountsHistory>()?;
tx.clear::<tables::StoragesHistory>()?;

View File

@@ -121,7 +121,16 @@ pub fn install() {
unsafe {
let alt_stack_size: usize = min_sigstack_size() + 64 * 1024;
let mut alt_stack: libc::stack_t = mem::zeroed();
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, 1).unwrap()).cast();
// Both SysV AMD64 ABI and aarch64 ABI require 16 bytes alignment. We are going to be
// generous here and just use a size of a page.
let raw_page_sz = libc::sysconf(libc::_SC_PAGESIZE);
let page_sz = if raw_page_sz == -1 {
// Fallback alignment in case sysconf fails.
4096_usize
} else {
raw_page_sz as usize
};
alt_stack.ss_sp = alloc(Layout::from_size_align(alt_stack_size, page_sz).unwrap()).cast();
alt_stack.ss_size = alt_stack_size;
libc::sigaltstack(&raw const alt_stack, ptr::null_mut());

View File

@@ -550,7 +550,6 @@ impl PruneConfig {
/// - `Option<PruneMode>` fields: set from `other` only if `self` is `None`.
/// - `block_interval`: set from `other` only if `self.block_interval ==
/// DEFAULT_BLOCK_INTERVAL`.
/// - `merkle_changesets`: always set from `other`.
/// - `receipts_log_filter`: set from `other` only if `self` is empty and `other` is non-empty.
pub fn merge(&mut self, other: Self) {
let Self {
@@ -563,7 +562,6 @@ impl PruneConfig {
account_history,
storage_history,
bodies_history,
merkle_changesets,
receipts_log_filter,
},
} = other;
@@ -580,8 +578,6 @@ impl PruneConfig {
self.segments.account_history = self.segments.account_history.or(account_history);
self.segments.storage_history = self.segments.storage_history.or(storage_history);
self.segments.bodies_history = self.segments.bodies_history.or(bodies_history);
// Merkle changesets is not optional; always take the value from `other`
self.segments.merkle_changesets = merkle_changesets;
if self.segments.receipts_log_filter.0.is_empty() && !receipts_log_filter.0.is_empty() {
self.segments.receipts_log_filter = receipts_log_filter;
@@ -1091,7 +1087,6 @@ receipts = { distance = 16384 }
account_history: None,
storage_history: Some(PruneMode::Before(5000)),
bodies_history: None,
merkle_changesets: PruneMode::Before(0),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([(
Address::random(),
PruneMode::Full,
@@ -1108,7 +1103,6 @@ receipts = { distance = 16384 }
account_history: Some(PruneMode::Distance(2000)),
storage_history: Some(PruneMode::Distance(3000)),
bodies_history: None,
merkle_changesets: PruneMode::Distance(10000),
receipts_log_filter: ReceiptsLogPruneConfig(BTreeMap::from([
(Address::random(), PruneMode::Distance(1000)),
(Address::random(), PruneMode::Before(2000)),
@@ -1127,7 +1121,6 @@ receipts = { distance = 16384 }
assert_eq!(config1.segments.receipts, Some(PruneMode::Distance(1000)));
assert_eq!(config1.segments.account_history, Some(PruneMode::Distance(2000)));
assert_eq!(config1.segments.storage_history, Some(PruneMode::Before(5000)));
assert_eq!(config1.segments.merkle_changesets, PruneMode::Distance(10000));
assert_eq!(config1.segments.receipts_log_filter, original_filter);
}

View File

@@ -14,14 +14,11 @@ workspace = true
# reth
reth-chainspec.workspace = true
reth-consensus.workspace = true
tracing.workspace = true
# ethereum
reth-primitives-traits.workspace = true
alloy-consensus.workspace = true
alloy-primitives.workspace = true
alloy-eips.workspace = true
alloy-rlp.workspace = true
[dev-dependencies]
alloy-primitives = { workspace = true, features = ["rand"] }
@@ -38,6 +35,4 @@ std = [
"reth-primitives-traits/std",
"reth-ethereum-primitives/std",
"alloy-primitives/std",
"alloy-rlp/std",
"tracing/std",
]

View File

@@ -69,29 +69,6 @@ pub fn validate_shanghai_withdrawals<B: Block>(
Ok(())
}
/// Validate that block access lists are present in Amsterdam
///
/// [EIP-7928]: https://eips.ethereum.org/EIPS/eip-7928
#[inline]
pub fn validate_amsterdam_block_access_lists<B: Block>(
block: &SealedBlock<B>,
) -> Result<(), ConsensusError> {
let bal = block.body().block_access_list().ok_or(ConsensusError::BlockAccessListMissing)?;
let bal_hash = alloy_primitives::keccak256(alloy_rlp::encode(bal));
let header_bal_hash =
block.block_access_list_hash().ok_or(ConsensusError::BlockAccessListHashMissing)?;
if bal_hash != header_bal_hash {
tracing::error!(
target: "consensus",
?header_bal_hash,
?bal,
"Block access list hash mismatch in validation.rs in L81"
);
return Err(ConsensusError::InvalidBalHash);
}
Ok(())
}
/// Validate that blob gas is present in the block if Cancun is active.
///
/// See [EIP-4844]: Shard Blob Transactions
@@ -154,21 +131,6 @@ where
}
_ => return Err(ConsensusError::WithdrawalsRootUnexpected),
}
if let (Some(expected_hash), Some(body_bal)) =
(header.block_access_list_hash(), body.block_access_list())
{
let got_hash = alloy_primitives::keccak256(alloy_rlp::encode(body_bal));
if got_hash != expected_hash {
tracing::error!(
target: "consensus",
?expected_hash,
?body_bal,
"Block access list hash mismatch in validation.rs in L164"
);
return Err(ConsensusError::InvalidBalHash);
}
}
Ok(())
}
@@ -255,10 +217,6 @@ where
})
}
if chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()) {
validate_amsterdam_block_access_lists(block)?;
}
Ok(())
}
@@ -534,7 +492,6 @@ mod tests {
transactions: vec![transaction],
ommers: vec![],
withdrawals: Some(Withdrawals::default()),
block_access_list: None,
};
let block = SealedBlock::seal_slow(alloy_consensus::Block { header, body });

View File

@@ -15,6 +15,12 @@ use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
use alloy_consensus::Header;
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
use core::error::Error;
/// Pre-computed receipt root and logs bloom.
///
/// When provided to [`FullConsensus::validate_block_post_execution`], this allows skipping
/// the receipt root computation and using the pre-computed values instead.
pub type ReceiptRootBloom = (B256, Bloom);
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{
constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK, MINIMUM_GAS_LIMIT},
@@ -39,11 +45,15 @@ pub trait FullConsensus<N: NodePrimitives>: Consensus<N::Block> {
///
/// See the Yellow Paper sections 4.3.2 "Holistic Validity".
///
/// If `receipt_root_bloom` is provided, the implementation should use the pre-computed
/// receipt root and logs bloom instead of computing them from the receipts.
///
/// Note: validating blocks does not include other validations of the Consensus
fn validate_block_post_execution(
&self,
block: &RecoveredBlock<N::Block>,
result: &BlockExecutionResult<N::Receipt>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError>;
}
@@ -402,41 +412,9 @@ pub enum ConsensusError {
/// The maximum allowed RLP length.
max_rlp_length: usize,
},
/// Error when the hash of block access list is different from the expected hash.
#[error("Block header's BAL hash does not match the computed BAL hash.")]
InvalidBalHash,
/// Error when the block access list hash is missing.
#[error("block access list hash missing")]
BlockAccessListHashMissing,
/// Error when the block access list is different from the expected access list.
#[error("Block's access list is invalid.")]
InvalidBlockAccessList,
/// Error when the block access list is missing.
#[error("block access list missing")]
BlockAccessListMissing,
/// Error when the block access list hash is unexpected.
#[error("block access list hash unexpected")]
BlockAccessListHashUnexpected,
/// Error when the block access list contains an account change that is not present in the
/// computed access list.
#[error("Block BAL contains an account change that is not present in the computed BAL.")]
InvalidBalExtraAccount,
/// Error when the block access list is missing an account change that is present in the
/// computed access list.
#[error("Block BAL is missing an account change that is present in the computed BAL.")]
InvalidBalMissingAccount,
/// EIP-7825: Transaction gas limit exceeds maximum allowed
#[error(transparent)]
TransactionGasLimitTooHigh(Box<TxGasLimitTooHighErr>),
/// Other, likely an injected L2 error.
#[error("{0}")]
Other(String),

View File

@@ -18,7 +18,7 @@
//!
//! **Not for production use** - provides no security guarantees or consensus validation.
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use alloc::sync::Arc;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
@@ -76,6 +76,7 @@ impl<N: NodePrimitives> FullConsensus<N> for NoopConsensus {
&self,
_block: &RecoveredBlock<N::Block>,
_result: &BlockExecutionResult<N::Receipt>,
_receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
Ok(())
}

View File

@@ -1,4 +1,4 @@
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use core::sync::atomic::{AtomicBool, Ordering};
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
@@ -51,6 +51,7 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
&self,
_block: &RecoveredBlock<N::Block>,
_result: &BlockExecutionResult<N::Receipt>,
_receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
if self.fail_validation() {
Err(ConsensusError::BaseFeeMissing)

View File

@@ -56,7 +56,6 @@ pub fn generate_test_blocks(chain_spec: &ChainSpec, count: u64) -> Vec<SealedBlo
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
block_access_list_hash: None,
};
// Set required fields based on chain spec
@@ -107,7 +106,6 @@ pub fn generate_test_blocks(chain_spec: &ChainSpec, count: u64) -> Vec<SealedBlo
transactions: vec![],
ommers: vec![],
withdrawals: header.withdrawals_root.is_some().then(Withdrawals::default),
block_access_list: None,
};
// Create the block

View File

@@ -227,7 +227,6 @@ where
suggested_fee_recipient: alloy_primitives::Address::random(),
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
};
env.active_node_state_mut()?
@@ -300,7 +299,6 @@ where
suggested_fee_recipient: alloy_primitives::Address::random(),
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
};
let fresh_fcu_result = EngineApiClient::<Engine>::fork_choice_updated_v3(

View File

@@ -254,7 +254,6 @@ where
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
};
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
};
@@ -285,7 +284,6 @@ where
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
};
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadBuilderAttributes::from(
EthPayloadBuilderAttributes::new(B256::ZERO, attributes),

View File

@@ -161,7 +161,6 @@ async fn test_testsuite_assert_mine_block() -> Result<()> {
suggested_fee_recipient: Address::random(),
withdrawals: None,
parent_beacon_block_root: None,
slot_number: None,
},
));

View File

@@ -839,7 +839,6 @@ mod tests {
receipts: vec![],
requests: Requests::default(),
gas_used: 0,
block_access_list: Default::default(),
blob_gas_used: 0,
},
};

View File

@@ -204,7 +204,7 @@ where
EngineApiMessageVersion::default(),
)
.await?;
tracing::debug!(target: "engine::local", "FCU result: {res:?}");
if !res.is_valid() {
eyre::bail!("Invalid payload status")
}

View File

@@ -57,7 +57,6 @@ where
.chain_spec
.is_cancun_active_at_timestamp(timestamp)
.then(B256::random),
slot_number: None,
}
}
}

View File

@@ -137,6 +137,8 @@ pub struct TreeConfig {
account_worker_count: usize,
/// Whether to enable V2 storage proofs.
enable_proof_v2: bool,
/// Whether to disable cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl Default for TreeConfig {
@@ -166,6 +168,7 @@ impl Default for TreeConfig {
storage_worker_count: default_storage_worker_count(),
account_worker_count: default_account_worker_count(),
enable_proof_v2: false,
disable_cache_metrics: false,
}
}
}
@@ -198,6 +201,7 @@ impl TreeConfig {
storage_worker_count: usize,
account_worker_count: usize,
enable_proof_v2: bool,
disable_cache_metrics: bool,
) -> Self {
Self {
persistence_threshold,
@@ -224,6 +228,7 @@ impl TreeConfig {
storage_worker_count,
account_worker_count,
enable_proof_v2,
disable_cache_metrics,
}
}
@@ -516,4 +521,15 @@ impl TreeConfig {
self.enable_proof_v2 = enable_proof_v2;
self
}
/// Returns whether cache metrics recording is disabled.
pub const fn disable_cache_metrics(&self) -> bool {
self.disable_cache_metrics
}
/// Setter for whether to disable cache metrics recording.
pub const fn without_cache_metrics(mut self, disable_cache_metrics: bool) -> Self {
self.disable_cache_metrics = disable_cache_metrics;
self
}
}

View File

@@ -62,8 +62,7 @@ pub trait EngineTypes:
+ TryInto<Self::ExecutionPayloadEnvelopeV2>
+ TryInto<Self::ExecutionPayloadEnvelopeV3>
+ TryInto<Self::ExecutionPayloadEnvelopeV4>
+ TryInto<Self::ExecutionPayloadEnvelopeV5>
+ TryInto<Self::ExecutionPayloadEnvelopeV6>,
+ TryInto<Self::ExecutionPayloadEnvelopeV5>,
> + DeserializeOwned
+ Serialize
{
@@ -107,14 +106,6 @@ pub trait EngineTypes:
+ Send
+ Sync
+ 'static;
/// Execution Payload V6 envelope type.
type ExecutionPayloadEnvelopeV6: DeserializeOwned
+ Serialize
+ Clone
+ Unpin
+ Send
+ Sync
+ 'static;
}
/// Type that validates the payloads processed by the engine API.

View File

@@ -25,6 +25,7 @@ reth-tasks.workspace = true
reth-node-types.workspace = true
reth-chainspec.workspace = true
reth-engine-primitives.workspace = true
reth-trie-db.workspace = true
# async
futures.workspace = true
@@ -40,6 +41,8 @@ reth-evm-ethereum.workspace = true
reth-exex-types.workspace = true
reth-primitives-traits.workspace = true
reth-node-ethereum.workspace = true
reth-trie-db.workspace = true
alloy-eips.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio-stream.workspace = true

View File

@@ -26,6 +26,7 @@ use reth_provider::{
use reth_prune::PrunerWithFactory;
use reth_stages_api::{MetricEventsSender, Pipeline};
use reth_tasks::TaskSpawner;
use reth_trie_db::ChangesetCache;
use std::{
pin::Pin,
sync::Arc,
@@ -84,6 +85,7 @@ where
tree_config: TreeConfig,
sync_metrics_tx: MetricEventsSender,
evm_config: C,
changeset_cache: ChangesetCache,
) -> Self
where
V: EngineValidator<N::Payload>,
@@ -109,6 +111,7 @@ where
tree_config,
engine_kind,
evm_config,
changeset_cache,
);
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
@@ -156,6 +159,7 @@ mod tests {
};
use reth_prune::Pruner;
use reth_tasks::TokioTaskExecutor;
use reth_trie_db::ChangesetCache;
use std::sync::Arc;
use tokio::sync::{mpsc::unbounded_channel, watch};
use tokio_stream::wrappers::UnboundedReceiverStream;
@@ -188,6 +192,8 @@ mod tests {
let pruner = Pruner::new_with_factory(provider_factory.clone(), vec![], 0, 0, None, rx);
let evm_config = EthEvmConfig::new(chain_spec.clone());
let changeset_cache = ChangesetCache::new();
let engine_validator = BasicEngineValidator::new(
blockchain_db.clone(),
consensus.clone(),
@@ -195,6 +201,7 @@ mod tests {
engine_payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache.clone(),
);
let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel();
@@ -214,6 +221,7 @@ mod tests {
TreeConfig::default(),
sync_metrics_tx,
evm_config,
changeset_cache,
);
}
}

View File

@@ -12,7 +12,7 @@ workspace = true
[dependencies]
# reth
reth-chain-state.workspace = true
reth-chain-state = { workspace = true, features = ["rayon"] }
reth-chainspec = { workspace = true, optional = true }
reth-consensus.workspace = true
reth-db.workspace = true
@@ -34,6 +34,8 @@ reth-trie-parallel.workspace = true
reth-trie-sparse = { workspace = true, features = ["std", "metrics"] }
reth-trie-sparse-parallel = { workspace = true, features = ["std"] }
reth-trie.workspace = true
reth-trie-common.workspace = true
reth-trie-db.workspace = true
# alloy
alloy-evm.workspace = true
@@ -94,7 +96,7 @@ reth-tracing.workspace = true
reth-node-ethereum.workspace = true
reth-e2e-test-utils.workspace = true
# alloy
# revm
revm-state.workspace = true
assert_matches.workspace = true
@@ -133,6 +135,8 @@ test-utils = [
"reth-static-file",
"reth-tracing",
"reth-trie/test-utils",
"reth-trie-common/test-utils",
"reth-trie-db/test-utils",
"reth-trie-sparse/test-utils",
"reth-prune-types?/test-utils",
"reth-trie-parallel/test-utils",

View File

@@ -28,10 +28,10 @@ fn create_bench_state(num_accounts: usize) -> EvmState {
code: Default::default(),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::empty(),
transaction_id: 0,
..Default::default()
};
let address = Address::with_last_byte(i as u8);

View File

@@ -62,7 +62,7 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
storage: HashMap::default(),
status: AccountStatus::SelfDestructed,
transaction_id: 0,
..Default::default()
original_info: Box::new(AccountInfo::default()),
}
} else {
RevmAccount {
@@ -86,8 +86,8 @@ fn create_bench_state_updates(params: &BenchParams) -> Vec<EvmState> {
})
.collect(),
status: AccountStatus::Touched,
original_info: Box::new(AccountInfo::default()),
transaction_id: 0,
..Default::default()
}
};
@@ -242,7 +242,10 @@ fn bench_state_root(c: &mut Criterion) {
std::convert::identity,
),
StateProviderBuilder::new(provider.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(provider),
OverlayStateProviderFactory::new(
provider,
reth_trie_db::ChangesetCache::new(),
),
&TreeConfig::default(),
None,
);

View File

@@ -159,6 +159,7 @@ where
self.metrics.save_blocks_block_count.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
Ok(last_block)
}
}

View File

@@ -606,12 +606,21 @@ pub(crate) struct SavedCache {
/// A guard to track in-flight usage of this cache.
/// The cache is considered available if the strong count is 1.
usage_guard: Arc<()>,
/// Whether to skip cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl SavedCache {
/// Creates a new instance with the internals
pub(super) fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
Self { hash, caches, metrics, usage_guard: Arc::new(()) }
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
}
/// Sets whether to disable cache metrics recording.
pub(super) const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
self.disable_cache_metrics = disable;
self
}
/// Returns the hash for this cache
@@ -619,9 +628,9 @@ impl SavedCache {
self.hash
}
/// Splits the cache into its caches and metrics, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics) {
(self.caches, self.metrics)
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
pub(crate) fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
(self.caches, self.metrics, self.disable_cache_metrics)
}
/// Returns true if the cache is available for use (no other tasks are currently using it).
@@ -645,7 +654,13 @@ impl SavedCache {
}
/// Updates the metrics for the [`ExecutionCache`].
///
/// Note: This can be expensive with large cached state as it iterates over
/// all storage entries. Use `with_disable_cache_metrics(true)` to skip.
pub(crate) fn update_metrics(&self) {
if self.disable_cache_metrics {
return;
}
self.metrics.storage_cache_size.set(self.caches.total_storage_slots() as f64);
self.metrics.account_cache_size.set(self.caches.account_cache.entry_count() as f64);
self.metrics.code_cache_size.set(self.caches.code_cache.entry_count() as f64);

View File

@@ -1,25 +1,15 @@
use crate::tree::{error::InsertBlockFatalError, MeteredStateHook, TreeOutcome};
use alloy_consensus::transaction::TxHashRef;
use alloy_evm::{
block::{BlockExecutor, ExecutableTx},
Evm,
};
use crate::tree::{error::InsertBlockFatalError, TreeOutcome};
use alloy_rpc_types_engine::{PayloadStatus, PayloadStatusEnum};
use core::borrow::BorrowMut;
use reth_engine_primitives::{ForkchoiceStatus, OnForkChoiceUpdated};
use reth_errors::{BlockExecutionError, ProviderError};
use reth_evm::{metrics::ExecutorMetrics, OnStateHook};
use reth_errors::ProviderError;
use reth_evm::metrics::ExecutorMetrics;
use reth_execution_types::BlockExecutionOutput;
use reth_metrics::{
metrics::{Counter, Gauge, Histogram},
Metrics,
};
use reth_primitives_traits::SignedTransaction;
use reth_trie::updates::TrieUpdates;
use revm::database::{states::bundle_state::BundleRetention, State};
use revm_primitives::Address;
use std::time::Instant;
use tracing::{debug_span, trace};
use std::time::{Duration, Instant};
/// Metrics for the `EngineApi`.
#[derive(Debug, Default)]
@@ -35,101 +25,24 @@ pub(crate) struct EngineApiMetrics {
}
impl EngineApiMetrics {
/// Helper function for metered execution
fn metered<F, R>(&self, f: F) -> R
where
F: FnOnce() -> (u64, R),
{
// Execute the block and record the elapsed time.
let execute_start = Instant::now();
let (gas_used, output) = f();
let execution_duration = execute_start.elapsed().as_secs_f64();
// Update gas metrics.
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_duration);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_duration);
self.executor.execution_duration.set(execution_duration);
output
}
/// Execute the given block using the provided [`BlockExecutor`] and update metrics for the
/// execution.
/// Records metrics for block execution.
///
/// This method updates metrics for execution time, gas usage, and the number
/// of accounts, storage slots and bytecodes loaded and updated.
pub(crate) fn execute_metered<E, DB>(
/// of accounts, storage slots and bytecodes updated.
pub(crate) fn record_block_execution<R>(
&self,
executor: E,
mut transactions: impl Iterator<Item = Result<impl ExecutableTx<E>, BlockExecutionError>>,
transaction_count: usize,
state_hook: Box<dyn OnStateHook>,
) -> Result<(BlockExecutionOutput<E::Receipt>, Vec<Address>), BlockExecutionError>
where
DB: alloy_evm::Database,
E: BlockExecutor<Evm: Evm<DB: BorrowMut<State<DB>>>, Transaction: SignedTransaction>,
{
// clone here is cheap, all the metrics are Option<Arc<_>>. additionally
// they are globally registered so that the data recorded in the hook will
// be accessible.
let wrapper = MeteredStateHook { metrics: self.executor.clone(), inner_hook: state_hook };
output: &BlockExecutionOutput<R>,
execution_duration: Duration,
) {
let execution_secs = execution_duration.as_secs_f64();
let gas_used = output.result.gas_used;
let mut senders = Vec::with_capacity(transaction_count);
let mut executor = executor.with_state_hook(Some(Box::new(wrapper)));
let f = || {
let start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.entered()
.in_scope(|| executor.apply_pre_execution_changes())?;
self.executor.pre_execution_histogram.record(start.elapsed());
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
loop {
let start = Instant::now();
let Some(tx) = transactions.next() else { break };
self.executor.transaction_wait_histogram.record(start.elapsed());
let tx = tx?;
senders.push(*tx.signer());
let span =
debug_span!(target: "engine::tree", "execute tx", tx_hash=?tx.tx().tx_hash());
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.executor.transaction_execution_histogram.record(start.elapsed());
// record the tx gas used
enter.record("gas_used", gas_used);
}
drop(exec_span);
let start = Instant::now();
let result = debug_span!(target: "engine::tree", "finish")
.entered()
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result));
self.executor.post_execution_histogram.record(start.elapsed());
result
};
// Use metered to execute and track timing/gas metrics
let (mut db, result) = self.metered(|| {
let res = f();
let gas_used = res.as_ref().map(|r| r.1.gas_used).unwrap_or(0);
(gas_used, res)
})?;
// merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.entered()
.in_scope(|| db.borrow_mut().merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.borrow_mut().take_bundle() };
// Update gas metrics
self.executor.gas_processed_total.increment(gas_used);
self.executor.gas_per_second.set(gas_used as f64 / execution_secs);
self.executor.gas_used_histogram.record(gas_used as f64);
self.executor.execution_histogram.record(execution_secs);
self.executor.execution_duration.set(execution_secs);
// Update the metrics for the number of accounts, storage slots and bytecodes updated
let accounts = output.state.state.len();
@@ -140,8 +53,31 @@ impl EngineApiMetrics {
self.executor.accounts_updated_histogram.record(accounts as f64);
self.executor.storage_slots_updated_histogram.record(storage_slots as f64);
self.executor.bytecodes_updated_histogram.record(bytecodes as f64);
}
Ok((output, senders))
/// Returns a reference to the executor metrics for use in state hooks.
pub(crate) const fn executor_metrics(&self) -> &ExecutorMetrics {
&self.executor
}
/// Records the duration of block pre-execution changes (e.g., beacon root update).
pub(crate) fn record_pre_execution(&self, elapsed: Duration) {
self.executor.pre_execution_histogram.record(elapsed);
}
/// Records the duration of block post-execution changes (e.g., finalization).
pub(crate) fn record_post_execution(&self, elapsed: Duration) {
self.executor.post_execution_histogram.record(elapsed);
}
/// Records the time spent waiting for the next transaction from the iterator.
pub(crate) fn record_transaction_wait(&self, elapsed: Duration) {
self.executor.transaction_wait_histogram.record(elapsed);
}
/// Records the duration of a single transaction execution.
pub(crate) fn record_transaction_execution(&self, elapsed: Duration) {
self.executor.transaction_execution_histogram.record(elapsed);
}
}
@@ -197,6 +133,12 @@ pub(crate) struct EngineMetrics {
#[derive(Metrics)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ForkchoiceUpdatedMetrics {
/// Finish time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest forkchoice updated call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of forkchoice updated messages received.
pub(crate) forkchoice_updated_messages: Counter,
/// The total count of forkchoice updated messages with payload received.
@@ -219,18 +161,35 @@ pub(crate) struct ForkchoiceUpdatedMetrics {
pub(crate) forkchoice_updated_last: Gauge,
/// Time diff between new payload call response and the next forkchoice updated call request.
pub(crate) new_payload_forkchoice_updated_time_diff: Histogram,
/// Time from previous forkchoice updated finish to current forkchoice updated start (idle
/// time).
pub(crate) time_between_forkchoice_updated: Histogram,
/// Time from previous forkchoice updated start to current forkchoice updated start (total
/// interval).
pub(crate) forkchoice_updated_interval: Histogram,
}
impl ForkchoiceUpdatedMetrics {
/// Increment the forkchoiceUpdated counter based on the given result
pub(crate) fn update_response_metrics(
&self,
&mut self,
start: Instant,
latest_new_payload_at: &mut Option<Instant>,
has_attrs: bool,
result: &Result<TreeOutcome<OnForkChoiceUpdated>, ProviderError>,
) {
let elapsed = start.elapsed();
let finish = Instant::now();
let elapsed = finish - start;
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_forkchoice_updated.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.forkchoice_updated_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.forkchoice_status() {
ForkchoiceStatus::Valid => self.forkchoice_updated_valid.increment(1),
@@ -257,7 +216,10 @@ impl ForkchoiceUpdatedMetrics {
pub(crate) struct NewPayloadStatusMetrics {
/// Finish time of the latest new payload call.
#[metric(skip)]
pub(crate) latest_at: Option<Instant>,
pub(crate) latest_finish_at: Option<Instant>,
/// Start time of the latest new payload call.
#[metric(skip)]
pub(crate) latest_start_at: Option<Instant>,
/// The total count of new payload messages received.
pub(crate) new_payload_messages: Counter,
/// The total count of new payload messages that we responded to with
@@ -285,6 +247,10 @@ pub(crate) struct NewPayloadStatusMetrics {
pub(crate) new_payload_latency: Histogram,
/// Latency for the last new payload call.
pub(crate) new_payload_last: Gauge,
/// Time from previous payload finish to current payload start (idle time).
pub(crate) time_between_new_payloads: Histogram,
/// Time from previous payload start to current payload start (total interval).
pub(crate) new_payload_interval: Histogram,
}
impl NewPayloadStatusMetrics {
@@ -298,7 +264,14 @@ impl NewPayloadStatusMetrics {
let finish = Instant::now();
let elapsed = finish - start;
self.latest_at = Some(finish);
if let Some(prev_finish) = self.latest_finish_at {
self.time_between_new_payloads.record(start - prev_finish);
}
if let Some(prev_start) = self.latest_start_at {
self.new_payload_interval.record(start - prev_start);
}
self.latest_finish_at = Some(finish);
self.latest_start_at = Some(start);
match result {
Ok(outcome) => match outcome.outcome.status {
PayloadStatusEnum::Valid => {
@@ -334,10 +307,6 @@ pub(crate) struct BlockValidationMetrics {
pub(crate) state_root_histogram: Histogram,
/// Histogram of deferred trie computation duration.
pub(crate) deferred_trie_compute_duration: Histogram,
/// Histogram of time spent waiting for deferred trie data to become available.
pub(crate) deferred_trie_wait_duration: Histogram,
/// Trie input computation duration
pub(crate) trie_input_duration: Histogram,
/// Payload conversion and validation latency
pub(crate) payload_validation_duration: Gauge,
/// Histogram of payload validation latency
@@ -387,134 +356,10 @@ pub(crate) struct BlockBufferMetrics {
mod tests {
use super::*;
use alloy_eips::eip7685::Requests;
use alloy_evm::block::StateChangeSource;
use alloy_primitives::{B256, U256};
use metrics_util::debugging::{DebuggingRecorder, Snapshotter};
use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_evm_ethereum::EthEvm;
use reth_ethereum_primitives::Receipt;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::RecoveredBlock;
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
database_interface::EmptyDB,
inspector::NoOpInspector,
state::{Account, AccountInfo, AccountStatus, EvmState, EvmStorage, EvmStorageSlot},
Context, MainBuilder, MainContext,
};
use revm_primitives::Bytes;
use std::sync::mpsc;
/// A simple mock executor for testing that doesn't require complex EVM setup
struct MockExecutor {
state: EvmState,
hook: Option<Box<dyn OnStateHook>>,
}
impl MockExecutor {
fn new(state: EvmState) -> Self {
Self { state, hook: None }
}
}
// Mock Evm type for testing
type MockEvm = EthEvm<State<EmptyDB>, NoOpInspector>;
impl BlockExecutor for MockExecutor {
type Transaction = TransactionSigned;
type Receipt = Receipt;
type Evm = MockEvm;
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
Ok(())
}
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
) -> Result<ResultAndState<<Self::Evm as Evm>::HaltReason>, BlockExecutionError> {
// Call hook with our mock state for each transaction
if let Some(hook) = self.hook.as_mut() {
hook.on_state(StateChangeSource::Transaction(0), &self.state);
}
Ok(ResultAndState::new(
ExecutionResult::Success {
reason: SuccessReason::Return,
gas_used: 1000, // Mock gas used
gas_refunded: 0,
logs: vec![],
output: Output::Call(Bytes::from(vec![])),
},
Default::default(),
))
}
fn commit_transaction(
&mut self,
_output: ResultAndState<<Self::Evm as Evm>::HaltReason>,
_tx: impl ExecutableTx<Self>,
) -> Result<u64, BlockExecutionError> {
Ok(1000)
}
fn finish(
self,
) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
let Self { hook, state, .. } = self;
// Call hook with our mock state
if let Some(mut hook) = hook {
hook.on_state(StateChangeSource::Transaction(0), &state);
}
// Create a mock EVM
let db = State::builder()
.with_database(EmptyDB::default())
.with_bundle_update()
.without_state_clear()
.build();
let evm = EthEvm::new(
Context::mainnet().with_db(db).build_mainnet_with_inspector(NoOpInspector {}),
false,
);
// Return successful result like the original tests
Ok((
evm,
BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 1000,
block_access_list: None,
blob_gas_used: 0,
},
))
}
fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
self.hook = hook;
}
fn evm(&self) -> &Self::Evm {
panic!("Mock executor evm() not implemented")
}
fn evm_mut(&mut self) -> &mut Self::Evm {
panic!("Mock executor evm_mut() not implemented")
}
}
struct ChannelStateHook {
output: i32,
sender: mpsc::Sender<i32>,
}
impl OnStateHook for ChannelStateHook {
fn on_state(&mut self, _source: StateChangeSource, _state: &EvmState) {
let _ = self.sender.send(self.output);
}
}
use reth_revm::db::BundleState;
fn setup_test_recorder() -> Snapshotter {
let recorder = DebuggingRecorder::new();
@@ -524,37 +369,7 @@ mod tests {
}
#[test]
fn test_executor_metrics_hook_called() {
let metrics = EngineApiMetrics::default();
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, rx) = mpsc::channel();
let expected_output = 42;
let state_hook = Box::new(ChannelStateHook { sender: tx, output: expected_output });
let state = EvmState::default();
let executor = MockExecutor::new(state);
// This will fail to create the EVM but should still call the hook
let _result = metrics.execute_metered::<_, EmptyDB>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
);
// Check if hook was called (it might not be if finish() fails early)
match rx.try_recv() {
Ok(actual_output) => assert_eq!(actual_output, expected_output),
Err(_) => {
// Hook wasn't called, which is expected if the mock fails early
// The test still validates that the code compiles and runs
}
}
}
#[test]
fn test_executor_metrics_hook_metrics_recorded() {
fn test_record_block_execution_metrics() {
let snapshotter = setup_test_recorder();
let metrics = EngineApiMetrics::default();
@@ -563,44 +378,17 @@ mod tests {
metrics.executor.gas_per_second.set(0.0);
metrics.executor.gas_used_histogram.record(0.0);
let input = RecoveredBlock::<reth_ethereum_primitives::Block>::default();
let (tx, _rx) = mpsc::channel();
let state_hook = Box::new(ChannelStateHook { sender: tx, output: 42 });
// Create a state with some data
let state = {
let mut state = EvmState::default();
let storage =
EvmStorage::from_iter([(U256::from(1), EvmStorageSlot::new(U256::from(2), 0))]);
state.insert(
Default::default(),
Account {
info: AccountInfo {
balance: U256::from(100),
nonce: 10,
code_hash: B256::random(),
code: Default::default(),
account_id: None,
},
storage,
status: AccountStatus::default(),
transaction_id: 0,
..Default::default()
},
);
state
let output = BlockExecutionOutput::<Receipt> {
state: BundleState::default(),
result: BlockExecutionResult {
receipts: vec![],
requests: Requests::default(),
gas_used: 21000,
blob_gas_used: 0,
},
};
let executor = MockExecutor::new(state);
// Execute (will fail but should still update some metrics)
let _result = metrics.execute_metered::<_, EmptyDB>(
executor,
input.clone_transactions_recovered().map(Ok::<_, BlockExecutionError>),
input.transaction_count(),
state_hook,
);
metrics.record_block_execution(&output, Duration::from_millis(100));
let snapshot = snapshotter.snapshot().into_vec();

View File

@@ -30,11 +30,13 @@ use reth_payload_primitives::{
};
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
use reth_provider::{
BlockReader, DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StateProviderBox,
StateProviderFactory, StateReader, TransactionVariant, TrieReader,
BlockExecutionOutput, BlockExecutionResult, BlockNumReader, BlockReader, ChangeSetReader,
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
StateProviderBox, StateProviderFactory, StateReader, TransactionVariant,
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
use reth_trie_db::ChangesetCache;
use revm::state::EvmState;
use state::TreeState;
use std::{fmt::Debug, ops, sync::Arc, time::Instant};
@@ -271,6 +273,8 @@ where
engine_kind: EngineApiKind,
/// The EVM configuration.
evm_config: C,
/// Changeset cache for in-memory trie changesets
changeset_cache: ChangesetCache,
}
impl<N, P: Debug, T: PayloadTypes + Debug, V: Debug, C> std::fmt::Debug
@@ -295,6 +299,7 @@ where
.field("metrics", &self.metrics)
.field("engine_kind", &self.engine_kind)
.field("evm_config", &self.evm_config)
.field("changeset_cache", &self.changeset_cache)
.finish()
}
}
@@ -307,11 +312,12 @@ where
+ StateProviderFactory
+ StateReader<Receipt = N::Receipt>
+ HashedPostStateProvider
+ TrieReader
+ Clone
+ 'static,
<P as DatabaseProviderFactory>::Provider:
BlockReader<Block = N::Block, Header = N::BlockHeader>,
<P as DatabaseProviderFactory>::Provider: BlockReader<Block = N::Block, Header = N::BlockHeader>
+ StageCheckpointReader
+ ChangeSetReader
+ BlockNumReader,
C: ConfigureEvm<Primitives = N> + 'static,
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
V: EngineValidator<T>,
@@ -331,6 +337,7 @@ where
config: TreeConfig,
engine_kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
) -> Self {
let (incoming_tx, incoming) = crossbeam_channel::unbounded();
@@ -351,6 +358,7 @@ where
incoming_tx,
engine_kind,
evm_config,
changeset_cache,
}
}
@@ -370,6 +378,7 @@ where
config: TreeConfig,
kind: EngineApiKind,
evm_config: C,
changeset_cache: ChangesetCache,
) -> (Sender<FromEngine<EngineApiRequest<T, N>, N::Block>>, UnboundedReceiver<EngineApiEvent<N>>)
{
let best_block_number = provider.best_block_number().unwrap_or(0);
@@ -401,6 +410,7 @@ where
config,
kind,
evm_config,
changeset_cache,
);
let incoming = task.incoming_tx.clone();
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
@@ -582,7 +592,7 @@ where
// null}` if the expected and the actual arrays don't match.
//
// This validation **MUST** be instantly run in all cases even during active sync process.
tracing::debug!("Payload received {:?}", payload);
let num_hash = payload.num_hash();
let engine_event = ConsensusEngineEvent::BlockReceived(num_hash);
self.emit_event(EngineApiEvent::BeaconConsensus(engine_event));
@@ -591,7 +601,6 @@ where
// Check for invalid ancestors
if let Some(invalid) = self.find_invalid_ancestor(&payload) {
tracing::debug!(target: "engine::tree", ?invalid, "found invalid ancestor for payload");
let status = self.handle_invalid_ancestor_payload(payload, invalid)?;
return Ok(TreeOutcome::new(status));
}
@@ -600,7 +609,6 @@ where
self.metrics.block_validation.record_payload_validation(start.elapsed().as_secs_f64());
let status = if self.backfill_sync_state.is_idle() {
tracing::debug!(target: "engine::tree", "inserting payload directly");
self.try_insert_payload(payload)?
} else {
self.try_buffer_payload(payload)?
@@ -639,7 +647,7 @@ where
let parent_hash = payload.parent_hash();
let mut latest_valid_hash = None;
match self.insert_payload(payload.clone()) {
match self.insert_payload(payload) {
Ok(status) => {
let status = match status {
InsertPayloadOk::Inserted(BlockStatus::Valid) => {
@@ -661,10 +669,7 @@ where
Ok(PayloadStatus::new(status, latest_valid_hash))
}
Err(error) => match error {
InsertPayloadError::Block(error) => {
tracing::debug!("payload in new payload l 617 {:?}", payload);
Ok(self.on_insert_block_error(error)?)
}
InsertPayloadError::Block(error) => Ok(self.on_insert_block_error(error)?),
InsertPayloadError::Payload(error) => {
Ok(self.on_new_payload_error(error, num_hash, parent_hash)?)
}
@@ -1370,6 +1375,21 @@ where
debug!(target: "engine::tree", ?last_persisted_block_hash, ?last_persisted_block_number, elapsed=?start_time.elapsed(), "Finished persisting, calling finish");
self.persistence_state.finish(last_persisted_block_hash, last_persisted_block_number);
// Evict trie changesets for blocks below the finalized block, but keep at least 64 blocks
if let Some(finalized) = self.canonical_in_memory_state.get_finalized_num_hash() {
let min_threshold = last_persisted_block_number.saturating_sub(64);
let eviction_threshold = finalized.number.min(min_threshold);
debug!(
target: "engine::tree",
last_persisted = last_persisted_block_number,
finalized_number = finalized.number,
eviction_threshold,
"Evicting changesets below threshold"
);
self.changeset_cache.evict(eviction_threshold);
}
self.on_new_persisted_block()?;
Ok(())
}
@@ -1458,7 +1478,7 @@ where
self.metrics.engine.forkchoice_updated.update_response_metrics(
start,
&mut self.metrics.engine.new_payload.latest_at,
&mut self.metrics.engine.new_payload.latest_finish_at,
has_attrs,
&output,
);
@@ -1656,6 +1676,18 @@ where
)));
return Ok(());
}
} else {
// We don't have the head block or any of its ancestors buffered. Request
// a download for the head block which will then trigger further sync.
debug!(
target: "engine::tree",
head_hash = %sync_target_state.head_block_hash,
"Backfill complete but head block not buffered, requesting download"
);
self.emit_event(EngineApiEvent::Download(DownloadRequest::single_block(
sync_target_state.head_block_hash,
)));
return Ok(());
}
// try to close the gap by executing buffered blocks that are child blocks of the new head
@@ -1823,6 +1855,7 @@ where
/// or the database. If the required historical data (such as trie change sets) has been
/// pruned for a given block, this operation will return an error. On archive nodes, it
/// can retrieve any block.
#[instrument(level = "debug", target = "engine::tree", skip(self))]
fn canonical_block_by_hash(&self, hash: B256) -> ProviderResult<Option<ExecutedBlock<N>>> {
trace!(target: "engine::tree", ?hash, "Fetching executed block by hash");
// check memory first
@@ -1835,12 +1868,23 @@ where
.sealed_block_with_senders(hash.into(), TransactionVariant::WithHash)?
.ok_or_else(|| ProviderError::HeaderNotFound(hash.into()))?
.split_sealed();
let execution_output = self
let mut execution_output = self
.provider
.get_state(block.header().number())?
.ok_or_else(|| ProviderError::StateForNumberNotFound(block.header().number()))?;
let hashed_state = self.provider.hashed_post_state(execution_output.state());
let trie_updates = self.provider.get_block_trie_updates(block.number())?;
debug!(
target: "engine::tree",
number = ?block.number(),
"computing block trie updates",
);
let db_provider = self.provider.database_provider_ro()?;
let trie_updates = reth_trie_db::compute_block_trie_updates(
&self.changeset_cache,
&db_provider,
block.number(),
)?;
let sorted_hashed_state = Arc::new(hashed_state.into_sorted());
let sorted_trie_updates = Arc::new(trie_updates);
@@ -1848,9 +1892,19 @@ where
let trie_data =
ComputedTrieData::without_trie_input(sorted_hashed_state, sorted_trie_updates);
let execution_output = Arc::new(BlockExecutionOutput {
state: execution_output.bundle,
result: BlockExecutionResult {
receipts: execution_output.receipts.pop().unwrap_or_default(),
requests: execution_output.requests.pop().unwrap_or_default(),
gas_used: block.gas_used(),
blob_gas_used: block.blob_gas_used().unwrap_or_default(),
},
});
Ok(Some(ExecutedBlock::new(
Arc::new(RecoveredBlock::new_sealed(block, senders)),
Arc::new(execution_output),
execution_output,
trie_data,
)))
}

View File

@@ -6,7 +6,6 @@ use alloy_primitives::{keccak256, Address, StorageKey, U256};
use reth_primitives_traits::Account;
use reth_provider::{AccountReader, ProviderError};
use reth_trie::{HashedPostState, HashedStorage};
use revm_primitives::B256;
use std::ops::Range;
/// Returns the total number of storage slots (both changed and read-only) across all accounts in
@@ -102,7 +101,7 @@ impl<'a> Iterator for BALSlotIter<'a> {
return None;
}
return Some((address, slot.into()));
return Some((address, StorageKey::from(slot)));
}
// Move to next account
@@ -178,7 +177,7 @@ where
let mut storage_map = HashedStorage::new(false);
for slot_changes in &account_changes.storage_changes {
let hashed_slot = keccak256(B256::from(slot_changes.slot));
let hashed_slot = keccak256(slot_changes.slot.to_be_bytes::<32>());
// Get the last change for this slot
if let Some(last_change) = slot_changes.changes.last() {
@@ -199,7 +198,7 @@ mod tests {
use alloy_eip7928::{
AccountChanges, BalanceChange, CodeChange, NonceChange, SlotChanges, StorageChange,
};
use alloy_primitives::{Address, Bytes, B256};
use alloy_primitives::{Address, Bytes, StorageKey, B256};
use reth_revm::test_utils::StateProviderTest;
#[test]
@@ -257,7 +256,7 @@ mod tests {
assert!(result.storages.contains_key(&hashed_address));
let storage = result.storages.get(&hashed_address).unwrap();
let hashed_slot = keccak256(B256::from(slot));
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
let stored_value = storage.storage.get(&hashed_slot).unwrap();
assert_eq!(*stored_value, value);
@@ -417,7 +416,7 @@ mod tests {
let hashed_address = keccak256(address);
let storage = result.storages.get(&hashed_address).unwrap();
let hashed_slot = keccak256(B256::from(slot));
let hashed_slot = keccak256(slot.to_be_bytes::<32>());
let stored_value = storage.storage.get(&hashed_slot).unwrap();

View File

@@ -19,6 +19,7 @@ use alloy_evm::{block::StateChangeSource, ToTxEnv};
use alloy_primitives::B256;
use crossbeam_channel::Sender as CrossbeamSender;
use executor::WorkloadExecutor;
use metrics::Counter;
use multiproof::{SparseTrieUpdate, *};
use parking_lot::RwLock;
use prewarm::PrewarmMetrics;
@@ -28,10 +29,11 @@ use reth_evm::{
ConfigureEvm, EvmEnvFor, ExecutableTxIterator, ExecutableTxTuple, OnStateHook, SpecFor,
TxEnvFor,
};
use reth_execution_types::ExecutionOutcome;
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
BlockReader, DatabaseProviderROFactory, StateProvider, StateProviderFactory, StateReader,
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
StateProviderFactory, StateReader,
};
use reth_revm::{db::BundleState, state::EvmState};
use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory};
@@ -61,6 +63,7 @@ mod configured_sparse_trie;
pub mod executor;
pub mod multiproof;
pub mod prewarm;
pub mod receipt_root_task;
pub mod sparse_trie;
use configured_sparse_trie::ConfiguredSparseTrie;
@@ -138,6 +141,8 @@ where
disable_parallel_sparse_trie: bool,
/// Maximum concurrency for prewarm task.
prewarm_max_concurrency: usize,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
impl<N, Evm> PayloadProcessor<Evm>
@@ -170,6 +175,7 @@ where
sparse_state_trie: Arc::default(),
disable_parallel_sparse_trie: config.disable_parallel_sparse_trie(),
prewarm_max_concurrency: config.prewarm_max_concurrency(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
}
@@ -299,7 +305,7 @@ where
// Build a state provider for the multiproof task
let provider = provider_builder.build().expect("failed to build provider");
let provider = if let Some(saved_cache) = saved_cache {
let (cache, metrics) = saved_cache.split();
let (cache, metrics, _) = saved_cache.split();
Box::new(CachedStateProvider::new(provider, cache, metrics))
as Box<dyn StateProvider>
} else {
@@ -476,6 +482,7 @@ where
debug!("creating new execution cache on cache miss");
let cache = ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size);
SavedCache::new(parent_hash, cache, CachedStateMetrics::zeroed())
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
@@ -557,6 +564,7 @@ where
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
) {
let disable_cache_metrics = self.disable_cache_metrics;
self.execution_cache.update_with_guard(|cached| {
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
debug!(
@@ -570,7 +578,8 @@ where
// Take existing cache (if any) or create fresh caches
let (caches, cache_metrics) = match cached.take() {
Some(existing) => {
existing.split()
let (c, m, _) = existing.split();
(c, m)
}
None => (
ExecutionCacheBuilder::default().build_caches(self.cross_block_cache_size),
@@ -579,7 +588,8 @@ where
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics);
let new_cache = SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
@@ -665,13 +675,15 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
/// Terminates the entire caching task.
///
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` allows sharing with the main execution
/// path without cloning the expensive `BundleState`.
///
/// Returns a sender for the channel that should be notified on block validation success.
pub(super) fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
) {
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
self.prewarm_handle.terminate_caching(execution_outcome)
}
@@ -707,15 +719,21 @@ impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
/// Terminates the entire pre-warming task.
///
/// If the [`ExecutionOutcome`] is provided it will update the shared cache using its
/// If the [`BlockExecutionOutput`] is provided it will update the shared cache using its
/// bundle state. Using `Arc<ExecutionOutcome>` avoids cloning the expensive `BundleState`.
#[must_use = "sender must be used and notified on block validation success"]
pub(super) fn terminate_caching(
&mut self,
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
) {
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
) -> Option<mpsc::Sender<()>> {
if let Some(tx) = self.to_prewarm_task.take() {
let event = PrewarmTaskEvent::Terminate { execution_outcome };
let (valid_block_tx, valid_block_rx) = mpsc::channel();
let event = PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx };
let _ = tx.send(event);
Some(valid_block_tx)
} else {
None
}
}
}
@@ -724,7 +742,10 @@ impl<R> Drop for CacheTaskHandle<R> {
fn drop(&mut self) {
// Ensure we always terminate on drop - send None without needing Send + Sync bounds
if let Some(tx) = self.to_prewarm_task.take() {
let _ = tx.send(PrewarmTaskEvent::Terminate { execution_outcome: None });
let _ = tx.send(PrewarmTaskEvent::Terminate {
execution_outcome: None,
valid_block_rx: mpsc::channel().1,
});
}
}
}
@@ -758,6 +779,8 @@ impl<R> Drop for CacheTaskHandle<R> {
struct ExecutionCache {
/// Guarded cloneable cache identified by a block hash.
inner: Arc<RwLock<Option<SavedCache>>>,
/// Metrics for cache operations.
metrics: ExecutionCacheMetrics,
}
impl ExecutionCache {
@@ -799,6 +822,10 @@ impl ExecutionCache {
if hash_matches && available {
return Some(c.clone());
}
if hash_matches && !available {
self.metrics.execution_cache_in_use.increment(1);
}
} else {
debug!(target: "engine::caching", %parent_hash, "No cache found");
}
@@ -834,6 +861,15 @@ impl ExecutionCache {
}
}
/// Metrics for execution cache operations.
#[derive(Metrics, Clone)]
#[metrics(scope = "consensus.engine.beacon")]
pub(crate) struct ExecutionCacheMetrics {
/// Counter for when the execution cache was unavailable because other threads
/// (e.g., prewarming) are still using it.
pub(crate) execution_cache_in_use: Counter,
}
/// EVM context required to execute a block.
#[derive(Debug, Clone)]
pub struct ExecutionEnv<Evm: ConfigureEvm> {
@@ -886,6 +922,7 @@ mod tests {
use reth_revm::db::BundleState;
use reth_testing_utils::generators;
use reth_trie::{test_utils::state_root, HashedPostState};
use reth_trie_db::ChangesetCache;
use revm_primitives::{Address, HashMap, B256, KECCAK_EMPTY, U256};
use revm_state::{AccountInfo, AccountStatus, EvmState, EvmStorageSlot};
use std::sync::Arc;
@@ -1060,10 +1097,10 @@ mod tests {
code: Some(Default::default()),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::Touched,
transaction_id: 0,
..Default::default()
};
state_update.insert(address, account);
@@ -1143,7 +1180,7 @@ mod tests {
std::convert::identity,
),
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(provider_factory),
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
&TreeConfig::default(),
None, // No BAL for test
);

View File

@@ -141,22 +141,27 @@ impl ProofSequencer {
/// Adds a proof with the corresponding state update and returns all sequential proofs and state
/// updates if we have a continuous sequence
fn add_proof(&mut self, sequence: u64, update: SparseTrieUpdate) -> Vec<SparseTrieUpdate> {
if sequence >= self.next_to_deliver {
// Optimization: fast path for in-order delivery to avoid BTreeMap overhead.
// If this is the expected sequence, return it immediately without buffering.
if sequence == self.next_to_deliver {
let mut consecutive_proofs = Vec::with_capacity(1);
consecutive_proofs.push(update);
self.next_to_deliver += 1;
// Check if we have subsequent proofs in the pending buffer
while let Some(pending) = self.pending_proofs.remove(&self.next_to_deliver) {
consecutive_proofs.push(pending);
self.next_to_deliver += 1;
}
return consecutive_proofs;
}
if sequence > self.next_to_deliver {
self.pending_proofs.insert(sequence, update);
}
let mut consecutive_proofs = Vec::with_capacity(self.pending_proofs.len());
let mut current_sequence = self.next_to_deliver;
// keep collecting proofs and state updates as long as we have consecutive sequence numbers
while let Some(pending) = self.pending_proofs.remove(&current_sequence) {
consecutive_proofs.push(pending);
current_sequence += 1;
}
self.next_to_deliver += consecutive_proofs.len() as u64;
consecutive_proofs
Vec::new()
}
/// Returns true if we still have pending proofs
@@ -1318,79 +1323,15 @@ mod tests {
use reth_provider::{
providers::OverlayStateProviderFactory, test_utils::create_test_provider_factory,
BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory, LatestStateProvider,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox, TrieReader,
PruneCheckpointReader, StageCheckpointReader, StateProviderBox,
};
use reth_trie::MultiProof;
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::proof_task::{ProofTaskCtx, ProofWorkerHandle};
use revm_primitives::{B256, U256};
use std::{
mem,
sync::{Arc, OnceLock},
};
use std::sync::{Arc, OnceLock};
use tokio::runtime::{Handle, Runtime};
/// Maximum number of targets to batch together for state update batching.
const STATE_UPDATE_MAX_BATCH_TARGETS: usize = 64;
/// Checks whether two `Source` values refer to the same origin.
fn same_source(lhs: Source, rhs: Source) -> bool {
match (lhs, rhs) {
(Source::Evm(a), Source::Evm(b)) => same_state_change_source(a, b),
(Source::BlockAccessList, Source::BlockAccessList) => true,
_ => false,
}
}
/// Checks whether two state change sources refer to the same origin.
fn same_state_change_source(lhs: StateChangeSource, rhs: StateChangeSource) -> bool {
match (lhs, rhs) {
(StateChangeSource::Transaction(a), StateChangeSource::Transaction(b)) => a == b,
(StateChangeSource::PreBlock(a), StateChangeSource::PreBlock(b)) => {
mem::discriminant(&a) == mem::discriminant(&b)
}
(StateChangeSource::PostBlock(a), StateChangeSource::PostBlock(b)) => {
mem::discriminant(&a) == mem::discriminant(&b)
}
_ => false,
}
}
/// Determines if a state update can be batched with the current batch.
fn can_batch_state_update(
batch_source: Source,
batch_update: &EvmState,
next_source: Source,
next_update: &EvmState,
) -> bool {
if !same_source(batch_source, next_source) {
return false;
}
match (batch_source, next_source) {
(
Source::Evm(StateChangeSource::PreBlock(_)),
Source::Evm(StateChangeSource::PreBlock(_)),
) |
(
Source::Evm(StateChangeSource::PostBlock(_)),
Source::Evm(StateChangeSource::PostBlock(_)),
) => batch_update == next_update,
_ => true,
}
}
/// Estimates target count from `EvmState` for batching decisions.
fn estimate_evm_state_targets(state: &EvmState) -> usize {
state
.values()
.filter(|account| account.is_touched())
.map(|account| {
let changed_slots = account.storage.iter().filter(|(_, v)| v.is_changed()).count();
1 + changed_slots
})
.sum()
}
/// Get a handle to the test runtime, creating it if necessary
fn get_test_runtime_handle() -> Handle {
static TEST_RT: OnceLock<Runtime> = OnceLock::new();
@@ -1406,7 +1347,6 @@ mod tests {
where
F: DatabaseProviderFactory<
Provider: BlockReader
+ TrieReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
@@ -1416,7 +1356,8 @@ mod tests {
+ 'static,
{
let rt_handle = get_test_runtime_handle();
let overlay_factory = OverlayStateProviderFactory::new(factory);
let changeset_cache = ChangesetCache::new();
let overlay_factory = OverlayStateProviderFactory::new(factory, changeset_cache);
let task_ctx = ProofTaskCtx::new(overlay_factory);
let proof_handle = ProofWorkerHandle::new(rt_handle, task_ctx, 1, 1, false);
let (to_sparse_trie, _receiver) = std::sync::mpsc::channel();
@@ -1428,7 +1369,7 @@ mod tests {
fn create_cached_provider<F>(factory: F) -> CachedStateProvider<StateProviderBox>
where
F: DatabaseProviderFactory<
Provider: BlockReader + TrieReader + StageCheckpointReader + PruneCheckpointReader,
Provider: BlockReader + StageCheckpointReader + PruneCheckpointReader,
> + Clone
+ Send
+ 'static,
@@ -1840,328 +1781,6 @@ mod tests {
assert_eq!(proofs_requested, 1);
}
/// Verifies that consecutive state update messages from the same source are batched together.
#[test]
fn test_state_update_batching() {
use alloy_evm::block::StateChangeSource;
use revm_state::Account;
let test_provider_factory = create_test_provider_factory();
let mut task = create_test_state_root_task(test_provider_factory);
// create multiple state updates
let addr1 = alloy_primitives::Address::random();
let addr2 = alloy_primitives::Address::random();
let mut update1 = EvmState::default();
update1.insert(
addr1,
Account {
info: revm_state::AccountInfo {
balance: U256::from(100),
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
},
original_info: Default::default(),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
},
);
let mut update2 = EvmState::default();
update2.insert(
addr2,
Account {
info: revm_state::AccountInfo {
balance: U256::from(200),
nonce: 2,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
},
original_info: Default::default(),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
},
);
let source = StateChangeSource::Transaction(0);
let tx = task.tx.clone();
tx.send(MultiProofMessage::StateUpdate(source.into(), update1.clone())).unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), update2.clone())).unwrap();
let proofs_requested =
if let Ok(MultiProofMessage::StateUpdate(_src, update)) = task.rx.recv() {
let mut merged_update = update;
let mut num_batched = 1;
while let Ok(MultiProofMessage::StateUpdate(_next_source, next_update)) =
task.rx.try_recv()
{
merged_update.extend(next_update);
num_batched += 1;
}
assert_eq!(num_batched, 2);
assert_eq!(merged_update.len(), 2);
assert!(merged_update.contains_key(&addr1));
assert!(merged_update.contains_key(&addr2));
task.on_state_update(source.into(), merged_update)
} else {
panic!("Expected StateUpdate message");
};
assert_eq!(proofs_requested, 1);
}
/// Verifies that state updates from different sources are not batched together.
#[test]
fn test_state_update_batching_separates_sources() {
use alloy_evm::block::StateChangeSource;
use revm_state::Account;
let test_provider_factory = create_test_provider_factory();
let task = create_test_state_root_task(test_provider_factory);
let addr_a1 = alloy_primitives::Address::random();
let addr_b1 = alloy_primitives::Address::random();
let addr_a2 = alloy_primitives::Address::random();
let create_state_update = |addr: alloy_primitives::Address, balance: u64| {
let mut state = EvmState::default();
state.insert(
addr,
Account {
info: revm_state::AccountInfo {
balance: U256::from(balance),
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
},
original_info: Default::default(),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
},
);
state
};
let source_a = StateChangeSource::Transaction(1);
let source_b = StateChangeSource::Transaction(2);
// Queue: A1 (immediate dispatch), B1 (batched), A2 (should become pending)
let tx = task.tx.clone();
tx.send(MultiProofMessage::StateUpdate(source_a.into(), create_state_update(addr_a1, 100)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source_b.into(), create_state_update(addr_b1, 200)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source_a.into(), create_state_update(addr_a2, 300)))
.unwrap();
let mut pending_msg: Option<MultiProofMessage> = None;
if let Ok(MultiProofMessage::StateUpdate(first_source, _)) = task.rx.recv() {
assert!(same_source(first_source, source_a.into()));
// Simulate batching loop for remaining messages
let mut accumulated_updates: Vec<(Source, EvmState)> = Vec::new();
let mut accumulated_targets = 0usize;
loop {
if accumulated_targets >= STATE_UPDATE_MAX_BATCH_TARGETS {
break;
}
match task.rx.try_recv() {
Ok(MultiProofMessage::StateUpdate(next_source, next_update)) => {
if let Some((batch_source, batch_update)) = accumulated_updates.first() &&
!can_batch_state_update(
*batch_source,
batch_update,
next_source,
&next_update,
)
{
pending_msg =
Some(MultiProofMessage::StateUpdate(next_source, next_update));
break;
}
let next_estimate = estimate_evm_state_targets(&next_update);
if next_estimate > STATE_UPDATE_MAX_BATCH_TARGETS {
pending_msg =
Some(MultiProofMessage::StateUpdate(next_source, next_update));
break;
}
if accumulated_targets + next_estimate > STATE_UPDATE_MAX_BATCH_TARGETS &&
!accumulated_updates.is_empty()
{
pending_msg =
Some(MultiProofMessage::StateUpdate(next_source, next_update));
break;
}
accumulated_targets += next_estimate;
accumulated_updates.push((next_source, next_update));
}
Ok(other_msg) => {
pending_msg = Some(other_msg);
break;
}
Err(_) => break,
}
}
assert_eq!(accumulated_updates.len(), 1, "Should only batch matching sources");
let batch_source = accumulated_updates[0].0;
assert!(same_source(batch_source, source_b.into()));
let batch_source = accumulated_updates[0].0;
let mut merged_update = accumulated_updates.remove(0).1;
for (_, next_update) in accumulated_updates {
merged_update.extend(next_update);
}
assert!(same_source(batch_source, source_b.into()), "Batch should use matching source");
assert!(merged_update.contains_key(&addr_b1));
assert!(!merged_update.contains_key(&addr_a1));
assert!(!merged_update.contains_key(&addr_a2));
} else {
panic!("Expected first StateUpdate");
}
match pending_msg {
Some(MultiProofMessage::StateUpdate(pending_source, pending_update)) => {
assert!(same_source(pending_source, source_a.into()));
assert!(pending_update.contains_key(&addr_a2));
}
other => panic!("Expected pending StateUpdate with source_a, got {:?}", other),
}
}
/// Verifies that pre-block updates only batch when their payloads are identical.
#[test]
fn test_pre_block_updates_require_payload_match_to_batch() {
use alloy_evm::block::{StateChangePreBlockSource, StateChangeSource};
use revm_state::Account;
let test_provider_factory = create_test_provider_factory();
let task = create_test_state_root_task(test_provider_factory);
let addr1 = alloy_primitives::Address::random();
let addr2 = alloy_primitives::Address::random();
let addr3 = alloy_primitives::Address::random();
let create_state_update = |addr: alloy_primitives::Address, balance: u64| {
let mut state = EvmState::default();
state.insert(
addr,
Account {
info: revm_state::AccountInfo {
balance: U256::from(balance),
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
},
original_info: Default::default(),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
},
);
state
};
let source = StateChangeSource::PreBlock(StateChangePreBlockSource::BeaconRootContract);
// Queue: first update dispatched immediately, next two should not merge
let tx = task.tx.clone();
tx.send(MultiProofMessage::StateUpdate(source.into(), create_state_update(addr1, 100)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), create_state_update(addr2, 200)))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(source.into(), create_state_update(addr3, 300)))
.unwrap();
let mut pending_msg: Option<MultiProofMessage> = None;
if let Ok(MultiProofMessage::StateUpdate(first_source, first_update)) = task.rx.recv() {
assert!(same_source(first_source, source.into()));
assert!(first_update.contains_key(&addr1));
let mut accumulated_updates: Vec<(Source, EvmState)> = Vec::new();
let mut accumulated_targets = 0usize;
loop {
if accumulated_targets >= STATE_UPDATE_MAX_BATCH_TARGETS {
break;
}
match task.rx.try_recv() {
Ok(MultiProofMessage::StateUpdate(next_source, next_update)) => {
if let Some((batch_source, batch_update)) = accumulated_updates.first() &&
!can_batch_state_update(
*batch_source,
batch_update,
next_source,
&next_update,
)
{
pending_msg =
Some(MultiProofMessage::StateUpdate(next_source, next_update));
break;
}
let next_estimate = estimate_evm_state_targets(&next_update);
if next_estimate > STATE_UPDATE_MAX_BATCH_TARGETS {
pending_msg =
Some(MultiProofMessage::StateUpdate(next_source, next_update));
break;
}
if accumulated_targets + next_estimate > STATE_UPDATE_MAX_BATCH_TARGETS &&
!accumulated_updates.is_empty()
{
pending_msg =
Some(MultiProofMessage::StateUpdate(next_source, next_update));
break;
}
accumulated_targets += next_estimate;
accumulated_updates.push((next_source, next_update));
}
Ok(other_msg) => {
pending_msg = Some(other_msg);
break;
}
Err(_) => break,
}
}
assert_eq!(
accumulated_updates.len(),
1,
"Second pre-block update should not merge with a different payload"
);
let (batched_source, batched_update) = accumulated_updates.remove(0);
assert!(same_source(batched_source, source.into()));
assert!(batched_update.contains_key(&addr2));
assert!(!batched_update.contains_key(&addr3));
match pending_msg {
Some(MultiProofMessage::StateUpdate(_, pending_update)) => {
assert!(pending_update.contains_key(&addr3));
}
other => panic!("Expected pending third pre-block update, got {:?}", other),
}
} else {
panic!("Expected first StateUpdate");
}
}
/// Verifies that different message types arriving mid-batch are not lost and preserve order.
#[test]
fn test_batching_preserves_ordering_with_different_message_type() {
@@ -2197,9 +1816,9 @@ mod tests {
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
account_id: None,
},
original_info: Default::default(),
original_info: Box::new(revm_state::AccountInfo::default()),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
@@ -2216,9 +1835,9 @@ mod tests {
nonce: 2,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
account_id: None,
},
original_info: Default::default(),
original_info: Box::new(revm_state::AccountInfo::default()),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
@@ -2320,9 +1939,9 @@ mod tests {
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
account_id: None,
},
original_info: Default::default(),
original_info: Box::new(revm_state::AccountInfo::default()),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
@@ -2369,157 +1988,7 @@ mod tests {
assert_eq!(targets.len(), 1);
assert!(targets.contains_key(&prefetch_addr2));
}
other => panic!("Expected remaining PrefetchProofs2 in pending_msg, got {:?}", other),
}
}
/// Verifies that pending messages from a previous batch drain get full batching treatment.
#[test]
fn test_pending_messages_get_full_batching_treatment() {
// Queue: [Prefetch1, State1, State2, State3, Prefetch2]
//
// Expected behavior:
// 1. recv() → Prefetch1
// 2. try_recv() → State1 is different type → pending = State1, break
// 3. Process Prefetch1
// 4. Next iteration: pending = State1 → process with batching
// 5. try_recv() → State2 same type → merge
// 6. try_recv() → State3 same type → merge
// 7. try_recv() → Prefetch2 different type → pending = Prefetch2, break
// 8. Process merged State (1+2+3)
//
// Without the state-machine fix, State1 would be processed alone (no batching).
use alloy_evm::block::StateChangeSource;
use revm_state::Account;
let test_provider_factory = create_test_provider_factory();
let task = create_test_state_root_task(test_provider_factory);
let prefetch_addr1 = B256::random();
let prefetch_addr2 = B256::random();
let state_addr1 = alloy_primitives::Address::random();
let state_addr2 = alloy_primitives::Address::random();
let state_addr3 = alloy_primitives::Address::random();
// Create Prefetch targets
let mut prefetch1 = MultiProofTargets::default();
prefetch1.insert(prefetch_addr1, HashSet::default());
let mut prefetch2 = MultiProofTargets::default();
prefetch2.insert(prefetch_addr2, HashSet::default());
// Create StateUpdates
let create_state_update = |addr: alloy_primitives::Address, balance: u64| {
let mut state = EvmState::default();
state.insert(
addr,
Account {
info: revm_state::AccountInfo {
balance: U256::from(balance),
nonce: 1,
code_hash: Default::default(),
code: Default::default(),
account_id: Some(0),
},
original_info: Default::default(),
transaction_id: Default::default(),
storage: Default::default(),
status: revm_state::AccountStatus::Touched,
},
);
state
};
let source = StateChangeSource::Transaction(42);
// Queue: [Prefetch1, State1, State2, State3, Prefetch2]
let tx = task.tx.clone();
tx.send(MultiProofMessage::PrefetchProofs(prefetch1.clone())).unwrap();
tx.send(MultiProofMessage::StateUpdate(
source.into(),
create_state_update(state_addr1, 100),
))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(
source.into(),
create_state_update(state_addr2, 200),
))
.unwrap();
tx.send(MultiProofMessage::StateUpdate(
source.into(),
create_state_update(state_addr3, 300),
))
.unwrap();
tx.send(MultiProofMessage::PrefetchProofs(prefetch2.clone())).unwrap();
// Simulate the state-machine loop behavior
let mut pending_msg: Option<MultiProofMessage> = None;
// First iteration: recv() gets Prefetch1, drains until State1
if let Ok(MultiProofMessage::PrefetchProofs(targets)) = task.rx.recv() {
let mut merged_targets = targets;
loop {
match task.rx.try_recv() {
Ok(MultiProofMessage::PrefetchProofs(next_targets)) => {
merged_targets.extend(next_targets);
}
Ok(other_msg) => {
pending_msg = Some(other_msg);
break;
}
Err(_) => break,
}
}
// Should have only Prefetch1 (State1 is different type)
assert_eq!(merged_targets.len(), 1);
assert!(merged_targets.contains_key(&prefetch_addr1));
} else {
panic!("Expected PrefetchProofs");
}
// Pending should be State1
assert!(matches!(pending_msg, Some(MultiProofMessage::StateUpdate(_, _))));
// Second iteration: process pending State1 WITH BATCHING
// This is the key test - the pending message should drain State2 and State3
if let Some(MultiProofMessage::StateUpdate(_src, first_update)) = pending_msg.take() {
let mut merged_update = first_update;
let mut num_batched = 1;
loop {
match task.rx.try_recv() {
Ok(MultiProofMessage::StateUpdate(_src, next_update)) => {
merged_update.extend(next_update);
num_batched += 1;
}
Ok(other_msg) => {
pending_msg = Some(other_msg);
break;
}
Err(_) => break,
}
}
// THE KEY ASSERTION: pending State1 should have batched with State2 and State3
assert_eq!(
num_batched, 3,
"Pending message should get full batching treatment and merge all 3 StateUpdates"
);
assert_eq!(merged_update.len(), 3, "Should have all 3 addresses in merged update");
assert!(merged_update.contains_key(&state_addr1));
assert!(merged_update.contains_key(&state_addr2));
assert!(merged_update.contains_key(&state_addr3));
} else {
panic!("Expected pending StateUpdate");
}
// Pending should now be Prefetch2
match pending_msg {
Some(MultiProofMessage::PrefetchProofs(targets)) => {
assert_eq!(targets.len(), 1);
assert!(targets.contains_key(&prefetch_addr2));
}
_ => panic!("Prefetch2 was lost!"),
other => panic!("Expected PrefetchProofs2 in channel, got {:?}", other),
}
}

View File

@@ -30,10 +30,12 @@ use alloy_primitives::{keccak256, map::B256Set, B256};
use crossbeam_channel::Sender as CrossbeamSender;
use metrics::{Counter, Gauge, Histogram};
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, SpecFor};
use reth_execution_types::ExecutionOutcome;
use reth_metrics::Metrics;
use reth_primitives_traits::NodePrimitives;
use reth_provider::{AccountReader, BlockReader, StateProvider, StateProviderFactory, StateReader};
use reth_provider::{
AccountReader, BlockExecutionOutput, BlockReader, StateProvider, StateProviderFactory,
StateReader,
};
use reth_revm::{database::StateProviderDatabase, state::EvmState};
use reth_trie::MultiProofTargets;
use std::{
@@ -259,7 +261,11 @@ where
///
/// This method is called from `run()` only after all execution tasks are complete.
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn save_cache(self, execution_outcome: Arc<ExecutionOutcome<N::Receipt>>) {
fn save_cache(
self,
execution_outcome: Arc<BlockExecutionOutput<N::Receipt>>,
valid_block_rx: mpsc::Receiver<()>,
) {
let start = Instant::now();
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
@@ -272,12 +278,13 @@ where
execution_cache.update_with_guard(|cached| {
// consumes the `SavedCache` held by the prewarming task, which releases its usage
// guard
let (caches, cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics);
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
// Insert state into cache while holding the lock
// Access the BundleState through the shared ExecutionOutcome
if new_cache.cache().insert_state(execution_outcome.state()).is_err() {
if new_cache.cache().insert_state(&execution_outcome.state).is_err() {
// Clear the cache on error to prevent having a polluted cache
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
@@ -286,9 +293,16 @@ where
new_cache.update_metrics();
// Replace the shared cache with the new one; the previous cache (if any) is
// dropped.
*cached = Some(new_cache);
if valid_block_rx.recv().is_ok() {
// Replace the shared cache with the new one; the previous cache (if any) is
// dropped.
*cached = Some(new_cache);
} else {
// Block was invalid; caches were already mutated by insert_state above,
// so we must clear to prevent using polluted state
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on invalid block");
}
});
let elapsed = start.elapsed();
@@ -419,9 +433,10 @@ where
// completed executing a set of transactions
self.send_multi_proof_targets(proof_targets);
}
PrewarmTaskEvent::Terminate { execution_outcome } => {
PrewarmTaskEvent::Terminate { execution_outcome, valid_block_rx } => {
trace!(target: "engine::tree::payload_processor::prewarm", "Received termination signal");
final_execution_outcome = Some(execution_outcome);
final_execution_outcome =
Some(execution_outcome.map(|outcome| (outcome, valid_block_rx)));
if finished_execution {
// all tasks are done, we can exit, which will save caches and exit
@@ -446,8 +461,8 @@ where
debug!(target: "engine::tree::payload_processor::prewarm", "Completed prewarm execution");
// save caches and finish using the shared ExecutionOutcome
if let Some(Some(execution_outcome)) = final_execution_outcome {
self.save_cache(execution_outcome);
if let Some(Some((execution_outcome, valid_block_rx))) = final_execution_outcome {
self.save_cache(execution_outcome, valid_block_rx);
}
}
}
@@ -567,9 +582,14 @@ where
.entered();
txs.recv()
} {
let enter =
debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm tx", index, tx_hash=%tx.tx().tx_hash())
.entered();
let enter = debug_span!(
target: "engine::tree::payload_processor::prewarm",
"prewarm tx",
index,
tx_hash = %tx.tx().tx_hash(),
is_success = tracing::field::Empty,
)
.entered();
// create the tx env
let start = Instant::now();
@@ -810,7 +830,12 @@ pub(super) enum PrewarmTaskEvent<R> {
Terminate {
/// The final execution outcome. Using `Arc` allows sharing with the main execution
/// path without cloning the expensive `BundleState`.
execution_outcome: Option<Arc<ExecutionOutcome<R>>>,
execution_outcome: Option<Arc<BlockExecutionOutput<R>>>,
/// Receiver for the block validation result.
///
/// Cache saving is racing the state root validation. We optimistically construct the
/// updated cache but only save it once we know the block is valid.
valid_block_rx: mpsc::Receiver<()>,
},
/// The outcome of a pre-warm task
Outcome {

View File

@@ -0,0 +1,259 @@
//! Receipt root computation in a background task.
//!
//! This module provides a streaming receipt root builder that computes the receipt trie root
//! in a background thread. Receipts are sent via a channel with their index, and for each
//! receipt received, the builder incrementally flushes leaves to the underlying
//! [`OrderedTrieRootEncodedBuilder`] when possible. When the channel closes, the task returns the
//! computed root.
use alloy_eips::Encodable2718;
use alloy_primitives::{Bloom, B256};
use crossbeam_channel::Receiver;
use reth_primitives_traits::Receipt;
use reth_trie_common::ordered_root::OrderedTrieRootEncodedBuilder;
use tokio::sync::oneshot;
/// Receipt with index, ready to be sent to the background task for encoding and trie building.
#[derive(Debug, Clone)]
pub struct IndexedReceipt<R> {
/// The transaction index within the block.
pub index: usize,
/// The receipt.
pub receipt: R,
}
impl<R> IndexedReceipt<R> {
/// Creates a new indexed receipt.
#[inline]
pub const fn new(index: usize, receipt: R) -> Self {
Self { index, receipt }
}
}
/// Handle for running the receipt root computation in a background task.
///
/// This struct holds the channels needed to receive receipts and send the result.
/// Use [`Self::run`] to execute the computation (typically in a spawned blocking task).
#[derive(Debug)]
pub struct ReceiptRootTaskHandle<R> {
/// Receiver for indexed receipts.
receipt_rx: Receiver<IndexedReceipt<R>>,
/// Sender for the computed result.
result_tx: oneshot::Sender<(B256, Bloom)>,
}
impl<R: Receipt> ReceiptRootTaskHandle<R> {
/// Creates a new handle from the receipt receiver and result sender channels.
pub const fn new(
receipt_rx: Receiver<IndexedReceipt<R>>,
result_tx: oneshot::Sender<(B256, Bloom)>,
) -> Self {
Self { receipt_rx, result_tx }
}
/// Runs the receipt root computation, consuming the handle.
///
/// This method receives indexed receipts from the channel, encodes them,
/// and builds the trie incrementally. When all receipts have been received
/// (channel closed), it sends the result through the oneshot channel.
///
/// This is designed to be called inside a blocking task (e.g., via
/// `executor.spawn_blocking(move || handle.run(receipts_len))`).
///
/// # Arguments
///
/// * `receipts_len` - The total number of receipts expected. This is needed to correctly order
/// the trie keys according to RLP encoding rules.
pub fn run(self, receipts_len: usize) {
let mut builder = OrderedTrieRootEncodedBuilder::new(receipts_len);
let mut aggregated_bloom = Bloom::ZERO;
let mut encode_buf = Vec::new();
let mut received_count = 0usize;
for indexed_receipt in self.receipt_rx {
let receipt_with_bloom = indexed_receipt.receipt.with_bloom_ref();
encode_buf.clear();
receipt_with_bloom.encode_2718(&mut encode_buf);
aggregated_bloom |= *receipt_with_bloom.bloom_ref();
builder.push_unchecked(indexed_receipt.index, &encode_buf);
received_count += 1;
}
let Ok(root) = builder.finalize() else {
// Finalize fails if we didn't receive exactly `receipts_len` receipts. This can
// happen if execution was aborted early (e.g., invalid transaction encountered).
// We return without sending a result, allowing the caller to handle the abort.
tracing::error!(
target: "engine::tree::payload_processor",
expected = receipts_len,
received = received_count,
"Receipt root task received incomplete receipts, execution likely aborted"
);
return;
};
let _ = self.result_tx.send((root, aggregated_bloom));
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_consensus::{proofs::calculate_receipt_root, TxReceipt};
use alloy_primitives::{b256, hex, Address, Bytes, Log};
use crossbeam_channel::bounded;
use reth_ethereum_primitives::{Receipt, TxType};
#[tokio::test]
async fn test_receipt_root_task_empty() {
let (_tx, rx) = bounded::<IndexedReceipt<Receipt>>(1);
let (result_tx, result_rx) = oneshot::channel();
drop(_tx);
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
tokio::task::spawn_blocking(move || handle.run(0)).await.unwrap();
let (root, bloom) = result_rx.await.unwrap();
// Empty trie root
assert_eq!(root, reth_trie_common::EMPTY_ROOT_HASH);
assert_eq!(bloom, Bloom::ZERO);
}
#[tokio::test]
async fn test_receipt_root_task_single_receipt() {
let receipts: Vec<Receipt> = vec![Receipt::default()];
let (tx, rx) = bounded(1);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
for (i, receipt) in receipts.clone().into_iter().enumerate() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (root, _bloom) = result_rx.await.unwrap();
// Verify against the standard calculation
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
let expected_root = calculate_receipt_root(&receipts_with_bloom);
assert_eq!(root, expected_root);
}
#[tokio::test]
async fn test_receipt_root_task_multiple_receipts() {
let receipts: Vec<Receipt> = vec![Receipt::default(); 5];
let (tx, rx) = bounded(4);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
for (i, receipt) in receipts.into_iter().enumerate() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (root, bloom) = result_rx.await.unwrap();
// Verify against expected values from existing test
assert_eq!(
root,
b256!("0x61353b4fb714dc1fccacbf7eafc4273e62f3d1eed716fe41b2a0cd2e12c63ebc")
);
assert_eq!(
bloom,
Bloom::from(hex!("00000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000"))
);
}
#[tokio::test]
async fn test_receipt_root_matches_standard_calculation() {
// Create some receipts with actual data
let receipts = vec![
Receipt {
tx_type: TxType::Legacy,
cumulative_gas_used: 21000,
success: true,
logs: vec![],
},
Receipt {
tx_type: TxType::Eip1559,
cumulative_gas_used: 42000,
success: true,
logs: vec![Log {
address: Address::ZERO,
data: alloy_primitives::LogData::new_unchecked(vec![B256::ZERO], Bytes::new()),
}],
},
Receipt {
tx_type: TxType::Eip2930,
cumulative_gas_used: 63000,
success: false,
logs: vec![],
},
];
// Calculate expected values first (before we move receipts)
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
let expected_root = calculate_receipt_root(&receipts_with_bloom);
let expected_bloom =
receipts_with_bloom.iter().fold(Bloom::ZERO, |bloom, r| bloom | r.bloom_ref());
// Calculate using the task
let (tx, rx) = bounded(4);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
for (i, receipt) in receipts.into_iter().enumerate() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (task_root, task_bloom) = result_rx.await.unwrap();
assert_eq!(task_root, expected_root);
assert_eq!(task_bloom, expected_bloom);
}
#[tokio::test]
async fn test_receipt_root_task_out_of_order() {
let receipts: Vec<Receipt> = vec![Receipt::default(); 5];
// Calculate expected values first (before we move receipts)
let receipts_with_bloom: Vec<_> = receipts.iter().map(|r| r.with_bloom_ref()).collect();
let expected_root = calculate_receipt_root(&receipts_with_bloom);
let (tx, rx) = bounded(4);
let (result_tx, result_rx) = oneshot::channel();
let receipts_len = receipts.len();
let handle = ReceiptRootTaskHandle::new(rx, result_tx);
let join_handle = tokio::task::spawn_blocking(move || handle.run(receipts_len));
// Send in reverse order to test out-of-order handling
for (i, receipt) in receipts.into_iter().enumerate().rev() {
tx.send(IndexedReceipt::new(i, receipt)).unwrap();
}
drop(tx);
join_handle.await.unwrap();
let (root, _bloom) = result_rx.await.unwrap();
assert_eq!(root, expected_root);
}
}

View File

@@ -1,11 +1,5 @@
//! Types and traits for validating blocks and payloads.
/// Threshold for switching from `extend_ref` loop to `merge_batch` in `merge_overlay_trie_input`.
///
/// Benchmarked crossover: `extend_ref` wins up to ~64 blocks, `merge_batch` wins beyond.
/// Using 64 as threshold since they're roughly equal there.
const MERGE_BATCH_THRESHOLD: usize = 64;
use crate::tree::{
cached_state::CachedStateProvider,
error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError},
@@ -13,17 +7,19 @@ use crate::tree::{
payload_processor::{executor::WorkloadExecutor, PayloadProcessor},
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
sparse_trie::StateRootComputeOutcome,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
StateProviderDatabase, TreeConfig,
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, MeteredStateHook, PayloadHandle,
StateProviderBuilder, StateProviderDatabase, TreeConfig,
};
use alloy_consensus::transaction::Either;
use alloy_consensus::transaction::{Either, TxHashRef};
use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip1898::BlockWithParent, NumHash};
use alloy_evm::Evm;
use alloy_primitives::B256;
use crate::tree::payload_processor::receipt_root_task::{IndexedReceipt, ReceiptRootTaskHandle};
use rayon::prelude::*;
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock};
use reth_consensus::{ConsensusError, FullConsensus};
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock, LazyOverlay};
use reth_consensus::{ConsensusError, FullConsensus, ReceiptRootBloom};
use reth_engine_primitives::{
ConfigureEngineEvm, ExecutableTxIterator, ExecutionPayload, InvalidBlockHook, PayloadValidator,
};
@@ -41,15 +37,13 @@ use reth_primitives_traits::{
};
use reth_provider::{
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, ExecutionOutcome,
HashedPostStateProvider, ProviderError, PruneCheckpointReader, StageCheckpointReader,
StateProvider, StateProviderFactory, StateReader, TrieReader,
};
use reth_revm::db::State;
use reth_trie::{
updates::{TrieUpdates, TrieUpdatesSorted},
HashedPostState, HashedPostStateSorted, StateRoot, TrieInputSorted,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader,
};
use reth_revm::db::{states::bundle_state::BundleRetention, State};
use reth_trie::{updates::TrieUpdates, HashedPostState, StateRoot};
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
use revm_primitives::Address;
use std::{
@@ -138,6 +132,8 @@ where
metrics: EngineApiMetrics,
/// Validator for the payload.
validator: V,
/// Changeset cache for in-memory trie changesets
changeset_cache: ChangesetCache,
}
impl<N, P, Evm, V> BasicEngineValidator<P, Evm, V>
@@ -145,7 +141,6 @@ where
N: NodePrimitives,
P: DatabaseProviderFactory<
Provider: BlockReader
+ TrieReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
@@ -169,6 +164,7 @@ where
validator: V,
config: TreeConfig,
invalid_block_hook: Box<dyn InvalidBlockHook<N>>,
changeset_cache: ChangesetCache,
) -> Self {
let precompile_cache_map = PrecompileCacheMap::default();
let payload_processor = PayloadProcessor::new(
@@ -188,6 +184,7 @@ where
invalid_block_hook,
metrics: EngineApiMetrics::default(),
validator,
changeset_cache,
}
}
@@ -372,7 +369,6 @@ where
}
let parent_hash = input.parent_hash();
let block_num_hash = input.num_hash();
trace!(target: "engine::tree::payload_validator", "Fetching block state provider");
let _enter =
@@ -427,13 +423,23 @@ where
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
.map(Arc::new);
// Create lazy overlay from ancestors - this doesn't block, allowing execution to start
// before the trie data is ready. The overlay will be computed on first access.
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state());
// Create overlay factory for payload processor (StateRootTask path needs it for
// multiproofs)
let overlay_factory =
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
.with_block_hash(Some(anchor_hash))
.with_lazy_overlay(lazy_overlay);
// Spawn the appropriate processor based on strategy
let mut handle = ensure_ok!(self.spawn_payload_processor(
env.clone(),
txs,
provider_builder,
parent_hash,
ctx.state(),
overlay_factory.clone(),
strategy,
block_access_list,
));
@@ -449,19 +455,48 @@ where
state_provider = Box::new(InstrumentedStateProvider::new(state_provider, "engine"));
}
// Execute the block and handle any execution errors
let (output, senders) = match self.execute_block(state_provider, env, &input, &mut handle) {
Ok(output) => output,
Err(err) => return self.handle_execution_error(input, err, &parent_block),
};
// Execute the block and handle any execution errors.
// The receipt root task is spawned before execution and receives receipts incrementally
// as transactions complete, allowing parallel computation during execution.
let (output, senders, receipt_root_rx) =
match self.execute_block(state_provider, env, &input, &mut handle) {
Ok(output) => output,
Err(err) => return self.handle_execution_error(input, err, &parent_block),
};
// After executing the block we can stop prewarming transactions
handle.stop_prewarming_execution();
// Create ExecutionOutcome early so we can terminate caching before validation and state
// root computation. Using Arc allows sharing with both the caching task and the deferred
// trie task without cloning the expensive BundleState.
let output = Arc::new(output);
// Terminate caching task early since execution is complete and caching is no longer
// needed. This frees up resources while state root computation continues.
let valid_block_tx = handle.terminate_caching(Some(output.clone()));
let block = self.convert_to_block(input)?.with_senders(senders);
// Wait for the receipt root computation to complete.
let receipt_root_bloom = receipt_root_rx
.blocking_recv()
.inspect_err(|_| {
tracing::error!(
target: "engine::tree::payload_validator",
"Receipt root task dropped sender without result, receipt root calculation likely aborted"
);
})
.ok();
let hashed_state = ensure_ok_post_block!(
self.validate_post_execution(&block, &parent_block, &output, &mut ctx),
self.validate_post_execution(
&block,
&parent_block,
&output,
&mut ctx,
receipt_root_bloom
),
block
);
@@ -494,11 +529,7 @@ where
}
StateRootStrategy::Parallel => {
debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm");
match self.compute_state_root_parallel(
block.parent_hash(),
&hashed_state,
ctx.state(),
) {
match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) {
Ok(result) => {
let elapsed = root_time.elapsed();
info!(
@@ -534,7 +565,7 @@ where
}
let (root, updates) = ensure_ok_post_block!(
self.compute_state_root_serial(block.parent_hash(), &hashed_state, ctx.state()),
self.compute_state_root_serial(overlay_factory.clone(), &hashed_state),
block
);
(root, updates, root_time.elapsed())
@@ -564,14 +595,18 @@ where
.into())
}
// Create ExecutionOutcome and wrap in Arc for sharing with both the caching task
// and the deferred trie task. This avoids cloning the expensive BundleState.
let execution_outcome = Arc::new(ExecutionOutcome::from((output, block_num_hash.number)));
if let Some(valid_block_tx) = valid_block_tx {
let _ = valid_block_tx.send(());
}
// Terminate prewarming task with the shared execution outcome
handle.terminate_caching(Some(Arc::clone(&execution_outcome)));
Ok(self.spawn_deferred_trie_task(block, execution_outcome, &ctx, hashed_state, trie_output))
Ok(self.spawn_deferred_trie_task(
block,
output,
&ctx,
hashed_state,
trie_output,
overlay_factory,
))
}
/// Return sealed block header from database or in-memory state by hash.
@@ -607,15 +642,29 @@ where
Ok(())
}
/// Executes a block with the given state provider
/// Executes a block with the given state provider.
///
/// This method orchestrates block execution:
/// 1. Sets up the EVM with state database and precompile caching
/// 2. Spawns a background task for incremental receipt root computation
/// 3. Executes transactions with metrics collection via state hooks
/// 4. Merges state transitions and records execution metrics
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
#[expect(clippy::type_complexity)]
fn execute_block<S, Err, T>(
&mut self,
state_provider: S,
env: ExecutionEnv<Evm>,
input: &BlockOrPayload<T>,
handle: &mut PayloadHandle<impl ExecutableTxFor<Evm>, Err, N::Receipt>,
) -> Result<(BlockExecutionOutput<N::Receipt>, Vec<Address>), InsertBlockErrorKind>
) -> Result<
(
BlockExecutionOutput<N::Receipt>,
Vec<Address>,
tokio::sync::oneshot::Receiver<(B256, alloy_primitives::Bloom)>,
),
InsertBlockErrorKind,
>
where
S: StateProvider + Send,
Err: core::error::Error + Send + Sync + 'static,
@@ -624,14 +673,12 @@ where
Evm: ConfigureEngineEvm<T::ExecutionData, Primitives = N>,
{
debug!(target: "engine::tree::payload_validator", "Executing block");
let mut db = State::builder()
.with_database(StateProviderDatabase::new(state_provider))
.with_bundle_update()
.with_bal_builder() //TODO
.without_state_clear()
.build();
db.bal_state.bal_index = 0;
db.bal_state.bal_builder = Some(revm::state::bal::Bal::new());
let spec_id = *env.evm_env.spec_id();
let evm = self.evm_config.evm_with_env(&mut db, env.evm_env);
@@ -656,22 +703,131 @@ where
});
}
// Spawn background task to compute receipt root and logs bloom incrementally.
// Unbounded channel is used since tx count bounds capacity anyway (max ~30k txs per block).
let receipts_len = input.transaction_count();
let (receipt_tx, receipt_rx) = crossbeam_channel::unbounded();
let (result_tx, result_rx) = tokio::sync::oneshot::channel();
let task_handle = ReceiptRootTaskHandle::new(receipt_rx, result_tx);
self.payload_processor.executor().spawn_blocking(move || task_handle.run(receipts_len));
// Wrap the state hook with metrics collection
let inner_hook = Box::new(handle.state_hook());
let state_hook =
MeteredStateHook { metrics: self.metrics.executor_metrics().clone(), inner_hook };
let transaction_count = input.transaction_count();
let executor = executor.with_state_hook(Some(Box::new(state_hook)));
let execution_start = Instant::now();
let state_hook = Box::new(handle.state_hook());
let (output, senders) = self.metrics.execute_metered(
// Execute all transactions and finalize
let (executor, senders) = self.execute_transactions(
executor,
handle.iter_transactions().map(|res| res.map_err(BlockExecutionError::other)),
input.transaction_count(),
state_hook,
transaction_count,
handle.iter_transactions(),
&receipt_tx,
)?;
let execution_finish = Instant::now();
let execution_time = execution_finish.duration_since(execution_start);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_time, "Executed block");
Ok((output, senders))
drop(receipt_tx);
// Finish execution and get the result
let post_exec_start = Instant::now();
let (_evm, result) = debug_span!(target: "engine::tree", "finish")
.in_scope(|| executor.finish())
.map(|(evm, result)| (evm.into_db(), result))?;
self.metrics.record_post_execution(post_exec_start.elapsed());
// Merge transitions into bundle state
debug_span!(target: "engine::tree", "merge transitions")
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
let output = BlockExecutionOutput { result, state: db.take_bundle() };
let execution_duration = execution_start.elapsed();
self.metrics.record_block_execution(&output, execution_duration);
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
Ok((output, senders, result_rx))
}
/// Executes transactions and collects senders, streaming receipts to a background task.
///
/// This method handles:
/// - Applying pre-execution changes (e.g., beacon root updates)
/// - Executing each transaction with timing metrics
/// - Streaming receipts to the receipt root computation task
/// - Collecting transaction senders for later use
///
/// Returns the executor (for finalization) and the collected senders.
fn execute_transactions<E, Tx, InnerTx, Err>(
&self,
mut executor: E,
transaction_count: usize,
transactions: impl Iterator<Item = Result<Tx, Err>>,
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
) -> Result<(E, Vec<Address>), BlockExecutionError>
where
E: BlockExecutor<Receipt = N::Receipt>,
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
InnerTx: TxHashRef,
Err: core::error::Error + Send + Sync + 'static,
{
let mut senders = Vec::with_capacity(transaction_count);
// Apply pre-execution changes (e.g., beacon root update)
let pre_exec_start = Instant::now();
debug_span!(target: "engine::tree", "pre execution")
.in_scope(|| executor.apply_pre_execution_changes())?;
self.metrics.record_pre_execution(pre_exec_start.elapsed());
// Execute transactions
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
let mut transactions = transactions.into_iter();
loop {
// Measure time spent waiting for next transaction from iterator
// (e.g., parallel signature recovery)
let wait_start = Instant::now();
let Some(tx_result) = transactions.next() else { break };
self.metrics.record_transaction_wait(wait_start.elapsed());
let tx = tx_result.map_err(BlockExecutionError::other)?;
let tx_signer = *<Tx as alloy_evm::RecoveredTx<InnerTx>>::signer(&tx);
let tx_hash = <Tx as alloy_evm::RecoveredTx<InnerTx>>::tx(&tx).tx_hash();
senders.push(tx_signer);
let span = debug_span!(
target: "engine::tree",
"execute tx",
?tx_hash,
gas_used = tracing::field::Empty,
);
let enter = span.entered();
trace!(target: "engine::tree", "Executing transaction");
let tx_start = Instant::now();
let gas_used = executor.execute_transaction(tx)?;
self.metrics.record_transaction_execution(tx_start.elapsed());
// Send the latest receipt to the background task for incremental root computation
if let Some(receipt) = executor.receipts().last() {
let tx_index = executor.receipts().len() - 1;
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
}
enter.record("gas_used", gas_used);
}
drop(exec_span);
Ok((executor, senders))
}
/// Compute state root for the given hashed post state in parallel.
///
/// Uses an overlay factory which provides the state of the parent block, along with the
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
/// trie updates for this block.
///
/// # Returns
///
/// Returns `Ok(_)` if computed successfully.
@@ -679,58 +835,39 @@ where
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
fn compute_state_root_parallel(
&self,
parent_hash: B256,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &HashedPostState,
state: &EngineApiTreeState<N>,
) -> Result<(B256, TrieUpdates), ParallelStateRootError> {
let (mut input, block_hash) = self.compute_trie_input(parent_hash, state)?;
// Extend state overlay with current block's sorted state.
input.prefix_sets.extend(hashed_state.construct_prefix_sets());
let sorted_hashed_state = hashed_state.clone_into_sorted();
Arc::make_mut(&mut input.state).extend_ref(&sorted_hashed_state);
let TrieInputSorted { nodes, state, prefix_sets: prefix_sets_mut } = input;
let factory = OverlayStateProviderFactory::new(self.provider.clone())
.with_block_hash(Some(block_hash))
.with_trie_overlay(Some(nodes))
.with_hashed_state_overlay(Some(state));
// The `hashed_state` argument is already taken into account as part of the overlay, but we
// The `hashed_state` argument will be taken into account as part of the overlay, but we
// need to use the prefix sets which were generated from it to indicate to the
// ParallelStateRoot which parts of the trie need to be recomputed.
let prefix_sets = prefix_sets_mut.freeze();
ParallelStateRoot::new(factory, prefix_sets).incremental_root_with_updates()
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
let overlay_factory =
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
ParallelStateRoot::new(overlay_factory, prefix_sets).incremental_root_with_updates()
}
/// Compute state root for the given hashed post state in serial.
///
/// Uses an overlay factory which provides the state of the parent block, along with the
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
/// trie updates for this block.
fn compute_state_root_serial(
&self,
parent_hash: B256,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &HashedPostState,
state: &EngineApiTreeState<N>,
) -> ProviderResult<(B256, TrieUpdates)> {
let (mut input, block_hash) = self.compute_trie_input(parent_hash, state)?;
// The `hashed_state` argument will be taken into account as part of the overlay, but we
// need to use the prefix sets which were generated from it to indicate to the
// StateRoot which parts of the trie need to be recomputed.
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
let overlay_factory =
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
// Extend state overlay with current block's sorted state.
input.prefix_sets.extend(hashed_state.construct_prefix_sets());
let sorted_hashed_state = hashed_state.clone_into_sorted();
Arc::make_mut(&mut input.state).extend_ref(&sorted_hashed_state);
let TrieInputSorted { nodes, state, .. } = input;
let prefix_sets = hashed_state.construct_prefix_sets();
let factory = OverlayStateProviderFactory::new(self.provider.clone())
.with_block_hash(Some(block_hash))
.with_trie_overlay(Some(nodes))
.with_hashed_state_overlay(Some(state));
let provider = factory.database_provider_ro()?;
let provider = overlay_factory.database_provider_ro()?;
Ok(StateRoot::new(&provider, &provider)
.with_prefix_sets(prefix_sets.freeze())
.with_prefix_sets(prefix_sets)
.root_with_updates()?)
}
@@ -740,6 +877,9 @@ where
/// - parent header validation
/// - post-execution consensus validation
/// - state-root based post-execution validation
///
/// If `receipt_root_bloom` is provided, it will be used instead of computing the receipt root
/// and logs bloom from the receipts.
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
fn validate_post_execution<T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>>(
&self,
@@ -747,6 +887,7 @@ where
parent_block: &SealedHeader<N::BlockHeader>,
output: &BlockExecutionOutput<N::Receipt>,
ctx: &mut TreeCtx<'_, N>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<HashedPostState, InsertBlockErrorKind>
where
V: PayloadValidator<T, Block = N::Block>,
@@ -773,7 +914,9 @@ where
let _enter =
debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution")
.entered();
if let Err(err) = self.consensus.validate_block_post_execution(block, output) {
if let Err(err) =
self.consensus.validate_block_post_execution(block, output, receipt_root_bloom)
{
// call post-block hook
self.on_invalid_block(parent_block, block, output, None, ctx.state_mut());
return Err(err.into())
@@ -813,6 +956,11 @@ where
///
/// The method handles strategy fallbacks if the preferred approach fails, ensuring
/// block execution always completes with a valid state root.
///
/// # Arguments
///
/// * `overlay_factory` - Pre-computed overlay factory for multiproof generation
/// (`StateRootTask`)
#[allow(clippy::too_many_arguments)]
#[instrument(
level = "debug",
@@ -825,8 +973,7 @@ where
env: ExecutionEnv<Evm>,
txs: T,
provider_builder: StateProviderBuilder<N, P>,
parent_hash: B256,
state: &EngineApiTreeState<N>,
overlay_factory: OverlayStateProviderFactory<P>,
strategy: StateRootStrategy,
block_access_list: Option<Arc<BlockAccessList>>,
) -> Result<
@@ -839,32 +986,14 @@ where
> {
match strategy {
StateRootStrategy::StateRootTask => {
// Compute trie input
let trie_input_start = Instant::now();
let (trie_input, block_hash) = self.compute_trie_input(parent_hash, state)?;
// Create OverlayStateProviderFactory with sorted trie data for multiproofs
let TrieInputSorted { nodes, state, .. } = trie_input;
let multiproof_provider_factory =
OverlayStateProviderFactory::new(self.provider.clone())
.with_block_hash(Some(block_hash))
.with_trie_overlay(Some(nodes))
.with_hashed_state_overlay(Some(state));
// Record trie input duration including OverlayStateProviderFactory setup
self.metrics
.block_validation
.trie_input_duration
.record(trie_input_start.elapsed().as_secs_f64());
let spawn_start = Instant::now();
// Use the pre-computed overlay factory for multiproofs
let handle = self.payload_processor.spawn(
env,
txs,
provider_builder,
multiproof_provider_factory,
overlay_factory,
&self.config,
block_access_list,
);
@@ -958,128 +1087,36 @@ where
self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates);
}
/// Computes [`TrieInputSorted`] for the provided parent hash by combining database state
/// with in-memory overlays.
/// Creates a [`LazyOverlay`] for the parent block without blocking.
///
/// The goal of this function is to take in-memory blocks and generate a [`TrieInputSorted`]
/// that extends from the highest persisted ancestor up through the parent. This enables state
/// root computation and proof generation without requiring all blocks to be persisted
/// first.
/// Returns a lazy overlay that will compute the trie input on first access, and the anchor
/// block hash (the highest persisted ancestor). This allows execution to start immediately
/// while the trie input computation is deferred until the overlay is actually needed.
///
/// It works as follows:
/// 1. Collect in-memory overlay blocks using [`crate::tree::TreeState::blocks_by_hash`]. This
/// returns the highest persisted ancestor hash (`block_hash`) and the list of in-memory
/// blocks building on top of it.
/// 2. Fast path: If the tip in-memory block's trie input is already anchored to `block_hash`
/// (its `anchor_hash` matches `block_hash`), reuse it directly.
/// 3. Slow path: Build a new [`TrieInputSorted`] by aggregating the overlay blocks (from oldest
/// to newest) on top of the database state at `block_hash`.
#[instrument(
level = "debug",
target = "engine::tree::payload_validator",
skip_all,
fields(parent_hash)
)]
fn compute_trie_input(
&self,
/// If parent is on disk (no in-memory blocks), returns `None` for the lazy overlay.
fn get_parent_lazy_overlay(
parent_hash: B256,
state: &EngineApiTreeState<N>,
) -> ProviderResult<(TrieInputSorted, B256)> {
let wait_start = Instant::now();
let (block_hash, blocks) =
) -> (Option<LazyOverlay>, B256) {
let (anchor_hash, blocks) =
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
// Fast path: if the tip block's anchor matches the persisted ancestor hash, reuse its
// TrieInput. This means the TrieInputSorted already aggregates all in-memory overlays
// from that ancestor, so we can avoid re-aggregation.
if let Some(tip_block) = blocks.first() {
let data = tip_block.trie_data();
if let (Some(anchor_hash), Some(trie_input)) =
(data.anchor_hash(), data.trie_input().cloned()) &&
anchor_hash == block_hash
{
trace!(target: "engine::tree::payload_validator", %block_hash,"Reusing trie input with matching anchor hash");
self.metrics
.block_validation
.deferred_trie_wait_duration
.record(wait_start.elapsed().as_secs_f64());
return Ok(((*trie_input).clone(), block_hash));
}
}
if blocks.is_empty() {
debug!(target: "engine::tree::payload_validator", "Parent found on disk");
} else {
debug!(target: "engine::tree::payload_validator", historical = ?block_hash, blocks = blocks.len(), "Parent found in memory");
debug!(target: "engine::tree::payload_validator", "Parent found on disk, no lazy overlay needed");
return (None, anchor_hash);
}
// Extend with contents of parent in-memory blocks directly in sorted form.
let input = Self::merge_overlay_trie_input(&blocks);
debug!(
target: "engine::tree::payload_validator",
%anchor_hash,
num_blocks = blocks.len(),
"Creating lazy overlay for in-memory blocks"
);
self.metrics
.block_validation
.deferred_trie_wait_duration
.record(wait_start.elapsed().as_secs_f64());
Ok((input, block_hash))
}
// Extract deferred trie data handles (non-blocking)
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
/// Aggregates in-memory blocks into a single [`TrieInputSorted`] by combining their
/// state changes.
///
/// The input `blocks` vector is ordered newest -> oldest (see `TreeState::blocks_by_hash`).
///
/// Uses `extend_ref` loop for small k, k-way `merge_batch` for large k.
/// See [`MERGE_BATCH_THRESHOLD`] for crossover point.
fn merge_overlay_trie_input(blocks: &[ExecutedBlock<N>]) -> TrieInputSorted {
if blocks.is_empty() {
return TrieInputSorted::default();
}
// Single block: return Arc directly without cloning
if blocks.len() == 1 {
let data = blocks[0].trie_data();
return TrieInputSorted {
state: Arc::clone(&data.hashed_state),
nodes: Arc::clone(&data.trie_updates),
prefix_sets: Default::default(),
};
}
if blocks.len() < MERGE_BATCH_THRESHOLD {
// Small k: extend_ref loop is faster
// Iterate oldest->newest so newer values override older ones
let mut blocks_iter = blocks.iter().rev();
let first = blocks_iter.next().expect("blocks is non-empty");
let data = first.trie_data();
let mut state = Arc::clone(&data.hashed_state);
let mut nodes = Arc::clone(&data.trie_updates);
let state_mut = Arc::make_mut(&mut state);
let nodes_mut = Arc::make_mut(&mut nodes);
for block in blocks_iter {
let data = block.trie_data();
state_mut.extend_ref(data.hashed_state.as_ref());
nodes_mut.extend_ref(data.trie_updates.as_ref());
}
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
} else {
// Large k: merge_batch is faster (O(n log k) via k-way merge)
let trie_data: Vec<_> = blocks.iter().map(|b| b.trie_data()).collect();
let merged_state = HashedPostStateSorted::merge_batch(
trie_data.iter().map(|d| d.hashed_state.as_ref()),
);
let merged_nodes =
TrieUpdatesSorted::merge_batch(trie_data.iter().map(|d| d.trie_updates.as_ref()));
TrieInputSorted {
state: Arc::new(merged_state),
nodes: Arc::new(merged_nodes),
prefix_sets: Default::default(),
}
}
(Some(LazyOverlay::new(anchor_hash, handles)), anchor_hash)
}
/// Spawns a background task to compute and sort trie data for the executed block.
@@ -1101,10 +1138,11 @@ where
fn spawn_deferred_trie_task(
&self,
block: RecoveredBlock<N::Block>,
execution_outcome: Arc<ExecutionOutcome<N::Receipt>>,
execution_outcome: Arc<BlockExecutionOutput<N::Receipt>>,
ctx: &TreeCtx<'_, N>,
hashed_state: HashedPostState,
trie_output: TrieUpdates,
overlay_factory: OverlayStateProviderFactory<P>,
) -> ExecutedBlock<N> {
// Capture parent hash and ancestor overlays for deferred trie input construction.
let (anchor_hash, overlay_blocks) = ctx
@@ -1128,9 +1166,21 @@ where
let deferred_handle_task = deferred_trie_data.clone();
let block_validation_metrics = self.metrics.block_validation.clone();
// Capture block info and cache handle for changeset computation
let block_hash = block.hash();
let block_number = block.number();
let changeset_cache = self.changeset_cache.clone();
// Spawn background task to compute trie data. Calling `wait_cloned` will compute from
// the stored inputs and cache the result, so subsequent calls return immediately.
let compute_trie_input_task = move || {
let _span = debug_span!(
target: "engine::tree::payload_validator",
"compute_trie_input_task",
block_number
)
.entered();
let result = panic::catch_unwind(AssertUnwindSafe(|| {
let compute_start = Instant::now();
let computed = deferred_handle_task.wait_cloned();
@@ -1153,6 +1203,40 @@ where
.anchored_overlay_hashed_state_size
.record(anchored.trie_input.state.total_len() as f64);
}
// Compute and cache changesets using the computed trie_updates
let changeset_start = Instant::now();
// Get a provider from the overlay factory for trie cursor access
let changeset_result =
overlay_factory.database_provider_ro().and_then(|provider| {
reth_trie::changesets::compute_trie_changesets(
&provider,
&computed.trie_updates,
)
.map_err(ProviderError::Database)
});
match changeset_result {
Ok(changesets) => {
debug!(
target: "engine::tree::changeset",
?block_number,
elapsed = ?changeset_start.elapsed(),
"Computed and caching changesets"
);
changeset_cache.insert(block_hash, block_number, Arc::new(changesets));
}
Err(e) => {
warn!(
target: "engine::tree::changeset",
?block_number,
?e,
"Failed to compute changesets in deferred trie task"
);
}
}
}));
if result.is_err() {
@@ -1249,7 +1333,6 @@ impl<N, Types, P, Evm, V> EngineValidator<Types> for BasicEngineValidator<P, Evm
where
P: DatabaseProviderFactory<
Provider: BlockReader
+ TrieReader
+ StageCheckpointReader
+ PruneCheckpointReader
+ ChangeSetReader
@@ -1302,7 +1385,7 @@ where
fn on_inserted_executed_block(&self, block: ExecutedBlock<N>) {
self.payload_processor.on_inserted_executed_block(
block.recovered_block.block_with_parent(),
block.execution_output.state(),
&block.execution_output.state,
);
}
}

View File

@@ -233,9 +233,9 @@ mod tests {
let dyn_precompile: DynPrecompile = (|_input: PrecompileInput<'_>| -> PrecompileResult {
Ok(PrecompileOutput {
gas_used: 0,
gas_refunded: 0,
bytes: Bytes::default(),
reverted: false,
gas_refunded: 0,
})
})
.into();
@@ -245,9 +245,9 @@ mod tests {
let output = PrecompileOutput {
gas_used: 50,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
reverted: false,
gas_refunded: 0,
};
let input = b"test_input";
@@ -277,9 +277,9 @@ mod tests {
Ok(PrecompileOutput {
gas_used: 5000,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
reverted: false,
gas_refunded: 0,
})
}
})
@@ -292,9 +292,9 @@ mod tests {
Ok(PrecompileOutput {
gas_used: 7000,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
reverted: false,
gas_refunded: 0,
})
}
})

View File

@@ -7,6 +7,7 @@ use crate::{
PersistTarget, TreeConfig,
},
};
use reth_trie_db::ChangesetCache;
use alloy_eips::eip1898::BlockWithParent;
use alloy_primitives::{
@@ -26,7 +27,7 @@ use reth_ethereum_engine_primitives::EthEngineTypes;
use reth_ethereum_primitives::{Block, EthPrimitives};
use reth_evm_ethereum::MockEvmConfig;
use reth_primitives_traits::Block as _;
use reth_provider::{test_utils::MockEthProvider, ExecutionOutcome};
use reth_provider::test_utils::MockEthProvider;
use std::{
collections::BTreeMap,
str::FromStr,
@@ -192,6 +193,7 @@ impl TestHarness {
let payload_builder = PayloadBuilderHandle::new(to_payload_service);
let evm_config = MockEvmConfig::default();
let changeset_cache = ChangesetCache::new();
let engine_validator = BasicEngineValidator::new(
provider.clone(),
consensus.clone(),
@@ -199,6 +201,7 @@ impl TestHarness {
payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache.clone(),
);
let tree = EngineApiTreeHandler::new(
@@ -215,6 +218,7 @@ impl TestHarness {
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
EngineApiKind::Ethereum,
evm_config,
changeset_cache,
);
let block_builder = TestBlockBuilder::default().with_chain_spec((*chain_spec).clone());
@@ -388,6 +392,7 @@ impl ValidatorTestHarness {
let provider = harness.provider.clone();
let payload_validator = MockEngineValidator;
let evm_config = MockEvmConfig::default();
let changeset_cache = ChangesetCache::new();
let validator = BasicEngineValidator::new(
provider,
@@ -396,6 +401,7 @@ impl ValidatorTestHarness {
payload_validator,
TreeConfig::default(),
Box::new(NoopInvalidBlockHook::default()),
changeset_cache,
);
Self { harness, validator, metrics: TestMetrics::default() }
@@ -832,7 +838,7 @@ fn test_tree_state_on_new_head_deep_fork() {
for block in &chain_a {
test_harness.tree.state.tree_state.insert_executed(ExecutedBlock::new(
Arc::new(block.clone()),
Arc::new(ExecutionOutcome::default()),
Arc::new(BlockExecutionOutput::default()),
empty_trie_data(),
));
}
@@ -841,7 +847,7 @@ fn test_tree_state_on_new_head_deep_fork() {
for block in &chain_b {
test_harness.tree.state.tree_state.insert_executed(ExecutedBlock::new(
Arc::new(block.clone()),
Arc::new(ExecutionOutcome::default()),
Arc::new(BlockExecutionOutput::default()),
empty_trie_data(),
));
}
@@ -1002,6 +1008,15 @@ async fn test_engine_tree_live_sync_transition_required_blocks_requested() {
_ => panic!("Unexpected event: {event:#?}"),
}
// After backfill completes with head not buffered, we also request head download
let event = test_harness.from_tree_rx.recv().await.unwrap();
match event {
EngineApiEvent::Download(DownloadRequest::BlockSet(hash_set)) => {
assert_eq!(hash_set, HashSet::from_iter([main_chain_last_hash]));
}
_ => panic!("Unexpected event: {event:#?}"),
}
let _ = test_harness
.tree
.on_engine_message(FromEngine::DownloadedBlocks(vec![main_chain

View File

@@ -27,9 +27,6 @@ reth-payload-primitives.workspace = true
alloy-rpc-types-engine.workspace = true
alloy-consensus.workspace = true
# revm
revm.workspace = true
# async
tokio = { workspace = true, default-features = false }
tokio-util.workspace = true

View File

@@ -283,12 +283,8 @@ where
let mut state = State::builder()
.with_database_ref(StateProviderDatabase::new(&state_provider))
.with_bundle_update()
.with_bal_builder()
.build();
state.bal_state.bal_index = 0;
state.bal_state.bal_builder = Some(revm::state::bal::Bal::new());
let ctx = evm_config.context_for_block(&reorg_target).map_err(RethError::other)?;
let evm = evm_config.evm_for_block(&mut state, &reorg_target).map_err(RethError::other)?;
let mut builder = evm_config.create_block_builder(evm, &reorg_target_parent, ctx);

View File

@@ -39,7 +39,6 @@
//! transactions: vec![Bytes::from(vec![1, 2, 3])],
//! ommers: vec![],
//! withdrawals: None,
//! block_access_list: None,
//! };
//! // Compress the body: rlp encoding and snappy compression
//! let compressed_body = CompressedBody::from_body(&body)?;
@@ -582,12 +581,8 @@ mod tests {
#[test]
fn test_block_body_conversion() {
let block_body: BlockBody<Bytes> = BlockBody {
transactions: vec![],
ommers: vec![],
withdrawals: None,
block_access_list: None,
};
let block_body: BlockBody<Bytes> =
BlockBody { transactions: vec![], ommers: vec![], withdrawals: None };
let compressed_body = CompressedBody::from_body(&block_body).unwrap();
@@ -642,8 +637,7 @@ mod tests {
let withdrawals = Some(Withdrawals(vec![]));
let block_body =
BlockBody { transactions, ommers: vec![], withdrawals, block_access_list: None };
let block_body = BlockBody { transactions, ommers: vec![], withdrawals };
let block = Block::new(header, block_body);

View File

@@ -34,7 +34,6 @@ pub(crate) fn create_header() -> Header {
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
block_access_list_hash: None,
}
}
@@ -139,7 +138,6 @@ pub(crate) fn create_test_block_with_compressed_data(number: BlockNumber) -> Blo
excess_blob_gas: None,
parent_beacon_block_root: None,
requests_hash: None,
block_access_list_hash: None,
};
// Create test body
@@ -147,7 +145,6 @@ pub(crate) fn create_test_block_with_compressed_data(number: BlockNumber) -> Blo
transactions: vec![Bytes::from(vec![(number % 256) as u8; 10])],
ommers: vec![],
withdrawals: Some(Withdrawals(vec![])),
block_access_list: None,
};
// Create test receipt list with bloom

View File

@@ -22,7 +22,6 @@ reth-consensus.workspace = true
alloy-eips.workspace = true
alloy-primitives.workspace = true
alloy-consensus.workspace = true
alloy-rlp.workspace = true
tracing.workspace = true
@@ -39,7 +38,6 @@ std = [
"reth-execution-types/std",
"reth-primitives-traits/std",
"tracing/std",
"alloy-rlp/std",
]
[dev-dependencies]

View File

@@ -15,7 +15,7 @@ use alloc::{fmt::Debug, sync::Arc};
use alloy_consensus::{constants::MAXIMUM_EXTRA_DATA_SIZE, EMPTY_OMMER_ROOT_HASH};
use alloy_eips::eip7840::BlobParams;
use reth_chainspec::{EthChainSpec, EthereumHardforks};
use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator};
use reth_consensus::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
use reth_consensus_common::validation::{
validate_4844_header_standalone, validate_against_parent_4844,
validate_against_parent_eip1559_base_fee, validate_against_parent_gas_limit,
@@ -74,13 +74,14 @@ where
&self,
block: &RecoveredBlock<N::Block>,
result: &BlockExecutionResult<N::Receipt>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
validate_block_post_execution(
block,
&self.chain_spec,
&result.receipts,
&result.requests,
&result.block_access_list,
receipt_root_bloom,
)
}
}
@@ -180,15 +181,6 @@ where
} else if header.requests_hash().is_some() {
return Err(ConsensusError::RequestsHashUnexpected)
}
// if self.chain_spec.is_amsterdam_active_at_timestamp(header.timestamp()) &&
// header.block_access_list_hash().is_none()
// {
// return Err(ConsensusError::BlockAccessListHashMissing)
// } else if !self.chain_spec.is_amsterdam_active_at_timestamp(header.timestamp()) &&
// header.block_access_list_hash().is_some()
// {
// return Err(ConsensusError::BlockAccessListHashUnexpected)
// }
Ok(())
}

View File

@@ -1,23 +1,26 @@
use alloc::vec::Vec;
use alloy_consensus::{proofs::calculate_receipt_root, BlockHeader, TxReceipt};
use alloy_eips::{eip7685::Requests, eip7928::BlockAccessList, Encodable2718};
use alloy_eips::{eip7685::Requests, Encodable2718};
use alloy_primitives::{Bloom, Bytes, B256};
use reth_chainspec::EthereumHardforks;
use reth_consensus::ConsensusError;
use reth_primitives_traits::{
receipt::gas_spent_by_transactions, Block, BlockBody, GotExpected, Receipt, RecoveredBlock,
receipt::gas_spent_by_transactions, Block, GotExpected, Receipt, RecoveredBlock,
};
/// Validate a block with regard to execution results:
///
/// - Compares the receipts root in the block header to the block body
/// - Compares the gas used in the block header to the actual gas usage after execution
///
/// If `receipt_root_bloom` is provided, the pre-computed receipt root and logs bloom are used
/// instead of computing them from the receipts.
pub fn validate_block_post_execution<B, R, ChainSpec>(
block: &RecoveredBlock<B>,
chain_spec: &ChainSpec,
receipts: &[R],
requests: &Requests,
block_access_list: &Option<BlockAccessList>,
receipt_root_bloom: Option<(B256, Bloom)>,
) -> Result<(), ConsensusError>
where
B: Block,
@@ -38,19 +41,26 @@ where
// operation as hashing that is required for state root got calculated in every
// transaction This was replaced with is_success flag.
// See more about EIP here: https://eips.ethereum.org/EIPS/eip-658
if chain_spec.is_byzantium_active_at_block(block.header().number()) &&
let Err(error) = verify_receipts(
block.header().receipts_root(),
block.header().logs_bloom(),
receipts,
)
{
let receipts = receipts
.iter()
.map(|r| Bytes::from(r.with_bloom_ref().encoded_2718()))
.collect::<Vec<_>>();
tracing::debug!(%error, ?receipts, "receipts verification failed");
return Err(error)
if chain_spec.is_byzantium_active_at_block(block.header().number()) {
let result = if let Some((receipts_root, logs_bloom)) = receipt_root_bloom {
compare_receipts_root_and_logs_bloom(
receipts_root,
logs_bloom,
block.header().receipts_root(),
block.header().logs_bloom(),
)
} else {
verify_receipts(block.header().receipts_root(), block.header().logs_bloom(), receipts)
};
if let Err(error) = result {
let receipts = receipts
.iter()
.map(|r| Bytes::from(r.with_bloom_ref().encoded_2718()))
.collect::<Vec<_>>();
tracing::debug!(%error, ?receipts, "receipts verification failed");
return Err(error)
}
}
// Validate that the header requests hash matches the calculated requests hash
@@ -66,33 +76,6 @@ where
}
}
// Validate bal hash matches the calculated hash
if chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()) {
let Some(header_block_access_list_hash) = block.header().block_access_list_hash() else {
return Err(ConsensusError::BlockAccessListHashMissing)
};
if let Some(bal) = block_access_list {
let bal_hash = alloy_primitives::keccak256(alloy_rlp::encode(bal));
let block_bal = block.body().block_access_list();
tracing::debug!("Block Bal :{:?}", block_bal);
if let Some(body_bal) = block_bal {
if body_bal.is_empty() {
tracing::debug!("Hit Empty BAL : Block is {:?}", block);
}
verify_bal(body_bal, bal)?;
}
if bal_hash != header_block_access_list_hash {
tracing::debug!(
?bal_hash,
?header_block_access_list_hash,
"block access list hash mismatch"
);
return Err(ConsensusError::InvalidBalHash);
}
}
}
Ok(())
}
@@ -141,47 +124,6 @@ fn compare_receipts_root_and_logs_bloom(
Ok(())
}
/// Validates that the block access list in the body matches the expected block access list.
fn verify_bal(
body_bal: &BlockAccessList,
expected_bal: &BlockAccessList,
) -> Result<(), ConsensusError> {
if body_bal == expected_bal {
return Ok(());
}
// Extract addresses
let body_addrs: Vec<_> = body_bal.iter().map(|a| a.address).collect();
let expected_addrs: Vec<_> = expected_bal.iter().map(|a| a.address).collect();
// Missing accounts (expected but not found in body)
for addr in &expected_addrs {
if !body_addrs.contains(addr) {
tracing::debug!("Missing acc : computed bal {:?},body bal{:?}", expected_bal, body_bal);
tracing::debug!("Missing Address: {:?}", addr);
return Err(ConsensusError::InvalidBalMissingAccount);
}
}
// Extra accounts (body has accounts not in expected)
for addr in &body_addrs {
if !expected_addrs.contains(addr) {
tracing::debug!("Extra acc : computed bal {:?},body bal{:?}", expected_bal, body_bal);
tracing::debug!("Extra Address: {:?}", addr);
return Err(ConsensusError::InvalidBalExtraAccount);
}
}
tracing::debug!(
?expected_bal,
?body_bal,
"block access list in body does not match the provided block access list"
);
// Fallback: mismatched access lists
Err(ConsensusError::InvalidBlockAccessList)
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -17,9 +17,7 @@ pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAt
mod error;
pub use error::*;
use alloy_rpc_types_engine::{
ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
};
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5};
pub use alloy_rpc_types_engine::{
ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4,
ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes,
@@ -68,15 +66,13 @@ where
+ TryInto<ExecutionPayloadEnvelopeV2>
+ TryInto<ExecutionPayloadEnvelopeV3>
+ TryInto<ExecutionPayloadEnvelopeV4>
+ TryInto<ExecutionPayloadEnvelopeV5>
+ TryInto<ExecutionPayloadEnvelopeV6>,
+ TryInto<ExecutionPayloadEnvelopeV5>,
{
type ExecutionPayloadEnvelopeV1 = ExecutionPayloadV1;
type ExecutionPayloadEnvelopeV2 = ExecutionPayloadEnvelopeV2;
type ExecutionPayloadEnvelopeV3 = ExecutionPayloadEnvelopeV3;
type ExecutionPayloadEnvelopeV4 = ExecutionPayloadEnvelopeV4;
type ExecutionPayloadEnvelopeV5 = ExecutionPayloadEnvelopeV5;
type ExecutionPayloadEnvelopeV6 = ExecutionPayloadEnvelopeV6;
}
/// A default payload type for [`EthEngineTypes`]

View File

@@ -11,9 +11,8 @@ use alloy_primitives::{Address, B256, U256};
use alloy_rlp::Encodable;
use alloy_rpc_types_engine::{
BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, ExecutionPayloadV4,
PayloadAttributes, PayloadId,
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadFieldV2,
ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
};
use core::convert::Infallible;
use reth_ethereum_primitives::EthPrimitives;
@@ -161,38 +160,6 @@ impl EthBuiltPayload {
execution_requests: requests.unwrap_or_default(),
})
}
/// Try converting built payload into [`ExecutionPayloadEnvelopeV6`].
pub fn try_into_v6(self) -> Result<ExecutionPayloadEnvelopeV6, BuiltPayloadConversionError> {
let Self { block, fees, sidecars, requests, .. } = self;
let blobs_bundle = match sidecars {
BlobSidecars::Empty => BlobsBundleV2::empty(),
BlobSidecars::Eip7594(sidecars) => BlobsBundleV2::from(sidecars),
BlobSidecars::Eip4844(_) => {
return Err(BuiltPayloadConversionError::UnexpectedEip4844Sidecars)
}
};
Ok(ExecutionPayloadEnvelopeV6 {
execution_payload: ExecutionPayloadV4::from_block_unchecked(
block.hash(),
&Arc::unwrap_or_clone(block).into_block(),
),
block_value: fees,
// From the engine API spec:
//
// > Client software **MAY** use any heuristics to decide whether to set
// `shouldOverrideBuilder` flag or not. If client software does not implement any
// heuristic this flag **SHOULD** be set to `false`.
//
// Spec:
// <https://github.com/ethereum/execution-apis/blob/fe8e13c288c592ec154ce25c534e26cb7ce0530d/src/engine/cancun.md#specification-2>
should_override_builder: false,
blobs_bundle,
execution_requests: requests.unwrap_or_default(),
})
}
}
impl<N: NodePrimitives> BuiltPayload for EthBuiltPayload<N> {
@@ -260,14 +227,6 @@ impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV5 {
}
}
impl TryFrom<EthBuiltPayload> for ExecutionPayloadEnvelopeV6 {
type Error = BuiltPayloadConversionError;
fn try_from(value: EthBuiltPayload) -> Result<Self, Self::Error> {
value.try_into_v6()
}
}
/// An enum representing blob transaction sidecars belonging to [`EthBuiltPayload`].
#[derive(Clone, Default, Debug)]
pub enum BlobSidecars {
@@ -503,7 +462,6 @@ mod tests {
.unwrap(),
withdrawals: None,
parent_beacon_block_root: None,
slot_number: None,
};
// Verify that the generated payload ID matches the expected value
@@ -541,7 +499,6 @@ mod tests {
},
]),
parent_beacon_block_root: None,
slot_number: None,
};
// Verify that the generated payload ID matches the expected value
@@ -574,7 +531,6 @@ mod tests {
)
.unwrap(),
),
slot_number: None,
};
// Verify that the generated payload ID matches the expected value

View File

@@ -27,7 +27,6 @@ alloy-eips.workspace = true
alloy-evm.workspace = true
alloy-consensus.workspace = true
alloy-rpc-types-engine.workspace = true
alloy-rlp.workspace = true
# Misc
parking_lot = { workspace = true, optional = true }
@@ -58,7 +57,6 @@ std = [
"derive_more?/std",
"alloy-rpc-types-engine/std",
"reth-storage-errors/std",
"alloy-rlp/std",
]
test-utils = [
"dep:parking_lot",

View File

@@ -45,8 +45,7 @@ where
execution_ctx: ctx,
parent,
transactions,
output:
BlockExecutionResult { receipts, requests, gas_used, blob_gas_used, block_access_list },
output: BlockExecutionResult { receipts, requests, gas_used, blob_gas_used },
state_root,
..
} = input;
@@ -91,18 +90,6 @@ where
};
}
let (built_block_access_list, block_access_list_hash) =
if self.chain_spec.is_amsterdam_active_at_timestamp(timestamp) {
if let Some(bal) = block_access_list {
let hash = alloy_primitives::keccak256(alloy_rlp::encode(bal));
(Some(bal), Some(hash))
} else {
(None, None)
}
} else {
(None, None)
};
let header = Header {
parent_hash: ctx.parent_hash,
ommers_hash: EMPTY_OMMER_ROOT_HASH,
@@ -125,17 +112,11 @@ where
blob_gas_used: block_blob_gas_used,
excess_blob_gas,
requests_hash,
block_access_list_hash,
};
Ok(Block {
header,
body: BlockBody {
transactions,
ommers: Default::default(),
withdrawals,
block_access_list: built_block_access_list.cloned(),
},
body: BlockBody { transactions, ommers: Default::default(), withdrawals },
})
}
}

View File

@@ -188,6 +188,7 @@ where
block: &'a SealedBlock<Block>,
) -> Result<EthBlockExecutionCtx<'a>, Self::Error> {
Ok(EthBlockExecutionCtx {
tx_count_hint: Some(block.transaction_count()),
parent_hash: block.header().parent_hash,
parent_beacon_block_root: block.header().parent_beacon_block_root,
ommers: &block.body().ommers,
@@ -202,6 +203,7 @@ where
attributes: Self::NextBlockEnvCtx,
) -> Result<EthBlockExecutionCtx<'_>, Self::Error> {
Ok(EthBlockExecutionCtx {
tx_count_hint: None,
parent_hash: parent.hash(),
parent_beacon_block_root: attributes.parent_beacon_block_root,
ommers: &[],
@@ -281,6 +283,7 @@ where
payload: &'a ExecutionData,
) -> Result<ExecutionCtxFor<'a, Self>, Self::Error> {
Ok(EthBlockExecutionCtx {
tx_count_hint: Some(payload.payload.transactions().len()),
parent_hash: payload.parent_hash(),
parent_beacon_block_root: payload.sidecar.parent_beacon_block_root(),
ommers: &[],

View File

@@ -2,7 +2,7 @@ use crate::EthEvmConfig;
use alloc::{boxed::Box, sync::Arc, vec, vec::Vec};
use alloy_consensus::Header;
use alloy_eips::eip7685::Requests;
use alloy_evm::{block::StateDB, precompiles::PrecompilesMap};
use alloy_evm::precompiles::PrecompilesMap;
use alloy_primitives::Bytes;
use alloy_rpc_types_engine::ExecutionData;
use parking_lot::Mutex;
@@ -19,6 +19,7 @@ use reth_execution_types::{BlockExecutionResult, ExecutionOutcome};
use reth_primitives_traits::{BlockTy, SealedBlock, SealedHeader};
use revm::{
context::result::{ExecutionResult, Output, ResultAndState, SuccessReason},
database::State,
Inspector,
};
@@ -57,30 +58,36 @@ impl BlockExecutorFactory for MockEvmConfig {
fn create_executor<'a, DB, I>(
&'a self,
evm: EthEvm<DB, I, PrecompilesMap>,
evm: EthEvm<&'a mut State<DB>, I, PrecompilesMap>,
_ctx: Self::ExecutionCtx<'a>,
) -> impl BlockExecutorFor<'a, Self, DB, I>
where
DB: StateDB + Database + 'a,
I: Inspector<<Self::EvmFactory as EvmFactory>::Context<DB>> + 'a,
DB: Database + 'a,
I: Inspector<<Self::EvmFactory as EvmFactory>::Context<&'a mut State<DB>>> + 'a,
{
MockExecutor { result: self.exec_results.lock().pop().unwrap(), evm, hook: None }
MockExecutor {
result: self.exec_results.lock().pop().unwrap(),
evm,
hook: None,
receipts: Vec::new(),
}
}
}
/// Mock executor that returns a fixed execution result.
#[derive(derive_more::Debug)]
pub struct MockExecutor<DB: Database, I> {
pub struct MockExecutor<'a, DB: Database, I> {
result: ExecutionOutcome,
evm: EthEvm<DB, I, PrecompilesMap>,
evm: EthEvm<&'a mut State<DB>, I, PrecompilesMap>,
#[debug(skip)]
hook: Option<Box<dyn reth_evm::OnStateHook>>,
receipts: Vec<Receipt>,
}
impl<DB: StateDB + Database, I: Inspector<EthEvmContext<DB>>> BlockExecutor
for MockExecutor<DB, I>
impl<'a, DB: Database, I: Inspector<EthEvmContext<&'a mut State<DB>>>> BlockExecutor
for MockExecutor<'a, DB, I>
{
type Evm = EthEvm<DB, I, PrecompilesMap>;
type Evm = EthEvm<&'a mut State<DB>, I, PrecompilesMap>;
type Transaction = TransactionSigned;
type Receipt = Receipt;
@@ -88,6 +95,10 @@ impl<DB: StateDB + Database, I: Inspector<EthEvmContext<DB>>> BlockExecutor
Ok(())
}
fn receipts(&self) -> &[Self::Receipt] {
&self.receipts
}
fn execute_transaction_without_commit(
&mut self,
_tx: impl ExecutableTx<Self>,
@@ -124,11 +135,10 @@ impl<DB: StateDB + Database, I: Inspector<EthEvmContext<DB>>> BlockExecutor
reqs
}),
gas_used: 0,
block_access_list: None,
blob_gas_used: 0,
};
*evm.db_mut().bundle_state_mut() = bundle;
evm.db_mut().bundle_state = bundle;
Ok((evm, result))
}

View File

@@ -88,12 +88,7 @@ fn eip_4788_non_genesis_call() {
.execute_one(&RecoveredBlock::new_unhashed(
Block {
header: header.clone(),
body: BlockBody {
transactions: vec![],
ommers: vec![],
withdrawals: None,
block_access_list: None,
},
body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None },
},
vec![],
))
@@ -112,12 +107,7 @@ fn eip_4788_non_genesis_call() {
.execute_one(&RecoveredBlock::new_unhashed(
Block {
header: header.clone(),
body: BlockBody {
transactions: vec![],
ommers: vec![],
withdrawals: None,
block_access_list: None,
},
body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None },
},
vec![],
))
@@ -177,12 +167,7 @@ fn eip_4788_no_code_cancun() {
.execute_one(&RecoveredBlock::new_unhashed(
Block {
header,
body: BlockBody {
transactions: vec![],
ommers: vec![],
withdrawals: None,
block_access_list: None,
},
body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None },
},
vec![],
))
@@ -224,12 +209,7 @@ fn eip_4788_empty_account_call() {
.execute_one(&RecoveredBlock::new_unhashed(
Block {
header,
body: BlockBody {
transactions: vec![],
ommers: vec![],
withdrawals: None,
block_access_list: None,
},
body: BlockBody { transactions: vec![], ommers: vec![], withdrawals: None },
},
vec![],
))
@@ -819,7 +799,6 @@ fn test_balance_increment_not_duplicated() {
transactions: vec![],
ommers: vec![],
withdrawals: Some(vec![withdrawal].into()),
block_access_list: None,
},
},
vec![],

View File

@@ -223,7 +223,6 @@ async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> {
suggested_fee_recipient: Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
};
let request = TestingBuildBlockRequestV1 {

View File

@@ -153,7 +153,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
w1.address(),
);
let pooled_tx1 = EthPooledTransaction::new(tx1.clone(), 200);
let tx_hash1 = *pooled_tx1.clone().hash();
let tx_hash1 = *pooled_tx1.hash();
// build tx2 from wallet2
let envelop2 = TransactionTestContext::transfer_tx(1, w2.clone()).await;
@@ -162,7 +162,7 @@ async fn maintain_txpool_reorg() -> eyre::Result<()> {
w2.address(),
);
let pooled_tx2 = EthPooledTransaction::new(tx2.clone(), 200);
let tx_hash2 = *pooled_tx2.clone().hash();
let tx_hash2 = *pooled_tx2.hash();
let block_info = BlockInfo {
block_gas_limit: ETHEREUM_BLOCK_GAS_LIMIT_30M,

View File

@@ -25,7 +25,6 @@ pub(crate) fn eth_payload_attributes(timestamp: u64) -> EthPayloadBuilderAttribu
suggested_fee_recipient: Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
};
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
}

View File

@@ -56,7 +56,6 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> {
suggested_fee_recipient: Address::ZERO,
withdrawals: None,
parent_beacon_block_root: None,
slot_number: None,
};
let request = TestingBuildBlockRequestV1 {

View File

@@ -153,14 +153,10 @@ where
let PayloadConfig { parent_header, attributes } = config;
let state_provider = client.state_by_block_hash(parent_header.hash())?;
let state = StateProviderDatabase::new(&state_provider);
let mut db = State::builder()
.with_database(cached_reads.as_db_mut(state))
.with_bundle_update()
.with_bal_builder()
.build();
db.bal_state.bal_index = 0;
db.bal_state.bal_builder = Some(revm::state::bal::Bal::new());
let state = StateProviderDatabase::new(state_provider.as_ref());
let mut db =
State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build();
let mut builder = evm_config
.builder_for_next_block(
&mut db,
@@ -251,7 +247,7 @@ where
limit: MAX_RLP_BLOCK_SIZE,
},
);
continue;
continue
}
// There's only limited amount of blob space available per block, so we need to check if

View File

@@ -3,7 +3,7 @@
use alloy_consensus::Block;
use alloy_rpc_types_engine::{ExecutionData, PayloadError};
use reth_chainspec::EthereumHardforks;
use reth_payload_validator::{amsterdam, cancun, prague, shanghai};
use reth_payload_validator::{cancun, prague, shanghai};
use reth_primitives_traits::{Block as _, SealedBlock, SignedTransaction};
use std::sync::Arc;
@@ -103,10 +103,5 @@ where
chain_spec.is_prague_active_at_timestamp(sealed_block.timestamp),
)?;
amsterdam::ensure_well_formed_fields(
sealed_block.body(),
chain_spec.is_amsterdam_active_at_timestamp(sealed_block.timestamp),
)?;
Ok(sealed_block)
}

View File

@@ -236,7 +236,7 @@ impl reth_codecs::Compact for Transaction {
// # Panics
//
// A panic will be triggered if an identifier larger than 3 is passed from the database. For
// optimism a identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
// optimism an identifier with value [`DEPOSIT_TX_TYPE_ID`] is allowed.
fn from_compact(buf: &[u8], identifier: usize) -> (Self, &[u8]) {
let (tx_type, buf) = TxType::from_compact(buf, identifier);

View File

@@ -1,12 +1,12 @@
//! Traits for execution.
use crate::{ConfigureEvm, Database, OnStateHook, TxEnvFor};
use alloc::{borrow::Cow, boxed::Box, sync::Arc, vec::Vec};
use alloc::{boxed::Box, sync::Arc, vec::Vec};
use alloy_consensus::{BlockHeader, Header};
use alloy_eips::eip2718::WithEncoded;
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
use alloy_evm::{
block::{CommitChanges, ExecutableTx, StateDB},
block::{CommitChanges, ExecutableTx},
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
};
use alloy_primitives::{Address, B256};
@@ -148,20 +148,6 @@ pub trait Executor<DB: Database>: Sized {
fn size_hint(&self) -> usize;
}
/// Helper type for the output of executing a block.
///
/// Deprecated: this type is unused within reth and will be removed in the next
/// major release. Use `reth_execution_types::BlockExecutionResult` or
/// `reth_execution_types::BlockExecutionOutput`.
#[deprecated(note = "Use reth_execution_types::BlockExecutionResult or BlockExecutionOutput")]
#[derive(Debug, Clone)]
pub struct ExecuteOutput<R> {
/// Receipts obtained after executing a block.
pub receipts: Vec<R>,
/// Cumulative gas used in the block execution.
pub gas_used: u64,
}
/// Input for block building. Consumed by [`BlockAssembler`].
///
/// This struct contains all the data needed by the [`BlockAssembler`] to create
@@ -214,7 +200,7 @@ pub struct BlockAssemblerInput<'a, 'b, F: BlockExecutorFactory, H = Header> {
/// Output of block execution.
pub output: &'b BlockExecutionResult<F::Receipt>,
/// [`BundleState`] after the block execution.
pub bundle_state: Cow<'a, BundleState>,
pub bundle_state: &'a BundleState,
/// Provider with access to state.
#[debug(skip)]
pub state_provider: &'b dyn StateProvider,
@@ -234,7 +220,7 @@ impl<'a, 'b, F: BlockExecutorFactory, H> BlockAssemblerInput<'a, 'b, F, H> {
parent: &'a SealedHeader<H>,
transactions: Vec<F::Transaction>,
output: &'b BlockExecutionResult<F::Receipt>,
bundle_state: impl Into<Cow<'a, BundleState>>,
bundle_state: &'a BundleState,
state_provider: &'b dyn StateProvider,
state_root: B256,
) -> Self {
@@ -244,7 +230,7 @@ impl<'a, 'b, F: BlockExecutorFactory, H> BlockAssemblerInput<'a, 'b, F, H> {
parent,
transactions,
output,
bundle_state: bundle_state.into(),
bundle_state,
state_provider,
state_root,
}
@@ -461,7 +447,8 @@ where
}
}
impl<'a, F, Executor, Builder, N> BlockBuilder for BasicBlockBuilder<'a, F, Executor, Builder, N>
impl<'a, F, DB, Executor, Builder, N> BlockBuilder
for BasicBlockBuilder<'a, F, Executor, Builder, N>
where
F: BlockExecutorFactory<Transaction = N::SignedTx, Receipt = N::Receipt>,
Executor: BlockExecutor<
@@ -469,11 +456,12 @@ where
Spec = <F::EvmFactory as EvmFactory>::Spec,
HaltReason = <F::EvmFactory as EvmFactory>::HaltReason,
BlockEnv = <F::EvmFactory as EvmFactory>::BlockEnv,
DB: StateDB + 'a,
DB = &'a mut State<DB>,
>,
Transaction = N::SignedTx,
Receipt = N::Receipt,
>,
DB: Database + 'a,
Builder: BlockAssembler<F, Block = N::Block>,
N: NodePrimitives,
{
@@ -506,13 +494,13 @@ where
state: impl StateProvider,
) -> Result<BlockBuilderOutcome<N>, BlockExecutionError> {
let (evm, result) = self.executor.finish()?;
let (mut db, evm_env) = evm.finish();
let (db, evm_env) = evm.finish();
// merge all transitions into bundle state
db.merge_transitions(BundleRetention::Reverts);
// calculate the state root
let hashed_state = state.hashed_post_state(db.bundle_state());
let hashed_state = state.hashed_post_state(&db.bundle_state);
let (state_root, trie_updates) = state
.state_root_with_updates(hashed_state.clone())
.map_err(BlockExecutionError::other)?;
@@ -526,7 +514,7 @@ where
parent: self.parent,
transactions,
output: &result,
bundle_state: Cow::Owned(db.take_bundle()),
bundle_state: &db.bundle_state,
state_provider: &state,
state_root,
})?;
@@ -562,14 +550,8 @@ pub struct BasicBlockExecutor<F, DB> {
impl<F, DB: Database> BasicBlockExecutor<F, DB> {
/// Creates a new `BasicBlockExecutor` with the given strategy.
pub fn new(strategy_factory: F, db: DB) -> Self {
let mut db = State::builder()
.with_database(db)
.with_bundle_update()
.with_bal_builder()
.without_state_clear()
.build();
db.bal_state.bal_index = 0;
db.bal_state.bal_builder = Some(revm::state::bal::Bal::new());
let db =
State::builder().with_database(db).with_bundle_update().without_state_clear().build();
Self { strategy_factory, db }
}
}

View File

@@ -18,7 +18,6 @@
extern crate alloc;
use crate::execute::{BasicBlockBuilder, Executor};
use ::revm::context::TxEnv;
use alloc::vec::Vec;
use alloy_eips::{
eip2718::{EIP2930_TX_TYPE_ID, LEGACY_TX_TYPE_ID},
@@ -26,7 +25,7 @@ use alloy_eips::{
eip4895::Withdrawals,
};
use alloy_evm::{
block::{BlockExecutorFactory, BlockExecutorFor, StateDB},
block::{BlockExecutorFactory, BlockExecutorFor},
precompiles::PrecompilesMap,
};
use alloy_primitives::{Address, Bytes, B256};
@@ -36,7 +35,7 @@ use reth_execution_errors::BlockExecutionError;
use reth_primitives_traits::{
BlockTy, HeaderTy, NodePrimitives, ReceiptTy, SealedBlock, SealedHeader, TxTy,
};
use revm::DatabaseCommit;
use revm::{context::TxEnv, database::State};
pub mod either;
/// EVM environment configuration.
@@ -313,20 +312,20 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
/// Creates a strategy with given EVM and execution context.
fn create_executor<'a, DB, I>(
&'a self,
evm: EvmFor<Self, DB, I>,
evm: EvmFor<Self, &'a mut State<DB>, I>,
ctx: <Self::BlockExecutorFactory as BlockExecutorFactory>::ExecutionCtx<'a>,
) -> impl BlockExecutorFor<'a, Self::BlockExecutorFactory, DB, I>
where
DB: StateDB + DatabaseCommit + Database + 'a,
I: InspectorFor<Self, DB> + 'a,
DB: Database,
I: InspectorFor<Self, &'a mut State<DB>> + 'a,
{
self.block_executor_factory().create_executor(evm, ctx)
}
/// Creates a strategy for execution of a given block.
fn executor_for_block<'a, DB: StateDB + DatabaseCommit + Database + 'a>(
fn executor_for_block<'a, DB: Database>(
&'a self,
db: DB,
db: &'a mut State<DB>,
block: &'a SealedBlock<<Self::Primitives as NodePrimitives>::Block>,
) -> Result<impl BlockExecutorFor<'a, Self::BlockExecutorFactory, DB>, Self::Error> {
let evm = self.evm_for_block(db, block.header())?;
@@ -351,7 +350,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
/// ```
fn create_block_builder<'a, DB, I>(
&'a self,
evm: EvmFor<Self, DB, I>,
evm: EvmFor<Self, &'a mut State<DB>, I>,
parent: &'a SealedHeader<HeaderTy<Self::Primitives>>,
ctx: <Self::BlockExecutorFactory as BlockExecutorFactory>::ExecutionCtx<'a>,
) -> impl BlockBuilder<
@@ -359,8 +358,8 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
Executor: BlockExecutorFor<'a, Self::BlockExecutorFactory, DB, I>,
>
where
DB: StateDB + DatabaseCommit + Database + 'a,
I: InspectorFor<Self, DB> + 'a,
DB: Database,
I: InspectorFor<Self, &'a mut State<DB>> + 'a,
{
BasicBlockBuilder {
executor: self.create_executor(evm, ctx.clone()),
@@ -400,9 +399,9 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
/// // Complete block building
/// let outcome = builder.finish(state_provider)?;
/// ```
fn builder_for_next_block<'a, DB: StateDB + DatabaseCommit + Database + 'a>(
fn builder_for_next_block<'a, DB: Database + 'a>(
&'a self,
db: DB,
db: &'a mut State<DB>,
parent: &'a SealedHeader<<Self::Primitives as NodePrimitives>::BlockHeader>,
attributes: Self::NextBlockEnvCtx,
) -> Result<

Some files were not shown because too many files have changed in this diff Show More