Compare commits

...

86 Commits

Author SHA1 Message Date
yongkangc
903c58749d refactor(rocksdb): extract shared unwind_history_shards_inner helper
Deduplicates the unwind shard processing logic between account and
storage history. Both unwind_account_history_to/batch and
unwind_storage_history_to/batch now delegate to a single generic
unwind_history_shards_inner method using closures, following the
same pattern as prune_history_shards_inner.

Amp-Thread-ID: https://ampcode.com/threads/T-019c436b-33ce-739d-8fa2-69ea27d4fc4a
2026-02-09 17:33:18 +00:00
yongkangc
d1dfce17ab docs: restore full doc comments for RocksDBRawIterEnum and decode_iter_item
Amp-Thread-ID: https://ampcode.com/threads/T-019c28b3-097d-741f-8a27-9af9a4f1ada1
2026-02-04 12:50:28 +00:00
yongkangc
abe603b94c docs: restore detailed flush_and_compact doc comment
Amp-Thread-ID: https://ampcode.com/threads/T-019c27a1-4820-715a-9b42-18fa2ce05c98
2026-02-04 08:11:28 +00:00
yongkangc
9258bcdc7b fix: check iter.status() after iteration loops
Amp-Thread-ID: https://ampcode.com/threads/T-019c27a1-4820-715a-9b42-18fa2ce05c98
2026-02-04 08:09:33 +00:00
yongkangc
7134ae4ec7 chore: fix clippy warnings and clean up comments
Amp-Thread-ID: https://ampcode.com/threads/T-019c2769-c7e2-7217-ae23-b189345c2c4b
2026-02-04 07:50:25 +00:00
yongkangc
d9f81bc104 perf(rocksdb): use single iterator for batch history unwind
Previously, unwind_account_history_indices and unwind_storage_history_indices
created a new iterator_cf for every address/storage key by calling
account_history_shards/storage_history_shards in a loop. This is the same
performance antipattern fixed in PR #21767 for pruning.

This commit adds:
- unwind_account_history_batch: batch version using single raw_iterator_cf
- unwind_storage_history_batch: batch version using single raw_iterator_cf
- RocksDBRawIterEnum: wrapper supporting seek() for iterator reuse

The batch methods reuse a single raw iterator and skip seeks when the
iterator is already positioned correctly (for sorted targets in key order).
This significantly reduces RocksDB seek overhead for large unwind operations.

Amp-Thread-ID: https://ampcode.com/threads/T-019c272c-eed5-751b-a0c4-8e260f96a3bc
2026-02-04 07:50:12 +00:00
Dan Cline
89be91de0e perf(pruner): do not create an iterator_cf for every address (#21767)
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-04 06:48:22 +00:00
Dan Cline
3af5a4a4e2 fix(pruner): implement pruning for rocksdb TransactionHashNumbers (#21782) 2026-02-04 04:11:37 +00:00
Dan Cline
95f6bbe922 chore(pruner): always flush and compact after reth prune command (#21783) 2026-02-04 03:07:55 +00:00
DaniPopes
abab83facd perf: spawn proof workers in a separate thread (#21780) 2026-02-04 01:20:43 +00:00
DaniPopes
9359e21f94 ci: enable debug assertions for statetests (#21775) 2026-02-04 00:53:28 +00:00
Huber
32d5ddfe40 fix(test): clean up test temp directories on drop (#21772) 2026-02-03 22:44:12 +00:00
Dan Cline
d7e740f96c chore(cli): expose static file metrics in cli (#21770) 2026-02-03 22:21:10 +00:00
DaniPopes
87bae74094 chore: decode MDBX error code (#21766) 2026-02-03 20:16:32 +00:00
DaniPopes
648f19fb56 perf: build for target-cpu=x86-64-v3 in docker by default (#21761) 2026-02-03 19:47:59 +00:00
DaniPopes
e6fc5ff54b perf(trie): use TrieMask iterator for efficient bit iteration (#21676) 2026-02-03 19:23:41 +00:00
YK
bc729671d9 perf(rocksdb): batch tx reads in TransactionLookupStage unwind (#21723)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 18:28:04 +00:00
joshieDo
eee27df27c fix: ensure transaction lookup can prune (#19553) 2026-02-03 18:11:13 +00:00
Dan Cline
6d02565c5e chore(prune): increase reth prune DELETE_LIMIT to 20M (#21762) 2026-02-03 17:47:50 +00:00
Dan Cline
e706d76aa9 chore(cli): support ctrl-C in reth prune (#21759) 2026-02-03 17:47:01 +00:00
DaniPopes
b9b7d092f6 perf: bump nybbles (#21725) 2026-02-03 17:15:30 +00:00
DaniPopes
d0fb5f31c2 chore: centralize thread::spawn to share tokio handles (#21754) 2026-02-03 16:58:46 +00:00
DaniPopes
9621b78586 chore: shorten thread names (#21751) 2026-02-03 16:40:35 +00:00
DaniPopes
3722071a7c chore(deps): bump bytes 1.11.1 (#21755) 2026-02-03 16:31:22 +00:00
DaniPopes
6273530501 perf: use alloy_primitives hasher for dashmaps (#21726)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 15:05:44 +00:00
Alexey Shekhirin
ce29101277 chore(static-files): proper segment writer scoped thread names (#21747) 2026-02-03 14:44:03 +00:00
John Chase
b1b95f9825 fix(discv5): add missing rand feature for test compilation (#21749) 2026-02-03 14:37:39 +00:00
YK
7f970e136a refactor(stages): use with_rocksdb_batch_auto_commit in tx_lookup (#21722) 2026-02-03 14:35:07 +00:00
YK
6b7cc00289 refactor(rocksdb): deduplicate first()/last() implementations (#21738)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:33:44 +00:00
YK
786140a99d perf(static-file): simplify stage checkpoint lookup to avoid allocs (#21730) 2026-02-03 14:32:43 +00:00
YK
ffcb486388 refactor(rocksdb): deduplicate iterator next() implementations (#21737) 2026-02-03 14:31:05 +00:00
YK
59d68f92c4 perf(static-file): hoist cursor creation outside block loop (#21731) 2026-02-03 14:29:07 +00:00
Matthias Seitz
0e0271a612 chore(deps): bump alloy 1.5.2 -> 1.6.1 (#21746)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 14:16:50 +00:00
Minhyuk Kim
df12fee965 feat(txpool): add is_transaction_ready to TransactionPool trait (#21742)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:13:52 +00:00
DaniPopes
11a4f65624 chore: misc tree cleanups (#21691)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 13:34:19 +00:00
Matthias Seitz
a782e1a18a chore: disable changelog workflow on PRs (#21748)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:12:43 +01:00
DaniPopes
2dc76f9abe chore: match statement order in ExecutionCache::new (#21712)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 12:47:15 +00:00
Nicolas SSS
65100971e5 fix(evm): remove unused reth-ethereum-forks (#21695)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 12:33:44 +00:00
Georgios Konstantopoulos
8e21afa9cc feat(trie): add memory_size heuristic for ParallelSparseTrie (#21745)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-03 12:29:57 +00:00
DaniPopes
46a9b9ad3d perf: replace RwLock<HashMap/HashSet> with DashMap/DashSet (#21692)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-03 13:31:05 +01:00
Georgios Konstantopoulos
3f77af4f98 feat: add AI-assisted changelog generation (#21743)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmail.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 12:19:49 +00:00
Arsenii Kulikov
79cabbf89c perf: optimize SparseTrieCacheTask (#21704) 2026-02-03 11:39:10 +00:00
drhgencer
e04afe6e0e fix(rpc): validate toBlock in trace_filter (#21718) 2026-02-03 11:02:57 +00:00
Arsenii Kulikov
ee224fe20f fix: update sparse trie masks (#21716) 2026-02-03 12:01:58 +01:00
DaniPopes
972f23745e chore: remove clone from in memory cursor (#21719) 2026-02-03 04:04:33 +00:00
Dan Cline
49f60822f7 chore: move TransactionLookup as first option (#21721) 2026-02-03 02:30:13 +00:00
Georgios Konstantopoulos
47ebc79c85 feat(rpc): add EIP-7928 eth_getBalanceWithProof and eth_getAccountWithProof (#21720)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 01:12:04 +00:00
Arsenii Kulikov
53f922927a feat: reintroduce --engine.state-root-task-compare-updates (#21717) 2026-02-02 23:48:54 +00:00
Dan Cline
f1f3980d29 fix(cli): actually enable reth-prune rocksdb feature in cli (#21715) 2026-02-02 23:39:04 +00:00
Dan Cline
6946f26d77 fix(cli): delete all static files when PruneModes::Full is configured (#21647) 2026-02-02 17:30:21 +00:00
Arsenii Kulikov
f663d1d110 fix: properly drain pending account updates (#21709) 2026-02-02 17:29:43 +00:00
Huber
f4943abf73 chore(ci): add consts to typos allowlist (#21708) 2026-02-02 17:02:16 +00:00
Matthias Seitz
102a6944ba perf(trie): avoid clearing already-cached sparse trie (#21702)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 13:03:07 +00:00
Alexey Shekhirin
1592e51d34 feat(engine): add CLI args for sparse trie pruning configuration (#21703) 2026-02-02 12:52:31 +00:00
Arsenii Kulikov
4280ccf470 fix: short-circuit in reveal_account_v2_proof_nodes on empty nodes (#21701) 2026-02-02 12:18:45 +00:00
Alexey Shekhirin
05ab98107c fix(reth-bench): gracefully stop when transaction source exhausted (#21700) 2026-02-02 11:10:58 +00:00
Brian Picciano
49128ed28f fix(trie): Return full_key from update_leaves unless it is not a child of the missing path (#21699)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
2026-02-02 11:07:56 +00:00
Huber
f74e594292 perf(trie): dispatch V2 storage proofs in lexicographical order (#21684) 2026-02-02 09:31:47 +00:00
Georgios Konstantopoulos
e7d4a05e36 perf(trie): fix allocation hot paths with capacity hints and buffer reuse (#21466)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-02 06:58:45 +00:00
Georgios Konstantopoulos
9382a4c713 fix(prune): use batched pruning loop with edge feature to prevent OOM (#21649)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-02 02:38:00 +00:00
DaniPopes
28409558f9 perf: add ParallelBridgeBuffered trait to replace par_bridge (#21674) 2026-02-02 00:58:43 +00:00
DaniPopes
5ef32726db refactor: add with_* compressor utility methods (#21680) 2026-02-01 20:43:25 +00:00
Snezhkko
60c3bef1e8 fix(zstd): use transaction dictionary for tx compressor (#21382) 2026-02-01 20:12:51 +00:00
iPLAY888
af96eeae56 refactor(provider): deduplicate segment-to-stage mapping in static file manager (#21670) 2026-02-01 20:09:32 +00:00
Georgios Konstantopoulos
5528aae8f6 fix(engine): wait for persistence service thread before RocksDB drop (#21640)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 19:55:45 +00:00
Georgios Konstantopoulos
83364aa2d6 fix(prune): migrate invalid receipts prune config to Distance(64) (#21677)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 19:44:14 +00:00
DaniPopes
749a742bcf chore(deps): update metrics-derive 0.1.1 (#21673) 2026-02-01 19:38:38 +00:00
ethfanWilliam
2970624413 chore: avoid eager evaluation in base_fee_params_at_timestamp (#21536) 2026-02-01 19:04:42 +00:00
Matthias Seitz
7e18aa4be8 fix(rpc): change debug_set_head number parameter to U64 (#21678)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 18:59:22 +00:00
YK
9f8c22e2c3 feat(prune): prune rocksdb account and storage history indices (#21331)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-01 18:42:17 +00:00
Georgios Konstantopoulos
3d699ac9c6 perf(trie): reuse account RLP buffer in SparseTrieCacheTask (#21644)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 15:20:11 +00:00
かりんとう
9be31d504d fix(trie): silence unused param warnings in sparse-parallel no_std build (#21657) 2026-02-01 13:05:39 +00:00
github-actions[bot]
34cc65cfe6 chore(deps): weekly cargo update (#21660)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
2026-02-01 13:03:13 +00:00
Matthias Seitz
6e161f0fc9 perf: batch finalized/safe block commits with SaveBlocks (#21663)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 13:02:59 +00:00
iPLAY888
63a3e18404 fix: remove unnecessary alloc (#21665) 2026-02-01 13:01:11 +00:00
Matthias Seitz
7d10e791b2 refactor(engine): improve payload processor tx iterator (#21658)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:44:10 +00:00
Georgios Konstantopoulos
a9b2c1d454 feat(rpc): make blob sidecar upcasting opt-in (#21624)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-01 12:25:46 +00:00
CPerezz
9127563914 fix: cleanup entire temp directory when using testing_node (#18399)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 16:46:11 +00:00
Georgios Konstantopoulos
a500fb22ba fix(metrics): rename save_blocks_block_count to save_blocks_batch_size (#21654)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 12:59:09 +00:00
Matthias Seitz
e869cd4670 perf(engine): skip DB lookup for new blocks in insert_block_or_payload (#21650)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 03:35:20 +00:00
DaniPopes
de69654b73 chore(deps): breaking bumps (#21584)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-01-31 00:44:09 +00:00
DaniPopes
8d28c4c8f2 chore(trie): add set_* methods alongside with_* builders (#21639)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:42:57 +00:00
Georgios Konstantopoulos
bfe778ab51 perf(trie): use Entry API to avoid empty Vec allocation in extend (#21645)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:29:21 +00:00
DaniPopes
e523a76fb8 chore(trie): clear RevealableSparseTrie in place (#21638)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 22:27:43 +00:00
DaniPopes
cd12ae58f2 docs(CLAUDE.md): tweaks (#21646) 2026-01-30 22:26:34 +00:00
Georgios Konstantopoulos
370a548f34 refactor(db): derive Clone for DatabaseEnv (#21641)
Co-authored-by: Amp <amp@ampcode.com>
2026-01-30 21:54:50 +00:00
168 changed files with 5477 additions and 1872 deletions

20
.changelog/config.toml Normal file
View File

@@ -0,0 +1,20 @@
# Changelogs configuration for reth
# https://github.com/wevm/changelogs
# How to bump packages that depend on changed packages
dependent_bump = "patch"
[changelog]
# Generate per-crate changelogs (vs single root changelog)
format = "per-crate"
# Fixed groups: all always share the same version
# reth binaries share version
[[fixed]]
members = ["reth", "op-reth"]
# Packages to ignore (internal/test-only crates)
ignore = [
"reth-testing-utils",
"reth-bench",
]

View File

@@ -0,0 +1,5 @@
---
reth-engine-tree: patch
---
Reordered cache size calculations in `ExecutionCache::new` to group related operations together.

View File

@@ -0,0 +1,6 @@
---
reth: patch
op-reth: patch
---
Added automated changelog generation infrastructure using wevm/changelogs-rs with Claude Code integration. Configured per-crate changelog format with fixed version groups for reth binaries and exclusions for internal test utilities.

View File

@@ -0,0 +1,5 @@
---
reth: patch
---
Updated Alloy dependencies from 1.5.2 to 1.6.1.

21
.github/workflows/changelog.yml vendored Normal file
View File

@@ -0,0 +1,21 @@
name: Changelog
on:
workflow_dispatch:
jobs:
changelog:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- run: npm install -g @anthropic-ai/claude-code
- uses: wevm/changelogs-rs/gen@master
with:
ai: 'claude -p'
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

View File

@@ -100,5 +100,7 @@ jobs:
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
optimism.tags=${{ steps.params.outputs.optimism_tags }}
ethereum-amd64.tags=${{ steps.params.outputs.ethereum_tags }}
ethereum-arm64.tags=${{ steps.params.outputs.ethereum_tags }}
optimism-amd64.tags=${{ steps.params.outputs.optimism_tags }}
optimism-arm64.tags=${{ steps.params.outputs.optimism_tags }}

View File

@@ -90,7 +90,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- run: cargo nextest run --release -p ef-tests --features "asm-keccak ef-tests"
- run: cargo nextest run --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
doc:
name: doc tests

View File

@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
2. **Linting**: Run clippy with all features
```bash
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
```
3. **Testing**: Use nextest for faster test execution
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
Before submitting changes, ensure:
1. **Format Check**: `cargo +nightly fmt --all --check`
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
2. **Clippy**: No warnings
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### Opening PRs against <https://github.com/paradigmxyz/reth>
Label PRs appropriately, first check the available labels and then apply the relevant ones:
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
}
```
5. **Run checks**:
5. **Run checks** (IMPORTANT!):
```bash
cargo +nightly fmt --all
cargo clippy --all-features
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
cargo test -p reth-discv4
```
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
cargo +nightly fmt --all
# Run lints
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
cargo +nightly clippy --workspace --all-features
# Run tests
cargo nextest run --workspace

810
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -490,42 +490,42 @@ alloy-sol-types = { version = "1.5.4", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
alloy-consensus = { version = "1.6.1", default-features = false }
alloy-contract = { version = "1.6.1", default-features = false }
alloy-eips = { version = "1.6.1", default-features = false }
alloy-genesis = { version = "1.6.1", default-features = false }
alloy-json-rpc = { version = "1.6.1", default-features = false }
alloy-network = { version = "1.6.1", default-features = false }
alloy-network-primitives = { version = "1.6.1", default-features = false }
alloy-provider = { version = "1.6.1", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.6.1", default-features = false }
alloy-rpc-client = { version = "1.6.1", default-features = false }
alloy-rpc-types = { version = "1.6.1", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.6.1", default-features = false }
alloy-rpc-types-anvil = { version = "1.6.1", default-features = false }
alloy-rpc-types-beacon = { version = "1.6.1", default-features = false }
alloy-rpc-types-debug = { version = "1.6.1", default-features = false }
alloy-rpc-types-engine = { version = "1.6.1", default-features = false }
alloy-rpc-types-eth = { version = "1.6.1", default-features = false }
alloy-rpc-types-mev = { version = "1.6.1", default-features = false }
alloy-rpc-types-trace = { version = "1.6.1", default-features = false }
alloy-rpc-types-txpool = { version = "1.6.1", default-features = false }
alloy-serde = { version = "1.6.1", default-features = false }
alloy-signer = { version = "1.6.1", default-features = false }
alloy-signer-local = { version = "1.6.1", default-features = false }
alloy-transport = { version = "1.6.1" }
alloy-transport-http = { version = "1.6.1", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.6.1", default-features = false }
alloy-transport-ws = { version = "1.6.1", default-features = false }
# op
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-evm = { version = "0.27.2", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -543,7 +543,7 @@ backon = { version = "1.2", default-features = false, features = ["std-blocking-
bincode = "1.3"
bitflags = "2.4"
boyer-moore-magiclen = "0.2.16"
bytes = { version = "1.5", default-features = false }
bytes = { version = "1.11.1", default-features = false }
brotli = "8"
cfg-if = "1.0"
clap = "4"
@@ -560,9 +560,9 @@ humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
lz4 = "1.28.1"
modular-bitfield = "0.11.2"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.2", default-features = false }
nybbles = { version = "0.4.8", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
parking_lot = "0.12"
paste = "1.0"
@@ -589,13 +589,13 @@ zstd = "0.13"
byteorder = "1"
fixed-cache = { version = "0.1.7", features = ["stats"] }
moka = "0.12"
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
tar-no-std = { version = "0.4.2", default-features = false }
miniz_oxide = { version = "0.9.0", default-features = false }
chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1"
metrics-derive = "0.1.1"
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.20.0" }
@@ -607,7 +607,7 @@ quote = "1.0"
# tokio
tokio = { version = "1.44.2", default-features = false }
tokio-stream = "0.1.11"
tokio-tungstenite = "0.26.2"
tokio-tungstenite = "0.28.0"
tokio-util = { version = "0.7.4", features = ["codec"] }
# async
@@ -620,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
hyper = "1.3"
hyper-util = "0.1.5"
pin-project = "1.0.12"
reqwest = { version = "0.12", default-features = false }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
tracing-futures = "0.2"
tower = "0.5"
tower-http = "0.6"
@@ -640,7 +640,6 @@ jsonrpsee-types = "0.26.0"
http = "1.0"
http-body = "1.0"
http-body-util = "0.1.2"
jsonwebtoken = "9"
proptest-arbitrary-interop = "0.1.0"
# crypto
@@ -654,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
c-kzg = "2.1.5"
# config
toml = "0.8"
toml = "0.9"
# rocksdb
rocksdb = { version = "0.24" }
@@ -673,16 +672,16 @@ assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.5"
proptest-derive = "0.7"
similar-asserts = { version = "1.5.0", features = ["serde"] }
tempfile = "3.20"
test-fuzz = "7"
rstest = "0.24.0"
rstest = "0.26.1"
test-case = "3"
# ssz encoding
ethereum_ssz = "0.9.0"
ethereum_ssz_derive = "0.9.0"
ethereum_ssz = "0.10.1"
ethereum_ssz_derive = "0.10.1"
# allocators
jemalloc_pprof = { version = "0.8", default-features = false }
@@ -694,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
aes = "0.8.1"
ahash = "0.8"
anyhow = "1.0"
bindgen = { version = "0.71", default-features = false }
block-padding = "0.3.2"
bindgen = { version = "0.72", default-features = false }
block-padding = "0.3"
cc = "1.2.15"
cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossterm = "0.28.0"
crossterm = "0.29.0"
csv = "1.3.0"
ctrlc = "3.4"
ctr = "0.9.2"
@@ -714,7 +713,7 @@ hmac = "0.12.1"
human_bytes = "0.4.1"
indexmap = "2"
interprocess = "2.2.0"
lz4_flex = { version = "0.11", default-features = false }
lz4_flex = { version = "0.12", default-features = false }
memmap2 = "0.9.4"
mev-share-sse = { version = "0.5.0", default-features = false }
num-traits = "0.2.15"
@@ -722,15 +721,15 @@ page_size = "0.6.0"
parity-scale-codec = "3.2.1"
plain_hasher = "0.2"
pretty_assertions = "1.4"
ratatui = { version = "0.29", default-features = false }
ringbuffer = "0.15.0"
ratatui = { version = "0.30", default-features = false }
ringbuffer = "0.16.0"
rmp-serde = "1.3"
roaring = "0.10.2"
roaring = "0.11.3"
rolling-file = "0.2.0"
sha3 = "0.10.5"
snap = "1.1.1"
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
socket2 = { version = "0.6", default-features = false }
sysinfo = { version = "0.38", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "=0.3.5"
tracing-samply = "0.1"

View File

@@ -56,7 +56,7 @@ ctrlc.workspace = true
shlex.workspace = true
[target.'cfg(unix)'.dependencies]
nix = { version = "0.29", features = ["signal", "process"] }
nix = { version = "0.31", features = ["signal", "process"] }
[features]
default = ["jemalloc"]

View File

@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
# reqwest
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
# tower
tower.workspace = true

View File

@@ -572,13 +572,22 @@ impl Command {
for i in 0..self.count {
// Get initial batch of transactions for this payload
let mut result = tx_buffer
.take_batch()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
let Some(mut result) = tx_buffer.take_batch().await else {
info!(
payloads_built = i,
payloads_requested = self.count,
"Transaction source exhausted, stopping"
);
break;
};
if result.transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
info!(
payloads_built = i,
payloads_requested = self.count,
"No more transactions available, stopping"
);
break;
}
// Build with retry - may need to request more transactions

View File

@@ -541,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
}
}
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
}
}
}

View File

@@ -133,4 +133,4 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]

View File

@@ -121,14 +121,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
let (db, sfp) = match access {
AccessRights::RW => (
Arc::new(init_db(db_path, self.db.database_args())?),
init_db(db_path, self.db.database_args())?,
StaticFileProviderBuilder::read_write(sf_path)
.with_metrics()
.with_genesis_block_number(genesis_block_number)
.build()?,
),
AccessRights::RO | AccessRights::RoInconsistent => {
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
(open_db_read_only(&db_path, self.db.database_args())?, {
let provider = StaticFileProviderBuilder::read_only(sf_path)
.with_metrics()
.with_genesis_block_number(genesis_block_number)
.build()?;
provider.watch_directory();
@@ -160,16 +162,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
fn create_provider_factory<N: CliNodeTypes>(
&self,
config: &Config,
db: Arc<DatabaseEnv>,
db: DatabaseEnv,
static_file_provider: StaticFileProvider<N::Primitives>,
rocksdb_provider: RocksDBProvider,
access: AccessRights,
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let prune_modes = config.prune.segments.clone();
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
db,
self.chain.clone(),
static_file_provider,
@@ -200,7 +202,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
// Builds and executes an unwind-only pipeline
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
.add_stages(DefaultStages::new(
factory.clone(),
tip_rx,
@@ -229,7 +231,7 @@ pub struct Environment<N: NodeTypes> {
/// Configuration for reth node
pub config: Config,
/// Provider factory.
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
/// Datadir path.
pub data_dir: ChainPath<DataDirPath>,
}
@@ -261,8 +263,8 @@ impl AccessRights {
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
type FullTypesAdapter<T> = FullNodeTypesAdapter<
T,
Arc<DatabaseEnv>,
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
DatabaseEnv,
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
>;
/// Helper trait with a common set of requirements for the

View File

@@ -17,7 +17,6 @@ use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProvider
use reth_static_file_types::StaticFileSegment;
use std::{
hash::{BuildHasher, Hasher},
sync::Arc,
time::{Duration, Instant},
};
use tracing::{info, warn};
@@ -90,7 +89,7 @@ impl Command {
/// Execute `db checksum` command
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("This command should be run without the node running!");
@@ -117,7 +116,7 @@ fn checksum_hasher() -> impl Hasher {
}
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
segment: StaticFileSegment,
start_block: Option<u64>,
end_block: Option<u64>,

View File

@@ -9,7 +9,7 @@ use reth_db_api::table::Table;
use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::RocksDBProviderFactory;
use std::{hash::Hasher, sync::Arc, time::Instant};
use std::{hash::Hasher, time::Instant};
use tracing::info;
/// RocksDB tables that can be checksummed.
@@ -36,7 +36,7 @@ impl RocksDbTable {
/// Computes a checksum for a RocksDB table.
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
table: RocksDbTable,
limit: Option<usize>,
) -> eyre::Result<()> {

View File

@@ -16,7 +16,6 @@ use std::{
hash::Hash,
io::Write,
path::{Path, PathBuf},
sync::Arc,
};
use tracing::{info, warn};
@@ -56,7 +55,7 @@ impl Command {
/// then written to a file in the output directory.
pub fn execute<T: NodeTypes>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
) -> eyre::Result<()> {
warn!("Make sure the node is not running when running `reth db diff`!");
// open second db

View File

@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
use reth_db_common::{DbTool, ListFilter};
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
use std::{cell::RefCell, sync::Arc};
use std::cell::RefCell;
use tracing::error;
#[derive(Parser, Debug)]
@@ -55,7 +55,7 @@ impl Command {
/// Execute `db list` command
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
self,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
self.table.view(&ListTableViewer { tool, args: &self })
}
@@ -89,7 +89,7 @@ impl Command {
}
struct ListTableViewer<'a, N: NodeTypes> {
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
args: &'a Command,
}

View File

@@ -11,6 +11,7 @@ use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_provider::providers::ProviderNodeTypes;
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
use reth_tasks::spawn_scoped_os_thread;
use std::{
collections::BTreeSet,
thread,
@@ -230,7 +231,7 @@ impl Command {
thread::scope(|s| {
let handles: Vec<_> = (0..num_threads)
.map(|thread_id| {
s.spawn(move || {
spawn_scoped_os_thread(s, "db-state-worker", move || {
loop {
// Get next chunk to process
let chunk_idx = {

View File

@@ -16,7 +16,7 @@ use reth_provider::{
RocksDBProviderFactory,
};
use reth_static_file_types::SegmentRangeInclusive;
use std::{sync::Arc, time::Duration};
use std::time::Duration;
#[derive(Parser, Debug)]
/// The arguments for the `reth db stats` command
@@ -48,7 +48,7 @@ impl Command {
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
self,
data_dir: ChainPath<DataDirPath>,
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()> {
if self.checksum {
let checksum_report = self.checksum_report(tool)?;
@@ -72,7 +72,7 @@ impl Command {
Ok(())
}
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
&self,
tool: &DbTool<N>,
) -> eyre::Result<ComfyTable> {

View File

@@ -227,8 +227,9 @@ where
// Handle errors
if let Err(err) = res {
error!("{:?}", err)
error!("{err}");
}
Ok(())
}
}
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
) -> io::Result<()>
where
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
io::Error: From<B::Error>,
{
let mut last_tick = Instant::now();
let mut running = true;

View File

@@ -2,7 +2,7 @@ use futures::Future;
use reth_cli::chainspec::ChainSpecParser;
use reth_db::DatabaseEnv;
use reth_node_builder::{NodeBuilder, WithLaunchContext};
use std::{fmt, sync::Arc};
use std::fmt;
/// A trait for launching a reth node with custom configuration strategies.
///
@@ -30,7 +30,7 @@ where
/// * `builder_args` - Extension arguments for configuration
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>>;
}
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
where
C: ChainSpecParser,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
@@ -77,13 +77,13 @@ where
C: ChainSpecParser,
Ext: clap::Args + fmt::Debug,
F: AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
{
fn entrypoint(
self,
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
builder_args: Ext,
) -> impl Future<Output = eyre::Result<()>> {
(self.func)(builder, builder_args)

View File

@@ -206,7 +206,7 @@ where
let db_path = data_dir.db();
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
if with_unused_ports {
node_config = node_config.with_unused_ports();

View File

@@ -1,9 +1,10 @@
//! Command that runs pruning without any limits.
//! Command that runs pruning.
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
use clap::Parser;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_cli_util::cancellation::CancellationToken;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{args::MetricArgs, version::version_metadata};
use reth_node_metrics::{
@@ -11,12 +12,14 @@ use reth_node_metrics::{
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
#[cfg(all(unix, feature = "edge"))]
use reth_provider::RocksDBProviderFactory;
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
use tracing::info;
/// Prunes according to the configuration without any limits
/// Prunes according to the configuration
#[derive(Debug, Parser)]
pub struct PruneCommand<C: ChainSpecParser> {
#[command(flatten)]
@@ -50,7 +53,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
build_profile: version_metadata().build_profile_name.as_ref(),
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
ctx.task_executor.clone(),
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
@@ -69,13 +72,66 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
// Delete data which has been copied to static files.
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Run the pruner according to the configuration, and don't enforce any limits on it
let mut pruner = PrunerBuilder::new(config)
.delete_limit(usize::MAX)
.build_with_provider_factory(provider_factory);
pruner.run(prune_tip)?;
info!(target: "reth::cli", "Pruned data from database");
// Set up cancellation token for graceful shutdown on Ctrl+C
let cancellation = CancellationToken::new();
let cancellation_clone = cancellation.clone();
ctx.task_executor.spawn_critical("prune-ctrl-c", async move {
tokio::signal::ctrl_c().await.expect("failed to listen for ctrl-c");
cancellation_clone.cancel();
});
// Use batched pruning with a limit to bound memory, running in a loop until complete.
//
// A limit of 20_000_000 results in a max memory usage of ~5G.
const DELETE_LIMIT: usize = 20_000_000;
let mut pruner = PrunerBuilder::new(config)
.delete_limit(DELETE_LIMIT)
.build_with_provider_factory(provider_factory.clone());
let mut total_pruned = 0usize;
loop {
if cancellation.is_cancelled() {
info!(target: "reth::cli", total_pruned, "Pruning interrupted by user");
break;
}
let output = pruner.run(prune_tip)?;
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
total_pruned = total_pruned.saturating_add(batch_pruned);
// Check if all segments are finished (not just the overall progress,
// since the pruner sets overall progress from the last segment only)
let all_segments_finished =
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
if all_segments_finished {
info!(target: "reth::cli", total_pruned, "Pruned data from database");
break;
}
if batch_pruned == 0 {
return Err(eyre::eyre!(
"pruner made no progress but reported more data remaining; \
aborting to prevent infinite loop"
));
}
info!(
target: "reth::cli",
batch_pruned,
total_pruned,
"Pruning batch complete, continuing..."
);
}
}
// Flush and compact RocksDB to reclaim disk space after pruning
#[cfg(all(unix, feature = "edge"))]
{
info!(target: "reth::cli", "Flushing and compacting RocksDB...");
provider_factory.rocksdb_provider().flush_and_compact()?;
info!(target: "reth::cli", "RocksDB compaction complete");
}
Ok(())

View File

@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
consensus: C,
) -> eyre::Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
C: FullConsensus<E::Primitives> + 'static,
{
@@ -39,7 +39,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -10,10 +10,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: BlockNumber,
to: BlockNumber,
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -9,10 +9,9 @@ use reth_provider::{
DatabaseProviderFactory, ProviderFactory,
};
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
use std::sync::Arc;
use tracing::info;
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
db_tool: &DbTool<N>,
from: u64,
to: u64,
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
consensus: impl FullConsensus<N::Primitives> + 'static,
) -> Result<()>
where
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
N: ProviderNodeTypes<DB = DatabaseEnv>,
{
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
@@ -59,7 +59,7 @@ where
if should_run {
dry_run(
ProviderFactory::<N>::new(
Arc::new(output_db),
output_db,
db_tool.chain(),
StaticFileProvider::read_write(output_datadir.static_files())?,
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,

View File

@@ -158,7 +158,7 @@ enum Subcommands {
impl Subcommands {
/// Returns the block to unwind to. The returned block will stay in database.
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
&self,
factory: ProviderFactory<N>,
) -> eyre::Result<u64> {

View File

@@ -83,22 +83,7 @@ impl CliRunner {
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
}
// `drop(tokio_runtime)` would block the current thread until its pools
// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
// it on a separate thread and wait for up to 5 seconds for this operation to
// complete.
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("tokio-runtime-shutdown".to_string())
.spawn(move || {
drop(tokio_runtime);
let _ = tx.send(());
})
.unwrap();
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
});
tokio_shutdown(tokio_runtime, true);
command_res
}
@@ -137,19 +122,7 @@ impl CliRunner {
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
}
// Shutdown the runtime on a separate thread
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("tokio-runtime-shutdown".to_string())
.spawn(move || {
drop(tokio_runtime);
let _ = tx.send(());
})
.unwrap();
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
});
tokio_shutdown(tokio_runtime, true);
command_res
}
@@ -179,13 +152,7 @@ impl CliRunner {
tokio_runtime
.block_on(run_until_ctrl_c(async move { fut.await.expect("Failed to join task") }))?;
// drop the tokio runtime on a separate thread because drop blocks until its pools
// (including blocking pool) are shutdown. In other words `drop(tokio_runtime)` would block
// the current thread but we want to exit right away.
std::thread::Builder::new()
.name("tokio-runtime-shutdown".to_string())
.spawn(move || drop(tokio_runtime))
.unwrap();
tokio_shutdown(tokio_runtime, false);
Ok(())
}
@@ -321,3 +288,27 @@ where
Ok(())
}
/// Shut down the given Tokio runtime, and wait for it if `wait` is set.
///
/// `drop(tokio_runtime)` would block the current thread until its pools
/// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
/// it on a separate thread and wait for up to 5 seconds for this operation to
/// complete.
fn tokio_shutdown(rt: tokio::runtime::Runtime, wait: bool) {
// Shutdown the runtime on a separate thread
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("tokio-shutdown".to_string())
.spawn(move || {
drop(rt);
let _ = tx.send(());
})
.unwrap();
if wait {
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
});
}
}

View File

@@ -29,7 +29,7 @@ auto_impl.workspace = true
derive_more.workspace = true
futures.workspace = true
eyre.workspace = true
reqwest = { workspace = true, features = ["rustls-tls"] }
reqwest.workspace = true
serde = { workspace = true, features = ["derive"] }
tokio = { workspace = true, features = ["time"] }
serde_json.workspace = true

View File

@@ -95,7 +95,7 @@ where
let block_hash = payload.block_hash();
let block_number = payload.block_number();
previous_block_hashes.push(block_hash);
previous_block_hashes.enqueue(block_hash);
// Send new events to execution client
let _ = self.engine_handle.new_payload(payload).await;
@@ -160,7 +160,7 @@ mod tests {
// Push hashes 0..65
for i in 0..65u8 {
buffer.push(B256::with_last_byte(i));
buffer.enqueue(B256::with_last_byte(i));
}
// offset=0 should return the most recent (64)
@@ -181,7 +181,7 @@ mod tests {
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
// With only 1 entry, only offset=0 works
buffer.push(B256::with_last_byte(1));
buffer.enqueue(B256::with_last_byte(1));
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
assert_eq!(get_hash_at_offset(&buffer, 1), None);
assert_eq!(get_hash_at_offset(&buffer, 32), None);
@@ -189,7 +189,7 @@ mod tests {
// With 33 entries, offset=32 works but offset=64 doesn't
for i in 2..=33u8 {
buffer.push(B256::with_last_byte(i));
buffer.enqueue(B256::with_last_byte(i));
}
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
assert_eq!(get_hash_at_offset(&buffer, 64), None);

View File

@@ -114,22 +114,22 @@ pub async fn setup_engine_with_chain_import(
// Initialize the database using init_db (same as CLI import command)
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args)?;
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, db_args)?;
// Create a provider factory with the initialized database (use regular DB, not
// TempDatabase) We need to specify the node types properly for the adapter
let provider_factory = ProviderFactory::<
NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
>::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)?;
let provider_factory =
ProviderFactory::<NodeTypesWithDBAdapter<EthereumNode, DatabaseEnv>>::new(
db.clone(),
chain_spec.clone(),
reth_provider::providers::StaticFileProvider::read_write(
static_files_path.clone(),
)?,
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
.with_default_tables()
.build()
.unwrap(),
)?;
// Initialize genesis if needed
reth_db_common::init::init_genesis(&provider_factory)?;
@@ -320,11 +320,10 @@ mod tests {
// Import the chain
{
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, db_args).unwrap();
let provider_factory: ProviderFactory<
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
> = ProviderFactory::new(
db.clone(),
chain_spec.clone(),
@@ -385,11 +384,10 @@ mod tests {
// Now reopen the database and verify checkpoints are still there
{
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let db = Arc::new(db_env);
let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
let provider_factory: ProviderFactory<
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
> = ProviderFactory::new(
db,
chain_spec.clone(),

View File

@@ -528,8 +528,12 @@ impl TreeConfig {
}
/// Setter for the number of storage proof worker threads.
pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self {
self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT);
///
/// No-op if it's [`None`].
pub fn with_storage_worker_count_opt(mut self, storage_worker_count: Option<usize>) -> Self {
if let Some(count) = storage_worker_count {
self.storage_worker_count = count.max(MIN_WORKER_COUNT);
}
self
}
@@ -539,8 +543,12 @@ impl TreeConfig {
}
/// Setter for the number of account proof worker threads.
pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self {
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
///
/// No-op if it's [`None`].
pub fn with_account_worker_count_opt(mut self, account_worker_count: Option<usize>) -> Self {
if let Some(count) = account_worker_count {
self.account_worker_count = count.max(MIN_WORKER_COUNT);
}
self
}

View File

@@ -17,7 +17,6 @@ reth-engine-tree.workspace = true
reth-evm.workspace = true
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
reth-stages-api.workspace = true

View File

@@ -14,7 +14,6 @@ pub use reth_engine_tree::{
chain::{ChainEvent, ChainOrchestrator},
engine::EngineApiEvent,
};
use reth_ethereum_primitives::EthPrimitives;
use reth_evm::ConfigureEvm;
use reth_network_p2p::BlockClient;
use reth_node_types::{BlockTy, NodeTypes};
@@ -97,7 +96,7 @@ where
let downloader = BasicBlockDownloader::new(client, consensus.clone());
let persistence_handle =
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx);
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();

View File

@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-primitives-traits.workspace = true
reth-primitives-traits = { workspace = true, features = ["rayon", "dashmap"] }
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
@@ -62,7 +62,6 @@ metrics.workspace = true
reth-metrics = { workspace = true, features = ["common"] }
# misc
dashmap.workspace = true
schnellru.workspace = true
rayon.workspace = true
tracing.workspace = true

View File

@@ -20,7 +20,7 @@ pub(crate) struct PersistenceMetrics {
/// How long it took for blocks to be saved
pub(crate) save_blocks_duration_seconds: Histogram,
/// How many blocks we persist at once.
pub(crate) save_blocks_block_count: Histogram,
pub(crate) save_blocks_batch_size: Histogram,
/// How long it took for blocks to be pruned
pub(crate) prune_before_duration_seconds: Histogram,
}

View File

@@ -11,8 +11,13 @@ use reth_provider::{
};
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
use reth_stages_api::{MetricEvent, MetricEventsSender};
use reth_tasks::spawn_os_thread;
use std::{
sync::mpsc::{Receiver, SendError, Sender},
sync::{
mpsc::{Receiver, SendError, Sender},
Arc,
},
thread::JoinHandle,
time::Instant,
};
use thiserror::Error;
@@ -40,6 +45,12 @@ where
metrics: PersistenceMetrics,
/// Sender for sync metrics - we only submit sync metrics for persisted blocks
sync_metrics_tx: MetricEventsSender,
/// Pending finalized block number to be committed with the next block save.
/// This avoids triggering a separate fsync for each finalized block update.
pending_finalized_block: Option<u64>,
/// Pending safe block number to be committed with the next block save.
/// This avoids triggering a separate fsync for each safe block update.
pending_safe_block: Option<u64>,
}
impl<N> PersistenceService<N>
@@ -53,7 +64,15 @@ where
pruner: PrunerWithFactory<ProviderFactory<N>>,
sync_metrics_tx: MetricEventsSender,
) -> Self {
Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx }
Self {
provider,
incoming,
pruner,
metrics: PersistenceMetrics::default(),
sync_metrics_tx,
pending_finalized_block: None,
pending_safe_block: None,
}
}
/// Prunes block data before the given block number according to the configured prune
@@ -106,14 +125,10 @@ where
}
}
PersistenceAction::SaveFinalizedBlock(finalized_block) => {
let provider = self.provider.database_provider_rw()?;
provider.save_finalized_block_number(finalized_block)?;
provider.commit()?;
self.pending_finalized_block = Some(finalized_block);
}
PersistenceAction::SaveSafeBlock(safe_block) => {
let provider = self.provider.database_provider_rw()?;
provider.save_safe_block_number(safe_block)?;
provider.commit()?;
self.pending_safe_block = Some(safe_block);
}
}
}
@@ -138,26 +153,39 @@ where
}
fn on_save_blocks(
&self,
&mut self,
blocks: Vec<ExecutedBlock<N::Primitives>>,
) -> Result<Option<BlockNumHash>, PersistenceError> {
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
let block_count = blocks.len();
// Take any pending finalized/safe block updates to commit together
let pending_finalized = self.pending_finalized_block.take();
let pending_safe = self.pending_safe_block.take();
debug!(target: "engine::persistence", ?block_count, first=?first_block, last=?last_block, "Saving range of blocks");
let start_time = Instant::now();
if last_block.is_some() {
let provider_rw = self.provider.database_provider_rw()?;
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
// Commit pending finalized/safe block updates in the same transaction
if let Some(finalized) = pending_finalized {
provider_rw.save_finalized_block_number(finalized)?;
}
if let Some(safe) = pending_safe {
provider_rw.save_safe_block_number(safe)?;
}
provider_rw.commit()?;
}
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
self.metrics.save_blocks_block_count.record(block_count as f64);
self.metrics.save_blocks_batch_size.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
Ok(last_block)
@@ -204,15 +232,25 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
pub struct PersistenceHandle<N: NodePrimitives = EthPrimitives> {
/// The channel used to communicate with the persistence service
sender: Sender<PersistenceAction<N>>,
/// Guard that joins the service thread when all handles are dropped.
/// Uses `Arc` so the handle remains `Clone`.
_service_guard: Arc<ServiceGuard>,
}
impl<T: NodePrimitives> PersistenceHandle<T> {
/// Create a new [`PersistenceHandle`] from a [`Sender<PersistenceAction>`].
pub const fn new(sender: Sender<PersistenceAction<T>>) -> Self {
Self { sender }
///
/// This is intended for testing purposes where you want to mock the persistence service.
/// For production use, prefer [`spawn_service`](Self::spawn_service).
pub fn new(sender: Sender<PersistenceAction<T>>) -> Self {
Self { sender, _service_guard: Arc::new(ServiceGuard(None)) }
}
/// Create a new [`PersistenceHandle`], and spawn the persistence service.
///
/// The returned handle can be cloned and shared. When all clones are dropped, the service
/// thread will be joined, ensuring graceful shutdown before resources (like `RocksDB`) are
/// released.
pub fn spawn_service<N>(
provider_factory: ProviderFactory<N>,
pruner: PrunerWithFactory<ProviderFactory<N>>,
@@ -224,22 +262,19 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
// create the initial channels
let (db_service_tx, db_service_rx) = std::sync::mpsc::channel();
// construct persistence handle
let persistence_handle = PersistenceHandle::new(db_service_tx);
// spawn the persistence service
let db_service =
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
std::thread::Builder::new()
.name("Persistence Service".to_string())
.spawn(|| {
if let Err(err) = db_service.run() {
error!(target: "engine::persistence", ?err, "Persistence service failed");
}
})
.unwrap();
let join_handle = spawn_os_thread("persistence", || {
if let Err(err) = db_service.run() {
error!(target: "engine::persistence", ?err, "Persistence service failed");
}
});
persistence_handle
PersistenceHandle {
sender: db_service_tx,
_service_guard: Arc::new(ServiceGuard(Some(join_handle))),
}
}
/// Sends a specific [`PersistenceAction`] in the contained channel. The caller is responsible
@@ -267,7 +302,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
}
/// Persists the finalized block number on disk.
/// Queues the finalized block number to be persisted on disk.
///
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
/// call to avoid triggering a separate fsync for each update.
pub fn save_finalized_block_number(
&self,
finalized_block: u64,
@@ -275,7 +313,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block))
}
/// Persists the safe block number on disk.
/// Queues the safe block number to be persisted on disk.
///
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
/// call to avoid triggering a separate fsync for each update.
pub fn save_safe_block_number(
&self,
safe_block: u64,
@@ -297,6 +338,27 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
}
}
/// Guard that joins the persistence service thread when dropped.
///
/// This ensures graceful shutdown - the service thread completes before resources like
/// `RocksDB` are released. Stored in an `Arc` inside [`PersistenceHandle`] so the handle
/// can be cloned while sharing the same guard.
struct ServiceGuard(Option<JoinHandle<()>>);
impl std::fmt::Debug for ServiceGuard {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.debug_tuple("ServiceGuard").field(&self.0.as_ref().map(|_| "...")).finish()
}
}
impl Drop for ServiceGuard {
fn drop(&mut self) {
if let Some(join_handle) = self.0.take() {
let _ = join_handle.join();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
@@ -323,12 +385,12 @@ mod tests {
#[test]
fn test_save_blocks_empty() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let blocks = vec![];
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let hash = rx.recv().unwrap();
assert_eq!(hash, None);
@@ -337,7 +399,7 @@ mod tests {
#[test]
fn test_save_blocks_single_block() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let block_number = 0;
let mut test_block_builder = TestBlockBuilder::eth();
let executed =
@@ -347,7 +409,7 @@ mod tests {
let blocks = vec![executed];
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx
.recv_timeout(std::time::Duration::from_secs(10))
@@ -360,14 +422,14 @@ mod tests {
#[test]
fn test_save_blocks_multiple_blocks() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let mut test_block_builder = TestBlockBuilder::eth();
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);
}
@@ -375,7 +437,7 @@ mod tests {
#[test]
fn test_save_blocks_multiple_calls() {
reth_tracing::init_test_tracing();
let persistence_handle = default_persistence_handle();
let handle = default_persistence_handle();
let ranges = [0..1, 1..2, 2..4, 4..5];
let mut test_block_builder = TestBlockBuilder::eth();
@@ -384,7 +446,7 @@ mod tests {
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
persistence_handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(blocks, tx).unwrap();
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
assert_eq!(last_hash, actual_hash);

View File

@@ -534,9 +534,9 @@ impl ExecutionCache {
/// Build an [`ExecutionCache`] struct, so that execution caches can be easily cloned.
pub fn new(total_cache_size: usize) -> Self {
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
let storage_cache_size = (total_cache_size * 8888) / 10000; // 88.88% of total
let account_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
let code_capacity = Self::bytes_to_entries(code_cache_size, CODE_CACHE_ENTRY_SIZE);
let storage_capacity = Self::bytes_to_entries(storage_cache_size, STORAGE_CACHE_ENTRY_SIZE);

View File

@@ -37,6 +37,7 @@ use reth_provider::{
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
use reth_tasks::spawn_os_thread;
use reth_trie_db::ChangesetCache;
use revm::state::EvmState;
use state::TreeState;
@@ -61,7 +62,6 @@ mod persistence_state;
pub mod precompile_cache;
#[cfg(test)]
mod tests;
#[expect(unused)]
mod trie_updates;
use crate::tree::error::AdvancePersistenceError;
@@ -432,7 +432,7 @@ where
changeset_cache,
);
let incoming = task.incoming_tx.clone();
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
spawn_os_thread("engine", || task.run());
(incoming, outgoing)
}
@@ -2613,19 +2613,27 @@ where
let block_num_hash = block_id.block;
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
match self.sealed_header_by_hash(block_num_hash.hash) {
Err(err) => {
let block = convert_to_block(self, input)?;
return Err(InsertBlockError::new(block, err.into()).into());
// Check if block already exists - first in memory, then DB only if it could be persisted
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
}
// Only query DB if block could be persisted (number <= last persisted block).
// New blocks from CL always have number > last persisted, so skip DB lookup for them.
if block_num_hash.number <= self.persistence_state.last_persisted_block.number {
match self.provider.sealed_header_by_hash(block_num_hash.hash) {
Err(err) => {
let block = convert_to_block(self, input)?;
return Err(InsertBlockError::new(block, err.into()).into());
}
Ok(Some(_)) => {
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
}
Ok(None) => {}
}
Ok(Some(_)) => {
// We now assume that we already have this block in the tree. However, we need to
// run the conversion to ensure that the block hash is valid.
convert_to_block(self, input)?;
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid))
}
_ => {}
};
}
// Ensure that the parent state is available.
match self.state_provider_builder(block_id.parent) {

View File

@@ -39,7 +39,7 @@ use reth_trie_parallel::{
proof_task::{ProofTaskCtx, ProofWorkerHandle},
root::ParallelStateRootError,
};
use reth_trie_sparse::{ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse::{RevealableSparseTrie, SparseStateTrie};
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
use std::{
collections::BTreeMap,
@@ -235,8 +235,7 @@ where
+ 'static,
{
// start preparing transactions immediately
let (prewarm_rx, execution_rx, transaction_count_hint) =
self.spawn_tx_iterator(transactions);
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
let span = Span::current();
let (to_sparse_trie, sparse_trie_rx) = channel();
@@ -260,7 +259,6 @@ where
self.spawn_caching_with(
env,
prewarm_rx,
transaction_count_hint,
provider_builder.clone(),
None, // Don't send proof targets when BAL is present
Some(bal),
@@ -271,7 +269,6 @@ where
self.spawn_caching_with(
env,
prewarm_rx,
transaction_count_hint,
provider_builder.clone(),
Some(to_multi_proof.clone()),
None,
@@ -355,10 +352,10 @@ where
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
let prewarm_handle =
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
PayloadHandle {
to_multi_proof: None,
prewarm_handle,
@@ -376,19 +373,15 @@ where
) -> (
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
usize,
) {
let (transactions, convert) = transactions.into();
let transactions = transactions.into_par_iter();
let transaction_count_hint = transactions.len();
let (ooo_tx, ooo_rx) = mpsc::channel();
let (prewarm_tx, prewarm_rx) = mpsc::channel();
let (execute_tx, execute_rx) = mpsc::channel();
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
self.executor.spawn_blocking(move || {
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
rayon::spawn(move || {
let (transactions, convert) = transactions.into();
transactions.into_par_iter().enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
let tx = convert(tx);
let tx = tx.map(|tx| {
let (tx_env, tx) = tx.into_parts();
@@ -424,16 +417,14 @@ where
}
});
(prewarm_rx, execute_rx, transaction_count_hint)
(prewarm_rx, execute_rx)
}
/// Spawn prewarming optionally wired to the multiproof task for target updates.
#[expect(clippy::too_many_arguments)]
fn spawn_caching_with<P>(
&self,
env: ExecutionEnv<Evm>,
mut transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
transaction_count_hint: usize,
provider_builder: StateProviderBuilder<N, P>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
bal: Option<Arc<BlockAccessList>>,
@@ -468,7 +459,6 @@ where
self.execution_cache.clone(),
prewarm_ctx,
to_multi_proof,
transaction_count_hint,
self.prewarm_max_concurrency,
);
@@ -527,6 +517,8 @@ where
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
let prune_depth = self.sparse_trie_prune_depth;
let max_storage_tries = self.sparse_trie_max_storage_tries;
let chunk_size =
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
self.executor.spawn_blocking(move || {
let _enter = span.entered();
@@ -562,11 +554,12 @@ where
sparse_state_trie,
))
} else {
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_cleared_trie(
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
from_multi_proof,
proof_worker_handle,
trie_metrics.clone(),
ClearedSparseStateTrie::from_state_trie(sparse_state_trie),
sparse_state_trie,
chunk_size,
))
};
@@ -961,6 +954,10 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
/// Used for sparse trie continuation: if the preserved trie's anchor matches this,
/// the trie can be reused directly.
pub parent_state_root: B256,
/// Number of transactions in the block.
/// Used to determine parallel worker count for prewarming.
/// A value of 0 indicates the count is unknown.
pub transaction_count: usize,
}
impl<Evm: ConfigureEvm> Default for ExecutionEnv<Evm>
@@ -973,6 +970,7 @@ where
hash: Default::default(),
parent_hash: Default::default(),
parent_state_root: Default::default(),
transaction_count: 0,
}
}
}

View File

@@ -22,7 +22,7 @@ use reth_trie_parallel::{
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
},
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
targets_v2::MultiProofTargetsV2,
};
use revm_primitives::map::{hash_map, B256Map};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
@@ -63,7 +63,7 @@ const PREFETCH_MAX_BATCH_MESSAGES: usize = 16;
/// The default max targets, for limiting the number of account and storage proof targets to be
/// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability.
const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
pub(crate) const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
@@ -311,11 +311,7 @@ impl VersionedMultiProofTargets {
fn chunking_length(&self) -> usize {
match self {
Self::Legacy(targets) => targets.chunking_length(),
Self::V2(targets) => {
// For V2, count accounts + storage slots
targets.account_targets.len() +
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
Self::V2(targets) => targets.chunking_length(),
}
}
@@ -367,9 +363,7 @@ impl VersionedMultiProofTargets {
Self::Legacy(targets) => {
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
}
Self::V2(targets) => {
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
}
Self::V2(targets) => Box::new(targets.chunks(chunk_size).map(Self::V2)),
}
}
}
@@ -1494,7 +1488,7 @@ fn get_proof_targets(
/// Dispatches work items as a single unit or in chunks based on target size and worker
/// availability.
#[allow(clippy::too_many_arguments)]
fn dispatch_with_chunking<T, I>(
pub(crate) fn dispatch_with_chunking<T, I>(
items: T,
chunking_len: usize,
chunk_size: Option<usize>,

View File

@@ -84,8 +84,6 @@ where
ctx: PrewarmContext<N, P, Evm>,
/// How many transactions should be executed in parallel
max_concurrency: usize,
/// The number of transactions to be processed
transaction_count_hint: usize,
/// Sender to emit evm state outcome messages, if any.
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
/// Receiver for events produced by tx execution
@@ -106,7 +104,6 @@ where
execution_cache: PayloadExecutionCache,
ctx: PrewarmContext<N, P, Evm>,
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
transaction_count_hint: usize,
max_concurrency: usize,
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
let (actions_tx, actions_rx) = channel();
@@ -114,7 +111,7 @@ where
trace!(
target: "engine::tree::payload_processor::prewarm",
max_concurrency,
transaction_count_hint,
transaction_count = ctx.env.transaction_count,
"Initialized prewarm task"
);
@@ -124,7 +121,6 @@ where
execution_cache,
ctx,
max_concurrency,
transaction_count_hint,
to_multi_proof,
actions_rx,
parent_span: Span::current(),
@@ -148,7 +144,6 @@ where
let executor = self.executor.clone();
let ctx = self.ctx.clone();
let max_concurrency = self.max_concurrency;
let transaction_count_hint = self.transaction_count_hint;
let span = Span::current();
self.executor.spawn_blocking(move || {
@@ -156,13 +151,14 @@ where
let (done_tx, done_rx) = mpsc::channel();
// When transaction_count_hint is 0, it means the count is unknown. In this case, spawn
// When transaction_count is 0, it means the count is unknown. In this case, spawn
// max workers to handle potentially many transactions in parallel rather
// than bottlenecking on a single worker.
let workers_needed = if transaction_count_hint == 0 {
let transaction_count = ctx.env.transaction_count;
let workers_needed = if transaction_count == 0 {
max_concurrency
} else {
transaction_count_hint.min(max_concurrency)
transaction_count.min(max_concurrency)
};
// Spawn workers

View File

@@ -1,18 +1,21 @@
//! Sparse Trie task related functionality.
use crate::tree::{
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
multiproof::{
dispatch_with_chunking, evm_state_to_hashed_post_state, MultiProofMessage,
VersionedMultiProofTargets, DEFAULT_MAX_TARGETS_FOR_CHUNKING,
},
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
};
use alloy_primitives::B256;
use alloy_rlp::Decodable;
use alloy_rlp::{Decodable, Encodable};
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use rayon::iter::{ParallelBridge, ParallelIterator};
use reth_errors::ProviderError;
use reth_primitives_traits::Account;
use rayon::iter::{IntoParallelRefMutIterator, ParallelBridge, ParallelIterator};
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
use reth_revm::state::EvmState;
use reth_trie::{
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
proof_v2::Target, updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, Nibbles,
TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE,
};
use reth_trie_parallel::{
proof_task::{
@@ -23,10 +26,9 @@ use reth_trie_parallel::{
targets_v2::MultiProofTargetsV2,
};
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
errors::{SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
ClearedSparseStateTrie, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie,
SparseTrieExt,
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
};
use revm_primitives::{hash_map::Entry, B256Map};
use smallvec::SmallVec;
@@ -34,7 +36,7 @@ use std::{
sync::mpsc,
time::{Duration, Instant},
};
use tracing::{debug, debug_span, instrument, trace};
use tracing::{debug, debug_span, error, instrument, trace};
#[expect(clippy::large_enum_variant)]
pub(super) enum SpawnedSparseTrieTask<BPF, A, S>
@@ -203,6 +205,9 @@ where
}
}
/// Maximum number of pending/prewarm updates that we accumulate in memory before actually applying.
const MAX_PENDING_UPDATES: usize = 100;
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
/// Sender for proof results.
@@ -215,6 +220,15 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
trie: SparseStateTrie<A, S>,
/// Handle to the proof worker pools (storage and account).
proof_worker_handle: ProofWorkerHandle,
/// The size of proof targets chunk to spawn in one calculation.
/// If None, chunking is disabled and all targets are processed in a single proof.
chunk_size: Option<usize>,
/// If this number is exceeded and chunking is enabled, then this will override whether or not
/// there are any active workers and force chunking across workers. This is to prevent tasks
/// which are very long from hitting a single worker.
max_targets_for_chunking: usize,
/// Account trie updates.
account_updates: B256Map<LeafUpdate>,
/// Storage trie updates. hashed address -> slot -> update.
@@ -239,6 +253,16 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
/// Cache of storage proof targets that have already been fetched/requested from the proof
/// workers. account -> slot -> lowest `min_len` requested.
fetched_storage_targets: B256Map<B256Map<u8>>,
/// Reusable buffer for RLP encoding of accounts.
account_rlp_buf: Vec<u8>,
/// Whether the last state update has been received.
finished_state_updates: bool,
/// Pending targets to be dispatched to the proof workers.
pending_targets: MultiProofTargetsV2,
/// Number of pending execution/prewarming updates received but not yet passed to
/// `update_leaves`.
pending_updates: usize,
/// Metrics for the sparse trie.
metrics: MultiProofTaskMetrics,
}
@@ -248,12 +272,13 @@ where
A: SparseTrieExt + Default,
S: SparseTrieExt + Default + Clone,
{
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
pub(super) fn new_with_cleared_trie(
/// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`].
pub(super) fn new_with_trie(
updates: CrossbeamReceiver<MultiProofMessage>,
proof_worker_handle: ProofWorkerHandle,
metrics: MultiProofTaskMetrics,
sparse_state_trie: ClearedSparseStateTrie<A, S>,
trie: SparseStateTrie<A, S>,
chunk_size: Option<usize>,
) -> Self {
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
Self {
@@ -261,12 +286,18 @@ where
proof_result_rx,
updates,
proof_worker_handle,
trie: sparse_state_trie.into_inner(),
trie,
chunk_size,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
account_updates: Default::default(),
storage_updates: Default::default(),
pending_account_updates: Default::default(),
fetched_account_targets: Default::default(),
fetched_storage_targets: Default::default(),
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
finished_state_updates: Default::default(),
pending_targets: Default::default(),
pending_updates: Default::default(),
metrics,
}
}
@@ -314,15 +345,8 @@ where
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
let now = Instant::now();
let mut finished_state_updates = false;
loop {
crossbeam_channel::select_biased! {
recv(self.proof_result_rx) -> message => {
let Ok(result) = message else {
unreachable!("we own the sender half")
};
self.on_proof_result(result)?;
},
recv(self.updates) -> message => {
let update = match message {
Ok(m) => m,
@@ -331,27 +355,48 @@ where
}
};
match update {
MultiProofMessage::PrefetchProofs(targets) => {
self.on_prewarm_targets(targets);
}
MultiProofMessage::StateUpdate(_, state) => {
self.on_state_update(state);
}
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
self.on_hashed_state_update(state);
}
MultiProofMessage::BlockAccessList(_) => todo!(),
MultiProofMessage::FinishedStateUpdates => {
finished_state_updates = true;
}
}
self.on_multiproof_message(update);
self.pending_updates += 1;
}
recv(self.proof_result_rx) -> message => {
let Ok(result) = message else {
unreachable!("we own the sender half")
};
let ProofResult::V2(mut result) = result.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
while let Ok(next) = self.proof_result_rx.try_recv() {
let ProofResult::V2(res) = next.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
result.extend(res);
}
self.on_proof_result(result)?;
},
}
self.process_updates()?;
if self.updates.is_empty() && self.proof_result_rx.is_empty() {
// If we don't have any pending messages, we can spend some time on computing
// storage roots and promoting account updates.
self.dispatch_pending_targets();
self.promote_pending_account_updates()?;
self.dispatch_pending_targets();
} else if self.updates.is_empty() || self.pending_updates > MAX_PENDING_UPDATES {
// If we don't have any pending updates OR we've accumulated a lot already, apply
// them to the trie,
self.process_leaf_updates()?;
self.dispatch_pending_targets();
} else if self.updates.is_empty() ||
self.pending_targets.chunking_length() > self.chunk_size.unwrap_or_default()
{
// Make sure to dispatch targets if we don't have any updates or if we've
// accumulated a lot of them.
self.dispatch_pending_targets();
}
if finished_state_updates &&
if self.finished_state_updates &&
self.account_updates.is_empty() &&
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
{
@@ -359,11 +404,6 @@ where
}
}
// Process any remaining pending account updates.
if !self.pending_account_updates.is_empty() {
self.process_updates()?;
}
debug!(target: "engine::root", "All proofs processed, ending calculation");
let start = Instant::now();
@@ -379,6 +419,22 @@ where
Ok(StateRootComputeOutcome { state_root, trie_updates })
}
/// Processes a [`MultiProofMessage`].
fn on_multiproof_message(&mut self, message: MultiProofMessage) {
match message {
MultiProofMessage::PrefetchProofs(targets) => self.on_prewarm_targets(targets),
MultiProofMessage::StateUpdate(_, state) => self.on_state_update(state),
MultiProofMessage::EmptyProof { .. } => unreachable!(),
MultiProofMessage::BlockAccessList(_) => todo!(),
MultiProofMessage::FinishedStateUpdates => self.finished_state_updates = true,
}
}
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
let VersionedMultiProofTargets::V2(targets) = targets else {
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
@@ -414,11 +470,7 @@ where
)]
fn on_state_update(&mut self, update: EvmState) {
let hashed_state_update = evm_state_to_hashed_post_state(update);
self.on_hashed_state_update(hashed_state_update)
}
/// Processes a hashed state update and encodes all state changes as trie updates.
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
for (address, storage) in hashed_state_update.storages {
for (slot, value) in storage.storage {
let encoded = if value.is_zero() {
@@ -456,150 +508,242 @@ where
fn on_proof_result(
&mut self,
result: ProofResultMessage,
result: DecodedMultiProofV2,
) -> Result<(), ParallelStateRootError> {
let ProofResult::V2(result) = result.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
})
}
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
fn process_updates(&mut self) -> Result<(), ProviderError> {
let mut targets = MultiProofTargetsV2::default();
/// Applies all account and storage leaf updates to corresponding tries and collects any new
/// multiproof targets.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn process_leaf_updates(&mut self) -> SparseTrieResult<()> {
self.pending_updates = 0;
for (addr, updates) in &mut self.storage_updates {
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
let fetched_storage = self.fetched_storage_targets.entry(*addr).or_default();
// Start with processing all storage updates in parallel.
let storage_results = self
.storage_updates
.iter_mut()
.map(|(address, updates)| {
let trie = self.trie.take_or_create_storage_trie(address);
let fetched = self.fetched_storage_targets.remove(address).unwrap_or_default();
trie.update_leaves(updates, |path, min_len| match fetched_storage.entry(path) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
targets
.storage_targets
.entry(*addr)
.or_default()
.push(Target::new(path).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets
.storage_targets
.entry(*addr)
.or_default()
.push(Target::new(path).with_min_len(min_len));
}
(address, updates, fetched, trie)
})
.map_err(ProviderError::other)?;
.par_bridge()
.map(|(address, updates, mut fetched, mut trie)| {
let mut targets = Vec::new();
// If all storage updates were processed, we can now compute the new storage root.
if updates.is_empty() {
let storage_root =
trie.root().expect("updates are drained, trie should be revealed by now");
// If there is a pending account update for this address with known info, we can
// encode it into proper update right away.
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
entry.get().is_some()
{
let account = entry.remove().expect("just checked, should be Some");
let encoded = if account.is_none_or(|account| account.is_empty()) &&
storage_root == EMPTY_ROOT_HASH
{
Vec::new()
} else {
// TODO: optimize allocation
alloy_rlp::encode(
account.unwrap_or_default().into_trie_account(storage_root),
)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
}
}
// Now handle pending account updates that can be upgraded to a proper update.
self.pending_account_updates.retain(|addr, account| {
// If account has pending storage updates, it is still pending.
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
return true;
}
// Get the current account state either from the trie or from latest account update.
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
Some(encoded).filter(|encoded| !encoded.is_empty())
} else if !self.account_updates.contains_key(addr) {
self.trie.get_account_value(addr)
} else {
// Needs to be revealed first
return true;
};
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
let (account, storage_root) = if let Some(account) = account.take() {
// If account is Some(_) here it means it didn't have any storage updates
// and we can fetch the storage root directly from the account trie.
//
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
(account, storage_root)
} else {
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
};
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
Vec::new()
} else {
let account = account.unwrap_or_default().into_trie_account(storage_root);
// TODO: optimize allocation
alloy_rlp::encode(account)
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
false
});
// Process account trie updates and fill the account targets.
self.trie
.trie_mut()
.update_leaves(&mut self.account_updates, |target, min_len| {
match self.fetched_account_targets.entry(target) {
trie.update_leaves(updates, |path, min_len| match fetched.entry(path) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
targets.account_targets.push(Target::new(target).with_min_len(min_len));
targets.push(Target::new(path).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets.account_targets.push(Target::new(target).with_min_len(min_len));
targets.push(Target::new(path).with_min_len(min_len));
}
})?;
SparseTrieResult::Ok((address, targets, fetched, trie))
})
.collect::<Result<Vec<_>, _>>()?;
for (address, targets, fetched, trie) in storage_results {
self.fetched_storage_targets.insert(*address, fetched);
self.trie.insert_storage_trie(*address, trie);
if !targets.is_empty() {
self.pending_targets.storage_targets.entry(*address).or_default().extend(targets);
}
}
// Process account trie updates and fill the account targets.
self.process_account_leaf_updates()?;
Ok(())
}
/// Invokes `update_leaves` for the accounts trie and collects any new targets.
///
/// Returns whether any updates were drained (applied to the trie).
fn process_account_leaf_updates(&mut self) -> SparseTrieResult<bool> {
let updates_len_before = self.account_updates.len();
self.trie.trie_mut().update_leaves(
&mut self.account_updates,
|target, min_len| match self.fetched_account_targets.entry(target) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
self.pending_targets
.account_targets
.push(Target::new(target).with_min_len(min_len));
}
}
})
.map_err(ProviderError::other)?;
Entry::Vacant(entry) => {
entry.insert(min_len);
self.pending_targets
.account_targets
.push(Target::new(target).with_min_len(min_len));
}
},
)?;
if !targets.is_empty() {
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
targets,
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
0,
HashedPostState::default(),
Instant::now(),
),
})?;
Ok(self.account_updates.len() < updates_len_before)
}
/// Iterates through all storage tries for which all updates were processed, computes their
/// storage roots, and promotes corresponding pending account updates into proper leaf updates
/// for accounts trie.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn promote_pending_account_updates(&mut self) -> SparseTrieResult<()> {
self.process_leaf_updates()?;
if self.pending_account_updates.is_empty() {
return Ok(());
}
let roots = self
.trie
.storage_tries_mut()
.par_iter_mut()
.filter(|(address, _)| {
self.storage_updates.get(*address).is_some_and(|updates| updates.is_empty())
})
.map(|(address, trie)| {
let root =
trie.root().expect("updates are drained, trie should be revealed by now");
(address, root)
})
.collect::<Vec<_>>();
for (addr, storage_root) in roots {
// If the storage root is known and we have a pending update for this account, encode it
// into a proper update.
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
entry.get().is_some()
{
let account = entry.remove().expect("just checked, should be Some");
let encoded = if account.is_none_or(|account| account.is_empty()) &&
storage_root == EMPTY_ROOT_HASH
{
Vec::new()
} else {
self.account_rlp_buf.clear();
account
.unwrap_or_default()
.into_trie_account(storage_root)
.encode(&mut self.account_rlp_buf);
self.account_rlp_buf.clone()
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
}
loop {
// Now handle pending account updates that can be upgraded to a proper update.
let account_rlp_buf = &mut self.account_rlp_buf;
self.pending_account_updates.retain(|addr, account| {
// If account has pending storage updates, it is still pending.
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
return true;
}
// Get the current account state either from the trie or from latest account update.
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
Some(encoded).filter(|encoded| !encoded.is_empty())
} else if !self.account_updates.contains_key(addr) {
self.trie.get_account_value(addr)
} else {
// Needs to be revealed first
return true;
};
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
let (account, storage_root) = if let Some(account) = account.take() {
// If account is Some(_) here it means it didn't have any storage updates
// and we can fetch the storage root directly from the account trie.
//
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
(account, storage_root)
} else {
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
};
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
Vec::new()
} else {
account_rlp_buf.clear();
account.unwrap_or_default().into_trie_account(storage_root).encode(account_rlp_buf);
account_rlp_buf.clone()
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
false
});
// Only exit when no new updates are processed.
//
// We need to keep iterating if any updates are being drained because that might
// indicate that more pending account updates can be promoted.
if !self.process_account_leaf_updates()? {
break
}
}
Ok(())
}
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn dispatch_pending_targets(&mut self) {
if !self.pending_targets.is_empty() {
let chunking_length = self.pending_targets.chunking_length();
dispatch_with_chunking(
std::mem::take(&mut self.pending_targets),
chunking_length,
self.chunk_size,
self.max_targets_for_chunking,
self.proof_worker_handle.available_account_workers(),
self.proof_worker_handle.available_storage_workers(),
MultiProofTargetsV2::chunks,
|proof_targets| {
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(
AccountMultiproofInput::V2 {
targets: proof_targets,
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
0,
HashedPostState::default(),
Instant::now(),
),
},
) {
error!("failed to dispatch account multiproof: {e:?}");
}
},
);
}
}
}
/// Outcome of the state root computation, including the state root itself with
@@ -651,7 +795,7 @@ where
.storages
.into_iter()
.map(|(address, storage)| (address, storage, trie.take_storage_trie(&address)))
.par_bridge()
.par_bridge_buffered()
.map(|(address, storage, storage_trie)| {
let _enter =
debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie", ?address)

View File

@@ -407,6 +407,7 @@ where
hash: input.hash(),
parent_hash: input.parent_hash(),
parent_state_root: parent_block.state_root(),
transaction_count: input.transaction_count(),
};
// Plan the strategy used for state root computation.
@@ -519,6 +520,14 @@ where
info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished");
// we double check the state root here for good measure
if state_root == block.header().state_root() {
// Compare trie updates with serial computation if configured
if self.config.always_compare_trie_updates() {
self.compare_trie_updates_with_serial(
overlay_factory.clone(),
&hashed_state,
trie_updates.clone(),
);
}
maybe_state_root = Some((state_root, trie_updates, elapsed))
} else {
warn!(
@@ -894,6 +903,62 @@ where
.root_with_updates()?)
}
/// Compares trie updates from the state root task with serial state root computation.
///
/// This is used for debugging and validating the correctness of the parallel state root
/// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this
/// method runs a separate serial state root computation and compares the resulting trie
/// updates.
fn compare_trie_updates_with_serial(
&self,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &HashedPostState,
task_trie_updates: TrieUpdates,
) {
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
match self.compute_state_root_serial(overlay_factory.clone(), hashed_state) {
Ok((serial_root, serial_trie_updates)) => {
debug!(
target: "engine::tree::payload_validator",
?serial_root,
"Serial state root computation finished for comparison"
);
// Get a database provider to use as trie cursor factory
match overlay_factory.database_provider_ro() {
Ok(provider) => {
if let Err(err) = super::trie_updates::compare_trie_updates(
&provider,
task_trie_updates,
serial_trie_updates,
) {
warn!(
target: "engine::tree::payload_validator",
%err,
"Error comparing trie updates"
);
}
}
Err(err) => {
warn!(
target: "engine::tree::payload_validator",
%err,
"Failed to get database provider for trie update comparison"
);
}
}
}
Err(err) => {
warn!(
target: "engine::tree::payload_validator",
%err,
"Failed to compute serial state root for comparison"
);
}
}
}
/// Validates the block after execution.
///
/// This performs:

View File

@@ -1,9 +1,9 @@
//! Contains a precompile cache backed by `schnellru::LruMap` (LRU by length).
use alloy_primitives::Bytes;
use dashmap::DashMap;
use moka::policy::EvictionPolicy;
use reth_evm::precompiles::{DynPrecompile, Precompile, PrecompileInput};
use reth_primitives_traits::dashmap::DashMap;
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
use revm_primitives::Address;
use std::{hash::Hash, sync::Arc};

View File

@@ -28,6 +28,7 @@ use reth_ethereum_primitives::{Block, EthPrimitives};
use reth_evm_ethereum::MockEvmConfig;
use reth_primitives_traits::Block as _;
use reth_provider::test_utils::MockEthProvider;
use reth_tasks::spawn_os_thread;
use std::{
collections::BTreeMap,
str::FromStr,
@@ -538,10 +539,7 @@ async fn test_tree_persist_blocks() {
.get_executed_blocks(1..tree_config.persistence_threshold() + 2)
.collect();
let test_harness = TestHarness::new(chain_spec).with_blocks(blocks.clone());
std::thread::Builder::new()
.name("Engine Task".to_string())
.spawn(|| test_harness.tree.run())
.unwrap();
spawn_os_thread("engine", || test_harness.tree.run());
// send a message to the tree to enter the main loop.
test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(vec![])).unwrap();
@@ -1989,10 +1987,7 @@ mod forkchoice_updated_tests {
let action_rx = test_harness.action_rx;
// Spawn tree in background thread
std::thread::Builder::new()
.name("Engine Task".to_string())
.spawn(|| test_harness.tree.run())
.unwrap();
spawn_os_thread("engine", || test_harness.tree.run());
// Send terminate request
to_tree_tx

View File

@@ -98,7 +98,7 @@ impl StorageTrieUpdatesDiff {
/// Compares the trie updates from state root task, regular state root calculation and database,
/// and logs the differences if there's any.
pub(super) fn compare_trie_updates(
pub(crate) fn compare_trie_updates(
trie_cursor_factory: impl TrieCursorFactory,
task: TrieUpdates,
regular: TrieUpdates,
@@ -186,7 +186,8 @@ fn compare_storage_trie_updates<C: TrieCursor>(
task: &mut StorageTrieUpdates,
regular: &mut StorageTrieUpdates,
) -> Result<StorageTrieUpdatesDiff, DatabaseError> {
let database_not_exists = trie_cursor()?.next()?.is_none();
// Check if the storage trie exists by seeking to the first entry
let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none();
let mut diff = StorageTrieUpdatesDiff {
// If the deletion is a no-op, meaning that the entry is not in the
// database, do not add it to the diff.

View File

@@ -20,8 +20,6 @@ reth-era.workspace = true
# http
bytes.workspace = true
reqwest.workspace = true
reqwest.default-features = false
reqwest.features = ["stream", "rustls-tls-native-roots"]
# async
tokio.workspace = true

View File

@@ -86,7 +86,7 @@ where
mut self,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> Result<()>,
) -> Result<()>
@@ -132,7 +132,7 @@ pub(crate) fn run_commands_with<C, Ext, Rpc, N, SubCmd>(
runner: CliRunner,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> Result<()>,
) -> Result<()>

View File

@@ -131,7 +131,7 @@ impl<
/// ````
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
C: ChainSpecParser<ChainSpec = ChainSpec>,
{
@@ -148,7 +148,7 @@ impl<
self,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
) -> eyre::Result<()>
@@ -180,7 +180,7 @@ impl<
/// ```
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
C: ChainSpecParser<ChainSpec = ChainSpec>,
{
@@ -196,7 +196,7 @@ impl<
runner: CliRunner,
components: impl CliComponentsBuilder<N>,
launcher: impl AsyncFnOnce(
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
Ext,
) -> eyre::Result<()>,
) -> eyre::Result<()>

View File

@@ -119,10 +119,9 @@ impl EthereumNode {
/// use reth_db::open_db_read_only;
/// use reth_node_ethereum::EthereumNode;
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
/// use std::sync::Arc;
///
/// let factory = EthereumNode::provider_factory_builder()
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
/// .db(open_db_read_only("db", Default::default()).unwrap())
/// .chainspec(ChainSpecBuilder::mainnet().build().into())
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())

View File

@@ -100,10 +100,12 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(),
);
let genesis_hash = chain_spec.genesis_hash();
let node_config = NodeConfig::test()
.with_chain(chain_spec)
.with_unused_ports()
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_force_blob_sidecar_upcasting(),
);
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
.testing_node(exec.clone())
.node(EthereumNode::default())
@@ -125,7 +127,7 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?;
// fetch it from rpc
let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?;
// assert that sidecar was converted to eip7594
// assert that sidecar was converted to eip7594 (force upcasting is enabled)
assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip7594());
// validate sidecar
TransactionTestContext::validate_sidecar(envelope);
@@ -161,10 +163,12 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> {
.build(),
);
let genesis_hash = chain_spec.genesis_hash();
let node_config = NodeConfig::test()
.with_chain(chain_spec)
.with_unused_ports()
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
RpcServerArgs::default()
.with_unused_ports()
.with_http()
.with_force_blob_sidecar_upcasting(),
);
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
.testing_node(exec.clone())
.node(EthereumNode::default())

View File

@@ -511,9 +511,8 @@ mod compact {
total_length += flags.len() + buffer.len();
buf.put_slice(&flags);
if zstd {
reth_zstd_compressors::RECEIPT_COMPRESSOR.with(|compressor| {
let compressed =
compressor.borrow_mut().compress(&buffer).expect("Failed to compress.");
reth_zstd_compressors::with_receipt_compressor(|compressor| {
let compressed = compressor.compress(&buffer).expect("Failed to compress.");
buf.put(compressed.as_slice());
});
} else {
@@ -525,8 +524,7 @@ mod compact {
fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) {
let (flags, mut buf) = ReceiptFlags::from(buf);
if flags.__zstd() != 0 {
reth_zstd_compressors::RECEIPT_DECOMPRESSOR.with(|decompressor| {
let decompressor = &mut decompressor.borrow_mut();
reth_zstd_compressors::with_receipt_decompressor(|decompressor| {
let decompressed = decompressor.decompress(buf);
let original_buf = buf;
let mut buf: &[u8] = decompressed;

View File

@@ -577,19 +577,11 @@ impl reth_codecs::Compact for TransactionSigned {
let tx_bits = if zstd_bit {
let mut tmp = Vec::with_capacity(256);
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
let mut compressor = compressor.borrow_mut();
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
})
} else {
let mut compressor = reth_zstd_compressors::create_tx_compressor();
reth_zstd_compressors::with_tx_compressor(|compressor| {
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
}
})
} else {
self.transaction.to_compact(buf) as u8
};
@@ -611,26 +603,13 @@ impl reth_codecs::Compact for TransactionSigned {
let zstd_bit = bitflags >> 3;
let (transaction, buf) = if zstd_bit != 0 {
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
let mut decompressor = decompressor.borrow_mut();
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) =
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
(transaction, buf)
})
} else {
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) =
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
(transaction, buf)
}
})
} else {
let transaction_type = bitflags >> 1;
Transaction::from_compact(buf, transaction_type)

View File

@@ -36,7 +36,6 @@ rayon = { workspace = true, optional = true }
[dev-dependencies]
reth-ethereum-primitives.workspace = true
reth-ethereum-forks.workspace = true
[features]
default = ["std"]
@@ -47,7 +46,6 @@ std = [
"alloy-primitives/std",
"alloy-consensus/std",
"revm/std",
"reth-ethereum-forks/std",
"alloy-evm/std",
"reth-execution-errors/std",
"reth-execution-types/std",

View File

@@ -41,6 +41,7 @@ metrics.workspace = true
[dev-dependencies]
reth-tracing.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
tokio = { workspace = true, features = ["rt-multi-thread"] }
secp256k1 = { workspace = true, features = ["std", "rand"] }
rand_08.workspace = true

View File

@@ -30,12 +30,12 @@ tokio-stream.workspace = true
hickory-resolver = { workspace = true, features = ["tokio"] }
# misc
dashmap = { workspace = true, features = ["inline"] }
data-encoding.workspace = true
linked_hash_set.workspace = true
schnellru.workspace = true
thiserror.workspace = true
tracing.workspace = true
parking_lot.workspace = true
serde = { workspace = true, optional = true }
serde_with = { workspace = true, optional = true }
@@ -56,9 +56,9 @@ serde = [
"alloy-primitives/serde",
"enr/serde",
"linked_hash_set/serde",
"parking_lot/serde",
"rand/serde",
"secp256k1/serde",
"hickory-resolver/serde",
"reth-ethereum-forks/serde",
"dashmap/serde",
]

View File

@@ -1,9 +1,9 @@
//! Perform DNS lookups
use dashmap::DashMap;
use hickory_resolver::name_server::ConnectionProvider;
pub use hickory_resolver::{ResolveError, TokioResolver};
use parking_lot::RwLock;
use std::{collections::HashMap, future::Future};
use std::future::Future;
use tracing::trace;
/// A type that can lookup DNS entries
@@ -72,25 +72,25 @@ impl Resolver for DnsResolver {
/// A [Resolver] that uses an in memory map to lookup entries
#[derive(Debug, Default)]
pub struct MapResolver(RwLock<HashMap<String, String>>);
pub struct MapResolver(DashMap<String, String>);
// === impl MapResolver ===
impl MapResolver {
/// Inserts a key-value pair into the map.
pub fn insert(&self, k: String, v: String) -> Option<String> {
self.0.write().insert(k, v)
self.0.insert(k, v)
}
/// Returns the value corresponding to the key
pub fn get(&self, k: &str) -> Option<String> {
self.0.read().get(k).cloned()
self.0.get(k).map(|entry| entry.value().clone())
}
/// Removes a key from the map, returning the value at the key if the key was previously in the
/// map.
pub fn remove(&self, k: &str) -> Option<String> {
self.0.write().remove(k)
self.0.remove(k).map(|(_, v)| v)
}
}

View File

@@ -251,6 +251,8 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
}
/// Creates a preconfigured node for testing purposes with a specific datadir.
///
/// The entire `datadir` will be cleaned up when the node is dropped.
#[cfg(feature = "test-utils")]
pub fn testing_node_with_datadir(
mut self,
@@ -268,7 +270,7 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
let data_dir =
path.unwrap_or_chain_default(self.config.chain.chain(), self.config.datadir.clone());
let db = reth_db::test_utils::create_test_rw_db_with_path(data_dir.db());
let db = reth_db::test_utils::create_test_rw_db_with_datadir(data_dir.data_dir());
WithLaunchContext { builder: self.with_database(db), task_executor }
}

View File

@@ -236,7 +236,7 @@ impl LaunchContext {
.map_or(0, |num| num.get().saturating_sub(reserved_cpu_cores).max(1));
if let Err(err) = ThreadPoolBuilder::new()
.num_threads(num_threads)
.thread_name(|i| format!("reth-rayon-{i}"))
.thread_name(|i| format!("rayon-{i}"))
.build_global()
{
warn!(%err, "Failed to build global thread pool")

View File

@@ -218,9 +218,9 @@ impl<Node: FullNodeComponents, AddOns: NodeAddOns<Node>> DerefMut for FullNode<N
}
/// Helper type alias to define [`FullNode`] for a given [`Node`].
pub type FullNodeFor<N, DB = Arc<DatabaseEnv>> =
pub type FullNodeFor<N, DB = DatabaseEnv> =
FullNode<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;
/// Helper type alias to define [`NodeHandle`] for a given [`Node`].
pub type NodeHandleFor<N, DB = Arc<DatabaseEnv>> =
pub type NodeHandleFor<N, DB = DatabaseEnv> =
NodeHandle<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;

View File

@@ -1192,6 +1192,7 @@ impl<'a, N: FullNodeComponents<Types: NodeTypes<ChainSpec: Hardforks + EthereumH
.pending_block_kind(self.config.pending_block_kind)
.raw_tx_forwarder(self.config.raw_tx_forwarder)
.evm_memory_limit(self.config.rpc_evm_memory_limit)
.force_blob_sidecar_upcasting(self.config.force_blob_sidecar_upcasting)
}
}

View File

@@ -1,7 +1,10 @@
//! clap [Args](clap::Args) for engine purposes
use clap::{builder::Resettable, Args};
use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE};
use reth_engine_primitives::{
TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
};
use std::sync::OnceLock;
use crate::node_config::{
@@ -38,6 +41,8 @@ pub struct DefaultEngineValues {
disable_proof_v2: bool,
cache_metrics_disabled: bool,
enable_sparse_trie_as_cache: bool,
sparse_trie_prune_depth: usize,
sparse_trie_max_storage_tries: usize,
}
impl DefaultEngineValues {
@@ -179,6 +184,18 @@ impl DefaultEngineValues {
self.enable_sparse_trie_as_cache = v;
self
}
/// Set the sparse trie prune depth by default
pub const fn with_sparse_trie_prune_depth(mut self, v: usize) -> Self {
self.sparse_trie_prune_depth = v;
self
}
/// Set the maximum number of storage tries to retain after sparse trie pruning by default
pub const fn with_sparse_trie_max_storage_tries(mut self, v: usize) -> Self {
self.sparse_trie_max_storage_tries = v;
self
}
}
impl Default for DefaultEngineValues {
@@ -205,6 +222,8 @@ impl Default for DefaultEngineValues {
disable_proof_v2: false,
cache_metrics_disabled: false,
enable_sparse_trie_as_cache: false,
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
}
}
}
@@ -336,6 +355,14 @@ pub struct EngineArgs {
/// Enable sparse trie as cache.
#[arg(long = "engine.enable-sparse-trie-as-cache", default_value_t = DefaultEngineValues::get_global().enable_sparse_trie_as_cache, conflicts_with = "disable_proof_v2")]
pub enable_sparse_trie_as_cache: bool,
/// Sparse trie prune depth.
#[arg(long = "engine.sparse-trie-prune-depth", default_value_t = DefaultEngineValues::get_global().sparse_trie_prune_depth, requires = "enable_sparse_trie_as_cache")]
pub sparse_trie_prune_depth: usize,
/// Maximum number of storage tries to retain after sparse trie pruning.
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries, requires = "enable_sparse_trie_as_cache")]
pub sparse_trie_max_storage_tries: usize,
}
#[allow(deprecated)]
@@ -363,6 +390,8 @@ impl Default for EngineArgs {
disable_proof_v2,
cache_metrics_disabled,
enable_sparse_trie_as_cache,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
} = DefaultEngineValues::get_global().clone();
Self {
persistence_threshold,
@@ -390,6 +419,8 @@ impl Default for EngineArgs {
disable_proof_v2,
cache_metrics_disabled,
enable_sparse_trie_as_cache,
sparse_trie_prune_depth,
sparse_trie_max_storage_tries,
}
}
}
@@ -397,7 +428,7 @@ impl Default for EngineArgs {
impl EngineArgs {
/// Creates a [`TreeConfig`] from the engine arguments.
pub fn tree_config(&self) -> TreeConfig {
let mut config = TreeConfig::default()
TreeConfig::default()
.with_persistence_threshold(self.persistence_threshold)
.with_memory_block_buffer_target(self.memory_block_buffer_target)
.with_legacy_state_root(self.legacy_state_root_task_enabled)
@@ -414,21 +445,14 @@ impl EngineArgs {
.with_always_process_payload_attributes_on_canonical_head(
self.always_process_payload_attributes_on_canonical_head,
)
.with_unwind_canonical_header(self.allow_unwind_canonical_header);
if let Some(count) = self.storage_worker_count {
config = config.with_storage_worker_count(count);
}
if let Some(count) = self.account_worker_count {
config = config.with_account_worker_count(count);
}
config = config.with_disable_proof_v2(self.disable_proof_v2);
config = config.without_cache_metrics(self.cache_metrics_disabled);
config = config.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache);
config
.with_unwind_canonical_header(self.allow_unwind_canonical_header)
.with_storage_worker_count_opt(self.storage_worker_count)
.with_account_worker_count_opt(self.account_worker_count)
.with_disable_proof_v2(self.disable_proof_v2)
.without_cache_metrics(self.cache_metrics_disabled)
.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache)
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
}
}
@@ -479,7 +503,9 @@ mod tests {
account_worker_count: Some(8),
disable_proof_v2: false,
cache_metrics_disabled: true,
enable_sparse_trie_as_cache: false,
enable_sparse_trie_as_cache: true,
sparse_trie_prune_depth: 10,
sparse_trie_max_storage_tries: 100,
};
let parsed_args = CommandParser::<EngineArgs>::parse_from([
@@ -510,6 +536,11 @@ mod tests {
"--engine.account-worker-count",
"8",
"--engine.disable-cache-metrics",
"--engine.enable-sparse-trie-as-cache",
"--engine.sparse-trie-prune-depth",
"10",
"--engine.sparse-trie-max-storage-tries",
"100",
])
.args;

View File

@@ -6,7 +6,7 @@ use clap::{builder::RangedU64ValueParser, Args};
use reth_chainspec::EthereumHardforks;
use reth_config::config::PruneConfig;
use reth_prune_types::{
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE,
};
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
@@ -81,7 +81,7 @@ impl Default for DefaultPruningValues {
minimal_prune_modes: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: Some(PruneMode::Full),
receipts: Some(PruneMode::Full),
receipts: Some(PruneMode::Distance(MINIMUM_DISTANCE)),
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
bodies_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),

View File

@@ -647,6 +647,14 @@ pub struct RpcServerArgs {
/// transactions from the same sender will also be skipped.
#[arg(long = "testing.skip-invalid-transactions", default_value_t = true)]
pub testing_skip_invalid_transactions: bool,
/// Force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
///
/// When enabled, blob transactions submitted via `eth_sendRawTransaction` with EIP-4844
/// sidecars will be automatically converted to EIP-7594 format if the next block is Osaka.
/// By default this is disabled, meaning transactions are submitted as-is.
#[arg(long = "rpc.force-blob-sidecar-upcasting", default_value_t = false)]
pub rpc_force_blob_sidecar_upcasting: bool,
}
impl RpcServerArgs {
@@ -768,6 +776,12 @@ impl RpcServerArgs {
self.rpc_send_raw_transaction_sync_timeout = timeout;
self
}
/// Enables forced blob sidecar upcasting from EIP-4844 to EIP-7594 format.
pub const fn with_force_blob_sidecar_upcasting(mut self) -> Self {
self.rpc_force_blob_sidecar_upcasting = true;
self
}
}
impl Default for RpcServerArgs {
@@ -860,6 +874,7 @@ impl Default for RpcServerArgs {
gas_price_oracle,
rpc_send_raw_transaction_sync_timeout,
testing_skip_invalid_transactions: true,
rpc_force_blob_sidecar_upcasting: false,
}
}
}
@@ -1036,6 +1051,7 @@ mod tests {
},
rpc_send_raw_transaction_sync_timeout: std::time::Duration::from_secs(30),
testing_skip_invalid_transactions: true,
rpc_force_blob_sidecar_upcasting: false,
};
let parsed_args = CommandParser::<RpcServerArgs>::parse_from([

View File

@@ -21,7 +21,7 @@ alloy-primitives.workspace = true
alloy-consensus.workspace = true
tokio.workspace = true
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
tokio-tungstenite.workspace = true
futures-util.workspace = true
tokio-stream.workspace = true

View File

@@ -37,7 +37,7 @@ tempfile = { workspace = true, optional = true }
tikv-jemalloc-ctl = { workspace = true, optional = true, features = ["stats"] }
[target.'cfg(target_os = "linux")'.dependencies]
procfs = "0.17.0"
procfs = "0.18.0"
[dev-dependencies]
reqwest.workspace = true

View File

@@ -37,7 +37,7 @@ pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCom
use reth_optimism_chainspec::OpChainSpec;
use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator};
use std::{ffi::OsString, fmt, marker::PhantomData, sync::Arc};
use std::{ffi::OsString, fmt, marker::PhantomData};
use chainspec::OpChainSpecParser;
use clap::Parser;
@@ -121,7 +121,7 @@ where
/// [`NodeCommand`](reth_cli_commands::node::NodeCommand).
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
{
self.with_runner(CliRunner::try_default_runtime()?, launcher)
@@ -130,7 +130,7 @@ where
/// Execute the configured cli command with the provided [`CliRunner`].
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
where
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
Fut: Future<Output = eyre::Result<()>>,
{
let mut this = self.configure();

View File

@@ -38,7 +38,7 @@ op-alloy-rpc-types-engine = { workspace = true, features = ["k256"] }
# io
tokio.workspace = true
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
tokio-tungstenite.workspace = true
serde_json.workspace = true
url.workspace = true
futures-util.workspace = true

View File

@@ -101,7 +101,7 @@ impl<T: SignedTransaction> SequenceManager<T> {
// Bundle completed sequence with its decoded transactions and push to cache
// Ring buffer automatically evicts oldest entry when full
let txs = std::mem::take(&mut self.pending_transactions);
self.completed_cache.push((completed, txs));
self.completed_cache.enqueue((completed, txs));
// ensure cache is wiped on new flashblock
let _ = self.pending.take_cached_reads();

View File

@@ -2,6 +2,7 @@ use futures_util::stream::StreamExt;
use reth_optimism_flashblocks::WsFlashBlockStream;
#[tokio::test]
#[ignore = "depends on external service availability"]
async fn test_streaming_flashblocks_from_remote_source_is_successful() {
let items = 3;
let ws_url = "wss://sepolia.flashblocks.base.org/ws".parse().unwrap();

View File

@@ -219,10 +219,9 @@ impl OpNode {
/// use reth_optimism_chainspec::OpChainSpecBuilder;
/// use reth_optimism_node::OpNode;
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
/// use std::sync::Arc;
///
/// let factory = OpNode::provider_factory_builder()
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
/// .db(open_db_read_only("db", Default::default()).unwrap())
/// .chainspec(OpChainSpecBuilder::base_mainnet().build().into())
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())

View File

@@ -435,19 +435,11 @@ impl reth_codecs::Compact for OpTransactionSigned {
let tx_bits = if zstd_bit {
let mut tmp = Vec::with_capacity(256);
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
let mut compressor = compressor.borrow_mut();
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
})
} else {
let mut compressor = reth_zstd_compressors::create_tx_compressor();
reth_zstd_compressors::with_tx_compressor(|compressor| {
let tx_bits = self.transaction.to_compact(&mut tmp);
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
tx_bits as u8
}
})
} else {
self.transaction.to_compact(buf) as u8
};
@@ -469,29 +461,15 @@ impl reth_codecs::Compact for OpTransactionSigned {
let zstd_bit = bitflags >> 3;
let (transaction, buf) = if zstd_bit != 0 {
if cfg!(feature = "std") {
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
let mut decompressor = decompressor.borrow_mut();
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) = OpTypedTransaction::from_compact(
decompressor.decompress(buf),
transaction_type,
);
(transaction, buf)
})
} else {
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
// TODO: enforce that zstd is only present at a "top" level type
let transaction_type = (bitflags & 0b110) >> 1;
let (transaction, _) = OpTypedTransaction::from_compact(
decompressor.decompress(buf),
transaction_type,
);
(transaction, buf)
}
})
} else {
let transaction_type = bitflags >> 1;
OpTypedTransaction::from_compact(buf, transaction_type)

View File

@@ -61,7 +61,7 @@ op-revm.workspace = true
tokio.workspace = true
futures.workspace = true
tokio-stream.workspace = true
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
async-trait.workspace = true
tower.workspace = true

View File

@@ -36,6 +36,7 @@ secp256k1 = { workspace = true, features = ["recovery"], optional = true }
auto_impl.workspace = true
byteorder = { workspace = true, optional = true }
bytes.workspace = true
dashmap = { workspace = true, features = ["inline"], optional = true }
derive_more.workspace = true
once_cell.workspace = true
serde_with = { workspace = true, optional = true }
@@ -116,6 +117,7 @@ arbitrary = [
"alloy-trie/arbitrary",
"reth-chainspec/arbitrary",
"alloy-rpc-types-eth?/arbitrary",
"dashmap?/arbitrary",
]
serde-bincode-compat = [
"serde",
@@ -144,6 +146,7 @@ serde = [
"revm-state/serde",
"rand_08/serde",
"alloy-rpc-types-eth?/serde",
"dashmap?/serde",
]
reth-codec = [
"dep:reth-codecs",
@@ -157,4 +160,5 @@ op = [
rayon = [
"dep:rayon",
]
dashmap = ["dep:dashmap"]
rpc-compat = ["alloy-rpc-types-eth"]

View File

@@ -188,6 +188,12 @@ pub mod serde_bincode_compat;
pub mod size;
pub use size::InMemorySize;
/// Rayon utilities
#[cfg(feature = "rayon")]
pub mod rayon;
#[cfg(feature = "rayon")]
pub use rayon::ParallelBridgeBuffered;
/// Node traits
pub mod node;
pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy};
@@ -239,3 +245,12 @@ pub mod test_utils {
#[cfg(any(test, feature = "test-utils"))]
pub use crate::{block::TestBlock, header::test_utils::TestHeader};
}
/// Re-exports of `dashmap` types with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
#[cfg(feature = "dashmap")]
pub mod dashmap {
pub use ::dashmap::{mapref, DashSet, Entry};
/// Re-export of `DashMap` with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
pub type DashMap<K, V, S = alloy_primitives::map::DefaultHashBuilder> =
::dashmap::DashMap<K, V, S>;
}

View File

@@ -0,0 +1,32 @@
//! Rayon parallel iterator utilities.
use alloc::vec::Vec;
use rayon::iter::IntoParallelIterator;
/// Extension trait for iterators to convert them to parallel iterators via collection.
///
/// This is an alternative to [`rayon::iter::ParallelBridge`] that first collects the iterator
/// into a `Vec`, then calls [`IntoParallelIterator`] on it. This avoids the mutex contention
/// that can occur with `par_bridge` when either the iterator's `next()` method is fast or the
/// parallel tasks are fast, as `par_bridge` wraps the iterator in a mutex.
///
/// # When to use
///
/// Use `par_bridge_buffered` instead of `par_bridge` when:
/// - The iterator produces items quickly
/// - The parallel work per item is relatively light
/// - The total number of items is known to be reasonable for memory
///
/// Stick with `par_bridge` when:
/// - The iterator is slow (e.g., I/O bound) and you want to overlap iteration with processing
/// - Memory is constrained and you cannot afford to collect all items upfront
pub trait ParallelBridgeBuffered: Iterator<Item: Send> + Sized {
/// Collects this iterator into a `Vec` and returns a parallel iterator over it.
///
/// See [this trait's documentation](ParallelBridgeBuffered) for more details.
fn par_bridge_buffered(self) -> rayon::vec::IntoIter<Self::Item> {
self.collect::<Vec<_>>().into_par_iter()
}
}
impl<I: Iterator<Item: Send>> ParallelBridgeBuffered for I {}

View File

@@ -42,11 +42,15 @@ rayon.workspace = true
tokio.workspace = true
rustc-hash.workspace = true
[features]
rocksdb = ["reth-provider/rocksdb"]
[dev-dependencies]
# reth
reth-db = { workspace = true, features = ["test-utils"] }
reth-stages = { workspace = true, features = ["test-utils"] }
reth-stages = { workspace = true, features = ["test-utils", "rocksdb"] }
reth-primitives-traits = { workspace = true, features = ["arbitrary"] }
reth-storage-api.workspace = true
reth-testing-utils.workspace = true
reth-tracing.workspace = true

View File

@@ -7,10 +7,10 @@ use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, PruneCheckpointWriter,
StageCheckpointReader, StaticFileProviderFactory, StorageSettingsCache,
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderFactory,
};
use reth_prune_types::PruneModes;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
use std::time::Duration;
use tokio::sync::watch;
@@ -85,6 +85,7 @@ impl PrunerBuilder {
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader
+ RocksDBProviderFactory
+ StaticFileProviderFactory<
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
>,
@@ -121,7 +122,8 @@ impl PrunerBuilder {
+ StorageSettingsCache
+ StageCheckpointReader
+ ChangeSetReader
+ StorageChangeSetReader,
+ StorageChangeSetReader
+ RocksDBProviderFactory,
{
let segments = SegmentSet::<Provider>::from_components(static_file_provider, self.segments);

View File

@@ -25,6 +25,10 @@ pub use user::{
///
/// This is a generic helper function used by both receipts and bodies pruning
/// when data is stored in static files.
///
/// The checkpoint block number is set to the highest block in the actually deleted files,
/// not `input.to_block`, since `to_block` might refer to a block in the middle of an
/// undeleted file.
pub(crate) fn prune_static_files<Provider>(
provider: &Provider,
input: PruneInput,
@@ -36,6 +40,51 @@ where
let deleted_headers =
provider.static_file_provider().delete_segment_below_block(segment, input.to_block + 1)?;
if deleted_headers.is_empty() {
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned: 0,
checkpoint: input
.previous_checkpoint
.map(SegmentOutputCheckpoint::from_prune_checkpoint),
})
}
let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range());
let pruned = tx_ranges.clone().map(|range| range.len()).sum::<u64>() as usize;
// The highest block number in the deleted files is the actual checkpoint.
let checkpoint_block = deleted_headers
.iter()
.filter_map(|header| header.block_range())
.map(|range| range.end())
.max();
Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: checkpoint_block,
tx_number: tx_ranges.map(|range| range.end()).max(),
}),
})
}
/// Deletes ALL static file jars for a given segment.
///
/// This is used for `PruneMode::Full` where all data should be removed, including the highest jar.
/// Unlike [`prune_static_files`], this does not preserve the most recent jar.
pub(crate) fn delete_static_files_segment<Provider>(
provider: &Provider,
input: PruneInput,
segment: StaticFileSegment,
) -> Result<SegmentOutput, PrunerError>
where
Provider: StaticFileProviderFactory,
{
let deleted_headers = provider.static_file_provider().delete_segment(segment)?;
if deleted_headers.is_empty() {
return Ok(SegmentOutput::done())
}

View File

@@ -7,10 +7,11 @@ use reth_db_api::{table::Value, transaction::DbTxMut};
use reth_primitives_traits::NodePrimitives;
use reth_provider::{
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory, StorageSettingsCache,
PruneCheckpointReader, PruneCheckpointWriter, RocksDBProviderFactory,
StaticFileProviderFactory,
};
use reth_prune_types::PruneModes;
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
/// Collection of [`Segment`]. Thread-safe, allocated on the heap.
#[derive(Debug)]
@@ -55,7 +56,8 @@ where
+ ChainStateBlockReader
+ StorageSettingsCache
+ ChangeSetReader
+ StorageChangeSetReader,
+ StorageChangeSetReader
+ RocksDBProviderFactory,
{
/// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and
/// [`PruneModes`].
@@ -74,8 +76,11 @@ where
} = prune_modes;
Self::default()
// Bodies - run first since file deletion is fast
.segment_opt(bodies_history.map(Bodies::new))
// Transaction lookup must run before bodies because it needs to read transaction
// data from static files before bodies deletes them.
.segment_opt(transaction_lookup.map(TransactionLookup::new))
// Bodies
.segment_opt(bodies_history.map(|mode| Bodies::new(mode, transaction_lookup)))
// Account history
.segment_opt(account_history.map(AccountHistory::new))
// Storage history
@@ -87,8 +92,6 @@ where
(!receipts_log_filter.is_empty())
.then(|| ReceiptsByLogs::new(receipts_log_filter.clone())),
)
// Transaction lookup
.segment_opt(transaction_lookup.map(TransactionLookup::new))
// Sender recovery
.segment_opt(sender_recovery.map(SenderRecovery::new))
}

View File

@@ -10,20 +10,20 @@ use alloy_primitives::BlockNumber;
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
use reth_provider::{
changeset_walker::StaticFileAccountChangesetWalker, DBProvider, EitherWriter,
StaticFileProviderFactory, StorageSettingsCache,
RocksDBProviderFactory, StaticFileProviderFactory,
};
use reth_prune_types::{
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::ChangeSetReader;
use reth_storage_api::{ChangeSetReader, StorageSettingsCache};
use rustc_hash::FxHashMap;
use tracing::{instrument, trace};
/// Number of account history tables to prune in one step.
///
/// Account History consists of two tables: [`tables::AccountChangeSets`] and
/// [`tables::AccountsHistory`]. We want to prune them to the same block number.
/// Account History consists of two tables: [`tables::AccountChangeSets`] (either in database or
/// static files) and [`tables::AccountsHistory`]. We want to prune them to the same block number.
const ACCOUNT_HISTORY_TABLES_TO_PRUNE: usize = 2;
#[derive(Debug)]
@@ -42,7 +42,8 @@ where
Provider: DBProvider<Tx: DbTxMut>
+ StaticFileProviderFactory
+ StorageSettingsCache
+ ChangeSetReader,
+ ChangeSetReader
+ RocksDBProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::AccountHistory
@@ -67,7 +68,13 @@ where
};
let range_end = *range.end();
// Check where account changesets are stored
// Check where account history indices are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().account_history_in_rocksdb {
return self.prune_rocksdb(provider, input, range, range_end);
}
// Check where account changesets are stored (MDBX path)
if EitherWriter::account_changesets_destination(provider).is_static_file() {
self.prune_static_files(provider, input, range, range_end)
} else {
@@ -94,6 +101,8 @@ impl AccountHistory {
input.limiter
};
// The limiter may already be exhausted from a previous segment in the same prune run.
// Early exit avoids unnecessary iteration when no budget remains.
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
@@ -101,11 +110,14 @@ impl AccountHistory {
))
}
// The size of this map it's limited by `prune_delete_limit * blocks_since_last_run /
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with the current defaults it's usually `3500 * 5 /
// 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total size should be up to
// ~0.25MB + some hashmap overhead. `blocks_since_last_run` is additionally limited by the
// `max_reorg_depth`, so no OOM is expected here.
// Deleted account changeset keys (account addresses) with the highest block number deleted
// for that key.
//
// The size of this map is limited by `prune_delete_limit * blocks_since_last_run /
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with current default it's usually `3500 * 5
// / 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total
// size should be up to ~0.25MB + some hashmap overhead. `blocks_since_last_run` is
// additionally limited by the `max_reorg_depth`, so no OOM is expected here.
let mut highest_deleted_accounts = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut pruned_changesets = 0;
@@ -124,8 +136,8 @@ impl AccountHistory {
limiter.increment_deleted_entries_count();
}
// Delete static file jars below the pruned block
if let Some(last_block) = last_changeset_pruned_block {
// Delete static file jars only when fully processed
if done && let Some(last_block) = last_changeset_pruned_block {
provider
.static_file_provider()
.delete_segment_below_block(StaticFileSegment::AccountChangeSets, last_block + 1)?;
@@ -210,6 +222,105 @@ impl AccountHistory {
)
.map_err(Into::into)
}
/// Prunes account history when indices are stored in `RocksDB`.
///
/// Reads account changesets from static files and prunes the corresponding
/// `RocksDB` history shards.
#[cfg(all(unix, feature = "rocksdb"))]
fn prune_rocksdb<Provider>(
&self,
provider: &Provider,
input: PruneInput,
range: std::ops::RangeInclusive<BlockNumber>,
range_end: BlockNumber,
) -> Result<SegmentOutput, PrunerError>
where
Provider: DBProvider + StaticFileProviderFactory + ChangeSetReader + RocksDBProviderFactory,
{
// Unlike MDBX path, we don't divide the limit by 2 because RocksDB path only prunes
// history shards (no separate changeset table to delete from). The changesets are in
// static files which are deleted separately.
let mut limiter = input.limiter;
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
))
}
let mut highest_deleted_accounts = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut changesets_processed = 0usize;
let mut done = true;
// Walk account changesets from static files using a streaming iterator.
// For each changeset, track the highest block number seen for each address
// to determine which history shard entries need pruning.
let walker = StaticFileAccountChangesetWalker::new(provider, range);
for result in walker {
if limiter.is_limit_reached() {
done = false;
break;
}
let (block_number, changeset) = result?;
highest_deleted_accounts.insert(changeset.address, block_number);
last_changeset_pruned_block = Some(block_number);
changesets_processed += 1;
limiter.increment_deleted_entries_count();
}
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned account changesets from static files");
let last_changeset_pruned_block = last_changeset_pruned_block
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
.unwrap_or(range_end);
// Prune RocksDB history shards for affected accounts
let mut deleted_shards = 0usize;
let mut updated_shards = 0usize;
// Sort by address for better RocksDB cache locality
let mut sorted_accounts: Vec<_> = highest_deleted_accounts.into_iter().collect();
sorted_accounts.sort_unstable_by_key(|(addr, _)| *addr);
provider.with_rocksdb_batch(|mut batch| {
let targets: Vec<_> = sorted_accounts
.iter()
.map(|(addr, highest)| (*addr, (*highest).min(last_changeset_pruned_block)))
.collect();
let outcomes = batch.prune_account_history_batch(&targets)?;
deleted_shards = outcomes.deleted;
updated_shards = outcomes.updated;
Ok(((), Some(batch.into_inner())))
})?;
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned account history (RocksDB indices)");
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
// is idempotent (re-pruning already-pruned shards is a no-op).
if done {
provider.static_file_provider().delete_segment_below_block(
StaticFileSegment::AccountChangeSets,
last_changeset_pruned_block + 1,
)?;
}
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: changesets_processed + deleted_shards + updated_shards,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_changeset_pruned_block),
tx_number: None,
}),
})
}
}
#[cfg(test)]
@@ -539,4 +650,272 @@ mod tests {
test_prune(998, 2, (PruneProgress::Finished, 1000));
test_prune(1400, 3, (PruneProgress::Finished, 804));
}
#[cfg(all(unix, feature = "rocksdb"))]
#[test]
fn prune_rocksdb_path() {
use reth_db_api::models::ShardedKey;
use reth_provider::{RocksDBProviderFactory, StaticFileProviderFactory};
let db = TestStageDB::default();
let mut rng = generators::rng();
let blocks = random_block_range(
&mut rng,
0..=100,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
let (changesets, _) = random_changeset_range(
&mut rng,
blocks.iter(),
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
0..0,
0..0,
);
db.insert_changesets_to_static_files(changesets.clone(), None)
.expect("insert changesets to static files");
let mut account_blocks: BTreeMap<_, Vec<u64>> = BTreeMap::new();
for (block, changeset) in changesets.iter().enumerate() {
for (address, _, _) in changeset {
account_blocks.entry(*address).or_default().push(block as u64);
}
}
let rocksdb = db.factory.rocksdb_provider();
let mut batch = rocksdb.batch();
for (address, block_numbers) in &account_blocks {
let shard = BlockNumberList::new_pre_sorted(block_numbers.iter().copied());
batch
.put::<tables::AccountsHistory>(ShardedKey::new(*address, u64::MAX), &shard)
.unwrap();
}
batch.commit().unwrap();
for (address, expected_blocks) in &account_blocks {
let shards = rocksdb.account_history_shards(*address).unwrap();
assert_eq!(shards.len(), 1);
assert_eq!(shards[0].1.iter().collect::<Vec<_>>(), *expected_blocks);
}
let to_block: BlockNumber = 50;
let prune_mode = PruneMode::Before(to_block);
let input =
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = AccountHistory::new(prune_mode);
db.factory.set_storage_settings_cache(
StorageSettings::default()
.with_account_changesets_in_static_files(true)
.with_account_history_in_rocksdb(true),
);
let provider = db.factory.database_provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");
assert_matches!(
result,
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
if pruned > 0
);
for (address, original_blocks) in &account_blocks {
let shards = rocksdb.account_history_shards(*address).unwrap();
let expected_blocks: Vec<u64> =
original_blocks.iter().copied().filter(|b| *b > to_block).collect();
if expected_blocks.is_empty() {
assert!(
shards.is_empty(),
"Expected no shards for address {address:?} after pruning"
);
} else {
assert_eq!(shards.len(), 1, "Expected 1 shard for address {address:?}");
assert_eq!(
shards[0].1.iter().collect::<Vec<_>>(),
expected_blocks,
"Shard blocks mismatch for address {address:?}"
);
}
}
let static_file_provider = db.factory.static_file_provider();
let highest_block = static_file_provider.get_highest_static_file_block(
reth_static_file_types::StaticFileSegment::AccountChangeSets,
);
if let Some(block) = highest_block {
assert!(
block > to_block,
"Static files should only contain blocks above to_block ({to_block}), got {block}"
);
}
}
/// Tests that when a limiter stops mid-block (with multiple changes for the same block),
/// the checkpoint is set to `block_number - 1` to avoid dangling index entries.
#[test]
fn prune_partial_progress_mid_block() {
use alloy_primitives::{Address, U256};
use reth_primitives_traits::Account;
use reth_testing_utils::generators::ChangeSet;
let db = TestStageDB::default();
let mut rng = generators::rng();
// Create blocks 0..=10
let blocks = random_block_range(
&mut rng,
0..=10,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
// Create specific changesets where block 5 has 4 account changes
let addr1 = Address::with_last_byte(1);
let addr2 = Address::with_last_byte(2);
let addr3 = Address::with_last_byte(3);
let addr4 = Address::with_last_byte(4);
let addr5 = Address::with_last_byte(5);
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
// Build changesets: blocks 0-4 have 1 change each, block 5 has 4 changes, block 6 has 1
let changesets: Vec<ChangeSet> = vec![
vec![(addr1, account, vec![])], // block 0
vec![(addr1, account, vec![])], // block 1
vec![(addr1, account, vec![])], // block 2
vec![(addr1, account, vec![])], // block 3
vec![(addr1, account, vec![])], // block 4
// block 5: 4 different account changes (sorted by address for consistency)
vec![
(addr1, account, vec![]),
(addr2, account, vec![]),
(addr3, account, vec![]),
(addr4, account, vec![]),
],
vec![(addr5, account, vec![])], // block 6
];
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
db.insert_history(changesets.clone(), None).expect("insert history");
// Total changesets: 5 (blocks 0-4) + 4 (block 5) + 1 (block 6) = 10
assert_eq!(
db.table::<tables::AccountChangeSets>().unwrap().len(),
changesets.iter().flatten().count()
);
let prune_mode = PruneMode::Before(10);
// Set limiter to stop after 7 entries (mid-block 5: 5 from blocks 0-4, then 2 of 4 from
// block 5). Due to ACCOUNT_HISTORY_TABLES_TO_PRUNE=2, actual limit is 7/2=3
// changesets. So we'll process blocks 0, 1, 2 (3 changesets), stopping before block
// 3. Actually, let's use a higher limit to reach block 5. With limit=14, we get 7
// changeset slots. Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so
// we stop mid-block 5.
let deleted_entries_limit = 14; // 14/2 = 7 changeset entries before limit
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
let segment = AccountHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
// Should report that there's more data
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
// Save checkpoint and commit
segment
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
.unwrap();
provider.commit().expect("commit");
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
let checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::AccountHistory)
.unwrap()
.expect("checkpoint should exist");
assert_eq!(
checkpoint.block_number,
Some(4),
"Checkpoint should be block 4 (block before incomplete block 5)"
);
// Verify remaining changesets (block 5 and 6 should still have entries)
let remaining_changesets = db.table::<tables::AccountChangeSets>().unwrap();
// After pruning blocks 0-4, remaining should be block 5 (4 entries) + block 6 (1 entry) = 5
// But since we stopped mid-block 5, some of block 5 might be pruned
// However, checkpoint is 4, so on re-run we should re-process from block 5
assert!(
!remaining_changesets.is_empty(),
"Should have remaining changesets for blocks 5-6"
);
// Verify no dangling history indices for blocks that weren't fully pruned
// The indices for block 5 should still reference blocks <= 5 appropriately
let history = db.table::<tables::AccountsHistory>().unwrap();
for (key, _blocks) in &history {
// All blocks in the history should be > checkpoint block number
// OR the shard's highest_block_number should be > checkpoint
assert!(
key.highest_block_number > 4,
"Found stale history shard with highest_block_number {} <= checkpoint 4",
key.highest_block_number
);
}
// Run prune again to complete - should finish processing block 5 and 6
let input2 = PruneInput {
previous_checkpoint: Some(checkpoint),
to_block: 10,
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
};
let provider2 = db.factory.database_provider_rw().unwrap();
provider2.set_storage_settings_cache(
StorageSettings::default().with_account_changesets_in_static_files(false),
);
let result2 = segment.prune(&provider2, input2).unwrap();
assert!(result2.progress.is_finished(), "Second run should complete");
segment
.save_checkpoint(
&provider2,
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
provider2.commit().expect("commit");
// Verify final checkpoint
let final_checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::AccountHistory)
.unwrap()
.expect("checkpoint should exist");
// Should now be at block 6 (the last block with changesets)
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
// All changesets should be pruned
let final_changesets = db.table::<tables::AccountChangeSets>().unwrap();
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
}
}

View File

@@ -2,9 +2,14 @@ use crate::{
segments::{self, PruneInput, Segment},
PrunerError,
};
use reth_provider::{BlockReader, StaticFileProviderFactory};
use reth_prune_types::{PruneMode, PrunePurpose, PruneSegment, SegmentOutput};
use alloy_primitives::BlockNumber;
use reth_provider::{BlockReader, PruneCheckpointReader, StaticFileProviderFactory};
use reth_prune_types::{
PruneInterruptReason, PruneMode, PrunePurpose, PruneSegment, SegmentOutput,
SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use tracing::debug;
/// Segment responsible for pruning transactions in static files.
///
@@ -12,18 +17,79 @@ use reth_static_file_types::StaticFileSegment;
#[derive(Debug)]
pub struct Bodies {
mode: PruneMode,
/// Transaction lookup prune mode. Used to determine if we need to wait for tx lookup pruning
/// before deleting transaction bodies.
tx_lookup_mode: Option<PruneMode>,
}
impl Bodies {
/// Creates a new [`Bodies`] segment with the given prune mode.
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
/// Creates a new [`Bodies`] segment with the given prune mode and optional transaction lookup
/// prune mode for coordination.
pub const fn new(mode: PruneMode, tx_lookup_mode: Option<PruneMode>) -> Self {
Self { mode, tx_lookup_mode }
}
/// Returns the next best block that bodies can prune up to considering the transaction lookup
/// pruning configuration (if any) and progress.
///
/// Returns `None` if there's no block available to prune (e.g., waiting on `tx_lookup`).
fn next_bodies_prune_target<Provider>(
&self,
provider: &Provider,
input: &PruneInput,
) -> Result<Option<BlockNumber>, PrunerError>
where
Provider: PruneCheckpointReader,
{
let Some(tx_lookup_mode) = self.tx_lookup_mode else { return Ok(Some(input.to_block)) };
let tx_lookup_checkpoint = provider
.get_prune_checkpoint(PruneSegment::TransactionLookup)?
.and_then(|cp| cp.block_number);
// Determine the safe prune target, if any.
// tx_lookup's next_pruned_block tells us what block it will prune next.
// - None: tx_lookup will never prune more blocks (e.g. Before(N) reached its target), so
// bodies can prune freely
// - Some(next) > to_block: tx_lookup is ahead of our target, so we're safe to prune
// to_block
// - Some(next) <= to_block: tx_lookup still needs to prune blocks we want to delete, so we
// must wait and only prune up to (next - 1) to preserve tx data it needs
let to_block = match tx_lookup_mode.next_pruned_block(tx_lookup_checkpoint) {
None => Some(input.to_block),
Some(tx_lookup_next) if tx_lookup_next > input.to_block => Some(input.to_block),
Some(tx_lookup_next) => {
// We can only prune bodies up to the block BEFORE tx_lookup's next target.
// tx_lookup_next is the next block tx_lookup will prune, meaning it still needs
// to read transactions from that block. We must preserve those transactions,
// so bodies can only safely delete up to (tx_lookup_next - 1).
let Some(safe) = tx_lookup_next.checked_sub(1) else {
return Ok(None);
};
if input.previous_checkpoint.is_some_and(|cp| cp.block_number.unwrap_or(0) >= safe)
{
// we have pruned what we can
return Ok(None)
}
debug!(
target: "pruner",
to_block = input.to_block,
safe,
"Bodies pruning limited by tx_lookup progress"
);
Some(safe)
}
};
Ok(to_block)
}
}
impl<Provider> Segment<Provider> for Bodies
where
Provider: StaticFileProviderFactory + BlockReader,
Provider: StaticFileProviderFactory + BlockReader + PruneCheckpointReader,
{
fn segment(&self) -> PruneSegment {
PruneSegment::Bodies
@@ -38,7 +104,20 @@ where
}
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
segments::prune_static_files(provider, input, StaticFileSegment::Transactions)
let Some(to_block) = self.next_bodies_prune_target(provider, &input)? else {
debug!(
to_block = input.to_block,
"Transaction lookup still has work to be done up to target block"
);
return Ok(SegmentOutput::not_done(
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
));
};
// Use the coordinated to_block instead of input.to_block
let adjusted_input = PruneInput { to_block, ..input };
segments::prune_static_files(provider, adjusted_input, StaticFileSegment::Transactions)
}
}
@@ -50,7 +129,8 @@ mod tests {
use reth_exex_types::FinishedExExHeight;
use reth_provider::{
test_utils::{create_test_provider_factory, MockNodeTypesWithDB},
ProviderFactory, StaticFileWriter,
DBProvider, DatabaseProviderFactory, ProviderFactory, PruneCheckpointWriter,
StaticFileWriter,
};
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
use reth_static_file_types::{
@@ -93,19 +173,83 @@ mod tests {
static_file_provider.initialize_index().expect("initialize index");
}
struct PruneTestCase {
prune_mode: PruneMode,
struct TestCase {
tx_lookup_mode: Option<PruneMode>,
tx_lookup_checkpoint_block: Option<BlockNumber>,
bodies_mode: PruneMode,
expected_pruned: usize,
expected_lowest_block: Option<BlockNumber>,
expected_progress: PruneProgress,
}
impl TestCase {
fn new() -> Self {
Self {
tx_lookup_mode: None,
tx_lookup_checkpoint_block: None,
bodies_mode: PruneMode::Full,
expected_pruned: 0,
expected_lowest_block: None,
expected_progress: PruneProgress::Finished,
}
}
fn with_bodies_mode(mut self, mode: PruneMode) -> Self {
self.bodies_mode = mode;
self
}
fn with_expected_pruned(mut self, pruned: usize) -> Self {
self.expected_pruned = pruned;
self
}
fn with_expected_progress(mut self, progress: PruneProgress) -> Self {
self.expected_progress = progress;
self
}
fn with_lowest_block(mut self, block: BlockNumber) -> Self {
self.expected_lowest_block = Some(block);
self
}
fn with_tx_lookup(mut self, mode: PruneMode, checkpoint: Option<BlockNumber>) -> Self {
self.tx_lookup_mode = Some(mode);
self.tx_lookup_checkpoint_block = checkpoint;
self
}
}
fn run_prune_test(
factory: &ProviderFactory<MockNodeTypesWithDB>,
finished_exex_height_rx: &tokio::sync::watch::Receiver<FinishedExExHeight>,
test_case: PruneTestCase,
test_case: TestCase,
tip: BlockNumber,
) {
let bodies = Bodies::new(test_case.prune_mode);
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
// Capture highest block before pruning
let static_provider = factory.static_file_provider();
let highest_before =
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions);
// Set up tx_lookup checkpoint if provided
if let Some(checkpoint_block) = test_case.tx_lookup_checkpoint_block {
let provider = factory.database_provider_rw().unwrap();
provider
.save_prune_checkpoint(
PruneSegment::TransactionLookup,
reth_prune_types::PruneCheckpoint {
block_number: Some(checkpoint_block),
tx_number: None,
prune_mode: test_case.tx_lookup_mode.unwrap(),
},
)
.unwrap();
provider.commit().unwrap();
}
let bodies = Bodies::new(test_case.bodies_mode, test_case.tx_lookup_mode);
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
let mut pruner = Pruner::new_with_factory(
@@ -114,27 +258,29 @@ mod tests {
5,
10000,
None,
finished_exex_height_rx.clone(),
finished_exex_height_rx,
);
let result = pruner.run(tip).expect("pruner run");
assert_eq!(result.progress, PruneProgress::Finished);
assert_eq!(result.progress, test_case.expected_progress);
assert_eq!(result.segments.len(), 1);
let (segment, output) = &result.segments[0];
assert_eq!(*segment, PruneSegment::Bodies);
assert_eq!(output.pruned, test_case.expected_pruned);
let static_provider = factory.static_file_provider();
assert_eq!(
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
test_case.expected_lowest_block
);
assert_eq!(
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
Some(tip)
);
if let Some(expected_lowest) = test_case.expected_lowest_block {
let static_provider = factory.static_file_provider();
assert_eq!(
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
Some(expected_lowest)
);
assert_eq!(
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
highest_before
);
}
}
#[test]
@@ -143,50 +289,82 @@ mod tests {
let tip = 2_499_999;
setup_static_file_jars(&factory, tip);
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
let (_, _finished_exex_height_rx) =
tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
let test_cases = vec![
// Test 1: PruneMode::Before(750_000) → deletes jar 1 (0-499_999)
PruneTestCase {
prune_mode: PruneMode::Before(750_000),
expected_pruned: 1000,
expected_lowest_block: Some(999_999),
},
// Test 2: PruneMode::Before(850_000) → no deletion (jar 2: 500_000-999_999 contains
// Test 1: PruneMode::Before(750_000) → deletes jar 0 (0-499_999)
// Checkpoint 499_999 != target 749_999 -> HasMoreData
TestCase::new()
.with_bodies_mode(PruneMode::Before(750_000))
.with_expected_pruned(1000)
.with_lowest_block(999_999),
// Test 2: PruneMode::Before(850_000) → no deletion (jar 1: 500_000-999_999 contains
// target)
PruneTestCase {
prune_mode: PruneMode::Before(850_000),
expected_pruned: 0,
expected_lowest_block: Some(999_999),
},
// Test 3: PruneMode::Before(1_599_999) → deletes jar 2 (500_000-999_999) and jar 3
// (1_000_000-1_499_999)
PruneTestCase {
prune_mode: PruneMode::Before(1_599_999),
expected_pruned: 2000,
expected_lowest_block: Some(1_999_999),
},
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 4
// (1_500_000-1_999_999)
PruneTestCase {
prune_mode: PruneMode::Distance(500_000),
expected_pruned: 1000,
expected_lowest_block: Some(2_499_999),
},
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 5: 2_000_000-2_499_999
TestCase::new().with_bodies_mode(PruneMode::Before(850_000)).with_lowest_block(999_999),
// Test 3: PruneMode::Before(1_599_999) → deletes jars 0 and 1 (0-999_999)
// Checkpoint 999_999 != target 1_599_998 -> HasMoreData
TestCase::new()
.with_bodies_mode(PruneMode::Before(1_599_999))
.with_expected_pruned(2000)
.with_lowest_block(1_999_999),
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 3
// (1_500_000-1_999_999) Checkpoint 1_999_999 == target 1_999_999 ->
// Finished
TestCase::new()
.with_bodies_mode(PruneMode::Distance(500_000))
.with_expected_pruned(1000)
.with_lowest_block(2_499_999),
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 4: 2_000_000-2_499_999
// contains target)
PruneTestCase {
prune_mode: PruneMode::Before(2_300_000),
expected_pruned: 0,
expected_lowest_block: Some(2_499_999),
},
TestCase::new()
.with_bodies_mode(PruneMode::Before(2_300_000))
.with_lowest_block(2_499_999),
];
for test_case in test_cases {
run_prune_test(&factory, &finished_exex_height_rx, test_case, tip);
run_prune_test(&factory, test_case, tip);
}
}
#[test]
fn checkpoint_reflects_deleted_files_not_target() {
// Test that checkpoint is set to the highest deleted block, not to_block.
// When to_block falls in the middle of an undeleted file, checkpoint should reflect
// what was actually deleted.
let factory = create_test_provider_factory();
let tip = 1_499_999;
setup_static_file_jars(&factory, tip);
// Use PruneMode::Before(900_000) which targets 899_999.
// This should delete jar 0 (0-499_999) since it's entirely below the target.
// Jar 1 (500_000-999_999) contains the target, so it won't be deleted.
// Checkpoint should be 499_999 (end of jar 0), not 899_999 (to_block).
let bodies = Bodies::new(PruneMode::Before(900_000), None);
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
let mut pruner =
Pruner::new_with_factory(factory, segments, 5, 10000, None, finished_exex_height_rx);
let result = pruner.run(tip).expect("pruner run");
assert_eq!(result.progress, PruneProgress::Finished);
assert_eq!(result.segments.len(), 1);
let (segment, output) = &result.segments[0];
assert_eq!(*segment, PruneSegment::Bodies);
// Verify checkpoint is set to the end of deleted jar (499_999), not to_block (899_999)
let checkpoint_block = output.checkpoint.as_ref().and_then(|cp| cp.block_number);
assert_eq!(
checkpoint_block,
Some(499_999),
"Checkpoint should be 499_999 (end of deleted jar 0), not 899_999 (to_block)"
);
}
#[test]
fn min_block_updated_on_sync() {
// Regression test: update_index must update min_block to prevent stale values
@@ -302,4 +480,100 @@ mod tests {
assert_eq!(deleted.len(), expected_deleted);
}
}
#[test]
fn bodies_with_tx_lookup_coordination() {
// Test that bodies pruning correctly coordinates with tx lookup pruning
// Using tip = 1_523_000 creates 4 static file jars:
// - Jar 0: blocks 0-499_999, txs 0-999
// - Jar 1: blocks 500_000-999_999, txs 1000-1999
// - Jar 2: blocks 1_000_000-1_499_999, txs 2000-2999
// - Jar 3: blocks 1_500_000-1_523_000, txs 3000-3999
let tip = 1_523_000;
let test_cases = vec![
// Scenario 1: tx_lookup disabled, bodies can prune freely (deletes jar 0)
// Checkpoint is 499_999 (end of jar 0), target is 599_999, so HasMoreData
TestCase::new()
.with_bodies_mode(PruneMode::Before(600_000))
.with_expected_pruned(1000)
.with_lowest_block(999_999),
// Scenario 2: tx_lookup enabled but not run yet, bodies cannot prune
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), None)
.with_bodies_mode(PruneMode::Before(600_000))
.with_expected_progress(PruneProgress::HasMoreData(
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
))
.with_lowest_block(499_999), // No jars deleted, jar 0 ends at 499_999
// Scenario 3: tx_lookup caught up to its target, bodies can prune freely
// Deletes jar 0, checkpoint is 499_999, target is 599_999 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
.with_bodies_mode(PruneMode::Before(600_000))
.with_expected_pruned(1000)
.with_lowest_block(999_999),
// Scenario 4: tx_lookup behind its target, bodies limited to tx_lookup checkpoint
// tx_lookup should prune up to 599_999, but checkpoint is only at 250_000
// bodies wants to prune up to 599_999, but limited to 250_000
// No jars deleted because jar 0 (0-499_999) ends beyond 250_000
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), Some(250_000))
.with_bodies_mode(PruneMode::Before(600_000))
.with_lowest_block(499_999), // No jars deleted
// Scenario 5: Both use Distance, tx_lookup caught up
// With tip=1_523_000, Distance(500_000) targets block 1_023_000
// Deletes jars 0 and 1, checkpoint is 999_999, target is 1_023_000 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Distance(500_000), Some(1_023_000))
.with_bodies_mode(PruneMode::Distance(500_000))
.with_expected_pruned(2000)
.with_lowest_block(1_499_999),
// Scenario 6: Both use Distance, tx_lookup less aggressive (bigger distance) than
// bodies With tip=1_523_000:
// - tx_lookup: Distance(1_000_000) targets block 523_000, checkpoint at 523_000
// - bodies: Distance(500_000) targets block 1_023_000
// Bodies can prune up to what tx_lookup has finished (523_000), deleting jar 0
// Checkpoint is 499_999, target is 1_023_000 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Distance(1_000_000), Some(523_000))
.with_bodies_mode(PruneMode::Distance(500_000))
.with_expected_pruned(1000) // Jar 0 deleted
.with_lowest_block(999_999), // Jar 0 (0-499_999) deleted
// Scenario 7: tx_lookup more aggressive than bodies (deletes jar 0 and 1)
// tx_lookup: Before(1_100_000) -> prune up to 1_099_999
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Before(1_100_000), Some(1_099_999))
.with_bodies_mode(PruneMode::Before(1_100_000))
.with_expected_pruned(2000)
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
// Scenario 8: tx_lookup has lower target than bodies, but is done
// tx_lookup: Before(600_000) -> prune up to 599_999 (checkpoint at 599_999, DONE)
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
// Since tx_lookup is done (next_pruned_block returns None), bodies can prune freely
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
.with_bodies_mode(PruneMode::Before(1_100_000))
.with_expected_pruned(2000)
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
// Scenario 9: Perfect alignment - checkpoint equals target
// bodies: Before(1_000_000) -> targets 999_999
// Deletes jars 0 and 1 (0-999_999), checkpoint is 999_999 which equals target ->
// Finished
TestCase::new()
.with_bodies_mode(PruneMode::Before(1_000_000))
.with_expected_pruned(2000)
.with_expected_progress(PruneProgress::Finished)
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
];
for test_case in test_cases {
let factory = create_test_provider_factory();
setup_static_file_jars(&factory, tip);
run_prune_test(&factory, test_case, tip);
}
}
}

View File

@@ -49,6 +49,16 @@ where
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
if EitherWriterDestination::senders(provider).is_static_file() {
debug!(target: "pruner", "Pruning transaction senders from static files.");
if self.mode.is_full() {
debug!(target: "pruner", "PruneMode::Full: deleting all transaction senders static files.");
return segments::delete_static_files_segment(
provider,
input,
StaticFileSegment::TransactionSenders,
)
}
return segments::prune_static_files(
provider,
input,

View File

@@ -12,7 +12,7 @@ use reth_db_api::{
tables,
transaction::DbTxMut,
};
use reth_provider::{DBProvider, EitherWriter, StaticFileProviderFactory};
use reth_provider::{DBProvider, EitherWriter, RocksDBProviderFactory, StaticFileProviderFactory};
use reth_prune_types::{
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
};
@@ -43,7 +43,8 @@ where
Provider: DBProvider<Tx: DbTxMut>
+ StaticFileProviderFactory
+ StorageChangeSetReader
+ StorageSettingsCache,
+ StorageSettingsCache
+ RocksDBProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::StorageHistory
@@ -68,6 +69,13 @@ where
};
let range_end = *range.end();
// Check where storage history indices are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().storages_history_in_rocksdb {
return self.prune_rocksdb(provider, input, range, range_end);
}
// Check where storage changesets are stored (MDBX path)
if EitherWriter::storage_changesets_destination(provider).is_static_file() {
self.prune_static_files(provider, input, range, range_end)
} else {
@@ -94,6 +102,8 @@ impl StorageHistory {
input.limiter
};
// The limiter may already be exhausted from a previous segment in the same prune run.
// Early exit avoids unnecessary iteration when no budget remains.
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
@@ -126,8 +136,8 @@ impl StorageHistory {
limiter.increment_deleted_entries_count();
}
// Delete static file jars below the pruned block
if let Some(last_block) = last_changeset_pruned_block {
// Delete static file jars only when fully processed
if done && let Some(last_block) = last_changeset_pruned_block {
provider
.static_file_provider()
.delete_segment_below_block(StaticFileSegment::StorageChangeSets, last_block + 1)?;
@@ -216,6 +226,108 @@ impl StorageHistory {
)
.map_err(Into::into)
}
/// Prunes storage history when indices are stored in `RocksDB`.
///
/// Reads storage changesets from static files and prunes the corresponding
/// `RocksDB` history shards.
#[cfg(all(unix, feature = "rocksdb"))]
fn prune_rocksdb<Provider>(
&self,
provider: &Provider,
input: PruneInput,
range: std::ops::RangeInclusive<BlockNumber>,
range_end: BlockNumber,
) -> Result<SegmentOutput, PrunerError>
where
Provider: DBProvider + StaticFileProviderFactory + RocksDBProviderFactory,
{
let mut limiter = input.limiter;
if limiter.is_limit_reached() {
return Ok(SegmentOutput::not_done(
limiter.interrupt_reason(),
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
))
}
let mut highest_deleted_storages: FxHashMap<_, _> = FxHashMap::default();
let mut last_changeset_pruned_block = None;
let mut changesets_processed = 0usize;
let mut done = true;
// Walk storage changesets from static files using a streaming iterator.
// For each changeset, track the highest block number seen for each (address, storage_key)
// pair to determine which history shard entries need pruning.
let walker = provider.static_file_provider().walk_storage_changeset_range(range);
for result in walker {
if limiter.is_limit_reached() {
done = false;
break;
}
let (block_address, entry) = result?;
let block_number = block_address.block_number();
let address = block_address.address();
highest_deleted_storages.insert((address, entry.key), block_number);
last_changeset_pruned_block = Some(block_number);
changesets_processed += 1;
limiter.increment_deleted_entries_count();
}
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned storage changesets from static files");
let last_changeset_pruned_block = last_changeset_pruned_block
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
.unwrap_or(range_end);
// Prune RocksDB history shards for affected storage slots
let mut deleted_shards = 0usize;
let mut updated_shards = 0usize;
// Sort by (address, storage_key) for better RocksDB cache locality
let mut sorted_storages: Vec<_> = highest_deleted_storages.into_iter().collect();
sorted_storages.sort_unstable_by_key(|((addr, key), _)| (*addr, *key));
provider.with_rocksdb_batch(|mut batch| {
let targets: Vec<_> = sorted_storages
.iter()
.map(|((addr, key), highest)| {
((*addr, *key), (*highest).min(last_changeset_pruned_block))
})
.collect();
let outcomes = batch.prune_storage_history_batch(&targets)?;
deleted_shards = outcomes.deleted;
updated_shards = outcomes.updated;
Ok(((), Some(batch.into_inner())))
})?;
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned storage history (RocksDB indices)");
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
// is idempotent (re-pruning already-pruned shards is a no-op).
if done {
provider.static_file_provider().delete_segment_below_block(
StaticFileSegment::StorageChangeSets,
last_changeset_pruned_block + 1,
)?;
}
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: changesets_processed + deleted_shards + updated_shards,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_changeset_pruned_block),
tx_number: None,
}),
})
}
}
#[cfg(test)]
@@ -553,4 +665,270 @@ mod tests {
test_prune(998, 2, (PruneProgress::Finished, 500));
test_prune(1200, 3, (PruneProgress::Finished, 202));
}
/// Tests that when a limiter stops mid-block (with multiple storage changes for the same
/// block), the checkpoint is set to `block_number - 1` to avoid dangling index entries.
#[test]
fn prune_partial_progress_mid_block() {
use alloy_primitives::{Address, U256};
use reth_primitives_traits::Account;
use reth_testing_utils::generators::ChangeSet;
let db = TestStageDB::default();
let mut rng = generators::rng();
// Create blocks 0..=10
let blocks = random_block_range(
&mut rng,
0..=10,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
// Create specific changesets where block 5 has 4 storage changes
let addr1 = Address::with_last_byte(1);
let addr2 = Address::with_last_byte(2);
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
// Create storage entries
let storage_entry = |key: u8| reth_primitives_traits::StorageEntry {
key: B256::with_last_byte(key),
value: U256::from(100),
};
// Build changesets: blocks 0-4 have 1 storage change each, block 5 has 4 changes, block 6
// has 1. Entries within each account must be sorted by key.
let changesets: Vec<ChangeSet> = vec![
vec![(addr1, account, vec![storage_entry(1)])], // block 0
vec![(addr1, account, vec![storage_entry(1)])], // block 1
vec![(addr1, account, vec![storage_entry(1)])], // block 2
vec![(addr1, account, vec![storage_entry(1)])], // block 3
vec![(addr1, account, vec![storage_entry(1)])], // block 4
// block 5: 4 different storage changes (2 addresses, each with 2 storage slots)
// Sorted by address, then by storage key within each address
vec![
(addr1, account, vec![storage_entry(1), storage_entry(2)]),
(addr2, account, vec![storage_entry(1), storage_entry(2)]),
],
vec![(addr1, account, vec![storage_entry(3)])], // block 6
];
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
db.insert_history(changesets.clone(), None).expect("insert history");
// Total storage changesets
let total_storage_entries: usize =
changesets.iter().flat_map(|c| c.iter()).map(|(_, _, entries)| entries.len()).sum();
assert_eq!(db.table::<tables::StorageChangeSets>().unwrap().len(), total_storage_entries);
let prune_mode = PruneMode::Before(10);
// Set limiter to stop mid-block 5
// With STORAGE_HISTORY_TABLES_TO_PRUNE=2, limit=14 gives us 7 storage entries before limit
// Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so we stop mid-block 5
let deleted_entries_limit = 14; // 14/2 = 7 storage entries before limit
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result = segment.prune(&provider, input).unwrap();
// Should report that there's more data
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
// Save checkpoint and commit
segment
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
.unwrap();
provider.commit().expect("commit");
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
let checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::StorageHistory)
.unwrap()
.expect("checkpoint should exist");
assert_eq!(
checkpoint.block_number,
Some(4),
"Checkpoint should be block 4 (block before incomplete block 5)"
);
// Verify remaining changesets
let remaining_changesets = db.table::<tables::StorageChangeSets>().unwrap();
assert!(
!remaining_changesets.is_empty(),
"Should have remaining changesets for blocks 5-6"
);
// Verify no dangling history indices for blocks that weren't fully pruned
let history = db.table::<tables::StoragesHistory>().unwrap();
for (key, _blocks) in &history {
assert!(
key.sharded_key.highest_block_number > 4,
"Found stale history shard with highest_block_number {} <= checkpoint 4",
key.sharded_key.highest_block_number
);
}
// Run prune again to complete - should finish processing block 5 and 6
let input2 = PruneInput {
previous_checkpoint: Some(checkpoint),
to_block: 10,
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
};
let provider2 = db.factory.database_provider_rw().unwrap();
provider2.set_storage_settings_cache(
StorageSettings::default().with_storage_changesets_in_static_files(false),
);
let result2 = segment.prune(&provider2, input2).unwrap();
assert!(result2.progress.is_finished(), "Second run should complete");
segment
.save_checkpoint(
&provider2,
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
)
.unwrap();
provider2.commit().expect("commit");
// Verify final checkpoint
let final_checkpoint = db
.factory
.provider()
.unwrap()
.get_prune_checkpoint(PruneSegment::StorageHistory)
.unwrap()
.expect("checkpoint should exist");
// Should now be at block 6 (the last block with changesets)
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
// All changesets should be pruned
let final_changesets = db.table::<tables::StorageChangeSets>().unwrap();
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
}
#[cfg(all(unix, feature = "rocksdb"))]
#[test]
fn prune_rocksdb() {
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettings;
let db = TestStageDB::default();
let mut rng = generators::rng();
let blocks = random_block_range(
&mut rng,
0..=100,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
let (changesets, _) = random_changeset_range(
&mut rng,
blocks.iter(),
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
1..2,
1..2,
);
db.insert_changesets_to_static_files(changesets.clone(), None)
.expect("insert changesets to static files");
let mut storage_indices: BTreeMap<(alloy_primitives::Address, B256), Vec<u64>> =
BTreeMap::new();
for (block, changeset) in changesets.iter().enumerate() {
for (address, _, storage_entries) in changeset {
for entry in storage_entries {
storage_indices.entry((*address, entry.key)).or_default().push(block as u64);
}
}
}
{
let rocksdb = db.factory.rocksdb_provider();
let mut batch = rocksdb.batch();
for ((address, storage_key), block_numbers) in &storage_indices {
let shard = BlockNumberList::new_pre_sorted(block_numbers.clone());
batch
.put::<tables::StoragesHistory>(
StorageShardedKey::last(*address, *storage_key),
&shard,
)
.expect("insert storage history shard");
}
batch.commit().expect("commit rocksdb batch");
}
{
let rocksdb = db.factory.rocksdb_provider();
for (address, storage_key) in storage_indices.keys() {
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
assert!(!shards.is_empty(), "RocksDB should contain storage history before prune");
}
}
let to_block = 50u64;
let prune_mode = PruneMode::Before(to_block);
let input =
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = StorageHistory::new(prune_mode);
let provider = db.factory.database_provider_rw().unwrap();
provider.set_storage_settings_cache(
StorageSettings::default()
.with_storage_changesets_in_static_files(true)
.with_storages_history_in_rocksdb(true),
);
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");
assert_matches!(
result,
SegmentOutput { progress: PruneProgress::Finished, checkpoint: Some(_), .. }
);
{
let rocksdb = db.factory.rocksdb_provider();
for ((address, storage_key), block_numbers) in &storage_indices {
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
let remaining_blocks: Vec<u64> =
block_numbers.iter().copied().filter(|&b| b > to_block).collect();
if remaining_blocks.is_empty() {
assert!(
shards.is_empty(),
"Shard for {:?}/{:?} should be deleted when all blocks pruned",
address,
storage_key
);
} else {
assert!(!shards.is_empty(), "Shard should exist with remaining blocks");
let actual_blocks: Vec<u64> =
shards.iter().flat_map(|(_, list)| list.iter()).collect();
assert_eq!(
actual_blocks, remaining_blocks,
"RocksDB shard should only contain blocks > {}",
to_block
);
}
}
}
}
}

View File

@@ -6,11 +6,15 @@ use crate::{
use alloy_eips::eip2718::Encodable2718;
use rayon::prelude::*;
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
use reth_provider::{
BlockReader, DBProvider, PruneCheckpointReader, RocksDBProviderFactory,
StaticFileProviderFactory,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageSettingsCache;
use tracing::{debug, instrument, trace};
#[derive(Debug)]
@@ -29,7 +33,9 @@ where
Provider: DBProvider<Tx: DbTxMut>
+ BlockReader<Transaction: Encodable2718>
+ PruneCheckpointReader
+ StaticFileProviderFactory,
+ StaticFileProviderFactory
+ StorageSettingsCache
+ RocksDBProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::TransactionLookup
@@ -83,6 +89,12 @@ where
}
.into_inner();
// Check where transaction hash numbers are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
return self.prune_rocksdb(provider, input, start, end);
}
// For PruneMode::Full, clear the entire table in one operation
if self.mode.is_full() {
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
@@ -174,6 +186,106 @@ where
}
}
impl TransactionLookup {
/// Prunes transaction lookup when indices are stored in `RocksDB`.
///
/// Reads transactions from static files and deletes corresponding entries
/// from the `RocksDB` `TransactionHashNumbers` table.
#[cfg(all(unix, feature = "rocksdb"))]
fn prune_rocksdb<Provider>(
&self,
provider: &Provider,
input: PruneInput,
start: alloy_primitives::TxNumber,
end: alloy_primitives::TxNumber,
) -> Result<SegmentOutput, PrunerError>
where
Provider: DBProvider
+ BlockReader<Transaction: Encodable2718>
+ StaticFileProviderFactory
+ RocksDBProviderFactory,
{
// For PruneMode::Full, clear the entire RocksDB table in one operation
if self.mode.is_full() {
let rocksdb = provider.rocksdb_provider();
rocksdb.clear::<tables::TransactionHashNumbers>()?;
trace!(target: "pruner", "Cleared transaction lookup table (RocksDB)");
let last_pruned_block = provider
.block_by_transaction_id(end)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned: 0, // RocksDB clear doesn't return count
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_pruned_block),
tx_number: Some(end),
}),
});
}
let tx_range_end = input
.limiter
.deleted_entries_limit_left()
.map(|left| start.saturating_add(left as u64).saturating_sub(1))
.map_or(end, |limited| limited.min(end));
let tx_range = start..=tx_range_end;
// Retrieve transactions in the range and calculate their hashes in parallel
let hashes: Vec<_> = provider
.transactions_by_tx_range(tx_range.clone())?
.into_par_iter()
.map(|transaction| transaction.trie_hash())
.collect();
// Number of transactions retrieved from the database should match the tx range count
let tx_count = tx_range.count();
if hashes.len() != tx_count {
return Err(PrunerError::InconsistentData(
"Unexpected number of transaction hashes retrieved by transaction number range",
))
}
let mut limiter = input.limiter;
// Delete transaction hash -> number mappings from RocksDB
let mut deleted = 0usize;
provider.with_rocksdb_batch(|mut batch| {
for hash in &hashes {
if limiter.is_limit_reached() {
break;
}
batch.delete::<tables::TransactionHashNumbers>(*hash)?;
limiter.increment_deleted_entries_count();
deleted += 1;
}
Ok(((), Some(batch.into_inner())))
})?;
let done = deleted == hashes.len() && tx_range_end == end;
trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup (RocksDB)");
let last_pruned_transaction = if deleted > 0 { start + deleted as u64 - 1 } else { start };
let last_pruned_block = provider
.block_by_transaction_id(last_pruned_transaction)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
.checked_sub(if done { 0 } else { 1 });
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: deleted,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: last_pruned_block,
tx_number: Some(last_pruned_transaction),
}),
})
}
}
#[cfg(test)]
mod tests {
use crate::segments::{PruneInput, PruneLimiter, Segment, SegmentOutput, TransactionLookup};
@@ -319,4 +431,100 @@ mod tests {
test_prune(6, (PruneProgress::Finished, 2));
test_prune(10, (PruneProgress::Finished, 8));
}
#[cfg(all(unix, feature = "rocksdb"))]
#[test]
fn prune_rocksdb() {
use reth_db_api::models::StorageSettings;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettingsCache;
let db = TestStageDB::default();
let mut rng = generators::rng();
let blocks = random_block_range(
&mut rng,
1..=10,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks");
// Collect transaction hashes and their tx numbers
let mut tx_hash_numbers = Vec::new();
for block in &blocks {
tx_hash_numbers.reserve_exact(block.transaction_count());
for transaction in &block.body().transactions {
tx_hash_numbers.push((*transaction.tx_hash(), tx_hash_numbers.len() as u64));
}
}
let tx_hash_numbers_len = tx_hash_numbers.len();
// Insert into RocksDB instead of MDBX
{
let rocksdb = db.factory.rocksdb_provider();
let mut batch = rocksdb.batch();
for (hash, tx_num) in &tx_hash_numbers {
batch.put::<tables::TransactionHashNumbers>(*hash, tx_num).unwrap();
}
batch.commit().expect("commit rocksdb batch");
}
// Verify RocksDB has all entries
{
let rocksdb = db.factory.rocksdb_provider();
for (hash, expected_tx_num) in &tx_hash_numbers {
let actual = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
assert_eq!(actual, Some(*expected_tx_num));
}
}
let to_block: BlockNumber = 6;
let prune_mode = PruneMode::Before(to_block);
let input =
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = TransactionLookup::new(prune_mode);
// Enable RocksDB storage for transaction hash numbers
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
);
let provider = db.factory.database_provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");
assert_matches!(
result,
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
if pruned > 0
);
// Calculate expected: blocks 1-6 should have their tx hashes pruned
let txs_up_to_block_6: usize = blocks.iter().take(6).map(|b| b.transaction_count()).sum();
// Verify RocksDB entries: first `txs_up_to_block_6` should be gone
{
let rocksdb = db.factory.rocksdb_provider();
for (i, (hash, _)) in tx_hash_numbers.iter().enumerate() {
let entry = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
if i < txs_up_to_block_6 {
assert!(entry.is_none(), "Entry {} (hash {:?}) should be pruned", i, hash);
} else {
assert!(entry.is_some(), "Entry {} (hash {:?}) should still exist", i, hash);
}
}
}
// Verify remaining count
{
let rocksdb = db.factory.rocksdb_provider();
let remaining: Vec<_> =
rocksdb.iter::<tables::TransactionHashNumbers>().unwrap().collect();
assert_eq!(
remaining.len(),
tx_hash_numbers_len - txs_up_to_block_6,
"Remaining RocksDB entries should match expected"
);
}
}
}

View File

@@ -84,6 +84,38 @@ impl PruneMode {
pub const fn is_distance(&self) -> bool {
matches!(self, Self::Distance(_))
}
/// Returns the next block number that will EVENTUALLY be pruned after the given checkpoint. It
/// should not be used to find if there are blocks to be pruned right now. For that, use
/// [`Self::prune_target_block`].
///
/// This is independent of the current tip and indicates what block is next in the pruning
/// sequence according to this mode's configuration. Returns `None` if no more blocks will
/// be pruned (i.e., the mode has reached its target).
///
/// # Examples
///
/// - `Before(10)` with checkpoint at block 5 returns `Some(6)`
/// - `Before(10)` with checkpoint at block 9 returns `None` (done)
/// - `Distance(100)` with checkpoint at block 1000 returns `Some(1001)` (always has more)
/// - `Full` always returns the next block after checkpoint
pub const fn next_pruned_block(&self, checkpoint: Option<BlockNumber>) -> Option<BlockNumber> {
let next = match checkpoint {
Some(c) => c + 1,
None => 0,
};
match self {
Self::Before(n) => {
if next < *n {
Some(next)
} else {
None
}
}
Self::Distance(_) | Self::Full => Some(next),
}
}
}
#[cfg(test)]

View File

@@ -93,7 +93,7 @@ impl SegmentOutput {
Self { progress: PruneProgress::Finished, pruned: 0, checkpoint: None }
}
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and `checkpoint = None`.
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and the given checkpoint.
/// Use when pruning is needed but cannot be done.
pub const fn not_done(
reason: PruneInterruptReason,
@@ -142,6 +142,8 @@ pub enum PruneInterruptReason {
Timeout,
/// Limit on the number of deleted entries (rows in the database) per prune run was reached.
DeletedEntriesLimitReached,
/// Waiting for another segment to finish pruning before this segment can proceed.
WaitingOnSegment(PruneSegment),
/// Unknown reason for stopping prune run.
Unknown,
}

View File

@@ -54,7 +54,7 @@ pub struct PruneModes {
pub transaction_lookup: Option<PruneMode>,
/// Receipts pruning configuration. This setting overrides `receipts_log_filter`
/// and offers improved performance.
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
pub receipts: Option<PruneMode>,
/// Account History pruning configuration.
#[cfg_attr(
@@ -75,7 +75,7 @@ pub struct PruneModes {
)]
pub storage_history: Option<PruneMode>,
/// Bodies History pruning configuration.
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
pub bodies_history: Option<PruneMode>,
/// Receipts pruning configuration by retaining only those receipts that contain logs emitted
/// by the specified addresses, discarding others. This setting is overridden by `receipts`.
@@ -112,7 +112,13 @@ impl PruneModes {
///
/// Returns `true` if any migration was performed.
pub const fn migrate(&mut self) -> bool {
false
match &self.receipts {
Some(PruneMode::Full | PruneMode::Distance(0..MINIMUM_DISTANCE)) => {
self.receipts = Some(PruneMode::Distance(MINIMUM_DISTANCE));
true
}
_ => false,
}
}
/// Returns an error if we can't unwind to the targeted block because the target block is

View File

@@ -2,7 +2,7 @@ use alloy_eip7928::BlockAccessList;
use alloy_eips::{BlockId, BlockNumberOrTag};
use alloy_genesis::ChainConfig;
use alloy_json_rpc::RpcObject;
use alloy_primitives::{Address, Bytes, B256};
use alloy_primitives::{Address, Bytes, B256, U64};
use alloy_rpc_types_debug::ExecutionWitness;
use alloy_rpc_types_eth::{Bundle, StateContext};
use alloy_rpc_types_trace::geth::{
@@ -325,7 +325,7 @@ pub trait DebugApi<TxReq: RpcObject> {
/// Sets the current head of the local chain by block number. Note, this is a destructive action
/// and may severely damage your chain. Use with extreme caution.
#[method(name = "setHead")]
async fn debug_set_head(&self, number: u64) -> RpcResult<()>;
async fn debug_set_head(&self, number: U64) -> RpcResult<()>;
/// Sets the rate of mutex profiling.
#[method(name = "setMutexProfileFraction")]

View File

@@ -107,6 +107,7 @@ impl RethRpcServerConfig for RpcServerArgs {
.pending_block_kind(self.rpc_pending_block)
.raw_tx_forwarder(self.rpc_forwarder.clone())
.rpc_evm_memory_limit(self.rpc_evm_memory_limit)
.force_blob_sidecar_upcasting(self.rpc_force_blob_sidecar_upcasting)
}
fn flashbots_config(&self) -> ValidationApiConfig {

View File

@@ -394,6 +394,17 @@ pub trait EthApi<
address: Address,
block: BlockId,
) -> RpcResult<alloy_rpc_types_eth::AccountInfo>;
/// Returns the EIP-7928 block access list for a block by hash.
#[method(name = "getBlockAccessListByBlockHash")]
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>>;
/// Returns the EIP-7928 block access list for a block by number.
#[method(name = "getBlockAccessListByBlockNumber")]
async fn block_access_list_by_block_number(
&self,
number: BlockNumberOrTag,
) -> RpcResult<Option<Bytes>>;
}
#[async_trait::async_trait]
@@ -881,4 +892,19 @@ where
trace!(target: "rpc::eth", "Serving eth_getAccountInfo");
Ok(EthState::get_account_info(self, address, block).await?)
}
/// Handler for: `eth_getBlockAccessListByBlockHash`
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>> {
trace!(target: "rpc::eth", ?hash, "Serving eth_getBlockAccessListByBlockHash");
Err(internal_rpc_err("unimplemented"))
}
/// Handler for: `eth_getBlockAccessListByBlockNumber`
async fn block_access_list_by_block_number(
&self,
number: BlockNumberOrTag,
) -> RpcResult<Option<Bytes>> {
trace!(target: "rpc::eth", ?number, "Serving eth_getBlockAccessListByBlockNumber");
Err(internal_rpc_err("unimplemented"))
}
}

View File

@@ -49,7 +49,7 @@ jsonrpsee-types.workspace = true
futures.workspace = true
tokio.workspace = true
tokio-stream.workspace = true
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
reqwest.workspace = true
# metrics
metrics.workspace = true

View File

@@ -107,6 +107,11 @@ pub struct EthConfig {
pub send_raw_transaction_sync_timeout: Duration,
/// Maximum memory the EVM can allocate per RPC request.
pub rpc_evm_memory_limit: u64,
/// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
///
/// This is disabled by default, allowing blob transactions with EIP-4844 sidecars to be
/// submitted without automatic conversion.
pub force_blob_sidecar_upcasting: bool,
}
impl EthConfig {
@@ -140,6 +145,7 @@ impl Default for EthConfig {
raw_tx_forwarder: ForwardConfig::default(),
send_raw_transaction_sync_timeout: RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS,
rpc_evm_memory_limit: (1 << 32) - 1,
force_blob_sidecar_upcasting: false,
}
}
}
@@ -242,6 +248,12 @@ impl EthConfig {
self.rpc_evm_memory_limit = memory_limit;
self
}
/// Configures whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
pub const fn force_blob_sidecar_upcasting(mut self, force: bool) -> Self {
self.force_blob_sidecar_upcasting = force;
self
}
}
/// Config for the filter

View File

@@ -71,10 +71,6 @@ revm-primitives = { workspace = true, features = ["serde"] }
# rpc
jsonrpsee.workspace = true
http.workspace = true
http-body.workspace = true
hyper.workspace = true
jsonwebtoken.workspace = true
serde_json.workspace = true
jsonrpsee-types.workspace = true
@@ -82,7 +78,6 @@ jsonrpsee-types.workspace = true
async-trait.workspace = true
tokio = { workspace = true, features = ["sync"] }
tokio-stream.workspace = true
tower.workspace = true
pin-project.workspace = true
parking_lot.workspace = true

View File

@@ -3,7 +3,7 @@ use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip2718::Encodable2718, BlockId, BlockNumberOrTag};
use alloy_evm::env::BlockEnvironment;
use alloy_genesis::ChainConfig;
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256};
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256, U64};
use alloy_rlp::{Decodable, Encodable};
use alloy_rpc_types::BlockTransactionsKind;
use alloy_rpc_types_debug::ExecutionWitness;
@@ -998,7 +998,7 @@ where
Ok(())
}
async fn debug_set_head(&self, _number: u64) -> RpcResult<()> {
async fn debug_set_head(&self, _number: U64) -> RpcResult<()> {
Ok(())
}

Some files were not shown because too many files have changed in this diff Show More