Compare commits

...

47 Commits

Author SHA1 Message Date
yongkangc
903c58749d refactor(rocksdb): extract shared unwind_history_shards_inner helper
Deduplicates the unwind shard processing logic between account and
storage history. Both unwind_account_history_to/batch and
unwind_storage_history_to/batch now delegate to a single generic
unwind_history_shards_inner method using closures, following the
same pattern as prune_history_shards_inner.

Amp-Thread-ID: https://ampcode.com/threads/T-019c436b-33ce-739d-8fa2-69ea27d4fc4a
2026-02-09 17:33:18 +00:00
yongkangc
d1dfce17ab docs: restore full doc comments for RocksDBRawIterEnum and decode_iter_item
Amp-Thread-ID: https://ampcode.com/threads/T-019c28b3-097d-741f-8a27-9af9a4f1ada1
2026-02-04 12:50:28 +00:00
yongkangc
abe603b94c docs: restore detailed flush_and_compact doc comment
Amp-Thread-ID: https://ampcode.com/threads/T-019c27a1-4820-715a-9b42-18fa2ce05c98
2026-02-04 08:11:28 +00:00
yongkangc
9258bcdc7b fix: check iter.status() after iteration loops
Amp-Thread-ID: https://ampcode.com/threads/T-019c27a1-4820-715a-9b42-18fa2ce05c98
2026-02-04 08:09:33 +00:00
yongkangc
7134ae4ec7 chore: fix clippy warnings and clean up comments
Amp-Thread-ID: https://ampcode.com/threads/T-019c2769-c7e2-7217-ae23-b189345c2c4b
2026-02-04 07:50:25 +00:00
yongkangc
d9f81bc104 perf(rocksdb): use single iterator for batch history unwind
Previously, unwind_account_history_indices and unwind_storage_history_indices
created a new iterator_cf for every address/storage key by calling
account_history_shards/storage_history_shards in a loop. This is the same
performance antipattern fixed in PR #21767 for pruning.

This commit adds:
- unwind_account_history_batch: batch version using single raw_iterator_cf
- unwind_storage_history_batch: batch version using single raw_iterator_cf
- RocksDBRawIterEnum: wrapper supporting seek() for iterator reuse

The batch methods reuse a single raw iterator and skip seeks when the
iterator is already positioned correctly (for sorted targets in key order).
This significantly reduces RocksDB seek overhead for large unwind operations.

Amp-Thread-ID: https://ampcode.com/threads/T-019c272c-eed5-751b-a0c4-8e260f96a3bc
2026-02-04 07:50:12 +00:00
Dan Cline
89be91de0e perf(pruner): do not create an iterator_cf for every address (#21767)
Co-authored-by: yongkangc <chiayongkang@hotmail.com>
2026-02-04 06:48:22 +00:00
Dan Cline
3af5a4a4e2 fix(pruner): implement pruning for rocksdb TransactionHashNumbers (#21782) 2026-02-04 04:11:37 +00:00
Dan Cline
95f6bbe922 chore(pruner): always flush and compact after reth prune command (#21783) 2026-02-04 03:07:55 +00:00
DaniPopes
abab83facd perf: spawn proof workers in a separate thread (#21780) 2026-02-04 01:20:43 +00:00
DaniPopes
9359e21f94 ci: enable debug assertions for statetests (#21775) 2026-02-04 00:53:28 +00:00
Huber
32d5ddfe40 fix(test): clean up test temp directories on drop (#21772) 2026-02-03 22:44:12 +00:00
Dan Cline
d7e740f96c chore(cli): expose static file metrics in cli (#21770) 2026-02-03 22:21:10 +00:00
DaniPopes
87bae74094 chore: decode MDBX error code (#21766) 2026-02-03 20:16:32 +00:00
DaniPopes
648f19fb56 perf: build for target-cpu=x86-64-v3 in docker by default (#21761) 2026-02-03 19:47:59 +00:00
DaniPopes
e6fc5ff54b perf(trie): use TrieMask iterator for efficient bit iteration (#21676) 2026-02-03 19:23:41 +00:00
YK
bc729671d9 perf(rocksdb): batch tx reads in TransactionLookupStage unwind (#21723)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 18:28:04 +00:00
joshieDo
eee27df27c fix: ensure transaction lookup can prune (#19553) 2026-02-03 18:11:13 +00:00
Dan Cline
6d02565c5e chore(prune): increase reth prune DELETE_LIMIT to 20M (#21762) 2026-02-03 17:47:50 +00:00
Dan Cline
e706d76aa9 chore(cli): support ctrl-C in reth prune (#21759) 2026-02-03 17:47:01 +00:00
DaniPopes
b9b7d092f6 perf: bump nybbles (#21725) 2026-02-03 17:15:30 +00:00
DaniPopes
d0fb5f31c2 chore: centralize thread::spawn to share tokio handles (#21754) 2026-02-03 16:58:46 +00:00
DaniPopes
9621b78586 chore: shorten thread names (#21751) 2026-02-03 16:40:35 +00:00
DaniPopes
3722071a7c chore(deps): bump bytes 1.11.1 (#21755) 2026-02-03 16:31:22 +00:00
DaniPopes
6273530501 perf: use alloy_primitives hasher for dashmaps (#21726)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 15:05:44 +00:00
Alexey Shekhirin
ce29101277 chore(static-files): proper segment writer scoped thread names (#21747) 2026-02-03 14:44:03 +00:00
John Chase
b1b95f9825 fix(discv5): add missing rand feature for test compilation (#21749) 2026-02-03 14:37:39 +00:00
YK
7f970e136a refactor(stages): use with_rocksdb_batch_auto_commit in tx_lookup (#21722) 2026-02-03 14:35:07 +00:00
YK
6b7cc00289 refactor(rocksdb): deduplicate first()/last() implementations (#21738)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:33:44 +00:00
YK
786140a99d perf(static-file): simplify stage checkpoint lookup to avoid allocs (#21730) 2026-02-03 14:32:43 +00:00
YK
ffcb486388 refactor(rocksdb): deduplicate iterator next() implementations (#21737) 2026-02-03 14:31:05 +00:00
YK
59d68f92c4 perf(static-file): hoist cursor creation outside block loop (#21731) 2026-02-03 14:29:07 +00:00
Matthias Seitz
0e0271a612 chore(deps): bump alloy 1.5.2 -> 1.6.1 (#21746)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 14:16:50 +00:00
Minhyuk Kim
df12fee965 feat(txpool): add is_transaction_ready to TransactionPool trait (#21742)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:13:52 +00:00
DaniPopes
11a4f65624 chore: misc tree cleanups (#21691)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 13:34:19 +00:00
Matthias Seitz
a782e1a18a chore: disable changelog workflow on PRs (#21748)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 14:12:43 +01:00
DaniPopes
2dc76f9abe chore: match statement order in ExecutionCache::new (#21712)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 12:47:15 +00:00
Nicolas SSS
65100971e5 fix(evm): remove unused reth-ethereum-forks (#21695)
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 12:33:44 +00:00
Georgios Konstantopoulos
8e21afa9cc feat(trie): add memory_size heuristic for ParallelSparseTrie (#21745)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-02-03 12:29:57 +00:00
DaniPopes
46a9b9ad3d perf: replace RwLock<HashMap/HashSet> with DashMap/DashSet (#21692)
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: joshieDo <93316087+joshieDo@users.noreply.github.com>
2026-02-03 13:31:05 +01:00
Georgios Konstantopoulos
3f77af4f98 feat: add AI-assisted changelog generation (#21743)
Co-authored-by: Amp <amp@ampcode.com>
Co-authored-by: Emma Jamieson-Hoare <emmajam@users.noreply.github.com>
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmail.com>
Co-authored-by: github-actions[bot] <github-actions[bot]@users.noreply.github.com>
2026-02-03 12:19:49 +00:00
Arsenii Kulikov
79cabbf89c perf: optimize SparseTrieCacheTask (#21704) 2026-02-03 11:39:10 +00:00
drhgencer
e04afe6e0e fix(rpc): validate toBlock in trace_filter (#21718) 2026-02-03 11:02:57 +00:00
Arsenii Kulikov
ee224fe20f fix: update sparse trie masks (#21716) 2026-02-03 12:01:58 +01:00
DaniPopes
972f23745e chore: remove clone from in memory cursor (#21719) 2026-02-03 04:04:33 +00:00
Dan Cline
49f60822f7 chore: move TransactionLookup as first option (#21721) 2026-02-03 02:30:13 +00:00
Georgios Konstantopoulos
47ebc79c85 feat(rpc): add EIP-7928 eth_getBalanceWithProof and eth_getAccountWithProof (#21720)
Co-authored-by: Amp <amp@ampcode.com>
2026-02-03 01:12:04 +00:00
81 changed files with 2714 additions and 975 deletions

20
.changelog/config.toml Normal file
View File

@@ -0,0 +1,20 @@
# Changelogs configuration for reth
# https://github.com/wevm/changelogs
# How to bump packages that depend on changed packages
dependent_bump = "patch"
[changelog]
# Generate per-crate changelogs (vs single root changelog)
format = "per-crate"
# Fixed groups: all always share the same version
# reth binaries share version
[[fixed]]
members = ["reth", "op-reth"]
# Packages to ignore (internal/test-only crates)
ignore = [
"reth-testing-utils",
"reth-bench",
]

View File

@@ -0,0 +1,5 @@
---
reth-engine-tree: patch
---
Reordered cache size calculations in `ExecutionCache::new` to group related operations together.

View File

@@ -0,0 +1,6 @@
---
reth: patch
op-reth: patch
---
Added automated changelog generation infrastructure using wevm/changelogs-rs with Claude Code integration. Configured per-crate changelog format with fixed version groups for reth binaries and exclusions for internal test utilities.

View File

@@ -0,0 +1,5 @@
---
reth: patch
---
Updated Alloy dependencies from 1.5.2 to 1.6.1.

21
.github/workflows/changelog.yml vendored Normal file
View File

@@ -0,0 +1,21 @@
name: Changelog
on:
workflow_dispatch:
jobs:
changelog:
runs-on: ubuntu-latest
permissions:
contents: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.head_ref }}
- run: npm install -g @anthropic-ai/claude-code
- uses: wevm/changelogs-rs/gen@master
with:
ai: 'claude -p'
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

View File

@@ -100,5 +100,7 @@ jobs:
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
optimism.tags=${{ steps.params.outputs.optimism_tags }}
ethereum-amd64.tags=${{ steps.params.outputs.ethereum_tags }}
ethereum-arm64.tags=${{ steps.params.outputs.ethereum_tags }}
optimism-amd64.tags=${{ steps.params.outputs.optimism_tags }}
optimism-arm64.tags=${{ steps.params.outputs.optimism_tags }}

View File

@@ -90,7 +90,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- run: cargo nextest run --release -p ef-tests --features "asm-keccak ef-tests"
- run: cargo nextest run --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
doc:
name: doc tests

181
Cargo.lock generated
View File

@@ -121,9 +121,9 @@ dependencies = [
[[package]]
name = "alloy-consensus"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ed1958f0294ecc05ebe7b3c9a8662a3e221c2523b7f2bcd94c7a651efbd510bf"
checksum = "86debde32d8dbb0ab29e7cc75ae1a98688ac7a4c9da54b3a9b14593b9b3c46d3"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -149,9 +149,9 @@ dependencies = [
[[package]]
name = "alloy-consensus-any"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f752e99497ddc39e22d547d7dfe516af10c979405a034ed90e69b914b7dddeae"
checksum = "8d6cb2e7efd385b333f5a77b71baaa2605f7e22f1d583f2879543b54cbce777c"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -164,9 +164,9 @@ dependencies = [
[[package]]
name = "alloy-contract"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f2140796bc79150b1b7375daeab99750f0ff5e27b1f8b0aa81ccde229c7f02a2"
checksum = "668859fcdb42eee289de22a9d01758c910955bb6ecda675b97276f99ce2e16b0"
dependencies = [
"alloy-consensus",
"alloy-dyn-abi",
@@ -262,9 +262,9 @@ dependencies = [
[[package]]
name = "alloy-eips"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "813a67f87e56b38554d18b182616ee5006e8e2bf9df96a0df8bf29dff1d52e3f"
checksum = "be47bf1b91674a5f394b9ed3c691d764fb58ba43937f1371550ff4bc8e59c295"
dependencies = [
"alloy-eip2124",
"alloy-eip2930",
@@ -289,9 +289,9 @@ dependencies = [
[[package]]
name = "alloy-evm"
version = "0.27.0"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1582933a9fc27c0953220eb4f18f6492ff577822e9a8d848890ff59f6b4f5beb"
checksum = "d2ccfe6d724ceabd5518350cfb34f17dd3a6c3cc33579eee94d98101d3a511ff"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -311,9 +311,9 @@ dependencies = [
[[package]]
name = "alloy-genesis"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "05864eef929c4d28895ae4b4d8ac9c6753c4df66e873b9c8fafc8089b59c1502"
checksum = "a59f6f520c323111650d319451de1edb1e32760029a468105b9d7b0f7c11bdf2"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -352,9 +352,9 @@ dependencies = [
[[package]]
name = "alloy-json-rpc"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d2dd146b3de349a6ffaa4e4e319ab3a90371fb159fb0bddeb1c7bbe8b1792eff"
checksum = "5a24c81a56d684f525cd1c012619815ad3a1dd13b0238f069356795d84647d3c"
dependencies = [
"alloy-primitives",
"alloy-sol-types",
@@ -367,9 +367,9 @@ dependencies = [
[[package]]
name = "alloy-network"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8c12278ffbb8872dfba3b2f17d8ea5e8503c2df5155d9bc5ee342794bde505c3"
checksum = "786c5b3ad530eaf43cda450f973fe7fb1c127b4c8990adf66709dafca25e3f6f"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -393,9 +393,9 @@ dependencies = [
[[package]]
name = "alloy-network-primitives"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "833037c04917bc2031541a60e8249e4ab5500e24c637c1c62e95e963a655d66f"
checksum = "c1ed40adf21ae4be786ef5eb62db9c692f6a30f86d34452ca3f849d6390ce319"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -406,9 +406,9 @@ dependencies = [
[[package]]
name = "alloy-op-evm"
version = "0.27.0"
version = "0.27.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6f19214adae08ea95600c3ede76bcbf0c40b36a263534a8f441a4c732f60e868"
checksum = "874bfd6cacd006d05e70560f3af7faa670e31166203f9ba14fae4b169227360b"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -468,9 +468,9 @@ dependencies = [
[[package]]
name = "alloy-provider"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "eafa840b0afe01c889a3012bb2fde770a544f74eab2e2870303eb0a5fb869c48"
checksum = "a3ca4c15818be7ac86208aff3a91b951d14c24e1426e66624e75f2215ba5e2cc"
dependencies = [
"alloy-chains",
"alloy-consensus",
@@ -513,9 +513,9 @@ dependencies = [
[[package]]
name = "alloy-pubsub"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57b3a3b3e4efc9f4d30e3326b6bd6811231d16ef94837e18a802b44ca55119e6"
checksum = "e9eb9c9371738ac47f589e40aae6e418cb5f7436ad25b87575a7f94a60ccf43b"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -557,9 +557,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-client"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "12768ae6303ec764905a8a7cd472aea9072f9f9c980d18151e26913da8ae0123"
checksum = "abe0addad5b8197e851062b49dc47157444bced173b601d91e3f9b561a060a50"
dependencies = [
"alloy-json-rpc",
"alloy-primitives",
@@ -583,9 +583,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0622d8bcac2f16727590aa33f4c3f05ea98130e7e4b4924bce8be85da5ad0dae"
checksum = "74d17d4645a717f0527e491f44f6f7a75c221b9c00ccf79ddba2d26c8e0df4c3"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-engine",
@@ -596,9 +596,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-admin"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c38c5ac70457ecc74e87fe1a5a19f936419224ded0eb0636241452412ca92733"
checksum = "ded79e60d8fd0d7c851044f8b2f2dd7fa8dfa467c577d620595d4de3c31eff7e"
dependencies = [
"alloy-genesis",
"alloy-primitives",
@@ -608,9 +608,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-anvil"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ae8eb0e5d6c48941b61ab76fabab4af66f7d88309a98aa14ad3dec7911c1eba3"
checksum = "2593ce5e1fd416e3b094e7671ef361f22e6944545e0e62ee309b6dfbd702225d"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -620,9 +620,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-any"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a1cf5a093e437dfd62df48e480f24e1a3807632358aad6816d7a52875f1c04aa"
checksum = "d0e98aabb013a71a4b67b52825f7b503e5bb6057fb3b7b2290d514b0b0574b57"
dependencies = [
"alloy-consensus-any",
"alloy-rpc-types-eth",
@@ -631,9 +631,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-beacon"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e07949e912479ef3b848e1cf8db54b534bdd7bc58e6c23f28ea9488960990c8c"
checksum = "3a647a4e3acf49182135c2333d6f9b11ab8684559ff43ef1958ed762cfe9fe0e"
dependencies = [
"alloy-eips",
"alloy-primitives",
@@ -651,9 +651,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-debug"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "925ff0f48c2169c050f0ae7a82769bdf3f45723d6742ebb6a5efb4ed2f491b26"
checksum = "2a1dd760b6a798ee045ab6a7bbd1a02ad8bd6a64d8e18d6e41732f4fc4a4fe5c"
dependencies = [
"alloy-primitives",
"derive_more",
@@ -663,9 +663,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-engine"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "336ef381c7409f23c69f6e79bddc1917b6e832cff23e7a5cf84b9381d53582e6"
checksum = "ddc871ae69688e358cf242a6a7ee6b6e0476a03fd0256434c68daedaec086ec4"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -684,9 +684,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-eth"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "28e97603095020543a019ab133e0e3dc38cd0819f19f19bdd70c642404a54751"
checksum = "5899af8417dcf89f40f88fa3bdb2f3f172605d8e167234311ee34811bbfdb0bf"
dependencies = [
"alloy-consensus",
"alloy-consensus-any",
@@ -706,9 +706,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-mev"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2805153975e25d38e37ee100880e642d5b24e421ed3014a7d2dae1d9be77562e"
checksum = "4d4c9229424e77bd97e629fba44dbfdadebe5bfadbb1e53ad4acbc955610b6c7"
dependencies = [
"alloy-consensus",
"alloy-eips",
@@ -721,9 +721,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-trace"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f1aec4e1c66505d067933ea1a949a4fb60a19c4cfc2f109aa65873ea99e62ea8"
checksum = "410a80e9ac786a2d885adfd7da3568e8f392da106cb5432f00eb4787689d281a"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -735,9 +735,9 @@ dependencies = [
[[package]]
name = "alloy-rpc-types-txpool"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "25b73c1d6e4f1737a20d246dad5a0abd6c1b76ec4c3d153684ef8c6f1b6bb4f4"
checksum = "3a8074654c0292783d504bfa1f2691a69f420154ee9a7883f9212eaf611e60cd"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -747,9 +747,9 @@ dependencies = [
[[package]]
name = "alloy-serde"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "946a0d413dbb5cd9adba0de5f8a1a34d5b77deda9b69c1d7feed8fc875a1aa26"
checksum = "feb73325ee881e42972a5a7bc85250f6af89f92c6ad1222285f74384a203abeb"
dependencies = [
"alloy-primitives",
"arbitrary",
@@ -759,9 +759,9 @@ dependencies = [
[[package]]
name = "alloy-signer"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2f7481dc8316768f042495eaf305d450c32defbc9bce09d8bf28afcd956895bb"
checksum = "1bea4c8f30eddb11d7ab56e83e49c814655daa78ca708df26c300c10d0189cbc"
dependencies = [
"alloy-primitives",
"async-trait",
@@ -774,9 +774,9 @@ dependencies = [
[[package]]
name = "alloy-signer-local"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1259dac1f534a4c66c1d65237c89915d0010a2a91d6c3b0bada24dc5ee0fb917"
checksum = "c28bd71507db58477151a6fe6988fa62a4b778df0f166c3e3e1ef11d059fe5fa"
dependencies = [
"alloy-consensus",
"alloy-network",
@@ -863,9 +863,9 @@ dependencies = [
[[package]]
name = "alloy-transport"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "78f169b85eb9334871db986e7eaf59c58a03d86a30cc68b846573d47ed0656bb"
checksum = "b321f506bd67a434aae8e8a7dfe5373bf66137c149a5f09c9e7dfb0ca43d7c91"
dependencies = [
"alloy-json-rpc",
"auto_impl",
@@ -886,9 +886,9 @@ dependencies = [
[[package]]
name = "alloy-transport-http"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "019821102e70603e2c141954418255bec539ef64ac4117f8e84fb493769acf73"
checksum = "30bf12879a20e1261cd39c3b101856f52d18886907a826e102538897f0d2b66e"
dependencies = [
"alloy-json-rpc",
"alloy-transport",
@@ -901,9 +901,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ipc"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e574ca2f490fb5961d2cdd78188897392c46615cd88b35c202d34bbc31571a81"
checksum = "b75f2334d400249e9672a1ec402536bab259e27a66201a94c3c9b3f1d3bae241"
dependencies = [
"alloy-json-rpc",
"alloy-pubsub",
@@ -921,9 +921,9 @@ dependencies = [
[[package]]
name = "alloy-transport-ws"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b92dea6996269769f74ae56475570e3586910661e037b7b52d50c9641f76c68f"
checksum = "527a0d9c8bbc5c3215b03ad465d4ae8775384ff5faec7c41f033f087c851a9f9"
dependencies = [
"alloy-pubsub",
"alloy-transport",
@@ -938,9 +938,9 @@ dependencies = [
[[package]]
name = "alloy-trie"
version = "0.9.3"
version = "0.9.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "428aa0f0e0658ff091f8f667c406e034b431cb10abd39de4f507520968acc499"
checksum = "4d7fd448ab0a017de542de1dcca7a58e7019fe0e7a34ed3f9543ebddf6aceffa"
dependencies = [
"alloy-primitives",
"alloy-rlp",
@@ -950,17 +950,18 @@ dependencies = [
"derive_more",
"nybbles",
"proptest",
"proptest-derive 0.5.1",
"proptest-derive 0.7.0",
"serde",
"smallvec",
"thiserror 2.0.18",
"tracing",
]
[[package]]
name = "alloy-tx-macros"
version = "1.5.2"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "45ceac797eb8a56bdf5ab1fab353072c17d472eab87645ca847afe720db3246d"
checksum = "6a91d6b4c2f6574fdbcb1611e460455c326667cf5b805c6bd1640dad8e8ee4d2"
dependencies = [
"darling 0.21.3",
"proc-macro2",
@@ -1019,7 +1020,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -1030,7 +1031,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.61.2",
"windows-sys 0.60.2",
]
[[package]]
@@ -1986,9 +1987,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
[[package]]
name = "bytes"
version = "1.11.0"
version = "1.11.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
dependencies = [
"serde",
]
@@ -2896,12 +2897,14 @@ version = "6.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
dependencies = [
"arbitrary",
"cfg-if",
"crossbeam-utils",
"hashbrown 0.14.5",
"lock_api",
"once_cell",
"parking_lot_core",
"serde",
]
[[package]]
@@ -3109,7 +3112,7 @@ dependencies = [
"libc",
"option-ext",
"redox_users 0.5.2",
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -3439,7 +3442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -5180,7 +5183,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -6144,7 +6147,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.59.0",
]
[[package]]
@@ -6271,9 +6274,9 @@ dependencies = [
[[package]]
name = "nybbles"
version = "0.4.7"
version = "0.4.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7b5676b5c379cf5b03da1df2b3061c4a4e2aa691086a56ac923e08c143f53f59"
checksum = "0d49ff0c0d00d4a502b39df9af3a525e1efeb14b9dabb5bb83335284c1309210"
dependencies = [
"alloy-rlp",
"arbitrary",
@@ -7078,9 +7081,9 @@ dependencies = [
[[package]]
name = "proptest-derive"
version = "0.5.1"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49"
checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30"
dependencies = [
"proc-macro2",
"quote",
@@ -7089,9 +7092,9 @@ dependencies = [
[[package]]
name = "proptest-derive"
version = "0.6.0"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30"
checksum = "fb6dc647500e84a25a85b100e76c85b8ace114c209432dc174f20aac11d4ed6c"
dependencies = [
"proc-macro2",
"quote",
@@ -8233,11 +8236,11 @@ dependencies = [
"alloy-chains",
"alloy-primitives",
"alloy-rlp",
"dashmap",
"data-encoding",
"enr",
"hickory-resolver",
"linked_hash_set",
"parking_lot",
"rand 0.9.2",
"reth-chainspec",
"reth-ethereum-forks",
@@ -8467,7 +8470,6 @@ dependencies = [
"assert_matches",
"codspeed-criterion-compat",
"crossbeam-channel",
"dashmap",
"derive_more",
"eyre",
"fixed-cache",
@@ -8881,7 +8883,6 @@ dependencies = [
"futures-util",
"metrics",
"rayon",
"reth-ethereum-forks",
"reth-ethereum-primitives",
"reth-execution-errors",
"reth-execution-types",
@@ -10192,6 +10193,7 @@ dependencies = [
"bincode 1.3.3",
"byteorder",
"bytes",
"dashmap",
"derive_more",
"modular-bitfield",
"once_cell",
@@ -10222,7 +10224,6 @@ dependencies = [
"alloy-primitives",
"alloy-rpc-types-engine",
"assert_matches",
"dashmap",
"eyre",
"itertools 0.14.0",
"metrics",
@@ -10248,6 +10249,7 @@ dependencies = [
"reth-static-file-types",
"reth-storage-api",
"reth-storage-errors",
"reth-tasks",
"reth-testing-utils",
"reth-tracing",
"reth-trie",
@@ -10999,7 +11001,7 @@ dependencies = [
"alloy-provider",
"alloy-rpc-types",
"alloy-rpc-types-engine",
"parking_lot",
"dashmap",
"reth-chainspec",
"reth-db-api",
"reth-errors",
@@ -11251,7 +11253,6 @@ dependencies = [
"alloy-rlp",
"codspeed-criterion-compat",
"crossbeam-channel",
"dashmap",
"derive_more",
"itertools 0.14.0",
"metrics",
@@ -11787,7 +11788,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -12647,7 +12648,7 @@ dependencies = [
"getrandom 0.3.4",
"once_cell",
"rustix",
"windows-sys 0.61.2",
"windows-sys 0.52.0",
]
[[package]]
@@ -13891,7 +13892,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.61.2",
"windows-sys 0.48.0",
]
[[package]]

View File

@@ -490,42 +490,42 @@ alloy-sol-types = { version = "1.5.4", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.0", default-features = false }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.5.2", default-features = false }
alloy-contract = { version = "1.5.2", default-features = false }
alloy-eips = { version = "1.5.2", default-features = false }
alloy-genesis = { version = "1.5.2", default-features = false }
alloy-json-rpc = { version = "1.5.2", default-features = false }
alloy-network = { version = "1.5.2", default-features = false }
alloy-network-primitives = { version = "1.5.2", default-features = false }
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.5.2", default-features = false }
alloy-rpc-client = { version = "1.5.2", default-features = false }
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
alloy-serde = { version = "1.5.2", default-features = false }
alloy-signer = { version = "1.5.2", default-features = false }
alloy-signer-local = { version = "1.5.2", default-features = false }
alloy-transport = { version = "1.5.2" }
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.5.2", default-features = false }
alloy-transport-ws = { version = "1.5.2", default-features = false }
alloy-consensus = { version = "1.6.1", default-features = false }
alloy-contract = { version = "1.6.1", default-features = false }
alloy-eips = { version = "1.6.1", default-features = false }
alloy-genesis = { version = "1.6.1", default-features = false }
alloy-json-rpc = { version = "1.6.1", default-features = false }
alloy-network = { version = "1.6.1", default-features = false }
alloy-network-primitives = { version = "1.6.1", default-features = false }
alloy-provider = { version = "1.6.1", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.6.1", default-features = false }
alloy-rpc-client = { version = "1.6.1", default-features = false }
alloy-rpc-types = { version = "1.6.1", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.6.1", default-features = false }
alloy-rpc-types-anvil = { version = "1.6.1", default-features = false }
alloy-rpc-types-beacon = { version = "1.6.1", default-features = false }
alloy-rpc-types-debug = { version = "1.6.1", default-features = false }
alloy-rpc-types-engine = { version = "1.6.1", default-features = false }
alloy-rpc-types-eth = { version = "1.6.1", default-features = false }
alloy-rpc-types-mev = { version = "1.6.1", default-features = false }
alloy-rpc-types-trace = { version = "1.6.1", default-features = false }
alloy-rpc-types-txpool = { version = "1.6.1", default-features = false }
alloy-serde = { version = "1.6.1", default-features = false }
alloy-signer = { version = "1.6.1", default-features = false }
alloy-signer-local = { version = "1.6.1", default-features = false }
alloy-transport = { version = "1.6.1" }
alloy-transport-http = { version = "1.6.1", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.6.1", default-features = false }
alloy-transport-ws = { version = "1.6.1", default-features = false }
# op
alloy-op-evm = { version = "0.27.0", default-features = false }
alloy-op-evm = { version = "0.27.2", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -543,7 +543,7 @@ backon = { version = "1.2", default-features = false, features = ["std-blocking-
bincode = "1.3"
bitflags = "2.4"
boyer-moore-magiclen = "0.2.16"
bytes = { version = "1.5", default-features = false }
bytes = { version = "1.11.1", default-features = false }
brotli = "8"
cfg-if = "1.0"
clap = "4"
@@ -562,7 +562,7 @@ linked_hash_set = "0.1"
lz4 = "1.28.1"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.2", default-features = false }
nybbles = { version = "0.4.8", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
parking_lot = "0.12"
paste = "1.0"

View File

@@ -123,12 +123,14 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
AccessRights::RW => (
init_db(db_path, self.db.database_args())?,
StaticFileProviderBuilder::read_write(sf_path)
.with_metrics()
.with_genesis_block_number(genesis_block_number)
.build()?,
),
AccessRights::RO | AccessRights::RoInconsistent => {
(open_db_read_only(&db_path, self.db.database_args())?, {
let provider = StaticFileProviderBuilder::read_only(sf_path)
.with_metrics()
.with_genesis_block_number(genesis_block_number)
.build()?;
provider.watch_directory();

View File

@@ -11,6 +11,7 @@ use reth_db_common::DbTool;
use reth_node_builder::NodeTypesWithDB;
use reth_provider::providers::ProviderNodeTypes;
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
use reth_tasks::spawn_scoped_os_thread;
use std::{
collections::BTreeSet,
thread,
@@ -230,7 +231,7 @@ impl Command {
thread::scope(|s| {
let handles: Vec<_> = (0..num_threads)
.map(|thread_id| {
s.spawn(move || {
spawn_scoped_os_thread(s, "db-state-worker", move || {
loop {
// Get next chunk to process
let chunk_idx = {

View File

@@ -4,6 +4,7 @@ use clap::Parser;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
use reth_cli::chainspec::ChainSpecParser;
use reth_cli_runner::CliContext;
use reth_cli_util::cancellation::CancellationToken;
use reth_node_builder::common::metrics_hooks;
use reth_node_core::{args::MetricArgs, version::version_metadata};
use reth_node_metrics::{
@@ -11,6 +12,8 @@ use reth_node_metrics::{
server::{MetricServer, MetricServerConfig},
version::VersionInfo,
};
#[cfg(all(unix, feature = "edge"))]
use reth_provider::RocksDBProviderFactory;
use reth_prune::PrunerBuilder;
use reth_static_file::StaticFileProducer;
use std::sync::Arc;
@@ -50,7 +53,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
build_profile: version_metadata().build_profile_name.as_ref(),
},
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
ctx.task_executor,
ctx.task_executor.clone(),
metrics_hooks(&provider_factory),
data_dir.pprof_dumps(),
);
@@ -70,14 +73,29 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
if let Some(prune_tip) = lowest_static_file_height {
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
// Set up cancellation token for graceful shutdown on Ctrl+C
let cancellation = CancellationToken::new();
let cancellation_clone = cancellation.clone();
ctx.task_executor.spawn_critical("prune-ctrl-c", async move {
tokio::signal::ctrl_c().await.expect("failed to listen for ctrl-c");
cancellation_clone.cancel();
});
// Use batched pruning with a limit to bound memory, running in a loop until complete.
const DELETE_LIMIT: usize = 200_000;
//
// A limit of 20_000_000 results in a max memory usage of ~5G.
const DELETE_LIMIT: usize = 20_000_000;
let mut pruner = PrunerBuilder::new(config)
.delete_limit(DELETE_LIMIT)
.build_with_provider_factory(provider_factory);
.build_with_provider_factory(provider_factory.clone());
let mut total_pruned = 0usize;
loop {
if cancellation.is_cancelled() {
info!(target: "reth::cli", total_pruned, "Pruning interrupted by user");
break;
}
let output = pruner.run(prune_tip)?;
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
total_pruned = total_pruned.saturating_add(batch_pruned);
@@ -88,6 +106,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
if all_segments_finished {
info!(target: "reth::cli", total_pruned, "Pruned data from database");
break;
}
@@ -105,7 +124,14 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
"Pruning batch complete, continuing..."
);
}
info!(target: "reth::cli", total_pruned, "Pruned data from database");
}
// Flush and compact RocksDB to reclaim disk space after pruning
#[cfg(all(unix, feature = "edge"))]
{
info!(target: "reth::cli", "Flushing and compacting RocksDB...");
provider_factory.rocksdb_provider().flush_and_compact()?;
info!(target: "reth::cli", "RocksDB compaction complete");
}
Ok(())

View File

@@ -83,22 +83,7 @@ impl CliRunner {
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
}
// `drop(tokio_runtime)` would block the current thread until its pools
// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
// it on a separate thread and wait for up to 5 seconds for this operation to
// complete.
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("tokio-runtime-shutdown".to_string())
.spawn(move || {
drop(tokio_runtime);
let _ = tx.send(());
})
.unwrap();
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
});
tokio_shutdown(tokio_runtime, true);
command_res
}
@@ -137,19 +122,7 @@ impl CliRunner {
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
}
// Shutdown the runtime on a separate thread
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("tokio-runtime-shutdown".to_string())
.spawn(move || {
drop(tokio_runtime);
let _ = tx.send(());
})
.unwrap();
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
});
tokio_shutdown(tokio_runtime, true);
command_res
}
@@ -179,13 +152,7 @@ impl CliRunner {
tokio_runtime
.block_on(run_until_ctrl_c(async move { fut.await.expect("Failed to join task") }))?;
// drop the tokio runtime on a separate thread because drop blocks until its pools
// (including blocking pool) are shutdown. In other words `drop(tokio_runtime)` would block
// the current thread but we want to exit right away.
std::thread::Builder::new()
.name("tokio-runtime-shutdown".to_string())
.spawn(move || drop(tokio_runtime))
.unwrap();
tokio_shutdown(tokio_runtime, false);
Ok(())
}
@@ -321,3 +288,27 @@ where
Ok(())
}
/// Shut down the given Tokio runtime, and wait for it if `wait` is set.
///
/// `drop(tokio_runtime)` would block the current thread until its pools
/// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
/// it on a separate thread and wait for up to 5 seconds for this operation to
/// complete.
fn tokio_shutdown(rt: tokio::runtime::Runtime, wait: bool) {
// Shutdown the runtime on a separate thread
let (tx, rx) = mpsc::channel();
std::thread::Builder::new()
.name("tokio-shutdown".to_string())
.spawn(move || {
drop(rt);
let _ = tx.send(());
})
.unwrap();
if wait {
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
});
}
}

View File

@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
reth-network-p2p.workspace = true
reth-payload-builder.workspace = true
reth-payload-primitives.workspace = true
reth-primitives-traits = { workspace = true, features = ["rayon"] }
reth-primitives-traits = { workspace = true, features = ["rayon", "dashmap"] }
reth-ethereum-primitives.workspace = true
reth-provider.workspace = true
reth-prune.workspace = true
@@ -62,7 +62,6 @@ metrics.workspace = true
reth-metrics = { workspace = true, features = ["common"] }
# misc
dashmap.workspace = true
schnellru.workspace = true
rayon.workspace = true
tracing.workspace = true

View File

@@ -11,6 +11,7 @@ use reth_provider::{
};
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
use reth_stages_api::{MetricEvent, MetricEventsSender};
use reth_tasks::spawn_os_thread;
use std::{
sync::{
mpsc::{Receiver, SendError, Sender},
@@ -264,14 +265,11 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
// spawn the persistence service
let db_service =
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
let join_handle = std::thread::Builder::new()
.name("Persistence Service".to_string())
.spawn(|| {
if let Err(err) = db_service.run() {
error!(target: "engine::persistence", ?err, "Persistence service failed");
}
})
.unwrap();
let join_handle = spawn_os_thread("persistence", || {
if let Err(err) = db_service.run() {
error!(target: "engine::persistence", ?err, "Persistence service failed");
}
});
PersistenceHandle {
sender: db_service_tx,

View File

@@ -534,9 +534,9 @@ impl ExecutionCache {
/// Build an [`ExecutionCache`] struct, so that execution caches can be easily cloned.
pub fn new(total_cache_size: usize) -> Self {
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
let storage_cache_size = (total_cache_size * 8888) / 10000; // 88.88% of total
let account_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
let code_capacity = Self::bytes_to_entries(code_cache_size, CODE_CACHE_ENTRY_SIZE);
let storage_capacity = Self::bytes_to_entries(storage_cache_size, STORAGE_CACHE_ENTRY_SIZE);

View File

@@ -37,6 +37,7 @@ use reth_provider::{
};
use reth_revm::database::StateProviderDatabase;
use reth_stages_api::ControlFlow;
use reth_tasks::spawn_os_thread;
use reth_trie_db::ChangesetCache;
use revm::state::EvmState;
use state::TreeState;
@@ -431,7 +432,7 @@ where
changeset_cache,
);
let incoming = task.incoming_tx.clone();
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
spawn_os_thread("engine", || task.run());
(incoming, outgoing)
}

View File

@@ -517,6 +517,8 @@ where
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
let prune_depth = self.sparse_trie_prune_depth;
let max_storage_tries = self.sparse_trie_max_storage_tries;
let chunk_size =
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
self.executor.spawn_blocking(move || {
let _enter = span.entered();
@@ -557,6 +559,7 @@ where
proof_worker_handle,
trie_metrics.clone(),
sparse_state_trie,
chunk_size,
))
};

View File

@@ -22,7 +22,7 @@ use reth_trie_parallel::{
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
ProofWorkerHandle,
},
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
targets_v2::MultiProofTargetsV2,
};
use revm_primitives::map::{hash_map, B256Map};
use std::{collections::BTreeMap, sync::Arc, time::Instant};
@@ -63,7 +63,7 @@ const PREFETCH_MAX_BATCH_MESSAGES: usize = 16;
/// The default max targets, for limiting the number of account and storage proof targets to be
/// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability.
const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
pub(crate) const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
/// state.
@@ -311,11 +311,7 @@ impl VersionedMultiProofTargets {
fn chunking_length(&self) -> usize {
match self {
Self::Legacy(targets) => targets.chunking_length(),
Self::V2(targets) => {
// For V2, count accounts + storage slots
targets.account_targets.len() +
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
Self::V2(targets) => targets.chunking_length(),
}
}
@@ -367,9 +363,7 @@ impl VersionedMultiProofTargets {
Self::Legacy(targets) => {
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
}
Self::V2(targets) => {
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
}
Self::V2(targets) => Box::new(targets.chunks(chunk_size).map(Self::V2)),
}
}
}
@@ -1494,7 +1488,7 @@ fn get_proof_targets(
/// Dispatches work items as a single unit or in chunks based on target size and worker
/// availability.
#[allow(clippy::too_many_arguments)]
fn dispatch_with_chunking<T, I>(
pub(crate) fn dispatch_with_chunking<T, I>(
items: T,
chunking_len: usize,
chunk_size: Option<usize>,

View File

@@ -1,19 +1,21 @@
//! Sparse Trie task related functionality.
use crate::tree::{
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
multiproof::{
dispatch_with_chunking, evm_state_to_hashed_post_state, MultiProofMessage,
VersionedMultiProofTargets, DEFAULT_MAX_TARGETS_FOR_CHUNKING,
},
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
};
use alloy_primitives::B256;
use alloy_rlp::{Decodable, Encodable};
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use rayon::iter::ParallelIterator;
use reth_errors::ProviderError;
use rayon::iter::{IntoParallelRefMutIterator, ParallelBridge, ParallelIterator};
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
use reth_revm::state::EvmState;
use reth_trie::{
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
TRIE_ACCOUNT_RLP_MAX_SIZE,
proof_v2::Target, updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, Nibbles,
TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE,
};
use reth_trie_parallel::{
proof_task::{
@@ -24,7 +26,7 @@ use reth_trie_parallel::{
targets_v2::MultiProofTargetsV2,
};
use reth_trie_sparse::{
errors::{SparseStateTrieResult, SparseTrieErrorKind},
errors::{SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult},
provider::{TrieNodeProvider, TrieNodeProviderFactory},
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
};
@@ -34,7 +36,7 @@ use std::{
sync::mpsc,
time::{Duration, Instant},
};
use tracing::{debug, debug_span, instrument, trace};
use tracing::{debug, debug_span, error, instrument, trace};
#[expect(clippy::large_enum_variant)]
pub(super) enum SpawnedSparseTrieTask<BPF, A, S>
@@ -203,6 +205,9 @@ where
}
}
/// Maximum number of pending/prewarm updates that we accumulate in memory before actually applying.
const MAX_PENDING_UPDATES: usize = 100;
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
/// Sender for proof results.
@@ -215,6 +220,15 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
trie: SparseStateTrie<A, S>,
/// Handle to the proof worker pools (storage and account).
proof_worker_handle: ProofWorkerHandle,
/// The size of proof targets chunk to spawn in one calculation.
/// If None, chunking is disabled and all targets are processed in a single proof.
chunk_size: Option<usize>,
/// If this number is exceeded and chunking is enabled, then this will override whether or not
/// there are any active workers and force chunking across workers. This is to prevent tasks
/// which are very long from hitting a single worker.
max_targets_for_chunking: usize,
/// Account trie updates.
account_updates: B256Map<LeafUpdate>,
/// Storage trie updates. hashed address -> slot -> update.
@@ -241,6 +255,14 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
fetched_storage_targets: B256Map<B256Map<u8>>,
/// Reusable buffer for RLP encoding of accounts.
account_rlp_buf: Vec<u8>,
/// Whether the last state update has been received.
finished_state_updates: bool,
/// Pending targets to be dispatched to the proof workers.
pending_targets: MultiProofTargetsV2,
/// Number of pending execution/prewarming updates received but not yet passed to
/// `update_leaves`.
pending_updates: usize,
/// Metrics for the sparse trie.
metrics: MultiProofTaskMetrics,
}
@@ -255,7 +277,8 @@ where
updates: CrossbeamReceiver<MultiProofMessage>,
proof_worker_handle: ProofWorkerHandle,
metrics: MultiProofTaskMetrics,
sparse_state_trie: SparseStateTrie<A, S>,
trie: SparseStateTrie<A, S>,
chunk_size: Option<usize>,
) -> Self {
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
Self {
@@ -263,13 +286,18 @@ where
proof_result_rx,
updates,
proof_worker_handle,
trie: sparse_state_trie,
trie,
chunk_size,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
account_updates: Default::default(),
storage_updates: Default::default(),
pending_account_updates: Default::default(),
fetched_account_targets: Default::default(),
fetched_storage_targets: Default::default(),
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
finished_state_updates: Default::default(),
pending_targets: Default::default(),
pending_updates: Default::default(),
metrics,
}
}
@@ -317,15 +345,8 @@ where
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
let now = Instant::now();
let mut finished_state_updates = false;
loop {
crossbeam_channel::select_biased! {
recv(self.proof_result_rx) -> message => {
let Ok(result) = message else {
unreachable!("we own the sender half")
};
self.on_proof_result(result)?;
},
recv(self.updates) -> message => {
let update = match message {
Ok(m) => m,
@@ -334,27 +355,48 @@ where
}
};
match update {
MultiProofMessage::PrefetchProofs(targets) => {
self.on_prewarm_targets(targets);
}
MultiProofMessage::StateUpdate(_, state) => {
self.on_state_update(state);
}
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
self.on_hashed_state_update(state);
}
MultiProofMessage::BlockAccessList(_) => todo!(),
MultiProofMessage::FinishedStateUpdates => {
finished_state_updates = true;
}
}
self.on_multiproof_message(update);
self.pending_updates += 1;
}
recv(self.proof_result_rx) -> message => {
let Ok(result) = message else {
unreachable!("we own the sender half")
};
let ProofResult::V2(mut result) = result.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
while let Ok(next) = self.proof_result_rx.try_recv() {
let ProofResult::V2(res) = next.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
result.extend(res);
}
self.on_proof_result(result)?;
},
}
self.process_updates()?;
if self.updates.is_empty() && self.proof_result_rx.is_empty() {
// If we don't have any pending messages, we can spend some time on computing
// storage roots and promoting account updates.
self.dispatch_pending_targets();
self.promote_pending_account_updates()?;
self.dispatch_pending_targets();
} else if self.updates.is_empty() || self.pending_updates > MAX_PENDING_UPDATES {
// If we don't have any pending updates OR we've accumulated a lot already, apply
// them to the trie,
self.process_leaf_updates()?;
self.dispatch_pending_targets();
} else if self.updates.is_empty() ||
self.pending_targets.chunking_length() > self.chunk_size.unwrap_or_default()
{
// Make sure to dispatch targets if we don't have any updates or if we've
// accumulated a lot of them.
self.dispatch_pending_targets();
}
if finished_state_updates &&
if self.finished_state_updates &&
self.account_updates.is_empty() &&
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
{
@@ -377,6 +419,22 @@ where
Ok(StateRootComputeOutcome { state_root, trie_updates })
}
/// Processes a [`MultiProofMessage`].
fn on_multiproof_message(&mut self, message: MultiProofMessage) {
match message {
MultiProofMessage::PrefetchProofs(targets) => self.on_prewarm_targets(targets),
MultiProofMessage::StateUpdate(_, state) => self.on_state_update(state),
MultiProofMessage::EmptyProof { .. } => unreachable!(),
MultiProofMessage::BlockAccessList(_) => todo!(),
MultiProofMessage::FinishedStateUpdates => self.finished_state_updates = true,
}
}
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
let VersionedMultiProofTargets::V2(targets) = targets else {
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
@@ -412,11 +470,7 @@ where
)]
fn on_state_update(&mut self, update: EvmState) {
let hashed_state_update = evm_state_to_hashed_post_state(update);
self.on_hashed_state_update(hashed_state_update)
}
/// Processes a hashed state update and encodes all state changes as trie updates.
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
for (address, storage) in hashed_state_update.storages {
for (slot, value) in storage.storage {
let encoded = if value.is_zero() {
@@ -454,72 +508,148 @@ where
fn on_proof_result(
&mut self,
result: ProofResultMessage,
result: DecodedMultiProofV2,
) -> Result<(), ParallelStateRootError> {
let ProofResult::V2(result) = result.result? else {
unreachable!("sparse trie as cache must only be used with multiproof v2");
};
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
})
}
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
fn process_updates(&mut self) -> Result<(), ProviderError> {
let mut targets = MultiProofTargetsV2::default();
/// Applies all account and storage leaf updates to corresponding tries and collects any new
/// multiproof targets.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn process_leaf_updates(&mut self) -> SparseTrieResult<()> {
self.pending_updates = 0;
for (addr, updates) in &mut self.storage_updates {
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
let fetched_storage = self.fetched_storage_targets.entry(*addr).or_default();
// Start with processing all storage updates in parallel.
let storage_results = self
.storage_updates
.iter_mut()
.map(|(address, updates)| {
let trie = self.trie.take_or_create_storage_trie(address);
let fetched = self.fetched_storage_targets.remove(address).unwrap_or_default();
trie.update_leaves(updates, |path, min_len| match fetched_storage.entry(path) {
(address, updates, fetched, trie)
})
.par_bridge()
.map(|(address, updates, mut fetched, mut trie)| {
let mut targets = Vec::new();
trie.update_leaves(updates, |path, min_len| match fetched.entry(path) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
targets.push(Target::new(path).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets.push(Target::new(path).with_min_len(min_len));
}
})?;
SparseTrieResult::Ok((address, targets, fetched, trie))
})
.collect::<Result<Vec<_>, _>>()?;
for (address, targets, fetched, trie) in storage_results {
self.fetched_storage_targets.insert(*address, fetched);
self.trie.insert_storage_trie(*address, trie);
if !targets.is_empty() {
self.pending_targets.storage_targets.entry(*address).or_default().extend(targets);
}
}
// Process account trie updates and fill the account targets.
self.process_account_leaf_updates()?;
Ok(())
}
/// Invokes `update_leaves` for the accounts trie and collects any new targets.
///
/// Returns whether any updates were drained (applied to the trie).
fn process_account_leaf_updates(&mut self) -> SparseTrieResult<bool> {
let updates_len_before = self.account_updates.len();
self.trie.trie_mut().update_leaves(
&mut self.account_updates,
|target, min_len| match self.fetched_account_targets.entry(target) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
targets
.storage_targets
.entry(*addr)
.or_default()
.push(Target::new(path).with_min_len(min_len));
self.pending_targets
.account_targets
.push(Target::new(target).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets
.storage_targets
.entry(*addr)
.or_default()
.push(Target::new(path).with_min_len(min_len));
self.pending_targets
.account_targets
.push(Target::new(target).with_min_len(min_len));
}
})
.map_err(ProviderError::other)?;
},
)?;
// If all storage updates were processed, we can now compute the new storage root.
if updates.is_empty() {
let storage_root =
Ok(self.account_updates.len() < updates_len_before)
}
/// Iterates through all storage tries for which all updates were processed, computes their
/// storage roots, and promotes corresponding pending account updates into proper leaf updates
/// for accounts trie.
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn promote_pending_account_updates(&mut self) -> SparseTrieResult<()> {
self.process_leaf_updates()?;
if self.pending_account_updates.is_empty() {
return Ok(());
}
let roots = self
.trie
.storage_tries_mut()
.par_iter_mut()
.filter(|(address, _)| {
self.storage_updates.get(*address).is_some_and(|updates| updates.is_empty())
})
.map(|(address, trie)| {
let root =
trie.root().expect("updates are drained, trie should be revealed by now");
// If there is a pending account update for this address with known info, we can
// encode it into proper update right away.
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
entry.get().is_some()
(address, root)
})
.collect::<Vec<_>>();
for (addr, storage_root) in roots {
// If the storage root is known and we have a pending update for this account, encode it
// into a proper update.
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
entry.get().is_some()
{
let account = entry.remove().expect("just checked, should be Some");
let encoded = if account.is_none_or(|account| account.is_empty()) &&
storage_root == EMPTY_ROOT_HASH
{
let account = entry.remove().expect("just checked, should be Some");
let encoded = if account.is_none_or(|account| account.is_empty()) &&
storage_root == EMPTY_ROOT_HASH
{
Vec::new()
} else {
self.account_rlp_buf.clear();
account
.unwrap_or_default()
.into_trie_account(storage_root)
.encode(&mut self.account_rlp_buf);
self.account_rlp_buf.clone()
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
Vec::new()
} else {
self.account_rlp_buf.clear();
account
.unwrap_or_default()
.into_trie_account(storage_root)
.encode(&mut self.account_rlp_buf);
self.account_rlp_buf.clone()
};
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
}
}
@@ -568,52 +698,52 @@ where
false
});
let updates_len_before = self.account_updates.len();
// Process account trie updates and fill the account targets.
self.trie
.trie_mut()
.update_leaves(&mut self.account_updates, |target, min_len| {
match self.fetched_account_targets.entry(target) {
Entry::Occupied(mut entry) => {
if min_len < *entry.get() {
entry.insert(min_len);
targets
.account_targets
.push(Target::new(target).with_min_len(min_len));
}
}
Entry::Vacant(entry) => {
entry.insert(min_len);
targets.account_targets.push(Target::new(target).with_min_len(min_len));
}
}
})
.map_err(ProviderError::other)?;
if updates_len_before == self.account_updates.len() {
// Only exit when no new updates are processed.
//
// We need to keep iterating if any updates are being drained because that might
// indicate that more pending account updates can be promoted.
break;
// Only exit when no new updates are processed.
//
// We need to keep iterating if any updates are being drained because that might
// indicate that more pending account updates can be promoted.
if !self.process_account_leaf_updates()? {
break
}
}
if !targets.is_empty() {
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
targets,
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
0,
HashedPostState::default(),
Instant::now(),
),
})?;
}
Ok(())
}
#[instrument(
level = "debug",
target = "engine::tree::payload_processor::sparse_trie",
skip_all
)]
fn dispatch_pending_targets(&mut self) {
if !self.pending_targets.is_empty() {
let chunking_length = self.pending_targets.chunking_length();
dispatch_with_chunking(
std::mem::take(&mut self.pending_targets),
chunking_length,
self.chunk_size,
self.max_targets_for_chunking,
self.proof_worker_handle.available_account_workers(),
self.proof_worker_handle.available_storage_workers(),
MultiProofTargetsV2::chunks,
|proof_targets| {
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(
AccountMultiproofInput::V2 {
targets: proof_targets,
proof_result_sender: ProofResultContext::new(
self.proof_result_tx.clone(),
0,
HashedPostState::default(),
Instant::now(),
),
},
) {
error!("failed to dispatch account multiproof: {e:?}");
}
},
);
}
}
}
/// Outcome of the state root computation, including the state root itself with

View File

@@ -1,9 +1,9 @@
//! Contains a precompile cache backed by `schnellru::LruMap` (LRU by length).
use alloy_primitives::Bytes;
use dashmap::DashMap;
use moka::policy::EvictionPolicy;
use reth_evm::precompiles::{DynPrecompile, Precompile, PrecompileInput};
use reth_primitives_traits::dashmap::DashMap;
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
use revm_primitives::Address;
use std::{hash::Hash, sync::Arc};

View File

@@ -28,6 +28,7 @@ use reth_ethereum_primitives::{Block, EthPrimitives};
use reth_evm_ethereum::MockEvmConfig;
use reth_primitives_traits::Block as _;
use reth_provider::test_utils::MockEthProvider;
use reth_tasks::spawn_os_thread;
use std::{
collections::BTreeMap,
str::FromStr,
@@ -538,10 +539,7 @@ async fn test_tree_persist_blocks() {
.get_executed_blocks(1..tree_config.persistence_threshold() + 2)
.collect();
let test_harness = TestHarness::new(chain_spec).with_blocks(blocks.clone());
std::thread::Builder::new()
.name("Engine Task".to_string())
.spawn(|| test_harness.tree.run())
.unwrap();
spawn_os_thread("engine", || test_harness.tree.run());
// send a message to the tree to enter the main loop.
test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(vec![])).unwrap();
@@ -1989,10 +1987,7 @@ mod forkchoice_updated_tests {
let action_rx = test_harness.action_rx;
// Spawn tree in background thread
std::thread::Builder::new()
.name("Engine Task".to_string())
.spawn(|| test_harness.tree.run())
.unwrap();
spawn_os_thread("engine", || test_harness.tree.run());
// Send terminate request
to_tree_tx

View File

@@ -36,7 +36,6 @@ rayon = { workspace = true, optional = true }
[dev-dependencies]
reth-ethereum-primitives.workspace = true
reth-ethereum-forks.workspace = true
[features]
default = ["std"]
@@ -47,7 +46,6 @@ std = [
"alloy-primitives/std",
"alloy-consensus/std",
"revm/std",
"reth-ethereum-forks/std",
"alloy-evm/std",
"reth-execution-errors/std",
"reth-execution-types/std",

View File

@@ -41,6 +41,7 @@ metrics.workspace = true
[dev-dependencies]
reth-tracing.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
tokio = { workspace = true, features = ["rt-multi-thread"] }
secp256k1 = { workspace = true, features = ["std", "rand"] }
rand_08.workspace = true

View File

@@ -30,12 +30,12 @@ tokio-stream.workspace = true
hickory-resolver = { workspace = true, features = ["tokio"] }
# misc
dashmap = { workspace = true, features = ["inline"] }
data-encoding.workspace = true
linked_hash_set.workspace = true
schnellru.workspace = true
thiserror.workspace = true
tracing.workspace = true
parking_lot.workspace = true
serde = { workspace = true, optional = true }
serde_with = { workspace = true, optional = true }
@@ -56,9 +56,9 @@ serde = [
"alloy-primitives/serde",
"enr/serde",
"linked_hash_set/serde",
"parking_lot/serde",
"rand/serde",
"secp256k1/serde",
"hickory-resolver/serde",
"reth-ethereum-forks/serde",
"dashmap/serde",
]

View File

@@ -1,9 +1,9 @@
//! Perform DNS lookups
use dashmap::DashMap;
use hickory_resolver::name_server::ConnectionProvider;
pub use hickory_resolver::{ResolveError, TokioResolver};
use parking_lot::RwLock;
use std::{collections::HashMap, future::Future};
use std::future::Future;
use tracing::trace;
/// A type that can lookup DNS entries
@@ -72,25 +72,25 @@ impl Resolver for DnsResolver {
/// A [Resolver] that uses an in memory map to lookup entries
#[derive(Debug, Default)]
pub struct MapResolver(RwLock<HashMap<String, String>>);
pub struct MapResolver(DashMap<String, String>);
// === impl MapResolver ===
impl MapResolver {
/// Inserts a key-value pair into the map.
pub fn insert(&self, k: String, v: String) -> Option<String> {
self.0.write().insert(k, v)
self.0.insert(k, v)
}
/// Returns the value corresponding to the key
pub fn get(&self, k: &str) -> Option<String> {
self.0.read().get(k).cloned()
self.0.get(k).map(|entry| entry.value().clone())
}
/// Removes a key from the map, returning the value at the key if the key was previously in the
/// map.
pub fn remove(&self, k: &str) -> Option<String> {
self.0.write().remove(k)
self.0.remove(k).map(|(_, v)| v)
}
}

View File

@@ -236,7 +236,7 @@ impl LaunchContext {
.map_or(0, |num| num.get().saturating_sub(reserved_cpu_cores).max(1));
if let Err(err) = ThreadPoolBuilder::new()
.num_threads(num_threads)
.thread_name(|i| format!("reth-rayon-{i}"))
.thread_name(|i| format!("rayon-{i}"))
.build_global()
{
warn!(%err, "Failed to build global thread pool")

View File

@@ -36,6 +36,7 @@ secp256k1 = { workspace = true, features = ["recovery"], optional = true }
auto_impl.workspace = true
byteorder = { workspace = true, optional = true }
bytes.workspace = true
dashmap = { workspace = true, features = ["inline"], optional = true }
derive_more.workspace = true
once_cell.workspace = true
serde_with = { workspace = true, optional = true }
@@ -116,6 +117,7 @@ arbitrary = [
"alloy-trie/arbitrary",
"reth-chainspec/arbitrary",
"alloy-rpc-types-eth?/arbitrary",
"dashmap?/arbitrary",
]
serde-bincode-compat = [
"serde",
@@ -144,6 +146,7 @@ serde = [
"revm-state/serde",
"rand_08/serde",
"alloy-rpc-types-eth?/serde",
"dashmap?/serde",
]
reth-codec = [
"dep:reth-codecs",
@@ -157,4 +160,5 @@ op = [
rayon = [
"dep:rayon",
]
dashmap = ["dep:dashmap"]
rpc-compat = ["alloy-rpc-types-eth"]

View File

@@ -245,3 +245,12 @@ pub mod test_utils {
#[cfg(any(test, feature = "test-utils"))]
pub use crate::{block::TestBlock, header::test_utils::TestHeader};
}
/// Re-exports of `dashmap` types with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
#[cfg(feature = "dashmap")]
pub mod dashmap {
pub use ::dashmap::{mapref, DashSet, Entry};
/// Re-export of `DashMap` with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
pub type DashMap<K, V, S = alloy_primitives::map::DefaultHashBuilder> =
::dashmap::DashMap<K, V, S>;
}

View File

@@ -25,6 +25,10 @@ pub use user::{
///
/// This is a generic helper function used by both receipts and bodies pruning
/// when data is stored in static files.
///
/// The checkpoint block number is set to the highest block in the actually deleted files,
/// not `input.to_block`, since `to_block` might refer to a block in the middle of an
/// undeleted file.
pub(crate) fn prune_static_files<Provider>(
provider: &Provider,
input: PruneInput,
@@ -37,18 +41,31 @@ where
provider.static_file_provider().delete_segment_below_block(segment, input.to_block + 1)?;
if deleted_headers.is_empty() {
return Ok(SegmentOutput::done())
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned: 0,
checkpoint: input
.previous_checkpoint
.map(SegmentOutputCheckpoint::from_prune_checkpoint),
})
}
let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range());
let pruned = tx_ranges.clone().map(|range| range.len()).sum::<u64>() as usize;
// The highest block number in the deleted files is the actual checkpoint.
let checkpoint_block = deleted_headers
.iter()
.filter_map(|header| header.block_range())
.map(|range| range.end())
.max();
Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(input.to_block),
block_number: checkpoint_block,
tx_number: tx_ranges.map(|range| range.end()).max(),
}),
})

View File

@@ -76,8 +76,11 @@ where
} = prune_modes;
Self::default()
// Bodies - run first since file deletion is fast
.segment_opt(bodies_history.map(Bodies::new))
// Transaction lookup must run before bodies because it needs to read transaction
// data from static files before bodies deletes them.
.segment_opt(transaction_lookup.map(TransactionLookup::new))
// Bodies
.segment_opt(bodies_history.map(|mode| Bodies::new(mode, transaction_lookup)))
// Account history
.segment_opt(account_history.map(AccountHistory::new))
// Storage history
@@ -89,8 +92,6 @@ where
(!receipts_log_filter.is_empty())
.then(|| ReceiptsByLogs::new(receipts_log_filter.clone())),
)
// Transaction lookup
.segment_opt(transaction_lookup.map(TransactionLookup::new))
// Sender recovery
.segment_opt(sender_recovery.map(SenderRecovery::new))
}

View File

@@ -238,8 +238,6 @@ impl AccountHistory {
where
Provider: DBProvider + StaticFileProviderFactory + ChangeSetReader + RocksDBProviderFactory,
{
use reth_provider::PruneShardOutcome;
// Unlike MDBX path, we don't divide the limit by 2 because RocksDB path only prunes
// history shards (no separate changeset table to delete from). The changesets are in
// static files which are deleted separately.
@@ -287,14 +285,15 @@ impl AccountHistory {
sorted_accounts.sort_unstable_by_key(|(addr, _)| *addr);
provider.with_rocksdb_batch(|mut batch| {
for (address, highest_block) in &sorted_accounts {
let prune_to = (*highest_block).min(last_changeset_pruned_block);
match batch.prune_account_history_to(*address, prune_to)? {
PruneShardOutcome::Deleted => deleted_shards += 1,
PruneShardOutcome::Updated => updated_shards += 1,
PruneShardOutcome::Unchanged => {}
}
}
let targets: Vec<_> = sorted_accounts
.iter()
.map(|(addr, highest)| (*addr, (*highest).min(last_changeset_pruned_block)))
.collect();
let outcomes = batch.prune_account_history_batch(&targets)?;
deleted_shards = outcomes.deleted;
updated_shards = outcomes.updated;
Ok(((), Some(batch.into_inner())))
})?;
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned account history (RocksDB indices)");

View File

@@ -2,9 +2,14 @@ use crate::{
segments::{self, PruneInput, Segment},
PrunerError,
};
use reth_provider::{BlockReader, StaticFileProviderFactory};
use reth_prune_types::{PruneMode, PrunePurpose, PruneSegment, SegmentOutput};
use alloy_primitives::BlockNumber;
use reth_provider::{BlockReader, PruneCheckpointReader, StaticFileProviderFactory};
use reth_prune_types::{
PruneInterruptReason, PruneMode, PrunePurpose, PruneSegment, SegmentOutput,
SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use tracing::debug;
/// Segment responsible for pruning transactions in static files.
///
@@ -12,18 +17,79 @@ use reth_static_file_types::StaticFileSegment;
#[derive(Debug)]
pub struct Bodies {
mode: PruneMode,
/// Transaction lookup prune mode. Used to determine if we need to wait for tx lookup pruning
/// before deleting transaction bodies.
tx_lookup_mode: Option<PruneMode>,
}
impl Bodies {
/// Creates a new [`Bodies`] segment with the given prune mode.
pub const fn new(mode: PruneMode) -> Self {
Self { mode }
/// Creates a new [`Bodies`] segment with the given prune mode and optional transaction lookup
/// prune mode for coordination.
pub const fn new(mode: PruneMode, tx_lookup_mode: Option<PruneMode>) -> Self {
Self { mode, tx_lookup_mode }
}
/// Returns the next best block that bodies can prune up to considering the transaction lookup
/// pruning configuration (if any) and progress.
///
/// Returns `None` if there's no block available to prune (e.g., waiting on `tx_lookup`).
fn next_bodies_prune_target<Provider>(
&self,
provider: &Provider,
input: &PruneInput,
) -> Result<Option<BlockNumber>, PrunerError>
where
Provider: PruneCheckpointReader,
{
let Some(tx_lookup_mode) = self.tx_lookup_mode else { return Ok(Some(input.to_block)) };
let tx_lookup_checkpoint = provider
.get_prune_checkpoint(PruneSegment::TransactionLookup)?
.and_then(|cp| cp.block_number);
// Determine the safe prune target, if any.
// tx_lookup's next_pruned_block tells us what block it will prune next.
// - None: tx_lookup will never prune more blocks (e.g. Before(N) reached its target), so
// bodies can prune freely
// - Some(next) > to_block: tx_lookup is ahead of our target, so we're safe to prune
// to_block
// - Some(next) <= to_block: tx_lookup still needs to prune blocks we want to delete, so we
// must wait and only prune up to (next - 1) to preserve tx data it needs
let to_block = match tx_lookup_mode.next_pruned_block(tx_lookup_checkpoint) {
None => Some(input.to_block),
Some(tx_lookup_next) if tx_lookup_next > input.to_block => Some(input.to_block),
Some(tx_lookup_next) => {
// We can only prune bodies up to the block BEFORE tx_lookup's next target.
// tx_lookup_next is the next block tx_lookup will prune, meaning it still needs
// to read transactions from that block. We must preserve those transactions,
// so bodies can only safely delete up to (tx_lookup_next - 1).
let Some(safe) = tx_lookup_next.checked_sub(1) else {
return Ok(None);
};
if input.previous_checkpoint.is_some_and(|cp| cp.block_number.unwrap_or(0) >= safe)
{
// we have pruned what we can
return Ok(None)
}
debug!(
target: "pruner",
to_block = input.to_block,
safe,
"Bodies pruning limited by tx_lookup progress"
);
Some(safe)
}
};
Ok(to_block)
}
}
impl<Provider> Segment<Provider> for Bodies
where
Provider: StaticFileProviderFactory + BlockReader,
Provider: StaticFileProviderFactory + BlockReader + PruneCheckpointReader,
{
fn segment(&self) -> PruneSegment {
PruneSegment::Bodies
@@ -38,7 +104,20 @@ where
}
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
segments::prune_static_files(provider, input, StaticFileSegment::Transactions)
let Some(to_block) = self.next_bodies_prune_target(provider, &input)? else {
debug!(
to_block = input.to_block,
"Transaction lookup still has work to be done up to target block"
);
return Ok(SegmentOutput::not_done(
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
));
};
// Use the coordinated to_block instead of input.to_block
let adjusted_input = PruneInput { to_block, ..input };
segments::prune_static_files(provider, adjusted_input, StaticFileSegment::Transactions)
}
}
@@ -50,7 +129,8 @@ mod tests {
use reth_exex_types::FinishedExExHeight;
use reth_provider::{
test_utils::{create_test_provider_factory, MockNodeTypesWithDB},
ProviderFactory, StaticFileWriter,
DBProvider, DatabaseProviderFactory, ProviderFactory, PruneCheckpointWriter,
StaticFileWriter,
};
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
use reth_static_file_types::{
@@ -93,19 +173,83 @@ mod tests {
static_file_provider.initialize_index().expect("initialize index");
}
struct PruneTestCase {
prune_mode: PruneMode,
struct TestCase {
tx_lookup_mode: Option<PruneMode>,
tx_lookup_checkpoint_block: Option<BlockNumber>,
bodies_mode: PruneMode,
expected_pruned: usize,
expected_lowest_block: Option<BlockNumber>,
expected_progress: PruneProgress,
}
impl TestCase {
fn new() -> Self {
Self {
tx_lookup_mode: None,
tx_lookup_checkpoint_block: None,
bodies_mode: PruneMode::Full,
expected_pruned: 0,
expected_lowest_block: None,
expected_progress: PruneProgress::Finished,
}
}
fn with_bodies_mode(mut self, mode: PruneMode) -> Self {
self.bodies_mode = mode;
self
}
fn with_expected_pruned(mut self, pruned: usize) -> Self {
self.expected_pruned = pruned;
self
}
fn with_expected_progress(mut self, progress: PruneProgress) -> Self {
self.expected_progress = progress;
self
}
fn with_lowest_block(mut self, block: BlockNumber) -> Self {
self.expected_lowest_block = Some(block);
self
}
fn with_tx_lookup(mut self, mode: PruneMode, checkpoint: Option<BlockNumber>) -> Self {
self.tx_lookup_mode = Some(mode);
self.tx_lookup_checkpoint_block = checkpoint;
self
}
}
fn run_prune_test(
factory: &ProviderFactory<MockNodeTypesWithDB>,
finished_exex_height_rx: &tokio::sync::watch::Receiver<FinishedExExHeight>,
test_case: PruneTestCase,
test_case: TestCase,
tip: BlockNumber,
) {
let bodies = Bodies::new(test_case.prune_mode);
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
// Capture highest block before pruning
let static_provider = factory.static_file_provider();
let highest_before =
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions);
// Set up tx_lookup checkpoint if provided
if let Some(checkpoint_block) = test_case.tx_lookup_checkpoint_block {
let provider = factory.database_provider_rw().unwrap();
provider
.save_prune_checkpoint(
PruneSegment::TransactionLookup,
reth_prune_types::PruneCheckpoint {
block_number: Some(checkpoint_block),
tx_number: None,
prune_mode: test_case.tx_lookup_mode.unwrap(),
},
)
.unwrap();
provider.commit().unwrap();
}
let bodies = Bodies::new(test_case.bodies_mode, test_case.tx_lookup_mode);
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
let mut pruner = Pruner::new_with_factory(
@@ -114,27 +258,29 @@ mod tests {
5,
10000,
None,
finished_exex_height_rx.clone(),
finished_exex_height_rx,
);
let result = pruner.run(tip).expect("pruner run");
assert_eq!(result.progress, PruneProgress::Finished);
assert_eq!(result.progress, test_case.expected_progress);
assert_eq!(result.segments.len(), 1);
let (segment, output) = &result.segments[0];
assert_eq!(*segment, PruneSegment::Bodies);
assert_eq!(output.pruned, test_case.expected_pruned);
let static_provider = factory.static_file_provider();
assert_eq!(
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
test_case.expected_lowest_block
);
assert_eq!(
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
Some(tip)
);
if let Some(expected_lowest) = test_case.expected_lowest_block {
let static_provider = factory.static_file_provider();
assert_eq!(
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
Some(expected_lowest)
);
assert_eq!(
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
highest_before
);
}
}
#[test]
@@ -143,50 +289,82 @@ mod tests {
let tip = 2_499_999;
setup_static_file_jars(&factory, tip);
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
let (_, _finished_exex_height_rx) =
tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
let test_cases = vec![
// Test 1: PruneMode::Before(750_000) → deletes jar 1 (0-499_999)
PruneTestCase {
prune_mode: PruneMode::Before(750_000),
expected_pruned: 1000,
expected_lowest_block: Some(999_999),
},
// Test 2: PruneMode::Before(850_000) → no deletion (jar 2: 500_000-999_999 contains
// Test 1: PruneMode::Before(750_000) → deletes jar 0 (0-499_999)
// Checkpoint 499_999 != target 749_999 -> HasMoreData
TestCase::new()
.with_bodies_mode(PruneMode::Before(750_000))
.with_expected_pruned(1000)
.with_lowest_block(999_999),
// Test 2: PruneMode::Before(850_000) → no deletion (jar 1: 500_000-999_999 contains
// target)
PruneTestCase {
prune_mode: PruneMode::Before(850_000),
expected_pruned: 0,
expected_lowest_block: Some(999_999),
},
// Test 3: PruneMode::Before(1_599_999) → deletes jar 2 (500_000-999_999) and jar 3
// (1_000_000-1_499_999)
PruneTestCase {
prune_mode: PruneMode::Before(1_599_999),
expected_pruned: 2000,
expected_lowest_block: Some(1_999_999),
},
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 4
// (1_500_000-1_999_999)
PruneTestCase {
prune_mode: PruneMode::Distance(500_000),
expected_pruned: 1000,
expected_lowest_block: Some(2_499_999),
},
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 5: 2_000_000-2_499_999
TestCase::new().with_bodies_mode(PruneMode::Before(850_000)).with_lowest_block(999_999),
// Test 3: PruneMode::Before(1_599_999) → deletes jars 0 and 1 (0-999_999)
// Checkpoint 999_999 != target 1_599_998 -> HasMoreData
TestCase::new()
.with_bodies_mode(PruneMode::Before(1_599_999))
.with_expected_pruned(2000)
.with_lowest_block(1_999_999),
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 3
// (1_500_000-1_999_999) Checkpoint 1_999_999 == target 1_999_999 ->
// Finished
TestCase::new()
.with_bodies_mode(PruneMode::Distance(500_000))
.with_expected_pruned(1000)
.with_lowest_block(2_499_999),
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 4: 2_000_000-2_499_999
// contains target)
PruneTestCase {
prune_mode: PruneMode::Before(2_300_000),
expected_pruned: 0,
expected_lowest_block: Some(2_499_999),
},
TestCase::new()
.with_bodies_mode(PruneMode::Before(2_300_000))
.with_lowest_block(2_499_999),
];
for test_case in test_cases {
run_prune_test(&factory, &finished_exex_height_rx, test_case, tip);
run_prune_test(&factory, test_case, tip);
}
}
#[test]
fn checkpoint_reflects_deleted_files_not_target() {
// Test that checkpoint is set to the highest deleted block, not to_block.
// When to_block falls in the middle of an undeleted file, checkpoint should reflect
// what was actually deleted.
let factory = create_test_provider_factory();
let tip = 1_499_999;
setup_static_file_jars(&factory, tip);
// Use PruneMode::Before(900_000) which targets 899_999.
// This should delete jar 0 (0-499_999) since it's entirely below the target.
// Jar 1 (500_000-999_999) contains the target, so it won't be deleted.
// Checkpoint should be 499_999 (end of jar 0), not 899_999 (to_block).
let bodies = Bodies::new(PruneMode::Before(900_000), None);
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
let mut pruner =
Pruner::new_with_factory(factory, segments, 5, 10000, None, finished_exex_height_rx);
let result = pruner.run(tip).expect("pruner run");
assert_eq!(result.progress, PruneProgress::Finished);
assert_eq!(result.segments.len(), 1);
let (segment, output) = &result.segments[0];
assert_eq!(*segment, PruneSegment::Bodies);
// Verify checkpoint is set to the end of deleted jar (499_999), not to_block (899_999)
let checkpoint_block = output.checkpoint.as_ref().and_then(|cp| cp.block_number);
assert_eq!(
checkpoint_block,
Some(499_999),
"Checkpoint should be 499_999 (end of deleted jar 0), not 899_999 (to_block)"
);
}
#[test]
fn min_block_updated_on_sync() {
// Regression test: update_index must update min_block to prevent stale values
@@ -302,4 +480,100 @@ mod tests {
assert_eq!(deleted.len(), expected_deleted);
}
}
#[test]
fn bodies_with_tx_lookup_coordination() {
// Test that bodies pruning correctly coordinates with tx lookup pruning
// Using tip = 1_523_000 creates 4 static file jars:
// - Jar 0: blocks 0-499_999, txs 0-999
// - Jar 1: blocks 500_000-999_999, txs 1000-1999
// - Jar 2: blocks 1_000_000-1_499_999, txs 2000-2999
// - Jar 3: blocks 1_500_000-1_523_000, txs 3000-3999
let tip = 1_523_000;
let test_cases = vec![
// Scenario 1: tx_lookup disabled, bodies can prune freely (deletes jar 0)
// Checkpoint is 499_999 (end of jar 0), target is 599_999, so HasMoreData
TestCase::new()
.with_bodies_mode(PruneMode::Before(600_000))
.with_expected_pruned(1000)
.with_lowest_block(999_999),
// Scenario 2: tx_lookup enabled but not run yet, bodies cannot prune
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), None)
.with_bodies_mode(PruneMode::Before(600_000))
.with_expected_progress(PruneProgress::HasMoreData(
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
))
.with_lowest_block(499_999), // No jars deleted, jar 0 ends at 499_999
// Scenario 3: tx_lookup caught up to its target, bodies can prune freely
// Deletes jar 0, checkpoint is 499_999, target is 599_999 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
.with_bodies_mode(PruneMode::Before(600_000))
.with_expected_pruned(1000)
.with_lowest_block(999_999),
// Scenario 4: tx_lookup behind its target, bodies limited to tx_lookup checkpoint
// tx_lookup should prune up to 599_999, but checkpoint is only at 250_000
// bodies wants to prune up to 599_999, but limited to 250_000
// No jars deleted because jar 0 (0-499_999) ends beyond 250_000
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), Some(250_000))
.with_bodies_mode(PruneMode::Before(600_000))
.with_lowest_block(499_999), // No jars deleted
// Scenario 5: Both use Distance, tx_lookup caught up
// With tip=1_523_000, Distance(500_000) targets block 1_023_000
// Deletes jars 0 and 1, checkpoint is 999_999, target is 1_023_000 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Distance(500_000), Some(1_023_000))
.with_bodies_mode(PruneMode::Distance(500_000))
.with_expected_pruned(2000)
.with_lowest_block(1_499_999),
// Scenario 6: Both use Distance, tx_lookup less aggressive (bigger distance) than
// bodies With tip=1_523_000:
// - tx_lookup: Distance(1_000_000) targets block 523_000, checkpoint at 523_000
// - bodies: Distance(500_000) targets block 1_023_000
// Bodies can prune up to what tx_lookup has finished (523_000), deleting jar 0
// Checkpoint is 499_999, target is 1_023_000 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Distance(1_000_000), Some(523_000))
.with_bodies_mode(PruneMode::Distance(500_000))
.with_expected_pruned(1000) // Jar 0 deleted
.with_lowest_block(999_999), // Jar 0 (0-499_999) deleted
// Scenario 7: tx_lookup more aggressive than bodies (deletes jar 0 and 1)
// tx_lookup: Before(1_100_000) -> prune up to 1_099_999
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Before(1_100_000), Some(1_099_999))
.with_bodies_mode(PruneMode::Before(1_100_000))
.with_expected_pruned(2000)
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
// Scenario 8: tx_lookup has lower target than bodies, but is done
// tx_lookup: Before(600_000) -> prune up to 599_999 (checkpoint at 599_999, DONE)
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
// Since tx_lookup is done (next_pruned_block returns None), bodies can prune freely
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
TestCase::new()
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
.with_bodies_mode(PruneMode::Before(1_100_000))
.with_expected_pruned(2000)
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
// Scenario 9: Perfect alignment - checkpoint equals target
// bodies: Before(1_000_000) -> targets 999_999
// Deletes jars 0 and 1 (0-999_999), checkpoint is 999_999 which equals target ->
// Finished
TestCase::new()
.with_bodies_mode(PruneMode::Before(1_000_000))
.with_expected_pruned(2000)
.with_expected_progress(PruneProgress::Finished)
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
];
for test_case in test_cases {
let factory = create_test_provider_factory();
setup_static_file_jars(&factory, tip);
run_prune_test(&factory, test_case, tip);
}
}
}

View File

@@ -242,8 +242,6 @@ impl StorageHistory {
where
Provider: DBProvider + StaticFileProviderFactory + RocksDBProviderFactory,
{
use reth_provider::PruneShardOutcome;
let mut limiter = input.limiter;
if limiter.is_limit_reached() {
@@ -291,14 +289,17 @@ impl StorageHistory {
sorted_storages.sort_unstable_by_key(|((addr, key), _)| (*addr, *key));
provider.with_rocksdb_batch(|mut batch| {
for ((address, storage_key), highest_block) in &sorted_storages {
let prune_to = (*highest_block).min(last_changeset_pruned_block);
match batch.prune_storage_history_to(*address, *storage_key, prune_to)? {
PruneShardOutcome::Deleted => deleted_shards += 1,
PruneShardOutcome::Updated => updated_shards += 1,
PruneShardOutcome::Unchanged => {}
}
}
let targets: Vec<_> = sorted_storages
.iter()
.map(|((addr, key), highest)| {
((*addr, *key), (*highest).min(last_changeset_pruned_block))
})
.collect();
let outcomes = batch.prune_storage_history_batch(&targets)?;
deleted_shards = outcomes.deleted;
updated_shards = outcomes.updated;
Ok(((), Some(batch.into_inner())))
})?;

View File

@@ -6,11 +6,15 @@ use crate::{
use alloy_eips::eip2718::Encodable2718;
use rayon::prelude::*;
use reth_db_api::{tables, transaction::DbTxMut};
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
use reth_provider::{
BlockReader, DBProvider, PruneCheckpointReader, RocksDBProviderFactory,
StaticFileProviderFactory,
};
use reth_prune_types::{
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageSettingsCache;
use tracing::{debug, instrument, trace};
#[derive(Debug)]
@@ -29,7 +33,9 @@ where
Provider: DBProvider<Tx: DbTxMut>
+ BlockReader<Transaction: Encodable2718>
+ PruneCheckpointReader
+ StaticFileProviderFactory,
+ StaticFileProviderFactory
+ StorageSettingsCache
+ RocksDBProviderFactory,
{
fn segment(&self) -> PruneSegment {
PruneSegment::TransactionLookup
@@ -83,6 +89,12 @@ where
}
.into_inner();
// Check where transaction hash numbers are stored
#[cfg(all(unix, feature = "rocksdb"))]
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
return self.prune_rocksdb(provider, input, start, end);
}
// For PruneMode::Full, clear the entire table in one operation
if self.mode.is_full() {
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
@@ -174,6 +186,106 @@ where
}
}
impl TransactionLookup {
/// Prunes transaction lookup when indices are stored in `RocksDB`.
///
/// Reads transactions from static files and deletes corresponding entries
/// from the `RocksDB` `TransactionHashNumbers` table.
#[cfg(all(unix, feature = "rocksdb"))]
fn prune_rocksdb<Provider>(
&self,
provider: &Provider,
input: PruneInput,
start: alloy_primitives::TxNumber,
end: alloy_primitives::TxNumber,
) -> Result<SegmentOutput, PrunerError>
where
Provider: DBProvider
+ BlockReader<Transaction: Encodable2718>
+ StaticFileProviderFactory
+ RocksDBProviderFactory,
{
// For PruneMode::Full, clear the entire RocksDB table in one operation
if self.mode.is_full() {
let rocksdb = provider.rocksdb_provider();
rocksdb.clear::<tables::TransactionHashNumbers>()?;
trace!(target: "pruner", "Cleared transaction lookup table (RocksDB)");
let last_pruned_block = provider
.block_by_transaction_id(end)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
return Ok(SegmentOutput {
progress: PruneProgress::Finished,
pruned: 0, // RocksDB clear doesn't return count
checkpoint: Some(SegmentOutputCheckpoint {
block_number: Some(last_pruned_block),
tx_number: Some(end),
}),
});
}
let tx_range_end = input
.limiter
.deleted_entries_limit_left()
.map(|left| start.saturating_add(left as u64).saturating_sub(1))
.map_or(end, |limited| limited.min(end));
let tx_range = start..=tx_range_end;
// Retrieve transactions in the range and calculate their hashes in parallel
let hashes: Vec<_> = provider
.transactions_by_tx_range(tx_range.clone())?
.into_par_iter()
.map(|transaction| transaction.trie_hash())
.collect();
// Number of transactions retrieved from the database should match the tx range count
let tx_count = tx_range.count();
if hashes.len() != tx_count {
return Err(PrunerError::InconsistentData(
"Unexpected number of transaction hashes retrieved by transaction number range",
))
}
let mut limiter = input.limiter;
// Delete transaction hash -> number mappings from RocksDB
let mut deleted = 0usize;
provider.with_rocksdb_batch(|mut batch| {
for hash in &hashes {
if limiter.is_limit_reached() {
break;
}
batch.delete::<tables::TransactionHashNumbers>(*hash)?;
limiter.increment_deleted_entries_count();
deleted += 1;
}
Ok(((), Some(batch.into_inner())))
})?;
let done = deleted == hashes.len() && tx_range_end == end;
trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup (RocksDB)");
let last_pruned_transaction = if deleted > 0 { start + deleted as u64 - 1 } else { start };
let last_pruned_block = provider
.block_by_transaction_id(last_pruned_transaction)?
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
.checked_sub(if done { 0 } else { 1 });
let progress = limiter.progress(done);
Ok(SegmentOutput {
progress,
pruned: deleted,
checkpoint: Some(SegmentOutputCheckpoint {
block_number: last_pruned_block,
tx_number: Some(last_pruned_transaction),
}),
})
}
}
#[cfg(test)]
mod tests {
use crate::segments::{PruneInput, PruneLimiter, Segment, SegmentOutput, TransactionLookup};
@@ -319,4 +431,100 @@ mod tests {
test_prune(6, (PruneProgress::Finished, 2));
test_prune(10, (PruneProgress::Finished, 8));
}
#[cfg(all(unix, feature = "rocksdb"))]
#[test]
fn prune_rocksdb() {
use reth_db_api::models::StorageSettings;
use reth_provider::RocksDBProviderFactory;
use reth_storage_api::StorageSettingsCache;
let db = TestStageDB::default();
let mut rng = generators::rng();
let blocks = random_block_range(
&mut rng,
1..=10,
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
);
db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks");
// Collect transaction hashes and their tx numbers
let mut tx_hash_numbers = Vec::new();
for block in &blocks {
tx_hash_numbers.reserve_exact(block.transaction_count());
for transaction in &block.body().transactions {
tx_hash_numbers.push((*transaction.tx_hash(), tx_hash_numbers.len() as u64));
}
}
let tx_hash_numbers_len = tx_hash_numbers.len();
// Insert into RocksDB instead of MDBX
{
let rocksdb = db.factory.rocksdb_provider();
let mut batch = rocksdb.batch();
for (hash, tx_num) in &tx_hash_numbers {
batch.put::<tables::TransactionHashNumbers>(*hash, tx_num).unwrap();
}
batch.commit().expect("commit rocksdb batch");
}
// Verify RocksDB has all entries
{
let rocksdb = db.factory.rocksdb_provider();
for (hash, expected_tx_num) in &tx_hash_numbers {
let actual = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
assert_eq!(actual, Some(*expected_tx_num));
}
}
let to_block: BlockNumber = 6;
let prune_mode = PruneMode::Before(to_block);
let input =
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
let segment = TransactionLookup::new(prune_mode);
// Enable RocksDB storage for transaction hash numbers
db.factory.set_storage_settings_cache(
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
);
let provider = db.factory.database_provider_rw().unwrap();
let result = segment.prune(&provider, input).unwrap();
provider.commit().expect("commit");
assert_matches!(
result,
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
if pruned > 0
);
// Calculate expected: blocks 1-6 should have their tx hashes pruned
let txs_up_to_block_6: usize = blocks.iter().take(6).map(|b| b.transaction_count()).sum();
// Verify RocksDB entries: first `txs_up_to_block_6` should be gone
{
let rocksdb = db.factory.rocksdb_provider();
for (i, (hash, _)) in tx_hash_numbers.iter().enumerate() {
let entry = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
if i < txs_up_to_block_6 {
assert!(entry.is_none(), "Entry {} (hash {:?}) should be pruned", i, hash);
} else {
assert!(entry.is_some(), "Entry {} (hash {:?}) should still exist", i, hash);
}
}
}
// Verify remaining count
{
let rocksdb = db.factory.rocksdb_provider();
let remaining: Vec<_> =
rocksdb.iter::<tables::TransactionHashNumbers>().unwrap().collect();
assert_eq!(
remaining.len(),
tx_hash_numbers_len - txs_up_to_block_6,
"Remaining RocksDB entries should match expected"
);
}
}
}

View File

@@ -84,6 +84,38 @@ impl PruneMode {
pub const fn is_distance(&self) -> bool {
matches!(self, Self::Distance(_))
}
/// Returns the next block number that will EVENTUALLY be pruned after the given checkpoint. It
/// should not be used to find if there are blocks to be pruned right now. For that, use
/// [`Self::prune_target_block`].
///
/// This is independent of the current tip and indicates what block is next in the pruning
/// sequence according to this mode's configuration. Returns `None` if no more blocks will
/// be pruned (i.e., the mode has reached its target).
///
/// # Examples
///
/// - `Before(10)` with checkpoint at block 5 returns `Some(6)`
/// - `Before(10)` with checkpoint at block 9 returns `None` (done)
/// - `Distance(100)` with checkpoint at block 1000 returns `Some(1001)` (always has more)
/// - `Full` always returns the next block after checkpoint
pub const fn next_pruned_block(&self, checkpoint: Option<BlockNumber>) -> Option<BlockNumber> {
let next = match checkpoint {
Some(c) => c + 1,
None => 0,
};
match self {
Self::Before(n) => {
if next < *n {
Some(next)
} else {
None
}
}
Self::Distance(_) | Self::Full => Some(next),
}
}
}
#[cfg(test)]

View File

@@ -93,7 +93,7 @@ impl SegmentOutput {
Self { progress: PruneProgress::Finished, pruned: 0, checkpoint: None }
}
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and `checkpoint = None`.
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and the given checkpoint.
/// Use when pruning is needed but cannot be done.
pub const fn not_done(
reason: PruneInterruptReason,
@@ -142,6 +142,8 @@ pub enum PruneInterruptReason {
Timeout,
/// Limit on the number of deleted entries (rows in the database) per prune run was reached.
DeletedEntriesLimitReached,
/// Waiting for another segment to finish pruning before this segment can proceed.
WaitingOnSegment(PruneSegment),
/// Unknown reason for stopping prune run.
Unknown,
}

View File

@@ -394,6 +394,17 @@ pub trait EthApi<
address: Address,
block: BlockId,
) -> RpcResult<alloy_rpc_types_eth::AccountInfo>;
/// Returns the EIP-7928 block access list for a block by hash.
#[method(name = "getBlockAccessListByBlockHash")]
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>>;
/// Returns the EIP-7928 block access list for a block by number.
#[method(name = "getBlockAccessListByBlockNumber")]
async fn block_access_list_by_block_number(
&self,
number: BlockNumberOrTag,
) -> RpcResult<Option<Bytes>>;
}
#[async_trait::async_trait]
@@ -881,4 +892,19 @@ where
trace!(target: "rpc::eth", "Serving eth_getAccountInfo");
Ok(EthState::get_account_info(self, address, block).await?)
}
/// Handler for: `eth_getBlockAccessListByBlockHash`
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>> {
trace!(target: "rpc::eth", ?hash, "Serving eth_getBlockAccessListByBlockHash");
Err(internal_rpc_err("unimplemented"))
}
/// Handler for: `eth_getBlockAccessListByBlockNumber`
async fn block_access_list_by_block_number(
&self,
number: BlockNumberOrTag,
) -> RpcResult<Option<Bytes>> {
trace!(target: "rpc::eth", ?number, "Serving eth_getBlockAccessListByBlockNumber");
Err(internal_rpc_err("unimplemented"))
}
}

View File

@@ -540,7 +540,11 @@ where
max_simulate_blocks,
eth_proof_window,
blocking_task_pool.unwrap_or_else(|| {
BlockingTaskPool::build().expect("failed to build blocking task pool")
BlockingTaskPool::builder()
.thread_name(|i| format!("blocking-{i}"))
.build()
.map(BlockingTaskPool::new)
.expect("failed to build blocking task pool")
}),
fee_history_cache,
task_spawner,

View File

@@ -353,6 +353,9 @@ where
return Err(EthApiError::HeaderNotFound(start.into()).into());
}
let end = to_block.unwrap_or(latest_block);
if end > latest_block {
return Err(EthApiError::HeaderNotFound(end.into()).into());
}
if start > end {
return Err(EthApiError::InvalidParams(

View File

@@ -161,37 +161,34 @@ where
provider.count_entries::<tables::TransactionHashNumbers>()?.is_zero();
// Auto-commits on threshold; consistency check heals any crash.
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch_with_auto_commit();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
let mut writer =
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
provider.with_rocksdb_batch_auto_commit(|rocksdb_batch| {
let mut writer =
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
for (index, hash_to_number) in hash_collector.iter()?.enumerate() {
let (hash_bytes, number_bytes) = hash_to_number?;
if index > 0 && index.is_multiple_of(interval) {
info!(
target: "sync::stages::transaction_lookup",
?append_only,
progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0),
"Inserting hashes"
);
let iter = hash_collector
.iter()
.map_err(|e| ProviderError::other(Box::new(e)))?;
for (index, hash_to_number) in iter.enumerate() {
let (hash_bytes, number_bytes) =
hash_to_number.map_err(|e| ProviderError::other(Box::new(e)))?;
if index > 0 && index.is_multiple_of(interval) {
info!(
target: "sync::stages::transaction_lookup",
?append_only,
progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0),
"Inserting hashes"
);
}
// Decode from raw ETL bytes
let hash = TxHash::decode(&hash_bytes)?;
let tx_num = TxNumber::decompress(&number_bytes)?;
writer.put_transaction_hash_number(hash, tx_num, append_only)?;
}
// Decode from raw ETL bytes
let hash = TxHash::decode(&hash_bytes)?;
let tx_num = TxNumber::decompress(&number_bytes)?;
writer.put_transaction_hash_number(hash, tx_num, append_only)?;
}
// Extract and register RocksDB batch for commit at provider level
#[cfg(all(unix, feature = "rocksdb"))]
if let Some(batch) = writer.into_raw_rocksdb_batch() {
provider.set_pending_rocksdb_batch(batch);
}
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
trace!(target: "sync::stages::transaction_lookup",
total_hashes,
@@ -223,36 +220,31 @@ where
) -> Result<UnwindOutput, StageError> {
let (range, unwind_to, _) = input.unwind_block_range_with_threshold(self.chunk_size);
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb = provider.rocksdb_provider();
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_batch = rocksdb.batch();
#[cfg(not(all(unix, feature = "rocksdb")))]
let rocksdb_batch = ();
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
provider.with_rocksdb_batch(|rocksdb_batch| {
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
let static_file_provider = provider.static_file_provider();
let rev_walker =
provider.block_body_indices_range(range.clone())?.into_iter().rev().zip(range.rev());
let static_file_provider = provider.static_file_provider();
let rev_walker = provider
.block_body_indices_range(range.clone())?
.into_iter()
.rev()
.zip(range.rev());
for (body, number) in rev_walker {
if number <= unwind_to {
break;
}
for (body, number) in rev_walker {
if number <= unwind_to {
break;
}
// Delete all transactions that belong to this block
for tx_id in body.tx_num_range() {
if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? {
// Delete all transactions that belong to this block
for transaction in
static_file_provider.transactions_by_tx_range(body.tx_num_range())?
{
writer.delete_transaction_hash_number(transaction.trie_hash())?;
}
}
}
// Extract and register RocksDB batch for commit at provider level
#[cfg(all(unix, feature = "rocksdb"))]
if let Some(batch) = writer.into_raw_rocksdb_batch() {
provider.set_pending_rocksdb_batch(batch);
}
Ok(((), writer.into_raw_rocksdb_batch()))
})?;
Ok(UnwindOutput {
checkpoint: StageCheckpoint::new(unwind_to)

View File

@@ -30,6 +30,10 @@ where
let mut static_file_writer =
provider.get_static_file_writer(*block_range.start(), StaticFileSegment::Receipts)?;
let mut receipts_cursor = provider
.tx_ref()
.cursor_read::<tables::Receipts<<Provider::Primitives as NodePrimitives>::Receipt>>()?;
for block in block_range {
static_file_writer.increment_block(block)?;
@@ -37,10 +41,6 @@ where
.block_body_indices(block)?
.ok_or(ProviderError::BlockBodyIndicesNotFound(block))?;
let mut receipts_cursor = provider
.tx_ref()
.cursor_read::<tables::Receipts<<Provider::Primitives as NodePrimitives>::Receipt>>(
)?;
let receipts_walker = receipts_cursor.walk_range(block_body_indices.tx_num_range())?;
static_file_writer.append_receipts(

View File

@@ -173,11 +173,10 @@ where
/// Returns highest block numbers for all static file segments.
pub fn copy_to_static_files(&self) -> ProviderResult<HighestStaticFiles> {
let provider = self.provider.database_provider_ro()?;
let stages_checkpoints = std::iter::once(StageId::Execution)
.map(|stage| provider.get_stage_checkpoint(stage).map(|c| c.map(|c| c.block_number)))
.collect::<Result<Vec<_>, _>>()?;
let execution_checkpoint =
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number);
let highest_static_files = HighestStaticFiles { receipts: stages_checkpoints[0] };
let highest_static_files = HighestStaticFiles { receipts: execution_checkpoint };
let targets = self.get_static_file_targets(highest_static_files)?;
self.run(targets)?;

View File

@@ -81,6 +81,20 @@ impl StaticFileSegment {
}
}
/// Returns a short string representation of the segment.
///
/// Linux threads have a length limit of 15 characters: <https://man7.org/linux/man-pages/man3/pthread_setname_np.3.html#DESCRIPTION>.
pub const fn as_short_str(&self) -> &'static str {
match self {
Self::Headers => "headers",
Self::Transactions => "transactions",
Self::Receipts => "receipts",
Self::TransactionSenders => "tx-senders",
Self::AccountChangeSets => "account-changes",
Self::StorageChangeSets => "storage-changes",
}
}
/// Returns an iterator over all segments.
pub fn iter() -> impl Iterator<Item = Self> {
// The order of segments is significant and must be maintained to ensure correctness.
@@ -861,4 +875,15 @@ mod tests {
assert_eq!(ser, format!("\"{expected_str}\""));
}
}
#[test]
fn test_static_file_segment_as_short_str() {
for segment in StaticFileSegment::iter() {
assert!(
segment.as_short_str().len() <= 15,
"{segment} segment name is too long: {}",
segment.as_short_str()
);
}
}
}

View File

@@ -643,9 +643,11 @@ mod tests {
use std::str::FromStr;
use tempfile::TempDir;
/// Create database for testing
fn create_test_db(kind: DatabaseEnvKind) -> DatabaseEnv {
create_test_db_with_path(kind, &tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep())
/// Create database for testing. Returns the `TempDir` to prevent cleanup until test ends.
fn create_test_db(kind: DatabaseEnvKind) -> (TempDir, DatabaseEnv) {
let tempdir = tempfile::TempDir::new().expect(ERROR_TEMPDIR);
let env = create_test_db_with_path(kind, tempdir.path());
(tempdir, env)
}
/// Create database for testing with specified path
@@ -670,13 +672,13 @@ mod tests {
#[test]
fn db_creation() {
create_test_db(DatabaseEnvKind::RW);
let _tempdir = create_test_db(DatabaseEnvKind::RW);
}
#[test]
fn db_drop_orphan_table() {
let path = tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep();
let db = create_test_db_with_path(DatabaseEnvKind::RW, &path);
let tempdir = tempfile::TempDir::new().expect(ERROR_TEMPDIR);
let db = create_test_db_with_path(DatabaseEnvKind::RW, tempdir.path());
// Create an orphan table by manually creating it
let orphan_table_name = "OrphanTestTable";
@@ -715,7 +717,7 @@ mod tests {
#[test]
fn db_manual_put_get() {
let env = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
let value = Header::default();
let key = 1u64;
@@ -734,7 +736,7 @@ mod tests {
#[test]
fn db_dup_cursor_delete_first() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
@@ -774,7 +776,7 @@ mod tests {
#[test]
fn db_cursor_walk() {
let env = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
let value = Header::default();
let key = 1u64;
@@ -799,7 +801,7 @@ mod tests {
#[test]
fn db_cursor_walk_range() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -863,7 +865,7 @@ mod tests {
#[test]
fn db_cursor_walk_range_on_dup_table() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let address0 = Address::ZERO;
let address1 = Address::with_last_byte(1);
@@ -923,7 +925,7 @@ mod tests {
#[expect(clippy::reversed_empty_ranges)]
#[test]
fn db_cursor_walk_range_invalid() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -951,7 +953,7 @@ mod tests {
#[test]
fn db_walker() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -981,7 +983,7 @@ mod tests {
#[test]
fn db_reverse_walker() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1011,7 +1013,7 @@ mod tests {
#[test]
fn db_walk_back() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT (0, 0), (1, 0), (3, 0)
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1050,7 +1052,7 @@ mod tests {
#[test]
fn db_cursor_seek_exact_or_previous_key() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1074,7 +1076,7 @@ mod tests {
#[test]
fn db_cursor_insert() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1114,7 +1116,7 @@ mod tests {
#[test]
fn db_cursor_insert_dup() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
@@ -1132,7 +1134,7 @@ mod tests {
#[test]
fn db_cursor_delete_current_non_existent() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let key1 = Address::with_last_byte(1);
@@ -1162,7 +1164,7 @@ mod tests {
#[test]
fn db_cursor_insert_wherever_cursor_is() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
// PUT
@@ -1195,7 +1197,7 @@ mod tests {
#[test]
fn db_cursor_append() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1222,7 +1224,7 @@ mod tests {
#[test]
fn db_cursor_append_failure() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
// PUT
let tx = db.tx_mut().expect(ERROR_INIT_TX);
@@ -1257,7 +1259,7 @@ mod tests {
#[test]
fn db_cursor_upsert() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let tx = db.tx_mut().expect(ERROR_INIT_TX);
let mut cursor = tx.cursor_write::<PlainAccountState>().unwrap();
@@ -1292,7 +1294,7 @@ mod tests {
#[test]
fn db_cursor_dupsort_append() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let transition_id = 2;
@@ -1356,7 +1358,8 @@ mod tests {
#[test]
fn db_closure_put_get() {
let path = TempDir::new().expect(ERROR_TEMPDIR).keep();
let tempdir = TempDir::new().expect(ERROR_TEMPDIR);
let path = tempdir.path();
let value = Account {
nonce: 18446744073709551615,
@@ -1367,7 +1370,7 @@ mod tests {
.expect(ERROR_ETH_ADDRESS);
{
let env = create_test_db_with_path(DatabaseEnvKind::RW, &path);
let env = create_test_db_with_path(DatabaseEnvKind::RW, path);
// PUT
let result = env.update(|tx| {
@@ -1378,7 +1381,7 @@ mod tests {
}
let env = DatabaseEnv::open(
&path,
path,
DatabaseEnvKind::RO,
DatabaseArguments::new(ClientVersion::default()),
)
@@ -1393,7 +1396,7 @@ mod tests {
#[test]
fn db_dup_sort() {
let env = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047")
.expect(ERROR_ETH_ADDRESS);
@@ -1437,7 +1440,7 @@ mod tests {
#[test]
fn db_walk_dup_with_not_existing_key() {
let env = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047")
.expect(ERROR_ETH_ADDRESS);
@@ -1465,7 +1468,7 @@ mod tests {
#[test]
fn db_iterate_over_all_dup_values() {
let env = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
let key1 = Address::from_str("0x1111111111111111111111111111111111111111")
.expect(ERROR_ETH_ADDRESS);
let key2 = Address::from_str("0x2222222222222222222222222222222222222222")
@@ -1511,7 +1514,7 @@ mod tests {
#[test]
fn dup_value_with_same_subkey() {
let env = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
let key1 = Address::new([0x11; 20]);
let key2 = Address::new([0x22; 20]);
@@ -1554,7 +1557,7 @@ mod tests {
#[test]
fn db_sharded_key() {
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
let real_key = address!("0xa2c122be93b0074270ebee7f6b7292c7deb45047");
let shards = 5;

View File

@@ -133,7 +133,7 @@ pub enum Error {
#[error("permission denied to setup database")]
Permission,
/// Unknown error code.
#[error("unknown error code: {0}")]
#[error("{}", Error::fmt_other(*.0))]
Other(i32),
}
@@ -215,6 +215,13 @@ impl Error {
Self::Other(err_code) => *err_code,
}
}
fn fmt_other(code: i32) -> String {
let mut s = String::with_capacity(1024);
let desc = unsafe { ffi::mdbx_strerror_r(code, s.as_mut_ptr().cast(), 1024) };
let desc = unsafe { std::ffi::CStr::from_ptr(desc) }.to_string_lossy();
desc.into_owned()
}
}
impl From<Error> for i32 {

View File

@@ -97,7 +97,7 @@ impl TxnManager {
}
}
};
std::thread::Builder::new().name("mdbx-rs-txn-manager".to_string()).spawn(task).unwrap();
std::thread::Builder::new().name("mdbx-rs-txn-mgr".to_string()).spawn(task).unwrap();
}
pub(crate) fn send_message(&self, message: TxnManagerMessage) {

View File

@@ -16,7 +16,7 @@ workspace = true
reth-chainspec.workspace = true
reth-execution-types.workspace = true
reth-ethereum-primitives = { workspace = true, features = ["reth-codec"] }
reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k1"] }
reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k1", "dashmap"] }
reth-errors.workspace = true
reth-storage-errors.workspace = true
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
@@ -31,6 +31,7 @@ reth-codecs.workspace = true
reth-chain-state.workspace = true
reth-node-types.workspace = true
reth-static-file-types.workspace = true
reth-tasks.workspace = true
# ethereum
alloy-eips.workspace = true
alloy-primitives.workspace = true
@@ -50,7 +51,6 @@ metrics.workspace = true
itertools.workspace = true
notify = { workspace = true, default-features = false, features = ["macos_fsevent"] }
parking_lot.workspace = true
dashmap = { workspace = true, features = ["inline"] }
strum.workspace = true
eyre.workspace = true

View File

@@ -21,7 +21,7 @@ pub mod providers;
pub use providers::{
DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, HistoricalStateProvider,
HistoricalStateProviderRef, LatestStateProvider, LatestStateProviderRef, ProviderFactory,
PruneShardOutcome, SaveBlocksMode, StaticFileAccess, StaticFileProviderBuilder,
PruneShardOutcome, PrunedIndices, SaveBlocksMode, StaticFileAccess, StaticFileProviderBuilder,
StaticFileWriteCtx, StaticFileWriter,
};

View File

@@ -745,9 +745,10 @@ mod tests {
fn provider_factory_with_database_path() {
let chain_spec = ChainSpecBuilder::mainnet().build();
let (_static_dir, static_dir_path) = create_test_static_files_dir();
let (_, rocksdb_path) = create_test_rocksdb_dir();
let (_rocksdb_dir, rocksdb_path) = create_test_rocksdb_dir();
let _db_tempdir = tempfile::TempDir::new().expect(ERROR_TEMPDIR);
let factory = ProviderFactory::<MockNodeTypesWithDB<DatabaseEnv>>::new_with_database_path(
tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(),
_db_tempdir.path(),
Arc::new(chain_spec),
DatabaseArguments::new(Default::default()),
StaticFileProvider::read_write(static_dir_path).unwrap(),
@@ -785,8 +786,9 @@ mod tests {
transaction_lookup: Some(PruneMode::Full),
..PruneModes::default()
};
let factory = create_test_provider_factory();
let provider = factory.with_prune_modes(prune_modes).provider_rw().unwrap();
// Keep factory alive until provider is dropped to prevent TempDatabase cleanup
let factory = create_test_provider_factory().with_prune_modes(prune_modes);
let provider = factory.provider_rw().unwrap();
assert_matches!(provider.insert_block(&block.clone().try_recover().unwrap()), Ok(_));
assert_matches!(provider.transaction_sender(0), Ok(None));
assert_matches!(

View File

@@ -64,6 +64,7 @@ use reth_storage_api::{
StorageSettingsCache, TryIntoHistoricalStateProvider, WriteStateInput,
};
use reth_storage_errors::provider::{ProviderResult, StaticFileWriterError};
use reth_tasks::spawn_scoped_os_thread;
use reth_trie::{
updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted},
HashedPostStateSorted, StoredNibbles,
@@ -552,7 +553,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
thread::scope(|s| {
// SF writes
let sf_handle = s.spawn(|| {
let sf_handle = spawn_scoped_os_thread(s, "static-files", || {
let start = Instant::now();
sf_provider.write_blocks_data(&blocks, &tx_nums, sf_ctx)?;
Ok::<_, ProviderError>(start.elapsed())
@@ -561,7 +562,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
// RocksDB writes
#[cfg(all(unix, feature = "rocksdb"))]
let rocksdb_handle = rocksdb_ctx.storage_settings.any_in_rocksdb().then(|| {
s.spawn(|| {
spawn_scoped_os_thread(s, "rocksdb", || {
let start = Instant::now();
rocksdb_provider.write_blocks_data(&blocks, &tx_nums, rocksdb_ctx)?;
Ok::<_, ProviderError>(start.elapsed())

View File

@@ -39,8 +39,8 @@ pub use consistent::ConsistentProvider;
pub(crate) mod rocksdb;
pub use rocksdb::{
PruneShardOutcome, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider, RocksDBRawIter,
RocksDBStats, RocksDBTableStats, RocksTx,
PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider,
RocksDBRawIter, RocksDBStats, RocksDBTableStats, RocksTx,
};
/// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy

View File

@@ -6,6 +6,6 @@ mod provider;
pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx};
pub use provider::{
PruneShardOutcome, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider, RocksDBRawIter,
RocksDBStats, RocksDBTableStats, RocksTx,
PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider,
RocksDBRawIter, RocksDBStats, RocksDBTableStats, RocksTx,
};

File diff suppressed because it is too large Load Diff

View File

@@ -219,3 +219,14 @@ pub enum PruneShardOutcome {
/// Shard was unchanged (no blocks <= `to_block`).
Unchanged,
}
/// Tracks pruning outcomes for batch operations (stub).
#[derive(Debug, Default, Clone, Copy)]
pub struct PrunedIndices {
/// Number of shards completely deleted.
pub deleted: usize,
/// Number of shards that were updated (filtered but still have entries).
pub updated: usize,
/// Number of shards that were unchanged.
pub unchanged: usize,
}

View File

@@ -1,10 +1,10 @@
use alloy_primitives::{BlockNumber, B256};
use metrics::{Counter, Histogram};
use parking_lot::RwLock;
use reth_chain_state::LazyOverlay;
use reth_db_api::DatabaseError;
use reth_errors::{ProviderError, ProviderResult};
use reth_metrics::Metrics;
use reth_primitives_traits::dashmap::{self, DashMap};
use reth_prune_types::PruneSegment;
use reth_stages_types::StageId;
use reth_storage_api::{
@@ -22,7 +22,6 @@ use reth_trie_db::{
ChangesetCache, DatabaseHashedCursorFactory, DatabaseHashedPostState, DatabaseTrieCursorFactory,
};
use std::{
collections::{hash_map::Entry, HashMap},
sync::Arc,
time::{Duration, Instant},
};
@@ -102,7 +101,7 @@ pub struct OverlayStateProviderFactory<F> {
metrics: OverlayStateProviderMetrics,
/// A cache which maps `db_tip -> Overlay`. If the db tip changes during usage of the factory
/// then a new entry will get added to this, but in most cases only one entry is present.
overlay_cache: Arc<RwLock<HashMap<BlockNumber, Overlay>>>,
overlay_cache: Arc<DashMap<BlockNumber, Overlay>>,
}
impl<F> OverlayStateProviderFactory<F> {
@@ -419,17 +418,16 @@ where
let db_tip_block = self.get_db_tip_block_number(provider)?;
// If the overlay is present in the cache then return it directly.
if let Some(overlay) = self.overlay_cache.as_ref().read().get(&db_tip_block) {
return Ok(overlay.clone());
if let Some(entry) = self.overlay_cache.get(&db_tip_block) {
return Ok(entry.value().clone());
}
// If the overlay is not present then we need to calculate a new one. We grab a write lock,
// and then check the cache again in case some other thread populated the cache since we
// checked with the read-lock. If still not present we calculate and populate.
// If the overlay is not present then we need to calculate a new one.
// DashMap's entry API handles the race condition internally.
let mut cache_miss = false;
let overlay = match self.overlay_cache.as_ref().write().entry(db_tip_block) {
Entry::Occupied(entry) => entry.get().clone(),
Entry::Vacant(entry) => {
let overlay = match self.overlay_cache.entry(db_tip_block) {
dashmap::Entry::Occupied(entry) => entry.get().clone(),
dashmap::Entry::Vacant(entry) => {
cache_miss = true;
let overlay = self.calculate_overlay(provider, db_tip_block)?;
entry.insert(overlay.clone());

View File

@@ -11,7 +11,6 @@ use crate::{
use alloy_consensus::{transaction::TransactionMeta, Header};
use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber};
use alloy_primitives::{b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256};
use dashmap::DashMap;
use notify::{RecommendedWatcher, RecursiveMode, Watcher};
use parking_lot::RwLock;
use reth_chain_state::ExecutedBlock;
@@ -34,8 +33,8 @@ use reth_ethereum_primitives::{Receipt, TransactionSigned};
use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION};
use reth_node_types::NodePrimitives;
use reth_primitives_traits::{
AlloyBlockHeader as _, BlockBody as _, RecoveredBlock, SealedHeader, SignedTransaction,
StorageEntry,
dashmap::DashMap, AlloyBlockHeader as _, BlockBody as _, RecoveredBlock, SealedHeader,
SignedTransaction, StorageEntry,
};
use reth_prune_types::PruneSegment;
use reth_stages_types::PipelineTarget;
@@ -48,6 +47,7 @@ use reth_storage_api::{
StorageChangeSetReader, StorageSettingsCache,
};
use reth_storage_errors::provider::{ProviderError, ProviderResult, StaticFileWriterError};
use reth_tasks::spawn_scoped_os_thread;
use std::{
collections::BTreeMap,
fmt::Debug,
@@ -670,7 +670,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
where
F: FnOnce(&mut StaticFileProviderRWRefMut<'_, N>) -> ProviderResult<()> + Send + 'env,
{
scope.spawn(move || {
spawn_scoped_os_thread(scope, segment.as_short_str(), move || {
let mut w = self.get_writer(first_block_number, segment)?;
f(&mut w)?;
w.sync_all()

View File

@@ -17,7 +17,8 @@ use reth_storage_errors::provider::{ProviderError, ProviderResult};
use std::{ops::Deref, sync::Arc};
/// Alias type for each specific `NippyJar`.
type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
type LoadedJarRef<'a> =
reth_primitives_traits::dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
/// Helper type to reuse an associated static file mmap handle on created cursors.
#[derive(Debug)]

View File

@@ -204,8 +204,9 @@ impl<N> std::ops::Deref for StaticFileProviderRWRefMut<'_, N> {
/// Extends `StaticFileProvider` with writing capabilities
pub struct StaticFileProviderRW<N> {
/// Reference back to the provider. We need [Weak] here because [`StaticFileProviderRW`] is
/// stored in a [`dashmap::DashMap`] inside the parent [`StaticFileProvider`].which is an
/// [Arc]. If we were to use an [Arc] here, we would create a reference cycle.
/// stored in a [`reth_primitives_traits::dashmap::DashMap`] inside the parent
/// [`StaticFileProvider`].which is an [Arc]. If we were to use an [Arc] here, we would
/// create a reference cycle.
reader: Weak<StaticFileProviderInner<N>>,
/// A [`NippyJarWriter`] instance.
writer: NippyJarWriter<SegmentHeader>,

View File

@@ -4,12 +4,7 @@ use crate::{
};
use alloy_primitives::B256;
use reth_chainspec::{ChainSpec, MAINNET};
use reth_db::{
test_utils::{
create_test_rocksdb_dir, create_test_rw_db, create_test_static_files_dir, TempDatabase,
},
DatabaseEnv,
};
use reth_db::{test_utils::TempDatabase, DatabaseEnv};
use reth_errors::ProviderResult;
use reth_ethereum_engine_primitives::EthEngineTypes;
use reth_node_types::NodeTypesWithDBAdapter;
@@ -55,14 +50,23 @@ pub fn create_test_provider_factory_with_chain_spec(
pub fn create_test_provider_factory_with_node_types<N: NodeTypesForProvider>(
chain_spec: Arc<N::ChainSpec>,
) -> ProviderFactory<NodeTypesWithDBAdapter<N, Arc<TempDatabase<DatabaseEnv>>>> {
let (static_dir, _) = create_test_static_files_dir();
let (rocksdb_dir, _) = create_test_rocksdb_dir();
let db = create_test_rw_db();
let rocksdb_path = rocksdb_dir.keep();
// Create a single temp directory that contains all data dirs (db, static_files, rocksdb).
// TempDatabase will clean up the entire directory on drop.
let datadir_path = reth_db::test_utils::tempdir_path();
let static_files_path = datadir_path.join("static_files");
let rocksdb_path = datadir_path.join("rocksdb");
// Create static_files directory
std::fs::create_dir_all(&static_files_path).expect("failed to create static_files dir");
// Create database with the datadir path so TempDatabase cleans up everything on drop
let db = reth_db::test_utils::create_test_rw_db_with_datadir(&datadir_path);
ProviderFactory::new(
db,
chain_spec,
StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"),
StaticFileProvider::read_write(static_files_path).expect("static file provider"),
RocksDBBuilder::new(&rocksdb_path)
.with_default_tables()
.build()

View File

@@ -40,7 +40,7 @@ tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
# other
tracing.workspace = true
parking_lot.workspace = true
dashmap = { workspace = true, features = ["inline"] }
# revm
revm.workspace = true

View File

@@ -27,13 +27,11 @@
use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader};
use alloy_eips::{BlockHashOrNumber, BlockNumberOrTag};
use alloy_network::{primitives::HeaderResponse, BlockResponse};
use alloy_primitives::{
map::HashMap, Address, BlockHash, BlockNumber, StorageKey, TxHash, TxNumber, B256, U256,
};
use alloy_primitives::{Address, BlockHash, BlockNumber, StorageKey, TxHash, TxNumber, B256, U256};
use alloy_provider::{ext::DebugApi, network::Network, Provider};
use alloy_rpc_types::{AccountInfo, BlockId};
use alloy_rpc_types_engine::ForkchoiceState;
use parking_lot::RwLock;
use dashmap::DashMap;
use reth_chainspec::{ChainInfo, ChainSpecProvider};
use reth_db_api::{
mock::{DatabaseMock, TxMock},
@@ -912,7 +910,7 @@ where
/// Cached bytecode for accounts
///
/// Since the state provider is short-lived, we don't worry about memory leaks.
code_store: RwLock<HashMap<B256, Bytecode>>,
code_store: DashMap<B256, Bytecode>,
/// Whether to use Reth-specific RPC methods for better performance
reth_rpc_support: bool,
}
@@ -942,7 +940,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
network: std::marker::PhantomData,
chain_spec: None,
compute_state_root: false,
code_store: RwLock::new(HashMap::default()),
code_store: Default::default(),
reth_rpc_support: true,
}
}
@@ -960,7 +958,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
network: std::marker::PhantomData,
chain_spec: Some(chain_spec),
compute_state_root: false,
code_store: RwLock::new(HashMap::default()),
code_store: Default::default(),
reth_rpc_support: true,
}
}
@@ -982,7 +980,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
network: self.network,
chain_spec: self.chain_spec.clone(),
compute_state_root: self.compute_state_root,
code_store: RwLock::new(HashMap::default()),
code_store: Default::default(),
reth_rpc_support: self.reth_rpc_support,
}
}
@@ -1038,9 +1036,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
let code_hash = account_info.code_hash();
if code_hash != KECCAK_EMPTY {
// Insert code into the cache
self.code_store
.write()
.insert(code_hash, Bytecode::new_raw(account_info.code.clone()));
self.code_store.insert(code_hash, Bytecode::new_raw(account_info.code.clone()));
}
Ok(account_info)
@@ -1119,7 +1115,7 @@ where
{
fn bytecode_by_hash(&self, code_hash: &B256) -> Result<Option<Bytecode>, ProviderError> {
if !self.reth_rpc_support {
return Ok(self.code_store.read().get(code_hash).cloned());
return Ok(self.code_store.get(code_hash).map(|entry| entry.value().clone()));
}
self.block_on_async(async {

View File

@@ -30,6 +30,7 @@ use std::{
Arc, OnceLock,
},
task::{ready, Context, Poll},
thread,
};
use tokio::{
runtime::Handle,
@@ -48,6 +49,51 @@ pub mod pool;
/// Global [`TaskExecutor`] instance that can be accessed from anywhere.
static GLOBAL_EXECUTOR: OnceLock<TaskExecutor> = OnceLock::new();
/// Spawns an OS thread with the current tokio runtime context propagated.
///
/// This function captures the current tokio runtime handle (if available) and enters it
/// in the newly spawned thread. This ensures that code running in the spawned thread can
/// use [`Handle::current()`], [`Handle::spawn_blocking()`], and other tokio utilities that
/// require a runtime context.
#[track_caller]
pub fn spawn_os_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
where
F: FnOnce() -> T + Send + 'static,
T: Send + 'static,
{
let handle = Handle::try_current().ok();
thread::Builder::new()
.name(name.to_string())
.spawn(move || {
let _guard = handle.as_ref().map(Handle::enter);
f()
})
.unwrap_or_else(|e| panic!("failed to spawn thread {name:?}: {e}"))
}
/// Spawns a scoped OS thread with the current tokio runtime context propagated.
///
/// This is the scoped thread version of [`spawn_os_thread`], for use with [`std::thread::scope`].
#[track_caller]
pub fn spawn_scoped_os_thread<'scope, 'env, F, T>(
scope: &'scope thread::Scope<'scope, 'env>,
name: &str,
f: F,
) -> thread::ScopedJoinHandle<'scope, T>
where
F: FnOnce() -> T + Send + 'scope,
T: Send + 'scope,
{
let handle = Handle::try_current().ok();
thread::Builder::new()
.name(name.to_string())
.spawn_scoped(scope, move || {
let _guard = handle.as_ref().map(Handle::enter);
f()
})
.unwrap_or_else(|e| panic!("failed to spawn scoped thread {name:?}: {e}"))
}
/// A type that can spawn tasks.
///
/// The main purpose of this type is to abstract over [`TaskExecutor`] so it's more convenient to

View File

@@ -610,6 +610,14 @@ where
self.pool.pending_transactions()
}
fn get_pending_transaction_by_sender_and_nonce(
&self,
sender: Address,
nonce: u64,
) -> Option<Arc<ValidPoolTransaction<Self::Transaction>>> {
self.pool.get_pending_transaction_by_sender_and_nonce(sender, nonce)
}
fn pending_transactions_max(
&self,
max: usize,

View File

@@ -1090,6 +1090,16 @@ where
self.get_pool_data().get_transactions_by_sender(sender_id)
}
/// Returns a pending transaction sent by the given sender with the given nonce.
pub fn get_pending_transaction_by_sender_and_nonce(
&self,
sender: Address,
nonce: u64,
) -> Option<Arc<ValidPoolTransaction<T::Transaction>>> {
let sender_id = self.get_sender_id(sender);
self.get_pool_data().get_pending_transaction_by_sender_and_nonce(sender_id, nonce)
}
/// Returns all queued transactions of the address by sender
pub fn get_queued_transactions_by_sender(
&self,

View File

@@ -560,6 +560,18 @@ impl<T: TransactionOrdering> TxPool<T> {
self.all_transactions.txs_iter(sender).map(|(_, tx)| Arc::clone(&tx.transaction)).collect()
}
/// Returns a pending transaction sent by the given sender with the given nonce.
pub(crate) fn get_pending_transaction_by_sender_and_nonce(
&self,
sender: SenderId,
nonce: u64,
) -> Option<Arc<ValidPoolTransaction<T::Transaction>>> {
self.all_transactions
.txs_iter(sender)
.find(|(id, tx)| id.nonce == nonce && tx.subpool == SubPool::Pending)
.map(|(_, tx)| Arc::clone(&tx.transaction))
}
/// Updates only the pending fees without triggering subpool updates.
/// Returns the previous base fee and blob fee values.
const fn update_pending_fees_only(

View File

@@ -405,6 +405,16 @@ pub trait TransactionPool: Clone + Debug + Send + Sync {
/// Consumer: RPC
fn pending_transactions(&self) -> Vec<Arc<ValidPoolTransaction<Self::Transaction>>>;
/// Returns a pending transaction if it exists and is ready for immediate execution
/// (i.e., has the lowest nonce among the sender's pending transactions).
fn get_pending_transaction_by_sender_and_nonce(
&self,
sender: Address,
nonce: u64,
) -> Option<Arc<ValidPoolTransaction<Self::Transaction>>> {
self.best_transactions().find(|tx| tx.sender() == sender && tx.nonce() == nonce)
}
/// Returns first `max` transactions that can be included in the next block.
/// See <https://github.com/paradigmxyz/reth/issues/12767#issuecomment-2493223579>
///

View File

@@ -85,4 +85,6 @@ pub mod serde_bincode_compat {
}
/// Re-export
pub use alloy_trie::{nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, EMPTY_ROOT_HASH};
pub use alloy_trie::{
nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, TrieMaskIter, EMPTY_ROOT_HASH,
};

View File

@@ -13,7 +13,7 @@ workspace = true
[dependencies]
# reth
reth-primitives-traits.workspace = true
reth-primitives-traits = { workspace = true, features = ["dashmap"] }
reth-execution-errors.workspace = true
reth-provider.workspace = true
reth-storage-errors.workspace = true
@@ -29,7 +29,6 @@ alloy-primitives.workspace = true
tracing.workspace = true
# misc
dashmap.workspace = true
thiserror.workspace = true
derive_more.workspace = true
rayon.workspace = true

View File

@@ -42,8 +42,8 @@ use alloy_primitives::{
};
use alloy_rlp::{BufMut, Encodable};
use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
use dashmap::DashMap;
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, StateProofError};
use reth_primitives_traits::dashmap::{self, DashMap};
use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult};
use reth_storage_errors::db::DatabaseError;
use reth_trie::{
@@ -155,10 +155,10 @@ impl ProofWorkerHandle {
// Initialize availability counters at zero. Each worker will increment when it
// successfully initializes, ensuring only healthy workers are counted.
let storage_available_workers = Arc::new(AtomicUsize::new(0));
let account_available_workers = Arc::new(AtomicUsize::new(0));
let storage_available_workers = Arc::<AtomicUsize>::default();
let account_available_workers = Arc::<AtomicUsize>::default();
let cached_storage_roots = Arc::new(DashMap::new());
let cached_storage_roots = Arc::<DashMap<_, _>>::default();
debug!(
target: "trie::proof_task",
@@ -168,101 +168,112 @@ impl ProofWorkerHandle {
"Spawning proof worker pools"
);
let parent_span =
debug_span!(target: "trie::proof_task", "storage proof workers", ?storage_worker_count)
.entered();
// Spawn storage workers
for worker_id in 0..storage_worker_count {
let span = debug_span!(target: "trie::proof_task", "storage worker", ?worker_id);
let task_ctx_clone = task_ctx.clone();
let work_rx_clone = storage_work_rx.clone();
let storage_available_workers_clone = storage_available_workers.clone();
let cached_storage_roots = cached_storage_roots.clone();
executor.spawn_blocking(move || {
#[cfg(feature = "metrics")]
let metrics = ProofTaskTrieMetrics::default();
#[cfg(feature = "metrics")]
let cursor_metrics = ProofTaskCursorMetrics::new();
let _guard = span.enter();
let worker = StorageProofWorker::new(
task_ctx_clone,
work_rx_clone,
worker_id,
storage_available_workers_clone,
cached_storage_roots,
#[cfg(feature = "metrics")]
metrics,
#[cfg(feature = "metrics")]
cursor_metrics,
)
.with_v2_proofs(v2_proofs_enabled);
if let Err(error) = worker.run() {
error!(
target: "trie::proof_task",
worker_id,
?error,
"Storage worker failed"
);
}
});
}
drop(parent_span);
let parent_span =
debug_span!(target: "trie::proof_task", "account proof workers", ?account_worker_count)
.entered();
// Spawn account workers
for worker_id in 0..account_worker_count {
let span = debug_span!(target: "trie::proof_task", "account worker", ?worker_id);
let task_ctx_clone = task_ctx.clone();
let work_rx_clone = account_work_rx.clone();
let storage_work_tx_clone = storage_work_tx.clone();
let account_available_workers_clone = account_available_workers.clone();
let cached_storage_roots = cached_storage_roots.clone();
executor.spawn_blocking(move || {
#[cfg(feature = "metrics")]
let metrics = ProofTaskTrieMetrics::default();
#[cfg(feature = "metrics")]
let cursor_metrics = ProofTaskCursorMetrics::new();
let _guard = span.enter();
let worker = AccountProofWorker::new(
task_ctx_clone,
work_rx_clone,
worker_id,
storage_work_tx_clone,
account_available_workers_clone,
cached_storage_roots,
#[cfg(feature = "metrics")]
metrics,
#[cfg(feature = "metrics")]
cursor_metrics,
)
.with_v2_proofs(v2_proofs_enabled);
if let Err(error) = worker.run() {
error!(
target: "trie::proof_task",
worker_id,
?error,
"Account worker failed"
);
}
});
}
drop(parent_span);
Self {
storage_work_tx,
let this = Self {
storage_work_tx: storage_work_tx.clone(),
account_work_tx,
storage_available_workers,
account_available_workers,
storage_available_workers: storage_available_workers.clone(),
account_available_workers: account_available_workers.clone(),
storage_worker_count,
account_worker_count,
v2_proofs_enabled,
}
};
// Clone for the first spawn_blocking (storage workers)
let task_ctx_for_storage = task_ctx.clone();
let executor_for_storage = executor.clone();
let cached_storage_roots_for_storage = cached_storage_roots.clone();
executor_for_storage.clone().spawn_blocking(move || {
let parent_span =
debug_span!(target: "trie::proof_task", "storage proof workers", ?storage_worker_count)
.entered();
// Spawn storage workers
for worker_id in 0..storage_worker_count {
let span = debug_span!(target: "trie::proof_task", "storage worker", ?worker_id);
let task_ctx_clone = task_ctx_for_storage.clone();
let work_rx_clone = storage_work_rx.clone();
let storage_available_workers_clone = storage_available_workers.clone();
let cached_storage_roots = cached_storage_roots_for_storage.clone();
executor_for_storage.spawn_blocking(move || {
#[cfg(feature = "metrics")]
let metrics = ProofTaskTrieMetrics::default();
#[cfg(feature = "metrics")]
let cursor_metrics = ProofTaskCursorMetrics::new();
let _guard = span.enter();
let worker = StorageProofWorker::new(
task_ctx_clone,
work_rx_clone,
worker_id,
storage_available_workers_clone,
cached_storage_roots,
#[cfg(feature = "metrics")]
metrics,
#[cfg(feature = "metrics")]
cursor_metrics,
)
.with_v2_proofs(v2_proofs_enabled);
if let Err(error) = worker.run() {
error!(
target: "trie::proof_task",
worker_id,
?error,
"Storage worker failed"
);
}
});
}
drop(parent_span);
});
executor.clone().spawn_blocking(move || {
let parent_span =
debug_span!(target: "trie::proof_task", "account proof workers", ?account_worker_count)
.entered();
// Spawn account workers
for worker_id in 0..account_worker_count {
let span = debug_span!(target: "trie::proof_task", "account worker", ?worker_id);
let task_ctx_clone = task_ctx.clone();
let work_rx_clone = account_work_rx.clone();
let storage_work_tx_clone = storage_work_tx.clone();
let account_available_workers_clone = account_available_workers.clone();
let cached_storage_roots = cached_storage_roots.clone();
executor.spawn_blocking(move || {
#[cfg(feature = "metrics")]
let metrics = ProofTaskTrieMetrics::default();
#[cfg(feature = "metrics")]
let cursor_metrics = ProofTaskCursorMetrics::new();
let _guard = span.enter();
let worker = AccountProofWorker::new(
task_ctx_clone,
work_rx_clone,
worker_id,
storage_work_tx_clone,
account_available_workers_clone,
cached_storage_roots,
#[cfg(feature = "metrics")]
metrics,
#[cfg(feature = "metrics")]
cursor_metrics,
)
.with_v2_proofs(v2_proofs_enabled);
if let Err(error) = worker.run() {
error!(
target: "trie::proof_task",
worker_id,
?error,
"Account worker failed"
);
}
});
}
drop(parent_span);
});
this
}
/// Returns whether V2 storage proofs are enabled for this worker pool.

View File

@@ -4,7 +4,7 @@ use crate::{stats::ParallelTrieTracker, storage_root_targets::StorageRootTargets
use alloy_primitives::B256;
use alloy_rlp::{BufMut, Encodable};
use itertools::Itertools;
use reth_execution_errors::{StateProofError, StorageRootError};
use reth_execution_errors::{SparseTrieError, StateProofError, StorageRootError};
use reth_provider::{DatabaseProviderROFactory, ProviderError};
use reth_storage_errors::db::DatabaseError;
use reth_trie::{
@@ -232,6 +232,9 @@ pub enum ParallelStateRootError {
/// Provider error.
#[error(transparent)]
Provider(#[from] ProviderError),
/// Sparse trie error.
#[error(transparent)]
SparseTrie(#[from] SparseTrieError),
/// Other unspecified error.
#[error("{_0}")]
Other(String),
@@ -244,6 +247,7 @@ impl From<ParallelStateRootError> for ProviderError {
ParallelStateRootError::StorageRoot(StorageRootError::Database(error)) => {
Self::Database(error)
}
ParallelStateRootError::SparseTrie(error) => Self::other(error),
ParallelStateRootError::Other(other) => Self::Database(DatabaseError::Other(other)),
}
}

View File

@@ -18,6 +18,17 @@ impl MultiProofTargetsV2 {
pub fn is_empty(&self) -> bool {
self.account_targets.is_empty() && self.storage_targets.is_empty()
}
/// Returns the number of items that will be considered during chunking.
pub fn chunking_length(&self) -> usize {
self.account_targets.len() +
self.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
}
/// Returns an iterator that yields chunks of the specified size.
pub fn chunks(self, chunk_size: usize) -> impl Iterator<Item = Self> {
ChunkedMultiProofTargetsV2::new(self, chunk_size)
}
}
/// An iterator that yields chunks of V2 proof targets of at most `size` account and storage

View File

@@ -3,9 +3,8 @@ use alloy_primitives::{map::B256Map, B256};
use alloy_rlp::Encodable;
use core::cell::RefCell;
use crossbeam_channel::Receiver as CrossbeamReceiver;
use dashmap::DashMap;
use reth_execution_errors::trie::StateProofError;
use reth_primitives_traits::Account;
use reth_primitives_traits::{dashmap::DashMap, Account};
use reth_storage_errors::db::DatabaseError;
use reth_trie::{
hashed_cursor::HashedStorageCursor,

View File

@@ -128,4 +128,16 @@ impl LowerSparseSubtrie {
Self::Blind(None) => {}
}
}
/// Returns a heuristic for the in-memory size of this subtrie in bytes.
pub(crate) fn memory_size(&self) -> usize {
match self {
Self::Revealed(subtrie) | Self::Blind(Some(subtrie)) => subtrie.memory_size(),
Self::Blind(None) => 0,
}
}
pub(crate) fn wipe(&mut self) {
*self = Self::default();
}
}

View File

@@ -10,7 +10,7 @@ use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, SparseTrieResu
use reth_trie_common::{
prefix_set::{PrefixSet, PrefixSetMut},
BranchNodeMasks, BranchNodeMasksMap, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, Nibbles,
ProofTrieNode, RlpNode, TrieNode, CHILD_INDEX_RANGE,
ProofTrieNode, RlpNode, TrieNode,
};
use reth_trie_sparse::{
provider::{RevealedNode, TrieNodeProvider},
@@ -106,7 +106,7 @@ pub struct ParallelSparseTrie {
/// This contains the trie nodes for the upper part of the trie.
upper_subtrie: Box<SparseSubtrie>,
/// An array containing the subtries at the second level of the trie.
lower_subtries: [LowerSparseSubtrie; NUM_LOWER_SUBTRIES],
lower_subtries: Box<[LowerSparseSubtrie; NUM_LOWER_SUBTRIES]>,
/// Set of prefixes (key paths) that have been marked as updated.
/// This is used to track which parts of the trie need to be recalculated.
prefix_set: PrefixSetMut,
@@ -138,7 +138,9 @@ impl Default for ParallelSparseTrie {
nodes: HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]),
..Default::default()
}),
lower_subtries: [const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES],
lower_subtries: Box::new(
[const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES],
),
prefix_set: PrefixSetMut::default(),
updates: None,
branch_node_masks: BranchNodeMasksMap::default(),
@@ -184,11 +186,12 @@ impl SparseTrie for ParallelSparseTrie {
|ProofTrieNode { path: path_a, .. }, ProofTrieNode { path: path_b, .. }| {
let subtrie_type_a = SparseSubtrieType::from_path(path_a);
let subtrie_type_b = SparseSubtrieType::from_path(path_b);
subtrie_type_a.cmp(&subtrie_type_b).then(path_a.cmp(path_b))
subtrie_type_a.cmp(&subtrie_type_b).then_with(|| path_a.cmp(path_b))
},
);
// Update the top-level branch node masks. This is simple and can't be done in parallel.
self.branch_node_masks.reserve(nodes.len());
for ProofTrieNode { path, masks, .. } in &nodes {
if let Some(branch_masks) = masks {
self.branch_node_masks.insert(*path, *branch_masks);
@@ -202,9 +205,7 @@ impl SparseTrie for ParallelSparseTrie {
.iter()
.position(|n| !SparseSubtrieType::path_len_is_upper(n.path.len()))
.unwrap_or(nodes.len());
let upper_nodes = &nodes[..num_upper_nodes];
let lower_nodes = &nodes[num_upper_nodes..];
let (upper_nodes, lower_nodes) = nodes.split_at(num_upper_nodes);
// Reserve the capacity of the upper subtrie's `nodes` HashMap before iterating, so we don't
// end up making many small capacity changes as we loop.
@@ -913,6 +914,19 @@ impl SparseTrie for ParallelSparseTrie {
fn take_updates(&mut self) -> SparseTrieUpdates {
match self.updates.take() {
Some(updates) => {
// Sync branch_node_masks with what's being committed to DB.
// This ensures that on subsequent root() calls, the masks reflect the actual
// DB state, which is needed for correct removal detection.
for (path, node) in &updates.updated_nodes {
self.branch_node_masks.insert(
*path,
BranchNodeMasks { tree_mask: node.tree_mask, hash_mask: node.hash_mask },
);
}
for path in &updates.removed_nodes {
self.branch_node_masks.remove(path);
}
// NOTE: we need to preserve Some case
self.updates = Some(SparseTrieUpdates::with_capacity(
updates.updated_nodes.len(),
@@ -926,7 +940,9 @@ impl SparseTrie for ParallelSparseTrie {
fn wipe(&mut self) {
self.upper_subtrie.wipe();
self.lower_subtries = [const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES];
for trie in &mut *self.lower_subtries {
trie.wipe();
}
self.prefix_set = PrefixSetMut::all();
self.updates = self.updates.is_some().then(SparseTrieUpdates::wiped);
self.subtrie_heat.clear();
@@ -935,7 +951,7 @@ impl SparseTrie for ParallelSparseTrie {
fn clear(&mut self) {
self.upper_subtrie.clear();
self.upper_subtrie.nodes.insert(Nibbles::default(), SparseNode::Empty);
for subtrie in &mut self.lower_subtries {
for subtrie in &mut *self.lower_subtries {
subtrie.clear();
}
self.prefix_set.clear();
@@ -1020,7 +1036,7 @@ impl SparseTrie for ParallelSparseTrie {
self.upper_subtrie.shrink_nodes_to(size_per_subtrie);
// Shrink lower subtries (works for both revealed and blind with allocation)
for subtrie in &mut self.lower_subtries {
for subtrie in &mut *self.lower_subtries {
subtrie.shrink_nodes_to(size_per_subtrie);
}
@@ -1039,7 +1055,7 @@ impl SparseTrie for ParallelSparseTrie {
self.upper_subtrie.shrink_values_to(size_per_subtrie);
// Shrink lower subtries (works for both revealed and blind with allocation)
for subtrie in &mut self.lower_subtries {
for subtrie in &mut *self.lower_subtries {
subtrie.shrink_values_to(size_per_subtrie);
}
}
@@ -1093,7 +1109,7 @@ impl SparseTrieExt for ParallelSparseTrie {
SparseNode::Extension { key, .. } => {
let mut child = path;
child.extend(key);
SmallVec::from_buf_and_len([child; 16], 1)
SmallVec::from_slice(&[child])
}
SparseNode::Branch { state_mask, .. } => {
let mut children = SmallVec::new();
@@ -1167,7 +1183,7 @@ impl SparseTrieExt for ParallelSparseTrie {
// Upper prune roots that are prefixes of lower subtrie root paths cause the entire
// subtrie to be cleared (preserving allocations for reuse).
if !roots_upper.is_empty() {
for subtrie in &mut self.lower_subtries {
for subtrie in &mut *self.lower_subtries {
let should_clear = subtrie.as_revealed_ref().is_some_and(|s| {
let search_idx = roots_upper.partition_point(|(root, _)| root <= &s.path);
search_idx > 0 && s.path.starts_with(&roots_upper[search_idx - 1].0)
@@ -1441,8 +1457,6 @@ impl ParallelSparseTrie {
///
/// Returns `None` if a lower subtrie does not exist for the given path.
fn subtrie_for_path(&self, path: &Nibbles) -> Option<&SparseSubtrie> {
// We can't just call `lower_subtrie_for_path` and return `upper_subtrie` if it returns
// None, because Rust complains about double mutable borrowing `self`.
if SparseSubtrieType::path_len_is_upper(path.len()) {
Some(&self.upper_subtrie)
} else {
@@ -2073,15 +2087,13 @@ impl ParallelSparseTrie {
// in the lower subtrie, and reveal accordingly.
if !SparseSubtrieType::path_len_is_upper(path.len() + 1) {
let mut stack_ptr = branch.as_ref().first_child_index();
for idx in CHILD_INDEX_RANGE {
if branch.state_mask.is_bit_set(idx) {
let mut child_path = path;
child_path.push_unchecked(idx);
self.lower_subtrie_for_path_mut(&child_path)
.expect("child_path must have a lower subtrie")
.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
stack_ptr += 1;
}
for idx in branch.state_mask.iter() {
let mut child_path = path;
child_path.push_unchecked(idx);
self.lower_subtrie_for_path_mut(&child_path)
.expect("child_path must have a lower subtrie")
.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
stack_ptr += 1;
}
}
}
@@ -2118,6 +2130,50 @@ impl ParallelSparseTrie {
self.subtrie_heat.mark_modified(index);
}
}
/// Returns a heuristic for the in-memory size of this trie in bytes.
///
/// This is an approximation that accounts for:
/// - The upper subtrie nodes and values
/// - All revealed lower subtries nodes and values
/// - The prefix set keys
/// - The branch node masks map
/// - Updates if retained
/// - Update action buffers
///
/// Note: Heap allocations for hash maps may be larger due to load factor overhead.
pub fn memory_size(&self) -> usize {
let mut size = core::mem::size_of::<Self>();
// Upper subtrie
size += self.upper_subtrie.memory_size();
// Lower subtries (both Revealed and Blind with allocation)
for subtrie in self.lower_subtries.iter() {
size += subtrie.memory_size();
}
// Prefix set keys
size += self.prefix_set.len() * core::mem::size_of::<Nibbles>();
// Branch node masks map
size += self.branch_node_masks.len() *
(core::mem::size_of::<Nibbles>() + core::mem::size_of::<BranchNodeMasks>());
// Updates if present
if let Some(updates) = &self.updates {
size += updates.updated_nodes.len() *
(core::mem::size_of::<Nibbles>() + core::mem::size_of::<BranchNodeCompact>());
size += updates.removed_nodes.len() * core::mem::size_of::<Nibbles>();
}
// Update actions buffers
for buf in &self.update_actions_buffers {
size += buf.capacity() * core::mem::size_of::<SparseTrieUpdatesAction>();
}
size
}
}
/// Bitset tracking which of the 256 lower subtries were modified in the current cycle.
@@ -2571,19 +2627,17 @@ impl SparseSubtrie {
self.nodes.insert(path, SparseNode::Empty);
}
TrieNode::Branch(branch) => {
// For a branch node, iterate over all potential children
// For a branch node, iterate over all children
let mut stack_ptr = branch.as_ref().first_child_index();
for idx in CHILD_INDEX_RANGE {
if branch.state_mask.is_bit_set(idx) {
let mut child_path = path;
child_path.push_unchecked(idx);
if Self::is_child_same_level(&path, &child_path) {
// Reveal each child node or hash it has, but only if the child is on
// the same level as the parent.
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
}
stack_ptr += 1;
for idx in branch.state_mask.iter() {
let mut child_path = path;
child_path.push_unchecked(idx);
if Self::is_child_same_level(&path, &child_path) {
// Reveal each child node or hash it has, but only if the child is on
// the same level as the parent.
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
}
stack_ptr += 1;
}
// Update the branch node entry in the nodes map, handling cases where a blinded
// node is now replaced with a revealed node.
@@ -2812,6 +2866,30 @@ impl SparseSubtrie {
pub(crate) fn shrink_values_to(&mut self, size: usize) {
self.inner.values.shrink_to(size);
}
/// Returns a heuristic for the in-memory size of this subtrie in bytes.
pub(crate) fn memory_size(&self) -> usize {
let mut size = core::mem::size_of::<Self>();
// Nodes map: key (Nibbles) + value (SparseNode)
for (path, node) in &self.nodes {
size += core::mem::size_of::<Nibbles>();
size += path.len(); // Nibbles heap allocation
size += node.memory_size();
}
// Values map: key (Nibbles) + value (Vec<u8>)
for (path, value) in &self.inner.values {
size += core::mem::size_of::<Nibbles>();
size += path.len(); // Nibbles heap allocation
size += core::mem::size_of::<Vec<u8>>() + value.capacity();
}
// Buffers
size += self.inner.buffers.memory_size();
size
}
}
/// Helper type for [`SparseSubtrie`] to mutably access only a subset of fields from the original
@@ -2989,12 +3067,10 @@ impl SparseSubtrieInner {
self.buffers.branch_child_buf.clear();
// Walk children in a reverse order from `f` to `0`, so we pop the `0` first
// from the stack and keep walking in the sorted order.
for bit in CHILD_INDEX_RANGE.rev() {
if state_mask.is_bit_set(bit) {
let mut child = path;
child.push_unchecked(bit);
self.buffers.branch_child_buf.push(child);
}
for bit in state_mask.iter().rev() {
let mut child = path;
child.push_unchecked(bit);
self.buffers.branch_child_buf.push(child);
}
self.buffers
@@ -3285,6 +3361,19 @@ impl SparseSubtrieBuffers {
self.branch_value_stack_buf.clear();
self.rlp_buf.clear();
}
/// Returns a heuristic for the in-memory size of these buffers in bytes.
const fn memory_size(&self) -> usize {
let mut size = core::mem::size_of::<Self>();
size += self.path_stack.capacity() * core::mem::size_of::<RlpNodePathStackItem>();
size += self.rlp_node_stack.capacity() * core::mem::size_of::<RlpNodeStackItem>();
size += self.branch_child_buf.capacity() * core::mem::size_of::<Nibbles>();
size += self.branch_value_stack_buf.capacity() * core::mem::size_of::<RlpNode>();
size += self.rlp_buf.capacity();
size
}
}
/// RLP node path stack item.
@@ -3318,7 +3407,10 @@ struct ChangedSubtrie {
/// If the path is shorter than [`UPPER_TRIE_MAX_DEPTH`] nibbles.
fn path_subtrie_index_unchecked(path: &Nibbles) -> usize {
debug_assert_eq!(UPPER_TRIE_MAX_DEPTH, 2);
path.get_byte_unchecked(0) as usize
let idx = path.get_byte_unchecked(0) as usize;
// SAFETY: always true.
unsafe { core::hint::assert_unchecked(idx < NUM_LOWER_SUBTRIES) };
idx
}
/// Checks if `path` is a strict descendant of any root in a sorted slice.
@@ -8895,4 +8987,41 @@ mod tests {
b256!("f000000000000000000000000000000000000000000000000000000000000000");
assert_eq!(ParallelSparseTrie::nibbles_to_padded_b256(&single), expected_single);
}
#[test]
fn test_memory_size() {
// Test that memory_size returns a reasonable value for an empty trie
let trie = ParallelSparseTrie::default();
let empty_size = trie.memory_size();
// Should at least be the size of the struct itself
assert!(empty_size >= core::mem::size_of::<ParallelSparseTrie>());
// Create a trie with some data
let mut trie = ParallelSparseTrie::default();
let nodes = vec![
ProofTrieNode {
path: Nibbles::from_nibbles_unchecked([0x1, 0x2]),
node: TrieNode::Leaf(LeafNode {
key: Nibbles::from_nibbles_unchecked([0x3, 0x4]),
value: vec![1, 2, 3],
}),
masks: None,
},
ProofTrieNode {
path: Nibbles::from_nibbles_unchecked([0x5, 0x6]),
node: TrieNode::Leaf(LeafNode {
key: Nibbles::from_nibbles_unchecked([0x7, 0x8]),
value: vec![4, 5, 6],
}),
masks: None,
},
];
trie.reveal_nodes(nodes).unwrap();
let populated_size = trie.memory_size();
// Populated trie should use more memory than an empty one
assert!(populated_size > empty_size);
}
}

View File

@@ -182,11 +182,23 @@ where
self.storage.tries.get_mut(address).and_then(|e| e.as_revealed_mut())
}
/// Returns mutable reference to storage tries.
pub const fn storage_tries_mut(&mut self) -> &mut B256Map<RevealableSparseTrie<S>> {
&mut self.storage.tries
}
/// Takes the storage trie for the provided address.
pub fn take_storage_trie(&mut self, address: &B256) -> Option<RevealableSparseTrie<S>> {
self.storage.tries.remove(address)
}
/// Takes the storage trie for the provided address, creating a blind one if it doesn't exist.
pub fn take_or_create_storage_trie(&mut self, address: &B256) -> RevealableSparseTrie<S> {
self.storage.tries.remove(address).unwrap_or_else(|| {
self.storage.cleared_tries.pop().unwrap_or_else(|| self.storage.default_trie.clone())
})
}
/// Inserts storage trie for the provided address.
pub fn insert_storage_trie(&mut self, address: B256, storage_trie: RevealableSparseTrie<S>) {
self.storage.tries.insert(address, storage_trie);

View File

@@ -21,8 +21,7 @@ use reth_execution_errors::{SparseTrieErrorKind, SparseTrieResult};
use reth_trie_common::{
prefix_set::{PrefixSet, PrefixSetMut},
BranchNodeCompact, BranchNodeMasks, BranchNodeMasksMap, BranchNodeRef, ExtensionNodeRef,
LeafNodeRef, Nibbles, ProofTrieNode, RlpNode, TrieMask, TrieNode, CHILD_INDEX_RANGE,
EMPTY_ROOT_HASH,
LeafNodeRef, Nibbles, ProofTrieNode, RlpNode, TrieMask, TrieNode, EMPTY_ROOT_HASH,
};
use tracing::{debug, instrument, trace};
@@ -422,13 +421,11 @@ impl fmt::Display for SerialSparseTrie {
SparseNode::Branch { state_mask, .. } => {
writeln!(f, "{packed_path} -> {node:?}")?;
for i in CHILD_INDEX_RANGE.rev() {
if state_mask.is_bit_set(i) {
let mut child_path = path;
child_path.push_unchecked(i);
if let Some(child_node) = self.nodes_ref().get(&child_path) {
stack.push((child_path, child_node, depth + 1));
}
for i in state_mask.iter().rev() {
let mut child_path = path;
child_path.push_unchecked(i);
if let Some(child_node) = self.nodes_ref().get(&child_path) {
stack.push((child_path, child_node, depth + 1));
}
}
}
@@ -503,16 +500,14 @@ impl SparseTrieTrait for SerialSparseTrie {
self.nodes.insert(path, SparseNode::Empty);
}
TrieNode::Branch(branch) => {
// For a branch node, iterate over all potential children
// For a branch node, iterate over all children
let mut stack_ptr = branch.as_ref().first_child_index();
for idx in CHILD_INDEX_RANGE {
if branch.state_mask.is_bit_set(idx) {
let mut child_path = path;
child_path.push_unchecked(idx);
// Reveal each child node or hash it has
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
stack_ptr += 1;
}
for idx in branch.state_mask.iter() {
let mut child_path = path;
child_path.push_unchecked(idx);
// Reveal each child node or hash it has
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
stack_ptr += 1;
}
// Update the branch node entry in the nodes map, handling cases where a blinded
// node is now replaced with a revealed node.
@@ -980,6 +975,19 @@ impl SparseTrieTrait for SerialSparseTrie {
fn take_updates(&mut self) -> SparseTrieUpdates {
match self.updates.take() {
Some(updates) => {
// Sync branch_node_masks with what's being committed to DB.
// This ensures that on subsequent root() calls, the masks reflect the actual
// DB state, which is needed for correct removal detection.
for (path, node) in &updates.updated_nodes {
self.branch_node_masks.insert(
*path,
BranchNodeMasks { tree_mask: node.tree_mask, hash_mask: node.hash_mask },
);
}
for path in &updates.removed_nodes {
self.branch_node_masks.remove(path);
}
// NOTE: we need to preserve Some case
self.updates = Some(SparseTrieUpdates::with_capacity(
updates.updated_nodes.len(),
@@ -1487,12 +1495,10 @@ impl SerialSparseTrie {
} else {
unchanged_prefix_set.insert(path);
for bit in CHILD_INDEX_RANGE.rev() {
if state_mask.is_bit_set(bit) {
let mut child_path = path;
child_path.push_unchecked(bit);
paths.push((child_path, level + 1));
}
for bit in state_mask.iter().rev() {
let mut child_path = path;
child_path.push_unchecked(bit);
paths.push((child_path, level + 1));
}
}
}
@@ -1646,12 +1652,10 @@ impl SerialSparseTrie {
buffers.branch_child_buf.clear();
// Walk children in a reverse order from `f` to `0`, so we pop the `0` first
// from the stack and keep walking in the sorted order.
for bit in CHILD_INDEX_RANGE.rev() {
if state_mask.is_bit_set(bit) {
let mut child = path;
child.push_unchecked(bit);
buffers.branch_child_buf.push(child);
}
for bit in state_mask.iter().rev() {
let mut child = path;
child.push_unchecked(bit);
buffers.branch_child_buf.push(child);
}
buffers
@@ -1971,6 +1975,16 @@ impl SparseNode {
}
}
}
/// Returns the memory size of this node in bytes.
pub const fn memory_size(&self) -> usize {
match self {
Self::Empty | Self::Hash(_) | Self::Branch { .. } => core::mem::size_of::<Self>(),
Self::Leaf { key, .. } | Self::Extension { key, .. } => {
core::mem::size_of::<Self>() + key.len()
}
}
}
}
/// A helper struct used to store information about a node that has been removed

View File

@@ -57,20 +57,16 @@ impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> {
/// For small slices, linear scan has better cache locality and lower overhead.
const BINARY_SEARCH_THRESHOLD: usize = 64;
impl<K, V> ForwardInMemoryCursor<'_, K, V>
where
K: Ord + Clone,
V: Clone,
{
impl<K: Ord, V> ForwardInMemoryCursor<'_, K, V> {
/// Returns the first entry from the current cursor position that's greater or equal to the
/// provided key. This method advances the cursor forward.
pub fn seek(&mut self, key: &K) -> Option<(K, V)> {
pub fn seek(&mut self, key: &K) -> Option<&(K, V)> {
self.advance_while(|k| k < key)
}
/// Returns the first entry from the current cursor position that's greater than the provided
/// key. This method advances the cursor forward.
pub fn first_after(&mut self, key: &K) -> Option<(K, V)> {
pub fn first_after(&mut self, key: &K) -> Option<&(K, V)> {
self.advance_while(|k| k <= key)
}
@@ -81,7 +77,7 @@ where
///
/// Returns the first entry for which `predicate` returns `false` or `None`. The cursor will
/// point to the returned entry.
fn advance_while(&mut self, predicate: impl Fn(&K) -> bool) -> Option<(K, V)> {
fn advance_while(&mut self, predicate: impl Fn(&K) -> bool) -> Option<&(K, V)> {
let remaining = self.entries.len().saturating_sub(self.idx);
if remaining >= BINARY_SEARCH_THRESHOLD {
let slice = &self.entries[self.idx..];
@@ -92,7 +88,7 @@ where
self.next();
}
}
self.current().cloned()
self.current()
}
}
@@ -105,16 +101,16 @@ mod tests {
let mut cursor = ForwardInMemoryCursor::new(&[(1, ()), (2, ()), (3, ()), (4, ()), (5, ())]);
assert_eq!(cursor.current(), Some(&(1, ())));
assert_eq!(cursor.seek(&0), Some((1, ())));
assert_eq!(cursor.seek(&0), Some(&(1, ())));
assert_eq!(cursor.current(), Some(&(1, ())));
assert_eq!(cursor.seek(&3), Some((3, ())));
assert_eq!(cursor.seek(&3), Some(&(3, ())));
assert_eq!(cursor.current(), Some(&(3, ())));
assert_eq!(cursor.seek(&3), Some((3, ())));
assert_eq!(cursor.seek(&3), Some(&(3, ())));
assert_eq!(cursor.current(), Some(&(3, ())));
assert_eq!(cursor.seek(&4), Some((4, ())));
assert_eq!(cursor.seek(&4), Some(&(4, ())));
assert_eq!(cursor.current(), Some(&(4, ())));
assert_eq!(cursor.seek(&6), None);
@@ -128,19 +124,19 @@ mod tests {
let mut cursor = ForwardInMemoryCursor::new(&entries);
// Seek to beginning
assert_eq!(cursor.seek(&0), Some((0, ())));
assert_eq!(cursor.seek(&0), Some(&(0, ())));
assert_eq!(cursor.idx, 0);
// Seek to middle (should use binary search)
assert_eq!(cursor.seek(&100), Some((100, ())));
assert_eq!(cursor.seek(&100), Some(&(100, ())));
assert_eq!(cursor.idx, 50);
// Seek to non-existent key (should find next greater)
assert_eq!(cursor.seek(&101), Some((102, ())));
assert_eq!(cursor.seek(&101), Some(&(102, ())));
assert_eq!(cursor.idx, 51);
// Seek to end
assert_eq!(cursor.seek(&398), Some((398, ())));
assert_eq!(cursor.seek(&398), Some(&(398, ())));
assert_eq!(cursor.idx, 199);
// Seek past end
@@ -153,16 +149,16 @@ mod tests {
let mut cursor = ForwardInMemoryCursor::new(&entries);
// first_after should find strictly greater
assert_eq!(cursor.first_after(&0), Some((2, ())));
assert_eq!(cursor.first_after(&0), Some(&(2, ())));
assert_eq!(cursor.idx, 1);
// Reset and test from beginning
cursor.reset();
assert_eq!(cursor.first_after(&99), Some((100, ())));
assert_eq!(cursor.first_after(&99), Some(&(100, ())));
// first_after on exact match
cursor.reset();
assert_eq!(cursor.first_after(&100), Some((102, ())));
assert_eq!(cursor.first_after(&100), Some(&(102, ())));
}
#[test]

View File

@@ -215,8 +215,8 @@ impl<C: TrieCursor> TrieCursor for InMemoryTrieCursor<'_, C> {
self.seeked = true;
let entry = match (mem_entry, &self.cursor_entry) {
(Some((mem_key, entry_inner)), _) if mem_key == key => {
entry_inner.map(|node| (key, node))
(Some((mem_key, entry_inner)), _) if *mem_key == key => {
entry_inner.clone().map(|node| (key, node))
}
(_, Some((db_key, node))) if db_key == &key => Some((key, node.clone())),
_ => None,

View File

@@ -1,4 +1,4 @@
use crate::{BranchNodeCompact, Nibbles, StoredSubNode, CHILD_INDEX_RANGE};
use crate::{BranchNodeCompact, Nibbles, StoredSubNode};
use alloy_primitives::B256;
use alloy_trie::proof::AddedRemovedKeys;
@@ -57,15 +57,12 @@ impl CursorSubNode {
/// Creates a new [`CursorSubNode`] from a key and an optional node.
pub fn new(key: Nibbles, node: Option<BranchNodeCompact>) -> Self {
// Find the first nibble that is set in the state mask of the node.
let position = node.as_ref().filter(|n| n.root_hash.is_none()).map_or(
SubNodePosition::ParentBranch,
|n| {
let mut child_index_range = CHILD_INDEX_RANGE;
SubNodePosition::Child(
child_index_range.find(|i| n.state_mask.is_bit_set(*i)).unwrap(),
)
},
);
let position = node
.as_ref()
.filter(|n| n.root_hash.is_none())
.map_or(SubNodePosition::ParentBranch, |n| {
SubNodePosition::Child(n.state_mask.iter().next().unwrap())
});
Self::new_with_full_key(key, node, position)
}

View File

@@ -1,8 +1,4 @@
// Docker Bake configuration for reth and op-reth images
// Usage:
// docker buildx bake ethereum # Build reth
// docker buildx bake optimism # Build op-reth
// docker buildx bake # Build all
// Docker Bake configuration for reth images
variable "REGISTRY" {
default = "ghcr.io/paradigmxyz"
@@ -35,23 +31,35 @@ variable "VERGEN_GIT_DIRTY" {
// Common settings for all targets
group "default" {
targets = ["ethereum", "optimism"]
targets = [
"ethereum-amd64",
"ethereum-arm64",
"optimism-amd64",
"optimism-arm64"
]
}
group "nightly" {
targets = ["ethereum", "ethereum-profiling", "ethereum-edge-profiling", "optimism", "optimism-profiling"]
targets = [
"ethereum-amd64",
"ethereum-arm64",
"ethereum-profiling",
"ethereum-edge-profiling",
"optimism-amd64",
"optimism-arm64",
"optimism-profiling"
]
}
// Base target with shared configuration
target "_base" {
dockerfile = "Dockerfile.depot"
platforms = ["linux/amd64", "linux/arm64"]
args = {
BUILD_PROFILE = "${BUILD_PROFILE}"
FEATURES = "${FEATURES}"
VERGEN_GIT_SHA = "${VERGEN_GIT_SHA}"
BUILD_PROFILE = "${BUILD_PROFILE}"
FEATURES = "${FEATURES}"
VERGEN_GIT_SHA = "${VERGEN_GIT_SHA}"
VERGEN_GIT_DESCRIBE = "${VERGEN_GIT_DESCRIBE}"
VERGEN_GIT_DIRTY = "${VERGEN_GIT_DIRTY}"
VERGEN_GIT_DIRTY = "${VERGEN_GIT_DIRTY}"
}
secret = [
{
@@ -60,14 +68,42 @@ target "_base" {
}
]
}
// amd64 base with x86-64-v3 optimizations
target "_base_amd64" {
inherits = ["_base"]
platforms = ["linux/amd64"]
args = {
# `x86-64-v3` features match the 2013 Intel Haswell architecture, excluding Intel-specific instructions;
# see: https://en.wikipedia.org/wiki/X86-64
#
# `pclmulqdq` is required for rocksdb: https://github.com/rust-rocksdb/rust-rocksdb/issues/1069
RUSTFLAGS = "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
}
}
// arm64 base
target "_base_arm64" {
inherits = ["_base"]
platforms = ["linux/arm64"]
}
target "_base_profiling" {
inherits = ["_base"]
platforms = ["linux/amd64"]
inherits = ["_base_amd64"]
}
// Ethereum (reth)
target "ethereum" {
inherits = ["_base"]
target "ethereum-amd64" {
inherits = ["_base_amd64"]
args = {
BINARY = "reth"
MANIFEST_PATH = "bin/reth"
}
tags = ["${REGISTRY}/reth:${TAG}"]
}
target "ethereum-arm64" {
inherits = ["_base_arm64"]
args = {
BINARY = "reth"
MANIFEST_PATH = "bin/reth"
@@ -98,8 +134,17 @@ target "ethereum-edge-profiling" {
}
// Optimism (op-reth)
target "optimism" {
inherits = ["_base"]
target "optimism-amd64" {
inherits = ["_base_amd64"]
args = {
BINARY = "op-reth"
MANIFEST_PATH = "crates/optimism/bin"
}
tags = ["${REGISTRY}/op-reth:${TAG}"]
}
target "optimism-arm64" {
inherits = ["_base_arm64"]
args = {
BINARY = "op-reth"
MANIFEST_PATH = "crates/optimism/bin"