mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
47 Commits
disable-pr
...
pr-21786
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
903c58749d | ||
|
|
d1dfce17ab | ||
|
|
abe603b94c | ||
|
|
9258bcdc7b | ||
|
|
7134ae4ec7 | ||
|
|
d9f81bc104 | ||
|
|
89be91de0e | ||
|
|
3af5a4a4e2 | ||
|
|
95f6bbe922 | ||
|
|
abab83facd | ||
|
|
9359e21f94 | ||
|
|
32d5ddfe40 | ||
|
|
d7e740f96c | ||
|
|
87bae74094 | ||
|
|
648f19fb56 | ||
|
|
e6fc5ff54b | ||
|
|
bc729671d9 | ||
|
|
eee27df27c | ||
|
|
6d02565c5e | ||
|
|
e706d76aa9 | ||
|
|
b9b7d092f6 | ||
|
|
d0fb5f31c2 | ||
|
|
9621b78586 | ||
|
|
3722071a7c | ||
|
|
6273530501 | ||
|
|
ce29101277 | ||
|
|
b1b95f9825 | ||
|
|
7f970e136a | ||
|
|
6b7cc00289 | ||
|
|
786140a99d | ||
|
|
ffcb486388 | ||
|
|
59d68f92c4 | ||
|
|
0e0271a612 | ||
|
|
df12fee965 | ||
|
|
11a4f65624 | ||
|
|
a782e1a18a | ||
|
|
2dc76f9abe | ||
|
|
65100971e5 | ||
|
|
8e21afa9cc | ||
|
|
46a9b9ad3d | ||
|
|
3f77af4f98 | ||
|
|
79cabbf89c | ||
|
|
e04afe6e0e | ||
|
|
ee224fe20f | ||
|
|
972f23745e | ||
|
|
49f60822f7 | ||
|
|
47ebc79c85 |
20
.changelog/config.toml
Normal file
20
.changelog/config.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
# Changelogs configuration for reth
|
||||
# https://github.com/wevm/changelogs
|
||||
|
||||
# How to bump packages that depend on changed packages
|
||||
dependent_bump = "patch"
|
||||
|
||||
[changelog]
|
||||
# Generate per-crate changelogs (vs single root changelog)
|
||||
format = "per-crate"
|
||||
|
||||
# Fixed groups: all always share the same version
|
||||
# reth binaries share version
|
||||
[[fixed]]
|
||||
members = ["reth", "op-reth"]
|
||||
|
||||
# Packages to ignore (internal/test-only crates)
|
||||
ignore = [
|
||||
"reth-testing-utils",
|
||||
"reth-bench",
|
||||
]
|
||||
5
.changelog/gentle-moons-cry.md
Normal file
5
.changelog/gentle-moons-cry.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-engine-tree: patch
|
||||
---
|
||||
|
||||
Reordered cache size calculations in `ExecutionCache::new` to group related operations together.
|
||||
6
.changelog/shy-tigers-dry.md
Normal file
6
.changelog/shy-tigers-dry.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth: patch
|
||||
op-reth: patch
|
||||
---
|
||||
|
||||
Added automated changelog generation infrastructure using wevm/changelogs-rs with Claude Code integration. Configured per-crate changelog format with fixed version groups for reth binaries and exclusions for internal test utilities.
|
||||
5
.changelog/vain-lakes-cry.md
Normal file
5
.changelog/vain-lakes-cry.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth: patch
|
||||
---
|
||||
|
||||
Updated Alloy dependencies from 1.5.2 to 1.6.1.
|
||||
21
.github/workflows/changelog.yml
vendored
Normal file
21
.github/workflows/changelog.yml
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
name: Changelog
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
changelog:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
- run: npm install -g @anthropic-ai/claude-code
|
||||
- uses: wevm/changelogs-rs/gen@master
|
||||
with:
|
||||
ai: 'claude -p'
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
6
.github/workflows/docker.yml
vendored
6
.github/workflows/docker.yml
vendored
@@ -100,5 +100,7 @@ jobs:
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
optimism.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
ethereum-amd64.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
ethereum-arm64.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
optimism-amd64.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
optimism-arm64.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
|
||||
2
.github/workflows/unit.yml
vendored
2
.github/workflows/unit.yml
vendored
@@ -90,7 +90,7 @@ jobs:
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo nextest run --release -p ef-tests --features "asm-keccak ef-tests"
|
||||
- run: cargo nextest run --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
|
||||
|
||||
doc:
|
||||
name: doc tests
|
||||
|
||||
181
Cargo.lock
generated
181
Cargo.lock
generated
@@ -121,9 +121,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-consensus"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ed1958f0294ecc05ebe7b3c9a8662a3e221c2523b7f2bcd94c7a651efbd510bf"
|
||||
checksum = "86debde32d8dbb0ab29e7cc75ae1a98688ac7a4c9da54b3a9b14593b9b3c46d3"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"alloy-primitives",
|
||||
@@ -149,9 +149,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-consensus-any"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f752e99497ddc39e22d547d7dfe516af10c979405a034ed90e69b914b7dddeae"
|
||||
checksum = "8d6cb2e7efd385b333f5a77b71baaa2605f7e22f1d583f2879543b54cbce777c"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -164,9 +164,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-contract"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2140796bc79150b1b7375daeab99750f0ff5e27b1f8b0aa81ccde229c7f02a2"
|
||||
checksum = "668859fcdb42eee289de22a9d01758c910955bb6ecda675b97276f99ce2e16b0"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-dyn-abi",
|
||||
@@ -262,9 +262,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-eips"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "813a67f87e56b38554d18b182616ee5006e8e2bf9df96a0df8bf29dff1d52e3f"
|
||||
checksum = "be47bf1b91674a5f394b9ed3c691d764fb58ba43937f1371550ff4bc8e59c295"
|
||||
dependencies = [
|
||||
"alloy-eip2124",
|
||||
"alloy-eip2930",
|
||||
@@ -289,9 +289,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-evm"
|
||||
version = "0.27.0"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1582933a9fc27c0953220eb4f18f6492ff577822e9a8d848890ff59f6b4f5beb"
|
||||
checksum = "d2ccfe6d724ceabd5518350cfb34f17dd3a6c3cc33579eee94d98101d3a511ff"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -311,9 +311,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-genesis"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "05864eef929c4d28895ae4b4d8ac9c6753c4df66e873b9c8fafc8089b59c1502"
|
||||
checksum = "a59f6f520c323111650d319451de1edb1e32760029a468105b9d7b0f7c11bdf2"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"alloy-primitives",
|
||||
@@ -352,9 +352,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-json-rpc"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d2dd146b3de349a6ffaa4e4e319ab3a90371fb159fb0bddeb1c7bbe8b1792eff"
|
||||
checksum = "5a24c81a56d684f525cd1c012619815ad3a1dd13b0238f069356795d84647d3c"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-sol-types",
|
||||
@@ -367,9 +367,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-network"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8c12278ffbb8872dfba3b2f17d8ea5e8503c2df5155d9bc5ee342794bde505c3"
|
||||
checksum = "786c5b3ad530eaf43cda450f973fe7fb1c127b4c8990adf66709dafca25e3f6f"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-consensus-any",
|
||||
@@ -393,9 +393,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-network-primitives"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "833037c04917bc2031541a60e8249e4ab5500e24c637c1c62e95e963a655d66f"
|
||||
checksum = "c1ed40adf21ae4be786ef5eb62db9c692f6a30f86d34452ca3f849d6390ce319"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -406,9 +406,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-op-evm"
|
||||
version = "0.27.0"
|
||||
version = "0.27.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6f19214adae08ea95600c3ede76bcbf0c40b36a263534a8f441a4c732f60e868"
|
||||
checksum = "874bfd6cacd006d05e70560f3af7faa670e31166203f9ba14fae4b169227360b"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -468,9 +468,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-provider"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "eafa840b0afe01c889a3012bb2fde770a544f74eab2e2870303eb0a5fb869c48"
|
||||
checksum = "a3ca4c15818be7ac86208aff3a91b951d14c24e1426e66624e75f2215ba5e2cc"
|
||||
dependencies = [
|
||||
"alloy-chains",
|
||||
"alloy-consensus",
|
||||
@@ -513,9 +513,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-pubsub"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57b3a3b3e4efc9f4d30e3326b6bd6811231d16ef94837e18a802b44ca55119e6"
|
||||
checksum = "e9eb9c9371738ac47f589e40aae6e418cb5f7436ad25b87575a7f94a60ccf43b"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-primitives",
|
||||
@@ -557,9 +557,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-client"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "12768ae6303ec764905a8a7cd472aea9072f9f9c980d18151e26913da8ae0123"
|
||||
checksum = "abe0addad5b8197e851062b49dc47157444bced173b601d91e3f9b561a060a50"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-primitives",
|
||||
@@ -583,9 +583,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "0622d8bcac2f16727590aa33f4c3f05ea98130e7e4b4924bce8be85da5ad0dae"
|
||||
checksum = "74d17d4645a717f0527e491f44f6f7a75c221b9c00ccf79ddba2d26c8e0df4c3"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-engine",
|
||||
@@ -596,9 +596,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-admin"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c38c5ac70457ecc74e87fe1a5a19f936419224ded0eb0636241452412ca92733"
|
||||
checksum = "ded79e60d8fd0d7c851044f8b2f2dd7fa8dfa467c577d620595d4de3c31eff7e"
|
||||
dependencies = [
|
||||
"alloy-genesis",
|
||||
"alloy-primitives",
|
||||
@@ -608,9 +608,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-anvil"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ae8eb0e5d6c48941b61ab76fabab4af66f7d88309a98aa14ad3dec7911c1eba3"
|
||||
checksum = "2593ce5e1fd416e3b094e7671ef361f22e6944545e0e62ee309b6dfbd702225d"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -620,9 +620,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-any"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a1cf5a093e437dfd62df48e480f24e1a3807632358aad6816d7a52875f1c04aa"
|
||||
checksum = "d0e98aabb013a71a4b67b52825f7b503e5bb6057fb3b7b2290d514b0b0574b57"
|
||||
dependencies = [
|
||||
"alloy-consensus-any",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -631,9 +631,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-beacon"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e07949e912479ef3b848e1cf8db54b534bdd7bc58e6c23f28ea9488960990c8c"
|
||||
checksum = "3a647a4e3acf49182135c2333d6f9b11ab8684559ff43ef1958ed762cfe9fe0e"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"alloy-primitives",
|
||||
@@ -651,9 +651,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-debug"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "925ff0f48c2169c050f0ae7a82769bdf3f45723d6742ebb6a5efb4ed2f491b26"
|
||||
checksum = "2a1dd760b6a798ee045ab6a7bbd1a02ad8bd6a64d8e18d6e41732f4fc4a4fe5c"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"derive_more",
|
||||
@@ -663,9 +663,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-engine"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "336ef381c7409f23c69f6e79bddc1917b6e832cff23e7a5cf84b9381d53582e6"
|
||||
checksum = "ddc871ae69688e358cf242a6a7ee6b6e0476a03fd0256434c68daedaec086ec4"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -684,9 +684,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-eth"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "28e97603095020543a019ab133e0e3dc38cd0819f19f19bdd70c642404a54751"
|
||||
checksum = "5899af8417dcf89f40f88fa3bdb2f3f172605d8e167234311ee34811bbfdb0bf"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-consensus-any",
|
||||
@@ -706,9 +706,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-mev"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2805153975e25d38e37ee100880e642d5b24e421ed3014a7d2dae1d9be77562e"
|
||||
checksum = "4d4c9229424e77bd97e629fba44dbfdadebe5bfadbb1e53ad4acbc955610b6c7"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -721,9 +721,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-trace"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f1aec4e1c66505d067933ea1a949a4fb60a19c4cfc2f109aa65873ea99e62ea8"
|
||||
checksum = "410a80e9ac786a2d885adfd7da3568e8f392da106cb5432f00eb4787689d281a"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -735,9 +735,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-txpool"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "25b73c1d6e4f1737a20d246dad5a0abd6c1b76ec4c3d153684ef8c6f1b6bb4f4"
|
||||
checksum = "3a8074654c0292783d504bfa1f2691a69f420154ee9a7883f9212eaf611e60cd"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -747,9 +747,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-serde"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "946a0d413dbb5cd9adba0de5f8a1a34d5b77deda9b69c1d7feed8fc875a1aa26"
|
||||
checksum = "feb73325ee881e42972a5a7bc85250f6af89f92c6ad1222285f74384a203abeb"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"arbitrary",
|
||||
@@ -759,9 +759,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-signer"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2f7481dc8316768f042495eaf305d450c32defbc9bce09d8bf28afcd956895bb"
|
||||
checksum = "1bea4c8f30eddb11d7ab56e83e49c814655daa78ca708df26c300c10d0189cbc"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"async-trait",
|
||||
@@ -774,9 +774,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-signer-local"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1259dac1f534a4c66c1d65237c89915d0010a2a91d6c3b0bada24dc5ee0fb917"
|
||||
checksum = "c28bd71507db58477151a6fe6988fa62a4b778df0f166c3e3e1ef11d059fe5fa"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-network",
|
||||
@@ -863,9 +863,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-transport"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "78f169b85eb9334871db986e7eaf59c58a03d86a30cc68b846573d47ed0656bb"
|
||||
checksum = "b321f506bd67a434aae8e8a7dfe5373bf66137c149a5f09c9e7dfb0ca43d7c91"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"auto_impl",
|
||||
@@ -886,9 +886,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-transport-http"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "019821102e70603e2c141954418255bec539ef64ac4117f8e84fb493769acf73"
|
||||
checksum = "30bf12879a20e1261cd39c3b101856f52d18886907a826e102538897f0d2b66e"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-transport",
|
||||
@@ -901,9 +901,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-transport-ipc"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e574ca2f490fb5961d2cdd78188897392c46615cd88b35c202d34bbc31571a81"
|
||||
checksum = "b75f2334d400249e9672a1ec402536bab259e27a66201a94c3c9b3f1d3bae241"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-pubsub",
|
||||
@@ -921,9 +921,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-transport-ws"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b92dea6996269769f74ae56475570e3586910661e037b7b52d50c9641f76c68f"
|
||||
checksum = "527a0d9c8bbc5c3215b03ad465d4ae8775384ff5faec7c41f033f087c851a9f9"
|
||||
dependencies = [
|
||||
"alloy-pubsub",
|
||||
"alloy-transport",
|
||||
@@ -938,9 +938,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "alloy-trie"
|
||||
version = "0.9.3"
|
||||
version = "0.9.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "428aa0f0e0658ff091f8f667c406e034b431cb10abd39de4f507520968acc499"
|
||||
checksum = "4d7fd448ab0a017de542de1dcca7a58e7019fe0e7a34ed3f9543ebddf6aceffa"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rlp",
|
||||
@@ -950,17 +950,18 @@ dependencies = [
|
||||
"derive_more",
|
||||
"nybbles",
|
||||
"proptest",
|
||||
"proptest-derive 0.5.1",
|
||||
"proptest-derive 0.7.0",
|
||||
"serde",
|
||||
"smallvec",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloy-tx-macros"
|
||||
version = "1.5.2"
|
||||
version = "1.6.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "45ceac797eb8a56bdf5ab1fab353072c17d472eab87645ca847afe720db3246d"
|
||||
checksum = "6a91d6b4c2f6574fdbcb1611e460455c326667cf5b805c6bd1640dad8e8ee4d2"
|
||||
dependencies = [
|
||||
"darling 0.21.3",
|
||||
"proc-macro2",
|
||||
@@ -1019,7 +1020,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1030,7 +1031,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.60.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1986,9 +1987,9 @@ checksum = "1fd0f2584146f6f2ef48085050886acf353beff7305ebd1ae69500e27c67f64b"
|
||||
|
||||
[[package]]
|
||||
name = "bytes"
|
||||
version = "1.11.0"
|
||||
version = "1.11.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3"
|
||||
checksum = "1e748733b7cbc798e1434b6ac524f0c1ff2ab456fe201501e6497c8417a4fc33"
|
||||
dependencies = [
|
||||
"serde",
|
||||
]
|
||||
@@ -2896,12 +2897,14 @@ version = "6.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5041cc499144891f3790297212f32a74fb938e5136a14943f338ef9e0ae276cf"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"cfg-if",
|
||||
"crossbeam-utils",
|
||||
"hashbrown 0.14.5",
|
||||
"lock_api",
|
||||
"once_cell",
|
||||
"parking_lot_core",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3109,7 +3112,7 @@ dependencies = [
|
||||
"libc",
|
||||
"option-ext",
|
||||
"redox_users 0.5.2",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3439,7 +3442,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5180,7 +5183,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6144,7 +6147,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6271,9 +6274,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "nybbles"
|
||||
version = "0.4.7"
|
||||
version = "0.4.8"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7b5676b5c379cf5b03da1df2b3061c4a4e2aa691086a56ac923e08c143f53f59"
|
||||
checksum = "0d49ff0c0d00d4a502b39df9af3a525e1efeb14b9dabb5bb83335284c1309210"
|
||||
dependencies = [
|
||||
"alloy-rlp",
|
||||
"arbitrary",
|
||||
@@ -7078,9 +7081,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proptest-derive"
|
||||
version = "0.5.1"
|
||||
version = "0.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4ee1c9ac207483d5e7db4940700de86a9aae46ef90c48b57f99fe7edb8345e49"
|
||||
checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -7089,9 +7092,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "proptest-derive"
|
||||
version = "0.6.0"
|
||||
version = "0.7.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "095a99f75c69734802359b682be8daaf8980296731f6470434ea2c652af1dd30"
|
||||
checksum = "fb6dc647500e84a25a85b100e76c85b8ace114c209432dc174f20aac11d4ed6c"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
@@ -8233,11 +8236,11 @@ dependencies = [
|
||||
"alloy-chains",
|
||||
"alloy-primitives",
|
||||
"alloy-rlp",
|
||||
"dashmap",
|
||||
"data-encoding",
|
||||
"enr",
|
||||
"hickory-resolver",
|
||||
"linked_hash_set",
|
||||
"parking_lot",
|
||||
"rand 0.9.2",
|
||||
"reth-chainspec",
|
||||
"reth-ethereum-forks",
|
||||
@@ -8467,7 +8470,6 @@ dependencies = [
|
||||
"assert_matches",
|
||||
"codspeed-criterion-compat",
|
||||
"crossbeam-channel",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
"eyre",
|
||||
"fixed-cache",
|
||||
@@ -8881,7 +8883,6 @@ dependencies = [
|
||||
"futures-util",
|
||||
"metrics",
|
||||
"rayon",
|
||||
"reth-ethereum-forks",
|
||||
"reth-ethereum-primitives",
|
||||
"reth-execution-errors",
|
||||
"reth-execution-types",
|
||||
@@ -10192,6 +10193,7 @@ dependencies = [
|
||||
"bincode 1.3.3",
|
||||
"byteorder",
|
||||
"bytes",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
"modular-bitfield",
|
||||
"once_cell",
|
||||
@@ -10222,7 +10224,6 @@ dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-engine",
|
||||
"assert_matches",
|
||||
"dashmap",
|
||||
"eyre",
|
||||
"itertools 0.14.0",
|
||||
"metrics",
|
||||
@@ -10248,6 +10249,7 @@ dependencies = [
|
||||
"reth-static-file-types",
|
||||
"reth-storage-api",
|
||||
"reth-storage-errors",
|
||||
"reth-tasks",
|
||||
"reth-testing-utils",
|
||||
"reth-tracing",
|
||||
"reth-trie",
|
||||
@@ -10999,7 +11001,7 @@ dependencies = [
|
||||
"alloy-provider",
|
||||
"alloy-rpc-types",
|
||||
"alloy-rpc-types-engine",
|
||||
"parking_lot",
|
||||
"dashmap",
|
||||
"reth-chainspec",
|
||||
"reth-db-api",
|
||||
"reth-errors",
|
||||
@@ -11251,7 +11253,6 @@ dependencies = [
|
||||
"alloy-rlp",
|
||||
"codspeed-criterion-compat",
|
||||
"crossbeam-channel",
|
||||
"dashmap",
|
||||
"derive_more",
|
||||
"itertools 0.14.0",
|
||||
"metrics",
|
||||
@@ -11787,7 +11788,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12647,7 +12648,7 @@ dependencies = [
|
||||
"getrandom 0.3.4",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -13891,7 +13892,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.61.2",
|
||||
"windows-sys 0.48.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
64
Cargo.toml
64
Cargo.toml
@@ -490,42 +490,42 @@ alloy-sol-types = { version = "1.5.4", default-features = false }
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.2", default-features = false }
|
||||
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.1", default-features = false }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
|
||||
alloy-consensus = { version = "1.5.2", default-features = false }
|
||||
alloy-contract = { version = "1.5.2", default-features = false }
|
||||
alloy-eips = { version = "1.5.2", default-features = false }
|
||||
alloy-genesis = { version = "1.5.2", default-features = false }
|
||||
alloy-json-rpc = { version = "1.5.2", default-features = false }
|
||||
alloy-network = { version = "1.5.2", default-features = false }
|
||||
alloy-network-primitives = { version = "1.5.2", default-features = false }
|
||||
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-client = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
|
||||
alloy-serde = { version = "1.5.2", default-features = false }
|
||||
alloy-signer = { version = "1.5.2", default-features = false }
|
||||
alloy-signer-local = { version = "1.5.2", default-features = false }
|
||||
alloy-transport = { version = "1.5.2" }
|
||||
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.5.2", default-features = false }
|
||||
alloy-transport-ws = { version = "1.5.2", default-features = false }
|
||||
alloy-consensus = { version = "1.6.1", default-features = false }
|
||||
alloy-contract = { version = "1.6.1", default-features = false }
|
||||
alloy-eips = { version = "1.6.1", default-features = false }
|
||||
alloy-genesis = { version = "1.6.1", default-features = false }
|
||||
alloy-json-rpc = { version = "1.6.1", default-features = false }
|
||||
alloy-network = { version = "1.6.1", default-features = false }
|
||||
alloy-network-primitives = { version = "1.6.1", default-features = false }
|
||||
alloy-provider = { version = "1.6.1", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-client = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types = { version = "1.6.1", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.6.1", default-features = false }
|
||||
alloy-serde = { version = "1.6.1", default-features = false }
|
||||
alloy-signer = { version = "1.6.1", default-features = false }
|
||||
alloy-signer-local = { version = "1.6.1", default-features = false }
|
||||
alloy-transport = { version = "1.6.1" }
|
||||
alloy-transport-http = { version = "1.6.1", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.6.1", default-features = false }
|
||||
alloy-transport-ws = { version = "1.6.1", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-op-evm = { version = "0.27.2", default-features = false }
|
||||
alloy-op-hardforks = "0.4.4"
|
||||
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
|
||||
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
|
||||
@@ -543,7 +543,7 @@ backon = { version = "1.2", default-features = false, features = ["std-blocking-
|
||||
bincode = "1.3"
|
||||
bitflags = "2.4"
|
||||
boyer-moore-magiclen = "0.2.16"
|
||||
bytes = { version = "1.5", default-features = false }
|
||||
bytes = { version = "1.11.1", default-features = false }
|
||||
brotli = "8"
|
||||
cfg-if = "1.0"
|
||||
clap = "4"
|
||||
@@ -562,7 +562,7 @@ linked_hash_set = "0.1"
|
||||
lz4 = "1.28.1"
|
||||
modular-bitfield = "0.13.1"
|
||||
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
|
||||
nybbles = { version = "0.4.2", default-features = false }
|
||||
nybbles = { version = "0.4.8", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
parking_lot = "0.12"
|
||||
paste = "1.0"
|
||||
|
||||
@@ -123,12 +123,14 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
AccessRights::RW => (
|
||||
init_db(db_path, self.db.database_args())?,
|
||||
StaticFileProviderBuilder::read_write(sf_path)
|
||||
.with_metrics()
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => {
|
||||
(open_db_read_only(&db_path, self.db.database_args())?, {
|
||||
let provider = StaticFileProviderBuilder::read_only(sf_path)
|
||||
.with_metrics()
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?;
|
||||
provider.watch_directory();
|
||||
|
||||
@@ -11,6 +11,7 @@ use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_provider::providers::ProviderNodeTypes;
|
||||
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
|
||||
use reth_tasks::spawn_scoped_os_thread;
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
thread,
|
||||
@@ -230,7 +231,7 @@ impl Command {
|
||||
thread::scope(|s| {
|
||||
let handles: Vec<_> = (0..num_threads)
|
||||
.map(|thread_id| {
|
||||
s.spawn(move || {
|
||||
spawn_scoped_os_thread(s, "db-state-worker", move || {
|
||||
loop {
|
||||
// Get next chunk to process
|
||||
let chunk_idx = {
|
||||
|
||||
@@ -4,6 +4,7 @@ use clap::Parser;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_node_builder::common::metrics_hooks;
|
||||
use reth_node_core::{args::MetricArgs, version::version_metadata};
|
||||
use reth_node_metrics::{
|
||||
@@ -11,6 +12,8 @@ use reth_node_metrics::{
|
||||
server::{MetricServer, MetricServerConfig},
|
||||
version::VersionInfo,
|
||||
};
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_prune::PrunerBuilder;
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use std::sync::Arc;
|
||||
@@ -50,7 +53,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
build_profile: version_metadata().build_profile_name.as_ref(),
|
||||
},
|
||||
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
|
||||
ctx.task_executor,
|
||||
ctx.task_executor.clone(),
|
||||
metrics_hooks(&provider_factory),
|
||||
data_dir.pprof_dumps(),
|
||||
);
|
||||
@@ -70,14 +73,29 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
if let Some(prune_tip) = lowest_static_file_height {
|
||||
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
|
||||
|
||||
// Set up cancellation token for graceful shutdown on Ctrl+C
|
||||
let cancellation = CancellationToken::new();
|
||||
let cancellation_clone = cancellation.clone();
|
||||
ctx.task_executor.spawn_critical("prune-ctrl-c", async move {
|
||||
tokio::signal::ctrl_c().await.expect("failed to listen for ctrl-c");
|
||||
cancellation_clone.cancel();
|
||||
});
|
||||
|
||||
// Use batched pruning with a limit to bound memory, running in a loop until complete.
|
||||
const DELETE_LIMIT: usize = 200_000;
|
||||
//
|
||||
// A limit of 20_000_000 results in a max memory usage of ~5G.
|
||||
const DELETE_LIMIT: usize = 20_000_000;
|
||||
let mut pruner = PrunerBuilder::new(config)
|
||||
.delete_limit(DELETE_LIMIT)
|
||||
.build_with_provider_factory(provider_factory);
|
||||
.build_with_provider_factory(provider_factory.clone());
|
||||
|
||||
let mut total_pruned = 0usize;
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
info!(target: "reth::cli", total_pruned, "Pruning interrupted by user");
|
||||
break;
|
||||
}
|
||||
|
||||
let output = pruner.run(prune_tip)?;
|
||||
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
|
||||
total_pruned = total_pruned.saturating_add(batch_pruned);
|
||||
@@ -88,6 +106,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
|
||||
|
||||
if all_segments_finished {
|
||||
info!(target: "reth::cli", total_pruned, "Pruned data from database");
|
||||
break;
|
||||
}
|
||||
|
||||
@@ -105,7 +124,14 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
"Pruning batch complete, continuing..."
|
||||
);
|
||||
}
|
||||
info!(target: "reth::cli", total_pruned, "Pruned data from database");
|
||||
}
|
||||
|
||||
// Flush and compact RocksDB to reclaim disk space after pruning
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
{
|
||||
info!(target: "reth::cli", "Flushing and compacting RocksDB...");
|
||||
provider_factory.rocksdb_provider().flush_and_compact()?;
|
||||
info!(target: "reth::cli", "RocksDB compaction complete");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -83,22 +83,7 @@ impl CliRunner {
|
||||
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
|
||||
}
|
||||
|
||||
// `drop(tokio_runtime)` would block the current thread until its pools
|
||||
// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
|
||||
// it on a separate thread and wait for up to 5 seconds for this operation to
|
||||
// complete.
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-runtime-shutdown".to_string())
|
||||
.spawn(move || {
|
||||
drop(tokio_runtime);
|
||||
let _ = tx.send(());
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
|
||||
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
|
||||
});
|
||||
tokio_shutdown(tokio_runtime, true);
|
||||
|
||||
command_res
|
||||
}
|
||||
@@ -137,19 +122,7 @@ impl CliRunner {
|
||||
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
|
||||
}
|
||||
|
||||
// Shutdown the runtime on a separate thread
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-runtime-shutdown".to_string())
|
||||
.spawn(move || {
|
||||
drop(tokio_runtime);
|
||||
let _ = tx.send(());
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
|
||||
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
|
||||
});
|
||||
tokio_shutdown(tokio_runtime, true);
|
||||
|
||||
command_res
|
||||
}
|
||||
@@ -179,13 +152,7 @@ impl CliRunner {
|
||||
tokio_runtime
|
||||
.block_on(run_until_ctrl_c(async move { fut.await.expect("Failed to join task") }))?;
|
||||
|
||||
// drop the tokio runtime on a separate thread because drop blocks until its pools
|
||||
// (including blocking pool) are shutdown. In other words `drop(tokio_runtime)` would block
|
||||
// the current thread but we want to exit right away.
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-runtime-shutdown".to_string())
|
||||
.spawn(move || drop(tokio_runtime))
|
||||
.unwrap();
|
||||
tokio_shutdown(tokio_runtime, false);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -321,3 +288,27 @@ where
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shut down the given Tokio runtime, and wait for it if `wait` is set.
|
||||
///
|
||||
/// `drop(tokio_runtime)` would block the current thread until its pools
|
||||
/// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
|
||||
/// it on a separate thread and wait for up to 5 seconds for this operation to
|
||||
/// complete.
|
||||
fn tokio_shutdown(rt: tokio::runtime::Runtime, wait: bool) {
|
||||
// Shutdown the runtime on a separate thread
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-shutdown".to_string())
|
||||
.spawn(move || {
|
||||
drop(rt);
|
||||
let _ = tx.send(());
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
if wait {
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
|
||||
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-payload-primitives.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["rayon"] }
|
||||
reth-primitives-traits = { workspace = true, features = ["rayon", "dashmap"] }
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
@@ -62,7 +62,6 @@ metrics.workspace = true
|
||||
reth-metrics = { workspace = true, features = ["common"] }
|
||||
|
||||
# misc
|
||||
dashmap.workspace = true
|
||||
schnellru.workspace = true
|
||||
rayon.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
@@ -11,6 +11,7 @@ use reth_provider::{
|
||||
};
|
||||
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
|
||||
use reth_stages_api::{MetricEvent, MetricEventsSender};
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use std::{
|
||||
sync::{
|
||||
mpsc::{Receiver, SendError, Sender},
|
||||
@@ -264,14 +265,11 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
// spawn the persistence service
|
||||
let db_service =
|
||||
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
|
||||
let join_handle = std::thread::Builder::new()
|
||||
.name("Persistence Service".to_string())
|
||||
.spawn(|| {
|
||||
if let Err(err) = db_service.run() {
|
||||
error!(target: "engine::persistence", ?err, "Persistence service failed");
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
let join_handle = spawn_os_thread("persistence", || {
|
||||
if let Err(err) = db_service.run() {
|
||||
error!(target: "engine::persistence", ?err, "Persistence service failed");
|
||||
}
|
||||
});
|
||||
|
||||
PersistenceHandle {
|
||||
sender: db_service_tx,
|
||||
|
||||
@@ -534,9 +534,9 @@ impl ExecutionCache {
|
||||
|
||||
/// Build an [`ExecutionCache`] struct, so that execution caches can be easily cloned.
|
||||
pub fn new(total_cache_size: usize) -> Self {
|
||||
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
|
||||
let storage_cache_size = (total_cache_size * 8888) / 10000; // 88.88% of total
|
||||
let account_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
|
||||
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
|
||||
|
||||
let code_capacity = Self::bytes_to_entries(code_cache_size, CODE_CACHE_ENTRY_SIZE);
|
||||
let storage_capacity = Self::bytes_to_entries(storage_cache_size, STORAGE_CACHE_ENTRY_SIZE);
|
||||
|
||||
@@ -37,6 +37,7 @@ use reth_provider::{
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages_api::ControlFlow;
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use revm::state::EvmState;
|
||||
use state::TreeState;
|
||||
@@ -431,7 +432,7 @@ where
|
||||
changeset_cache,
|
||||
);
|
||||
let incoming = task.incoming_tx.clone();
|
||||
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
|
||||
spawn_os_thread("engine", || task.run());
|
||||
(incoming, outgoing)
|
||||
}
|
||||
|
||||
|
||||
@@ -517,6 +517,8 @@ where
|
||||
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
|
||||
let prune_depth = self.sparse_trie_prune_depth;
|
||||
let max_storage_tries = self.sparse_trie_max_storage_tries;
|
||||
let chunk_size =
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
@@ -557,6 +559,7 @@ where
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
chunk_size,
|
||||
))
|
||||
};
|
||||
|
||||
|
||||
@@ -22,7 +22,7 @@ use reth_trie_parallel::{
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
},
|
||||
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
};
|
||||
use revm_primitives::map::{hash_map, B256Map};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
@@ -63,7 +63,7 @@ const PREFETCH_MAX_BATCH_MESSAGES: usize = 16;
|
||||
|
||||
/// The default max targets, for limiting the number of account and storage proof targets to be
|
||||
/// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability.
|
||||
const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
pub(crate) const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
|
||||
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
|
||||
/// state.
|
||||
@@ -311,11 +311,7 @@ impl VersionedMultiProofTargets {
|
||||
fn chunking_length(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.chunking_length(),
|
||||
Self::V2(targets) => {
|
||||
// For V2, count accounts + storage slots
|
||||
targets.account_targets.len() +
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
Self::V2(targets) => targets.chunking_length(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,9 +363,7 @@ impl VersionedMultiProofTargets {
|
||||
Self::Legacy(targets) => {
|
||||
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
|
||||
}
|
||||
Self::V2(targets) => Box::new(targets.chunks(chunk_size).map(Self::V2)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1494,7 +1488,7 @@ fn get_proof_targets(
|
||||
/// Dispatches work items as a single unit or in chunks based on target size and worker
|
||||
/// availability.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn dispatch_with_chunking<T, I>(
|
||||
pub(crate) fn dispatch_with_chunking<T, I>(
|
||||
items: T,
|
||||
chunking_len: usize,
|
||||
chunk_size: Option<usize>,
|
||||
|
||||
@@ -1,19 +1,21 @@
|
||||
//! Sparse Trie task related functionality.
|
||||
|
||||
use crate::tree::{
|
||||
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
|
||||
multiproof::{
|
||||
dispatch_with_chunking, evm_state_to_hashed_post_state, MultiProofMessage,
|
||||
VersionedMultiProofTargets, DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
},
|
||||
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use rayon::iter::ParallelIterator;
|
||||
use reth_errors::ProviderError;
|
||||
use rayon::iter::{IntoParallelRefMutIterator, ParallelBridge, ParallelIterator};
|
||||
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
|
||||
TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
proof_v2::Target, updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, Nibbles,
|
||||
TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use reth_trie_parallel::{
|
||||
proof_task::{
|
||||
@@ -24,7 +26,7 @@ use reth_trie_parallel::{
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
|
||||
};
|
||||
@@ -34,7 +36,7 @@ use std::{
|
||||
sync::mpsc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{debug, debug_span, instrument, trace};
|
||||
use tracing::{debug, debug_span, error, instrument, trace};
|
||||
|
||||
#[expect(clippy::large_enum_variant)]
|
||||
pub(super) enum SpawnedSparseTrieTask<BPF, A, S>
|
||||
@@ -203,6 +205,9 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum number of pending/prewarm updates that we accumulate in memory before actually applying.
|
||||
const MAX_PENDING_UPDATES: usize = 100;
|
||||
|
||||
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
|
||||
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
|
||||
/// Sender for proof results.
|
||||
@@ -215,6 +220,15 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
|
||||
trie: SparseStateTrie<A, S>,
|
||||
/// Handle to the proof worker pools (storage and account).
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
|
||||
/// The size of proof targets chunk to spawn in one calculation.
|
||||
/// If None, chunking is disabled and all targets are processed in a single proof.
|
||||
chunk_size: Option<usize>,
|
||||
/// If this number is exceeded and chunking is enabled, then this will override whether or not
|
||||
/// there are any active workers and force chunking across workers. This is to prevent tasks
|
||||
/// which are very long from hitting a single worker.
|
||||
max_targets_for_chunking: usize,
|
||||
|
||||
/// Account trie updates.
|
||||
account_updates: B256Map<LeafUpdate>,
|
||||
/// Storage trie updates. hashed address -> slot -> update.
|
||||
@@ -241,6 +255,14 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
|
||||
fetched_storage_targets: B256Map<B256Map<u8>>,
|
||||
/// Reusable buffer for RLP encoding of accounts.
|
||||
account_rlp_buf: Vec<u8>,
|
||||
/// Whether the last state update has been received.
|
||||
finished_state_updates: bool,
|
||||
/// Pending targets to be dispatched to the proof workers.
|
||||
pending_targets: MultiProofTargetsV2,
|
||||
/// Number of pending execution/prewarming updates received but not yet passed to
|
||||
/// `update_leaves`.
|
||||
pending_updates: usize,
|
||||
|
||||
/// Metrics for the sparse trie.
|
||||
metrics: MultiProofTaskMetrics,
|
||||
}
|
||||
@@ -255,7 +277,8 @@ where
|
||||
updates: CrossbeamReceiver<MultiProofMessage>,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
sparse_state_trie: SparseStateTrie<A, S>,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
chunk_size: Option<usize>,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
Self {
|
||||
@@ -263,13 +286,18 @@ where
|
||||
proof_result_rx,
|
||||
updates,
|
||||
proof_worker_handle,
|
||||
trie: sparse_state_trie,
|
||||
trie,
|
||||
chunk_size,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
account_updates: Default::default(),
|
||||
storage_updates: Default::default(),
|
||||
pending_account_updates: Default::default(),
|
||||
fetched_account_targets: Default::default(),
|
||||
fetched_storage_targets: Default::default(),
|
||||
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
|
||||
finished_state_updates: Default::default(),
|
||||
pending_targets: Default::default(),
|
||||
pending_updates: Default::default(),
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
@@ -317,15 +345,8 @@ where
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
let now = Instant::now();
|
||||
|
||||
let mut finished_state_updates = false;
|
||||
loop {
|
||||
crossbeam_channel::select_biased! {
|
||||
recv(self.proof_result_rx) -> message => {
|
||||
let Ok(result) = message else {
|
||||
unreachable!("we own the sender half")
|
||||
};
|
||||
self.on_proof_result(result)?;
|
||||
},
|
||||
recv(self.updates) -> message => {
|
||||
let update = match message {
|
||||
Ok(m) => m,
|
||||
@@ -334,27 +355,48 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
match update {
|
||||
MultiProofMessage::PrefetchProofs(targets) => {
|
||||
self.on_prewarm_targets(targets);
|
||||
}
|
||||
MultiProofMessage::StateUpdate(_, state) => {
|
||||
self.on_state_update(state);
|
||||
}
|
||||
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
|
||||
self.on_hashed_state_update(state);
|
||||
}
|
||||
MultiProofMessage::BlockAccessList(_) => todo!(),
|
||||
MultiProofMessage::FinishedStateUpdates => {
|
||||
finished_state_updates = true;
|
||||
}
|
||||
}
|
||||
self.on_multiproof_message(update);
|
||||
self.pending_updates += 1;
|
||||
}
|
||||
recv(self.proof_result_rx) -> message => {
|
||||
let Ok(result) = message else {
|
||||
unreachable!("we own the sender half")
|
||||
};
|
||||
let ProofResult::V2(mut result) = result.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
|
||||
while let Ok(next) = self.proof_result_rx.try_recv() {
|
||||
let ProofResult::V2(res) = next.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
result.extend(res);
|
||||
}
|
||||
|
||||
self.on_proof_result(result)?;
|
||||
},
|
||||
}
|
||||
|
||||
self.process_updates()?;
|
||||
if self.updates.is_empty() && self.proof_result_rx.is_empty() {
|
||||
// If we don't have any pending messages, we can spend some time on computing
|
||||
// storage roots and promoting account updates.
|
||||
self.dispatch_pending_targets();
|
||||
self.promote_pending_account_updates()?;
|
||||
self.dispatch_pending_targets();
|
||||
} else if self.updates.is_empty() || self.pending_updates > MAX_PENDING_UPDATES {
|
||||
// If we don't have any pending updates OR we've accumulated a lot already, apply
|
||||
// them to the trie,
|
||||
self.process_leaf_updates()?;
|
||||
self.dispatch_pending_targets();
|
||||
} else if self.updates.is_empty() ||
|
||||
self.pending_targets.chunking_length() > self.chunk_size.unwrap_or_default()
|
||||
{
|
||||
// Make sure to dispatch targets if we don't have any updates or if we've
|
||||
// accumulated a lot of them.
|
||||
self.dispatch_pending_targets();
|
||||
}
|
||||
|
||||
if finished_state_updates &&
|
||||
if self.finished_state_updates &&
|
||||
self.account_updates.is_empty() &&
|
||||
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
|
||||
{
|
||||
@@ -377,6 +419,22 @@ where
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
}
|
||||
|
||||
/// Processes a [`MultiProofMessage`].
|
||||
fn on_multiproof_message(&mut self, message: MultiProofMessage) {
|
||||
match message {
|
||||
MultiProofMessage::PrefetchProofs(targets) => self.on_prewarm_targets(targets),
|
||||
MultiProofMessage::StateUpdate(_, state) => self.on_state_update(state),
|
||||
MultiProofMessage::EmptyProof { .. } => unreachable!(),
|
||||
MultiProofMessage::BlockAccessList(_) => todo!(),
|
||||
MultiProofMessage::FinishedStateUpdates => self.finished_state_updates = true,
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
|
||||
let VersionedMultiProofTargets::V2(targets) = targets else {
|
||||
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
|
||||
@@ -412,11 +470,7 @@ where
|
||||
)]
|
||||
fn on_state_update(&mut self, update: EvmState) {
|
||||
let hashed_state_update = evm_state_to_hashed_post_state(update);
|
||||
self.on_hashed_state_update(hashed_state_update)
|
||||
}
|
||||
|
||||
/// Processes a hashed state update and encodes all state changes as trie updates.
|
||||
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
|
||||
for (address, storage) in hashed_state_update.storages {
|
||||
for (slot, value) in storage.storage {
|
||||
let encoded = if value.is_zero() {
|
||||
@@ -454,72 +508,148 @@ where
|
||||
|
||||
fn on_proof_result(
|
||||
&mut self,
|
||||
result: ProofResultMessage,
|
||||
result: DecodedMultiProofV2,
|
||||
) -> Result<(), ParallelStateRootError> {
|
||||
let ProofResult::V2(result) = result.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
|
||||
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
|
||||
fn process_updates(&mut self) -> Result<(), ProviderError> {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
/// Applies all account and storage leaf updates to corresponding tries and collects any new
|
||||
/// multiproof targets.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn process_leaf_updates(&mut self) -> SparseTrieResult<()> {
|
||||
self.pending_updates = 0;
|
||||
|
||||
for (addr, updates) in &mut self.storage_updates {
|
||||
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
|
||||
let fetched_storage = self.fetched_storage_targets.entry(*addr).or_default();
|
||||
// Start with processing all storage updates in parallel.
|
||||
let storage_results = self
|
||||
.storage_updates
|
||||
.iter_mut()
|
||||
.map(|(address, updates)| {
|
||||
let trie = self.trie.take_or_create_storage_trie(address);
|
||||
let fetched = self.fetched_storage_targets.remove(address).unwrap_or_default();
|
||||
|
||||
trie.update_leaves(updates, |path, min_len| match fetched_storage.entry(path) {
|
||||
(address, updates, fetched, trie)
|
||||
})
|
||||
.par_bridge()
|
||||
.map(|(address, updates, mut fetched, mut trie)| {
|
||||
let mut targets = Vec::new();
|
||||
|
||||
trie.update_leaves(updates, |path, min_len| match fetched.entry(path) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
})?;
|
||||
|
||||
SparseTrieResult::Ok((address, targets, fetched, trie))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
for (address, targets, fetched, trie) in storage_results {
|
||||
self.fetched_storage_targets.insert(*address, fetched);
|
||||
self.trie.insert_storage_trie(*address, trie);
|
||||
|
||||
if !targets.is_empty() {
|
||||
self.pending_targets.storage_targets.entry(*address).or_default().extend(targets);
|
||||
}
|
||||
}
|
||||
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.process_account_leaf_updates()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Invokes `update_leaves` for the accounts trie and collects any new targets.
|
||||
///
|
||||
/// Returns whether any updates were drained (applied to the trie).
|
||||
fn process_account_leaf_updates(&mut self) -> SparseTrieResult<bool> {
|
||||
let updates_len_before = self.account_updates.len();
|
||||
|
||||
self.trie.trie_mut().update_leaves(
|
||||
&mut self.account_updates,
|
||||
|target, min_len| match self.fetched_account_targets.entry(target) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(*addr)
|
||||
.or_default()
|
||||
.push(Target::new(path).with_min_len(min_len));
|
||||
self.pending_targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(*addr)
|
||||
.or_default()
|
||||
.push(Target::new(path).with_min_len(min_len));
|
||||
self.pending_targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
},
|
||||
)?;
|
||||
|
||||
// If all storage updates were processed, we can now compute the new storage root.
|
||||
if updates.is_empty() {
|
||||
let storage_root =
|
||||
Ok(self.account_updates.len() < updates_len_before)
|
||||
}
|
||||
|
||||
/// Iterates through all storage tries for which all updates were processed, computes their
|
||||
/// storage roots, and promotes corresponding pending account updates into proper leaf updates
|
||||
/// for accounts trie.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn promote_pending_account_updates(&mut self) -> SparseTrieResult<()> {
|
||||
self.process_leaf_updates()?;
|
||||
|
||||
if self.pending_account_updates.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let roots = self
|
||||
.trie
|
||||
.storage_tries_mut()
|
||||
.par_iter_mut()
|
||||
.filter(|(address, _)| {
|
||||
self.storage_updates.get(*address).is_some_and(|updates| updates.is_empty())
|
||||
})
|
||||
.map(|(address, trie)| {
|
||||
let root =
|
||||
trie.root().expect("updates are drained, trie should be revealed by now");
|
||||
|
||||
// If there is a pending account update for this address with known info, we can
|
||||
// encode it into proper update right away.
|
||||
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
|
||||
entry.get().is_some()
|
||||
(address, root)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for (addr, storage_root) in roots {
|
||||
// If the storage root is known and we have a pending update for this account, encode it
|
||||
// into a proper update.
|
||||
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
|
||||
entry.get().is_some()
|
||||
{
|
||||
let account = entry.remove().expect("just checked, should be Some");
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) &&
|
||||
storage_root == EMPTY_ROOT_HASH
|
||||
{
|
||||
let account = entry.remove().expect("just checked, should be Some");
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) &&
|
||||
storage_root == EMPTY_ROOT_HASH
|
||||
{
|
||||
Vec::new()
|
||||
} else {
|
||||
self.account_rlp_buf.clear();
|
||||
account
|
||||
.unwrap_or_default()
|
||||
.into_trie_account(storage_root)
|
||||
.encode(&mut self.account_rlp_buf);
|
||||
self.account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
Vec::new()
|
||||
} else {
|
||||
self.account_rlp_buf.clear();
|
||||
account
|
||||
.unwrap_or_default()
|
||||
.into_trie_account(storage_root)
|
||||
.encode(&mut self.account_rlp_buf);
|
||||
self.account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -568,52 +698,52 @@ where
|
||||
false
|
||||
});
|
||||
|
||||
let updates_len_before = self.account_updates.len();
|
||||
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.trie
|
||||
.trie_mut()
|
||||
.update_leaves(&mut self.account_updates, |target, min_len| {
|
||||
match self.fetched_account_targets.entry(target) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets.account_targets.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
|
||||
if updates_len_before == self.account_updates.len() {
|
||||
// Only exit when no new updates are processed.
|
||||
//
|
||||
// We need to keep iterating if any updates are being drained because that might
|
||||
// indicate that more pending account updates can be promoted.
|
||||
break;
|
||||
// Only exit when no new updates are processed.
|
||||
//
|
||||
// We need to keep iterating if any updates are being drained because that might
|
||||
// indicate that more pending account updates can be promoted.
|
||||
if !self.process_account_leaf_updates()? {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if !targets.is_empty() {
|
||||
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
|
||||
targets,
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
0,
|
||||
HashedPostState::default(),
|
||||
Instant::now(),
|
||||
),
|
||||
})?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn dispatch_pending_targets(&mut self) {
|
||||
if !self.pending_targets.is_empty() {
|
||||
let chunking_length = self.pending_targets.chunking_length();
|
||||
dispatch_with_chunking(
|
||||
std::mem::take(&mut self.pending_targets),
|
||||
chunking_length,
|
||||
self.chunk_size,
|
||||
self.max_targets_for_chunking,
|
||||
self.proof_worker_handle.available_account_workers(),
|
||||
self.proof_worker_handle.available_storage_workers(),
|
||||
MultiProofTargetsV2::chunks,
|
||||
|proof_targets| {
|
||||
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(
|
||||
AccountMultiproofInput::V2 {
|
||||
targets: proof_targets,
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
0,
|
||||
HashedPostState::default(),
|
||||
Instant::now(),
|
||||
),
|
||||
},
|
||||
) {
|
||||
error!("failed to dispatch account multiproof: {e:?}");
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of the state root computation, including the state root itself with
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
//! Contains a precompile cache backed by `schnellru::LruMap` (LRU by length).
|
||||
|
||||
use alloy_primitives::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use moka::policy::EvictionPolicy;
|
||||
use reth_evm::precompiles::{DynPrecompile, Precompile, PrecompileInput};
|
||||
use reth_primitives_traits::dashmap::DashMap;
|
||||
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
|
||||
use revm_primitives::Address;
|
||||
use std::{hash::Hash, sync::Arc};
|
||||
|
||||
@@ -28,6 +28,7 @@ use reth_ethereum_primitives::{Block, EthPrimitives};
|
||||
use reth_evm_ethereum::MockEvmConfig;
|
||||
use reth_primitives_traits::Block as _;
|
||||
use reth_provider::test_utils::MockEthProvider;
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
str::FromStr,
|
||||
@@ -538,10 +539,7 @@ async fn test_tree_persist_blocks() {
|
||||
.get_executed_blocks(1..tree_config.persistence_threshold() + 2)
|
||||
.collect();
|
||||
let test_harness = TestHarness::new(chain_spec).with_blocks(blocks.clone());
|
||||
std::thread::Builder::new()
|
||||
.name("Engine Task".to_string())
|
||||
.spawn(|| test_harness.tree.run())
|
||||
.unwrap();
|
||||
spawn_os_thread("engine", || test_harness.tree.run());
|
||||
|
||||
// send a message to the tree to enter the main loop.
|
||||
test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(vec![])).unwrap();
|
||||
@@ -1989,10 +1987,7 @@ mod forkchoice_updated_tests {
|
||||
let action_rx = test_harness.action_rx;
|
||||
|
||||
// Spawn tree in background thread
|
||||
std::thread::Builder::new()
|
||||
.name("Engine Task".to_string())
|
||||
.spawn(|| test_harness.tree.run())
|
||||
.unwrap();
|
||||
spawn_os_thread("engine", || test_harness.tree.run());
|
||||
|
||||
// Send terminate request
|
||||
to_tree_tx
|
||||
|
||||
@@ -36,7 +36,6 @@ rayon = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-ethereum-forks.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
@@ -47,7 +46,6 @@ std = [
|
||||
"alloy-primitives/std",
|
||||
"alloy-consensus/std",
|
||||
"revm/std",
|
||||
"reth-ethereum-forks/std",
|
||||
"alloy-evm/std",
|
||||
"reth-execution-errors/std",
|
||||
"reth-execution-types/std",
|
||||
|
||||
@@ -41,6 +41,7 @@ metrics.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
reth-tracing.workspace = true
|
||||
alloy-primitives = { workspace = true, features = ["rand"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread"] }
|
||||
secp256k1 = { workspace = true, features = ["std", "rand"] }
|
||||
rand_08.workspace = true
|
||||
|
||||
@@ -30,12 +30,12 @@ tokio-stream.workspace = true
|
||||
hickory-resolver = { workspace = true, features = ["tokio"] }
|
||||
|
||||
# misc
|
||||
dashmap = { workspace = true, features = ["inline"] }
|
||||
data-encoding.workspace = true
|
||||
linked_hash_set.workspace = true
|
||||
schnellru.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
parking_lot.workspace = true
|
||||
serde = { workspace = true, optional = true }
|
||||
serde_with = { workspace = true, optional = true }
|
||||
|
||||
@@ -56,9 +56,9 @@ serde = [
|
||||
"alloy-primitives/serde",
|
||||
"enr/serde",
|
||||
"linked_hash_set/serde",
|
||||
"parking_lot/serde",
|
||||
"rand/serde",
|
||||
"secp256k1/serde",
|
||||
"hickory-resolver/serde",
|
||||
"reth-ethereum-forks/serde",
|
||||
"dashmap/serde",
|
||||
]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
//! Perform DNS lookups
|
||||
|
||||
use dashmap::DashMap;
|
||||
use hickory_resolver::name_server::ConnectionProvider;
|
||||
pub use hickory_resolver::{ResolveError, TokioResolver};
|
||||
use parking_lot::RwLock;
|
||||
use std::{collections::HashMap, future::Future};
|
||||
use std::future::Future;
|
||||
use tracing::trace;
|
||||
|
||||
/// A type that can lookup DNS entries
|
||||
@@ -72,25 +72,25 @@ impl Resolver for DnsResolver {
|
||||
|
||||
/// A [Resolver] that uses an in memory map to lookup entries
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MapResolver(RwLock<HashMap<String, String>>);
|
||||
pub struct MapResolver(DashMap<String, String>);
|
||||
|
||||
// === impl MapResolver ===
|
||||
|
||||
impl MapResolver {
|
||||
/// Inserts a key-value pair into the map.
|
||||
pub fn insert(&self, k: String, v: String) -> Option<String> {
|
||||
self.0.write().insert(k, v)
|
||||
self.0.insert(k, v)
|
||||
}
|
||||
|
||||
/// Returns the value corresponding to the key
|
||||
pub fn get(&self, k: &str) -> Option<String> {
|
||||
self.0.read().get(k).cloned()
|
||||
self.0.get(k).map(|entry| entry.value().clone())
|
||||
}
|
||||
|
||||
/// Removes a key from the map, returning the value at the key if the key was previously in the
|
||||
/// map.
|
||||
pub fn remove(&self, k: &str) -> Option<String> {
|
||||
self.0.write().remove(k)
|
||||
self.0.remove(k).map(|(_, v)| v)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -236,7 +236,7 @@ impl LaunchContext {
|
||||
.map_or(0, |num| num.get().saturating_sub(reserved_cpu_cores).max(1));
|
||||
if let Err(err) = ThreadPoolBuilder::new()
|
||||
.num_threads(num_threads)
|
||||
.thread_name(|i| format!("reth-rayon-{i}"))
|
||||
.thread_name(|i| format!("rayon-{i}"))
|
||||
.build_global()
|
||||
{
|
||||
warn!(%err, "Failed to build global thread pool")
|
||||
|
||||
@@ -36,6 +36,7 @@ secp256k1 = { workspace = true, features = ["recovery"], optional = true }
|
||||
auto_impl.workspace = true
|
||||
byteorder = { workspace = true, optional = true }
|
||||
bytes.workspace = true
|
||||
dashmap = { workspace = true, features = ["inline"], optional = true }
|
||||
derive_more.workspace = true
|
||||
once_cell.workspace = true
|
||||
serde_with = { workspace = true, optional = true }
|
||||
@@ -116,6 +117,7 @@ arbitrary = [
|
||||
"alloy-trie/arbitrary",
|
||||
"reth-chainspec/arbitrary",
|
||||
"alloy-rpc-types-eth?/arbitrary",
|
||||
"dashmap?/arbitrary",
|
||||
]
|
||||
serde-bincode-compat = [
|
||||
"serde",
|
||||
@@ -144,6 +146,7 @@ serde = [
|
||||
"revm-state/serde",
|
||||
"rand_08/serde",
|
||||
"alloy-rpc-types-eth?/serde",
|
||||
"dashmap?/serde",
|
||||
]
|
||||
reth-codec = [
|
||||
"dep:reth-codecs",
|
||||
@@ -157,4 +160,5 @@ op = [
|
||||
rayon = [
|
||||
"dep:rayon",
|
||||
]
|
||||
dashmap = ["dep:dashmap"]
|
||||
rpc-compat = ["alloy-rpc-types-eth"]
|
||||
|
||||
@@ -245,3 +245,12 @@ pub mod test_utils {
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
pub use crate::{block::TestBlock, header::test_utils::TestHeader};
|
||||
}
|
||||
|
||||
/// Re-exports of `dashmap` types with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
|
||||
#[cfg(feature = "dashmap")]
|
||||
pub mod dashmap {
|
||||
pub use ::dashmap::{mapref, DashSet, Entry};
|
||||
/// Re-export of `DashMap` with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
|
||||
pub type DashMap<K, V, S = alloy_primitives::map::DefaultHashBuilder> =
|
||||
::dashmap::DashMap<K, V, S>;
|
||||
}
|
||||
|
||||
@@ -25,6 +25,10 @@ pub use user::{
|
||||
///
|
||||
/// This is a generic helper function used by both receipts and bodies pruning
|
||||
/// when data is stored in static files.
|
||||
///
|
||||
/// The checkpoint block number is set to the highest block in the actually deleted files,
|
||||
/// not `input.to_block`, since `to_block` might refer to a block in the middle of an
|
||||
/// undeleted file.
|
||||
pub(crate) fn prune_static_files<Provider>(
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
@@ -37,18 +41,31 @@ where
|
||||
provider.static_file_provider().delete_segment_below_block(segment, input.to_block + 1)?;
|
||||
|
||||
if deleted_headers.is_empty() {
|
||||
return Ok(SegmentOutput::done())
|
||||
return Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned: 0,
|
||||
checkpoint: input
|
||||
.previous_checkpoint
|
||||
.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
})
|
||||
}
|
||||
|
||||
let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range());
|
||||
|
||||
let pruned = tx_ranges.clone().map(|range| range.len()).sum::<u64>() as usize;
|
||||
|
||||
// The highest block number in the deleted files is the actual checkpoint.
|
||||
let checkpoint_block = deleted_headers
|
||||
.iter()
|
||||
.filter_map(|header| header.block_range())
|
||||
.map(|range| range.end())
|
||||
.max();
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(input.to_block),
|
||||
block_number: checkpoint_block,
|
||||
tx_number: tx_ranges.map(|range| range.end()).max(),
|
||||
}),
|
||||
})
|
||||
|
||||
@@ -76,8 +76,11 @@ where
|
||||
} = prune_modes;
|
||||
|
||||
Self::default()
|
||||
// Bodies - run first since file deletion is fast
|
||||
.segment_opt(bodies_history.map(Bodies::new))
|
||||
// Transaction lookup must run before bodies because it needs to read transaction
|
||||
// data from static files before bodies deletes them.
|
||||
.segment_opt(transaction_lookup.map(TransactionLookup::new))
|
||||
// Bodies
|
||||
.segment_opt(bodies_history.map(|mode| Bodies::new(mode, transaction_lookup)))
|
||||
// Account history
|
||||
.segment_opt(account_history.map(AccountHistory::new))
|
||||
// Storage history
|
||||
@@ -89,8 +92,6 @@ where
|
||||
(!receipts_log_filter.is_empty())
|
||||
.then(|| ReceiptsByLogs::new(receipts_log_filter.clone())),
|
||||
)
|
||||
// Transaction lookup
|
||||
.segment_opt(transaction_lookup.map(TransactionLookup::new))
|
||||
// Sender recovery
|
||||
.segment_opt(sender_recovery.map(SenderRecovery::new))
|
||||
}
|
||||
|
||||
@@ -238,8 +238,6 @@ impl AccountHistory {
|
||||
where
|
||||
Provider: DBProvider + StaticFileProviderFactory + ChangeSetReader + RocksDBProviderFactory,
|
||||
{
|
||||
use reth_provider::PruneShardOutcome;
|
||||
|
||||
// Unlike MDBX path, we don't divide the limit by 2 because RocksDB path only prunes
|
||||
// history shards (no separate changeset table to delete from). The changesets are in
|
||||
// static files which are deleted separately.
|
||||
@@ -287,14 +285,15 @@ impl AccountHistory {
|
||||
sorted_accounts.sort_unstable_by_key(|(addr, _)| *addr);
|
||||
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
for (address, highest_block) in &sorted_accounts {
|
||||
let prune_to = (*highest_block).min(last_changeset_pruned_block);
|
||||
match batch.prune_account_history_to(*address, prune_to)? {
|
||||
PruneShardOutcome::Deleted => deleted_shards += 1,
|
||||
PruneShardOutcome::Updated => updated_shards += 1,
|
||||
PruneShardOutcome::Unchanged => {}
|
||||
}
|
||||
}
|
||||
let targets: Vec<_> = sorted_accounts
|
||||
.iter()
|
||||
.map(|(addr, highest)| (*addr, (*highest).min(last_changeset_pruned_block)))
|
||||
.collect();
|
||||
|
||||
let outcomes = batch.prune_account_history_batch(&targets)?;
|
||||
deleted_shards = outcomes.deleted;
|
||||
updated_shards = outcomes.updated;
|
||||
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned account history (RocksDB indices)");
|
||||
|
||||
@@ -2,9 +2,14 @@ use crate::{
|
||||
segments::{self, PruneInput, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_provider::{BlockReader, StaticFileProviderFactory};
|
||||
use reth_prune_types::{PruneMode, PrunePurpose, PruneSegment, SegmentOutput};
|
||||
use alloy_primitives::BlockNumber;
|
||||
use reth_provider::{BlockReader, PruneCheckpointReader, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneInterruptReason, PruneMode, PrunePurpose, PruneSegment, SegmentOutput,
|
||||
SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use tracing::debug;
|
||||
|
||||
/// Segment responsible for pruning transactions in static files.
|
||||
///
|
||||
@@ -12,18 +17,79 @@ use reth_static_file_types::StaticFileSegment;
|
||||
#[derive(Debug)]
|
||||
pub struct Bodies {
|
||||
mode: PruneMode,
|
||||
/// Transaction lookup prune mode. Used to determine if we need to wait for tx lookup pruning
|
||||
/// before deleting transaction bodies.
|
||||
tx_lookup_mode: Option<PruneMode>,
|
||||
}
|
||||
|
||||
impl Bodies {
|
||||
/// Creates a new [`Bodies`] segment with the given prune mode.
|
||||
pub const fn new(mode: PruneMode) -> Self {
|
||||
Self { mode }
|
||||
/// Creates a new [`Bodies`] segment with the given prune mode and optional transaction lookup
|
||||
/// prune mode for coordination.
|
||||
pub const fn new(mode: PruneMode, tx_lookup_mode: Option<PruneMode>) -> Self {
|
||||
Self { mode, tx_lookup_mode }
|
||||
}
|
||||
|
||||
/// Returns the next best block that bodies can prune up to considering the transaction lookup
|
||||
/// pruning configuration (if any) and progress.
|
||||
///
|
||||
/// Returns `None` if there's no block available to prune (e.g., waiting on `tx_lookup`).
|
||||
fn next_bodies_prune_target<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: &PruneInput,
|
||||
) -> Result<Option<BlockNumber>, PrunerError>
|
||||
where
|
||||
Provider: PruneCheckpointReader,
|
||||
{
|
||||
let Some(tx_lookup_mode) = self.tx_lookup_mode else { return Ok(Some(input.to_block)) };
|
||||
|
||||
let tx_lookup_checkpoint = provider
|
||||
.get_prune_checkpoint(PruneSegment::TransactionLookup)?
|
||||
.and_then(|cp| cp.block_number);
|
||||
|
||||
// Determine the safe prune target, if any.
|
||||
// tx_lookup's next_pruned_block tells us what block it will prune next.
|
||||
// - None: tx_lookup will never prune more blocks (e.g. Before(N) reached its target), so
|
||||
// bodies can prune freely
|
||||
// - Some(next) > to_block: tx_lookup is ahead of our target, so we're safe to prune
|
||||
// to_block
|
||||
// - Some(next) <= to_block: tx_lookup still needs to prune blocks we want to delete, so we
|
||||
// must wait and only prune up to (next - 1) to preserve tx data it needs
|
||||
let to_block = match tx_lookup_mode.next_pruned_block(tx_lookup_checkpoint) {
|
||||
None => Some(input.to_block),
|
||||
Some(tx_lookup_next) if tx_lookup_next > input.to_block => Some(input.to_block),
|
||||
Some(tx_lookup_next) => {
|
||||
// We can only prune bodies up to the block BEFORE tx_lookup's next target.
|
||||
// tx_lookup_next is the next block tx_lookup will prune, meaning it still needs
|
||||
// to read transactions from that block. We must preserve those transactions,
|
||||
// so bodies can only safely delete up to (tx_lookup_next - 1).
|
||||
let Some(safe) = tx_lookup_next.checked_sub(1) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if input.previous_checkpoint.is_some_and(|cp| cp.block_number.unwrap_or(0) >= safe)
|
||||
{
|
||||
// we have pruned what we can
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "pruner",
|
||||
to_block = input.to_block,
|
||||
safe,
|
||||
"Bodies pruning limited by tx_lookup progress"
|
||||
);
|
||||
Some(safe)
|
||||
}
|
||||
};
|
||||
|
||||
Ok(to_block)
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider> Segment<Provider> for Bodies
|
||||
where
|
||||
Provider: StaticFileProviderFactory + BlockReader,
|
||||
Provider: StaticFileProviderFactory + BlockReader + PruneCheckpointReader,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::Bodies
|
||||
@@ -38,7 +104,20 @@ where
|
||||
}
|
||||
|
||||
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
|
||||
segments::prune_static_files(provider, input, StaticFileSegment::Transactions)
|
||||
let Some(to_block) = self.next_bodies_prune_target(provider, &input)? else {
|
||||
debug!(
|
||||
to_block = input.to_block,
|
||||
"Transaction lookup still has work to be done up to target block"
|
||||
);
|
||||
return Ok(SegmentOutput::not_done(
|
||||
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
|
||||
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
));
|
||||
};
|
||||
|
||||
// Use the coordinated to_block instead of input.to_block
|
||||
let adjusted_input = PruneInput { to_block, ..input };
|
||||
segments::prune_static_files(provider, adjusted_input, StaticFileSegment::Transactions)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +129,8 @@ mod tests {
|
||||
use reth_exex_types::FinishedExExHeight;
|
||||
use reth_provider::{
|
||||
test_utils::{create_test_provider_factory, MockNodeTypesWithDB},
|
||||
ProviderFactory, StaticFileWriter,
|
||||
DBProvider, DatabaseProviderFactory, ProviderFactory, PruneCheckpointWriter,
|
||||
StaticFileWriter,
|
||||
};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_static_file_types::{
|
||||
@@ -93,19 +173,83 @@ mod tests {
|
||||
static_file_provider.initialize_index().expect("initialize index");
|
||||
}
|
||||
|
||||
struct PruneTestCase {
|
||||
prune_mode: PruneMode,
|
||||
struct TestCase {
|
||||
tx_lookup_mode: Option<PruneMode>,
|
||||
tx_lookup_checkpoint_block: Option<BlockNumber>,
|
||||
bodies_mode: PruneMode,
|
||||
expected_pruned: usize,
|
||||
expected_lowest_block: Option<BlockNumber>,
|
||||
expected_progress: PruneProgress,
|
||||
}
|
||||
|
||||
impl TestCase {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
tx_lookup_mode: None,
|
||||
tx_lookup_checkpoint_block: None,
|
||||
bodies_mode: PruneMode::Full,
|
||||
expected_pruned: 0,
|
||||
expected_lowest_block: None,
|
||||
expected_progress: PruneProgress::Finished,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_bodies_mode(mut self, mode: PruneMode) -> Self {
|
||||
self.bodies_mode = mode;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_expected_pruned(mut self, pruned: usize) -> Self {
|
||||
self.expected_pruned = pruned;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_expected_progress(mut self, progress: PruneProgress) -> Self {
|
||||
self.expected_progress = progress;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_lowest_block(mut self, block: BlockNumber) -> Self {
|
||||
self.expected_lowest_block = Some(block);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_tx_lookup(mut self, mode: PruneMode, checkpoint: Option<BlockNumber>) -> Self {
|
||||
self.tx_lookup_mode = Some(mode);
|
||||
self.tx_lookup_checkpoint_block = checkpoint;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn run_prune_test(
|
||||
factory: &ProviderFactory<MockNodeTypesWithDB>,
|
||||
finished_exex_height_rx: &tokio::sync::watch::Receiver<FinishedExExHeight>,
|
||||
test_case: PruneTestCase,
|
||||
test_case: TestCase,
|
||||
tip: BlockNumber,
|
||||
) {
|
||||
let bodies = Bodies::new(test_case.prune_mode);
|
||||
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
// Capture highest block before pruning
|
||||
let static_provider = factory.static_file_provider();
|
||||
let highest_before =
|
||||
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions);
|
||||
|
||||
// Set up tx_lookup checkpoint if provided
|
||||
if let Some(checkpoint_block) = test_case.tx_lookup_checkpoint_block {
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_prune_checkpoint(
|
||||
PruneSegment::TransactionLookup,
|
||||
reth_prune_types::PruneCheckpoint {
|
||||
block_number: Some(checkpoint_block),
|
||||
tx_number: None,
|
||||
prune_mode: test_case.tx_lookup_mode.unwrap(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let bodies = Bodies::new(test_case.bodies_mode, test_case.tx_lookup_mode);
|
||||
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
|
||||
|
||||
let mut pruner = Pruner::new_with_factory(
|
||||
@@ -114,27 +258,29 @@ mod tests {
|
||||
5,
|
||||
10000,
|
||||
None,
|
||||
finished_exex_height_rx.clone(),
|
||||
finished_exex_height_rx,
|
||||
);
|
||||
|
||||
let result = pruner.run(tip).expect("pruner run");
|
||||
|
||||
assert_eq!(result.progress, PruneProgress::Finished);
|
||||
assert_eq!(result.progress, test_case.expected_progress);
|
||||
assert_eq!(result.segments.len(), 1);
|
||||
|
||||
let (segment, output) = &result.segments[0];
|
||||
assert_eq!(*segment, PruneSegment::Bodies);
|
||||
assert_eq!(output.pruned, test_case.expected_pruned);
|
||||
|
||||
let static_provider = factory.static_file_provider();
|
||||
assert_eq!(
|
||||
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
|
||||
test_case.expected_lowest_block
|
||||
);
|
||||
assert_eq!(
|
||||
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
|
||||
Some(tip)
|
||||
);
|
||||
if let Some(expected_lowest) = test_case.expected_lowest_block {
|
||||
let static_provider = factory.static_file_provider();
|
||||
assert_eq!(
|
||||
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
|
||||
Some(expected_lowest)
|
||||
);
|
||||
assert_eq!(
|
||||
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
|
||||
highest_before
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -143,50 +289,82 @@ mod tests {
|
||||
let tip = 2_499_999;
|
||||
setup_static_file_jars(&factory, tip);
|
||||
|
||||
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
let (_, _finished_exex_height_rx) =
|
||||
tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
let test_cases = vec![
|
||||
// Test 1: PruneMode::Before(750_000) → deletes jar 1 (0-499_999)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(750_000),
|
||||
expected_pruned: 1000,
|
||||
expected_lowest_block: Some(999_999),
|
||||
},
|
||||
// Test 2: PruneMode::Before(850_000) → no deletion (jar 2: 500_000-999_999 contains
|
||||
// Test 1: PruneMode::Before(750_000) → deletes jar 0 (0-499_999)
|
||||
// Checkpoint 499_999 != target 749_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(750_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(999_999),
|
||||
// Test 2: PruneMode::Before(850_000) → no deletion (jar 1: 500_000-999_999 contains
|
||||
// target)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(850_000),
|
||||
expected_pruned: 0,
|
||||
expected_lowest_block: Some(999_999),
|
||||
},
|
||||
// Test 3: PruneMode::Before(1_599_999) → deletes jar 2 (500_000-999_999) and jar 3
|
||||
// (1_000_000-1_499_999)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(1_599_999),
|
||||
expected_pruned: 2000,
|
||||
expected_lowest_block: Some(1_999_999),
|
||||
},
|
||||
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 4
|
||||
// (1_500_000-1_999_999)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Distance(500_000),
|
||||
expected_pruned: 1000,
|
||||
expected_lowest_block: Some(2_499_999),
|
||||
},
|
||||
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 5: 2_000_000-2_499_999
|
||||
TestCase::new().with_bodies_mode(PruneMode::Before(850_000)).with_lowest_block(999_999),
|
||||
// Test 3: PruneMode::Before(1_599_999) → deletes jars 0 and 1 (0-999_999)
|
||||
// Checkpoint 999_999 != target 1_599_998 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(1_599_999))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_999_999),
|
||||
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 3
|
||||
// (1_500_000-1_999_999) Checkpoint 1_999_999 == target 1_999_999 ->
|
||||
// Finished
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Distance(500_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(2_499_999),
|
||||
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 4: 2_000_000-2_499_999
|
||||
// contains target)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(2_300_000),
|
||||
expected_pruned: 0,
|
||||
expected_lowest_block: Some(2_499_999),
|
||||
},
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(2_300_000))
|
||||
.with_lowest_block(2_499_999),
|
||||
];
|
||||
|
||||
for test_case in test_cases {
|
||||
run_prune_test(&factory, &finished_exex_height_rx, test_case, tip);
|
||||
run_prune_test(&factory, test_case, tip);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_reflects_deleted_files_not_target() {
|
||||
// Test that checkpoint is set to the highest deleted block, not to_block.
|
||||
// When to_block falls in the middle of an undeleted file, checkpoint should reflect
|
||||
// what was actually deleted.
|
||||
let factory = create_test_provider_factory();
|
||||
let tip = 1_499_999;
|
||||
setup_static_file_jars(&factory, tip);
|
||||
|
||||
// Use PruneMode::Before(900_000) which targets 899_999.
|
||||
// This should delete jar 0 (0-499_999) since it's entirely below the target.
|
||||
// Jar 1 (500_000-999_999) contains the target, so it won't be deleted.
|
||||
// Checkpoint should be 499_999 (end of jar 0), not 899_999 (to_block).
|
||||
let bodies = Bodies::new(PruneMode::Before(900_000), None);
|
||||
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
|
||||
|
||||
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
let mut pruner =
|
||||
Pruner::new_with_factory(factory, segments, 5, 10000, None, finished_exex_height_rx);
|
||||
|
||||
let result = pruner.run(tip).expect("pruner run");
|
||||
|
||||
assert_eq!(result.progress, PruneProgress::Finished);
|
||||
assert_eq!(result.segments.len(), 1);
|
||||
|
||||
let (segment, output) = &result.segments[0];
|
||||
assert_eq!(*segment, PruneSegment::Bodies);
|
||||
|
||||
// Verify checkpoint is set to the end of deleted jar (499_999), not to_block (899_999)
|
||||
let checkpoint_block = output.checkpoint.as_ref().and_then(|cp| cp.block_number);
|
||||
assert_eq!(
|
||||
checkpoint_block,
|
||||
Some(499_999),
|
||||
"Checkpoint should be 499_999 (end of deleted jar 0), not 899_999 (to_block)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_block_updated_on_sync() {
|
||||
// Regression test: update_index must update min_block to prevent stale values
|
||||
@@ -302,4 +480,100 @@ mod tests {
|
||||
assert_eq!(deleted.len(), expected_deleted);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bodies_with_tx_lookup_coordination() {
|
||||
// Test that bodies pruning correctly coordinates with tx lookup pruning
|
||||
// Using tip = 1_523_000 creates 4 static file jars:
|
||||
// - Jar 0: blocks 0-499_999, txs 0-999
|
||||
// - Jar 1: blocks 500_000-999_999, txs 1000-1999
|
||||
// - Jar 2: blocks 1_000_000-1_499_999, txs 2000-2999
|
||||
// - Jar 3: blocks 1_500_000-1_523_000, txs 3000-3999
|
||||
let tip = 1_523_000;
|
||||
|
||||
let test_cases = vec![
|
||||
// Scenario 1: tx_lookup disabled, bodies can prune freely (deletes jar 0)
|
||||
// Checkpoint is 499_999 (end of jar 0), target is 599_999, so HasMoreData
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(999_999),
|
||||
// Scenario 2: tx_lookup enabled but not run yet, bodies cannot prune
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), None)
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_expected_progress(PruneProgress::HasMoreData(
|
||||
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
|
||||
))
|
||||
.with_lowest_block(499_999), // No jars deleted, jar 0 ends at 499_999
|
||||
// Scenario 3: tx_lookup caught up to its target, bodies can prune freely
|
||||
// Deletes jar 0, checkpoint is 499_999, target is 599_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(999_999),
|
||||
// Scenario 4: tx_lookup behind its target, bodies limited to tx_lookup checkpoint
|
||||
// tx_lookup should prune up to 599_999, but checkpoint is only at 250_000
|
||||
// bodies wants to prune up to 599_999, but limited to 250_000
|
||||
// No jars deleted because jar 0 (0-499_999) ends beyond 250_000
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), Some(250_000))
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_lowest_block(499_999), // No jars deleted
|
||||
// Scenario 5: Both use Distance, tx_lookup caught up
|
||||
// With tip=1_523_000, Distance(500_000) targets block 1_023_000
|
||||
// Deletes jars 0 and 1, checkpoint is 999_999, target is 1_023_000 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Distance(500_000), Some(1_023_000))
|
||||
.with_bodies_mode(PruneMode::Distance(500_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_499_999),
|
||||
// Scenario 6: Both use Distance, tx_lookup less aggressive (bigger distance) than
|
||||
// bodies With tip=1_523_000:
|
||||
// - tx_lookup: Distance(1_000_000) targets block 523_000, checkpoint at 523_000
|
||||
// - bodies: Distance(500_000) targets block 1_023_000
|
||||
// Bodies can prune up to what tx_lookup has finished (523_000), deleting jar 0
|
||||
// Checkpoint is 499_999, target is 1_023_000 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Distance(1_000_000), Some(523_000))
|
||||
.with_bodies_mode(PruneMode::Distance(500_000))
|
||||
.with_expected_pruned(1000) // Jar 0 deleted
|
||||
.with_lowest_block(999_999), // Jar 0 (0-499_999) deleted
|
||||
// Scenario 7: tx_lookup more aggressive than bodies (deletes jar 0 and 1)
|
||||
// tx_lookup: Before(1_100_000) -> prune up to 1_099_999
|
||||
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
|
||||
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(1_100_000), Some(1_099_999))
|
||||
.with_bodies_mode(PruneMode::Before(1_100_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
|
||||
// Scenario 8: tx_lookup has lower target than bodies, but is done
|
||||
// tx_lookup: Before(600_000) -> prune up to 599_999 (checkpoint at 599_999, DONE)
|
||||
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
|
||||
// Since tx_lookup is done (next_pruned_block returns None), bodies can prune freely
|
||||
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
|
||||
.with_bodies_mode(PruneMode::Before(1_100_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
|
||||
// Scenario 9: Perfect alignment - checkpoint equals target
|
||||
// bodies: Before(1_000_000) -> targets 999_999
|
||||
// Deletes jars 0 and 1 (0-999_999), checkpoint is 999_999 which equals target ->
|
||||
// Finished
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(1_000_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_expected_progress(PruneProgress::Finished)
|
||||
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
|
||||
];
|
||||
|
||||
for test_case in test_cases {
|
||||
let factory = create_test_provider_factory();
|
||||
setup_static_file_jars(&factory, tip);
|
||||
run_prune_test(&factory, test_case, tip);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -242,8 +242,6 @@ impl StorageHistory {
|
||||
where
|
||||
Provider: DBProvider + StaticFileProviderFactory + RocksDBProviderFactory,
|
||||
{
|
||||
use reth_provider::PruneShardOutcome;
|
||||
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
if limiter.is_limit_reached() {
|
||||
@@ -291,14 +289,17 @@ impl StorageHistory {
|
||||
sorted_storages.sort_unstable_by_key(|((addr, key), _)| (*addr, *key));
|
||||
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
for ((address, storage_key), highest_block) in &sorted_storages {
|
||||
let prune_to = (*highest_block).min(last_changeset_pruned_block);
|
||||
match batch.prune_storage_history_to(*address, *storage_key, prune_to)? {
|
||||
PruneShardOutcome::Deleted => deleted_shards += 1,
|
||||
PruneShardOutcome::Updated => updated_shards += 1,
|
||||
PruneShardOutcome::Unchanged => {}
|
||||
}
|
||||
}
|
||||
let targets: Vec<_> = sorted_storages
|
||||
.iter()
|
||||
.map(|((addr, key), highest)| {
|
||||
((*addr, *key), (*highest).min(last_changeset_pruned_block))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let outcomes = batch.prune_storage_history_batch(&targets)?;
|
||||
deleted_shards = outcomes.deleted;
|
||||
updated_shards = outcomes.updated;
|
||||
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
|
||||
|
||||
@@ -6,11 +6,15 @@ use crate::{
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use rayon::prelude::*;
|
||||
use reth_db_api::{tables, transaction::DbTxMut};
|
||||
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
|
||||
use reth_provider::{
|
||||
BlockReader, DBProvider, PruneCheckpointReader, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use tracing::{debug, instrument, trace};
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -29,7 +33,9 @@ where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ BlockReader<Transaction: Encodable2718>
|
||||
+ PruneCheckpointReader
|
||||
+ StaticFileProviderFactory,
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::TransactionLookup
|
||||
@@ -83,6 +89,12 @@ where
|
||||
}
|
||||
.into_inner();
|
||||
|
||||
// Check where transaction hash numbers are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, start, end);
|
||||
}
|
||||
|
||||
// For PruneMode::Full, clear the entire table in one operation
|
||||
if self.mode.is_full() {
|
||||
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
|
||||
@@ -174,6 +186,106 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl TransactionLookup {
|
||||
/// Prunes transaction lookup when indices are stored in `RocksDB`.
|
||||
///
|
||||
/// Reads transactions from static files and deletes corresponding entries
|
||||
/// from the `RocksDB` `TransactionHashNumbers` table.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn prune_rocksdb<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
start: alloy_primitives::TxNumber,
|
||||
end: alloy_primitives::TxNumber,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ BlockReader<Transaction: Encodable2718>
|
||||
+ StaticFileProviderFactory
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
// For PruneMode::Full, clear the entire RocksDB table in one operation
|
||||
if self.mode.is_full() {
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
rocksdb.clear::<tables::TransactionHashNumbers>()?;
|
||||
trace!(target: "pruner", "Cleared transaction lookup table (RocksDB)");
|
||||
|
||||
let last_pruned_block = provider
|
||||
.block_by_transaction_id(end)?
|
||||
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
|
||||
|
||||
return Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned: 0, // RocksDB clear doesn't return count
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_pruned_block),
|
||||
tx_number: Some(end),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
let tx_range_end = input
|
||||
.limiter
|
||||
.deleted_entries_limit_left()
|
||||
.map(|left| start.saturating_add(left as u64).saturating_sub(1))
|
||||
.map_or(end, |limited| limited.min(end));
|
||||
let tx_range = start..=tx_range_end;
|
||||
|
||||
// Retrieve transactions in the range and calculate their hashes in parallel
|
||||
let hashes: Vec<_> = provider
|
||||
.transactions_by_tx_range(tx_range.clone())?
|
||||
.into_par_iter()
|
||||
.map(|transaction| transaction.trie_hash())
|
||||
.collect();
|
||||
|
||||
// Number of transactions retrieved from the database should match the tx range count
|
||||
let tx_count = tx_range.count();
|
||||
if hashes.len() != tx_count {
|
||||
return Err(PrunerError::InconsistentData(
|
||||
"Unexpected number of transaction hashes retrieved by transaction number range",
|
||||
))
|
||||
}
|
||||
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
// Delete transaction hash -> number mappings from RocksDB
|
||||
let mut deleted = 0usize;
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
for hash in &hashes {
|
||||
if limiter.is_limit_reached() {
|
||||
break;
|
||||
}
|
||||
batch.delete::<tables::TransactionHashNumbers>(*hash)?;
|
||||
limiter.increment_deleted_entries_count();
|
||||
deleted += 1;
|
||||
}
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
|
||||
let done = deleted == hashes.len() && tx_range_end == end;
|
||||
trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup (RocksDB)");
|
||||
|
||||
let last_pruned_transaction = if deleted > 0 { start + deleted as u64 - 1 } else { start };
|
||||
|
||||
let last_pruned_block = provider
|
||||
.block_by_transaction_id(last_pruned_transaction)?
|
||||
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
|
||||
.checked_sub(if done { 0 } else { 1 });
|
||||
|
||||
let progress = limiter.progress(done);
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress,
|
||||
pruned: deleted,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: last_pruned_block,
|
||||
tx_number: Some(last_pruned_transaction),
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{PruneInput, PruneLimiter, Segment, SegmentOutput, TransactionLookup};
|
||||
@@ -319,4 +431,100 @@ mod tests {
|
||||
test_prune(6, (PruneProgress::Finished, 2));
|
||||
test_prune(10, (PruneProgress::Finished, 8));
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
#[test]
|
||||
fn prune_rocksdb() {
|
||||
use reth_db_api::models::StorageSettings;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
1..=10,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks");
|
||||
|
||||
// Collect transaction hashes and their tx numbers
|
||||
let mut tx_hash_numbers = Vec::new();
|
||||
for block in &blocks {
|
||||
tx_hash_numbers.reserve_exact(block.transaction_count());
|
||||
for transaction in &block.body().transactions {
|
||||
tx_hash_numbers.push((*transaction.tx_hash(), tx_hash_numbers.len() as u64));
|
||||
}
|
||||
}
|
||||
let tx_hash_numbers_len = tx_hash_numbers.len();
|
||||
|
||||
// Insert into RocksDB instead of MDBX
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let mut batch = rocksdb.batch();
|
||||
for (hash, tx_num) in &tx_hash_numbers {
|
||||
batch.put::<tables::TransactionHashNumbers>(*hash, tx_num).unwrap();
|
||||
}
|
||||
batch.commit().expect("commit rocksdb batch");
|
||||
}
|
||||
|
||||
// Verify RocksDB has all entries
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for (hash, expected_tx_num) in &tx_hash_numbers {
|
||||
let actual = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
|
||||
assert_eq!(actual, Some(*expected_tx_num));
|
||||
}
|
||||
}
|
||||
|
||||
let to_block: BlockNumber = 6;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let input =
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = TransactionLookup::new(prune_mode);
|
||||
|
||||
// Enable RocksDB storage for transaction hash numbers
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
assert_matches!(
|
||||
result,
|
||||
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
|
||||
if pruned > 0
|
||||
);
|
||||
|
||||
// Calculate expected: blocks 1-6 should have their tx hashes pruned
|
||||
let txs_up_to_block_6: usize = blocks.iter().take(6).map(|b| b.transaction_count()).sum();
|
||||
|
||||
// Verify RocksDB entries: first `txs_up_to_block_6` should be gone
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for (i, (hash, _)) in tx_hash_numbers.iter().enumerate() {
|
||||
let entry = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
|
||||
if i < txs_up_to_block_6 {
|
||||
assert!(entry.is_none(), "Entry {} (hash {:?}) should be pruned", i, hash);
|
||||
} else {
|
||||
assert!(entry.is_some(), "Entry {} (hash {:?}) should still exist", i, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify remaining count
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let remaining: Vec<_> =
|
||||
rocksdb.iter::<tables::TransactionHashNumbers>().unwrap().collect();
|
||||
assert_eq!(
|
||||
remaining.len(),
|
||||
tx_hash_numbers_len - txs_up_to_block_6,
|
||||
"Remaining RocksDB entries should match expected"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +84,38 @@ impl PruneMode {
|
||||
pub const fn is_distance(&self) -> bool {
|
||||
matches!(self, Self::Distance(_))
|
||||
}
|
||||
|
||||
/// Returns the next block number that will EVENTUALLY be pruned after the given checkpoint. It
|
||||
/// should not be used to find if there are blocks to be pruned right now. For that, use
|
||||
/// [`Self::prune_target_block`].
|
||||
///
|
||||
/// This is independent of the current tip and indicates what block is next in the pruning
|
||||
/// sequence according to this mode's configuration. Returns `None` if no more blocks will
|
||||
/// be pruned (i.e., the mode has reached its target).
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// - `Before(10)` with checkpoint at block 5 returns `Some(6)`
|
||||
/// - `Before(10)` with checkpoint at block 9 returns `None` (done)
|
||||
/// - `Distance(100)` with checkpoint at block 1000 returns `Some(1001)` (always has more)
|
||||
/// - `Full` always returns the next block after checkpoint
|
||||
pub const fn next_pruned_block(&self, checkpoint: Option<BlockNumber>) -> Option<BlockNumber> {
|
||||
let next = match checkpoint {
|
||||
Some(c) => c + 1,
|
||||
None => 0,
|
||||
};
|
||||
|
||||
match self {
|
||||
Self::Before(n) => {
|
||||
if next < *n {
|
||||
Some(next)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Self::Distance(_) | Self::Full => Some(next),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -93,7 +93,7 @@ impl SegmentOutput {
|
||||
Self { progress: PruneProgress::Finished, pruned: 0, checkpoint: None }
|
||||
}
|
||||
|
||||
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and `checkpoint = None`.
|
||||
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and the given checkpoint.
|
||||
/// Use when pruning is needed but cannot be done.
|
||||
pub const fn not_done(
|
||||
reason: PruneInterruptReason,
|
||||
@@ -142,6 +142,8 @@ pub enum PruneInterruptReason {
|
||||
Timeout,
|
||||
/// Limit on the number of deleted entries (rows in the database) per prune run was reached.
|
||||
DeletedEntriesLimitReached,
|
||||
/// Waiting for another segment to finish pruning before this segment can proceed.
|
||||
WaitingOnSegment(PruneSegment),
|
||||
/// Unknown reason for stopping prune run.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
@@ -394,6 +394,17 @@ pub trait EthApi<
|
||||
address: Address,
|
||||
block: BlockId,
|
||||
) -> RpcResult<alloy_rpc_types_eth::AccountInfo>;
|
||||
|
||||
/// Returns the EIP-7928 block access list for a block by hash.
|
||||
#[method(name = "getBlockAccessListByBlockHash")]
|
||||
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>>;
|
||||
|
||||
/// Returns the EIP-7928 block access list for a block by number.
|
||||
#[method(name = "getBlockAccessListByBlockNumber")]
|
||||
async fn block_access_list_by_block_number(
|
||||
&self,
|
||||
number: BlockNumberOrTag,
|
||||
) -> RpcResult<Option<Bytes>>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -881,4 +892,19 @@ where
|
||||
trace!(target: "rpc::eth", "Serving eth_getAccountInfo");
|
||||
Ok(EthState::get_account_info(self, address, block).await?)
|
||||
}
|
||||
|
||||
/// Handler for: `eth_getBlockAccessListByBlockHash`
|
||||
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>> {
|
||||
trace!(target: "rpc::eth", ?hash, "Serving eth_getBlockAccessListByBlockHash");
|
||||
Err(internal_rpc_err("unimplemented"))
|
||||
}
|
||||
|
||||
/// Handler for: `eth_getBlockAccessListByBlockNumber`
|
||||
async fn block_access_list_by_block_number(
|
||||
&self,
|
||||
number: BlockNumberOrTag,
|
||||
) -> RpcResult<Option<Bytes>> {
|
||||
trace!(target: "rpc::eth", ?number, "Serving eth_getBlockAccessListByBlockNumber");
|
||||
Err(internal_rpc_err("unimplemented"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -540,7 +540,11 @@ where
|
||||
max_simulate_blocks,
|
||||
eth_proof_window,
|
||||
blocking_task_pool.unwrap_or_else(|| {
|
||||
BlockingTaskPool::build().expect("failed to build blocking task pool")
|
||||
BlockingTaskPool::builder()
|
||||
.thread_name(|i| format!("blocking-{i}"))
|
||||
.build()
|
||||
.map(BlockingTaskPool::new)
|
||||
.expect("failed to build blocking task pool")
|
||||
}),
|
||||
fee_history_cache,
|
||||
task_spawner,
|
||||
|
||||
@@ -353,6 +353,9 @@ where
|
||||
return Err(EthApiError::HeaderNotFound(start.into()).into());
|
||||
}
|
||||
let end = to_block.unwrap_or(latest_block);
|
||||
if end > latest_block {
|
||||
return Err(EthApiError::HeaderNotFound(end.into()).into());
|
||||
}
|
||||
|
||||
if start > end {
|
||||
return Err(EthApiError::InvalidParams(
|
||||
|
||||
@@ -161,37 +161,34 @@ where
|
||||
provider.count_entries::<tables::TransactionHashNumbers>()?.is_zero();
|
||||
|
||||
// Auto-commits on threshold; consistency check heals any crash.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb_batch = rocksdb.batch_with_auto_commit();
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
let rocksdb_batch = ();
|
||||
let mut writer =
|
||||
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
provider.with_rocksdb_batch_auto_commit(|rocksdb_batch| {
|
||||
let mut writer =
|
||||
EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
|
||||
for (index, hash_to_number) in hash_collector.iter()?.enumerate() {
|
||||
let (hash_bytes, number_bytes) = hash_to_number?;
|
||||
if index > 0 && index.is_multiple_of(interval) {
|
||||
info!(
|
||||
target: "sync::stages::transaction_lookup",
|
||||
?append_only,
|
||||
progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0),
|
||||
"Inserting hashes"
|
||||
);
|
||||
let iter = hash_collector
|
||||
.iter()
|
||||
.map_err(|e| ProviderError::other(Box::new(e)))?;
|
||||
|
||||
for (index, hash_to_number) in iter.enumerate() {
|
||||
let (hash_bytes, number_bytes) =
|
||||
hash_to_number.map_err(|e| ProviderError::other(Box::new(e)))?;
|
||||
if index > 0 && index.is_multiple_of(interval) {
|
||||
info!(
|
||||
target: "sync::stages::transaction_lookup",
|
||||
?append_only,
|
||||
progress = %format!("{:.2}%", (index as f64 / total_hashes as f64) * 100.0),
|
||||
"Inserting hashes"
|
||||
);
|
||||
}
|
||||
|
||||
// Decode from raw ETL bytes
|
||||
let hash = TxHash::decode(&hash_bytes)?;
|
||||
let tx_num = TxNumber::decompress(&number_bytes)?;
|
||||
writer.put_transaction_hash_number(hash, tx_num, append_only)?;
|
||||
}
|
||||
|
||||
// Decode from raw ETL bytes
|
||||
let hash = TxHash::decode(&hash_bytes)?;
|
||||
let tx_num = TxNumber::decompress(&number_bytes)?;
|
||||
writer.put_transaction_hash_number(hash, tx_num, append_only)?;
|
||||
}
|
||||
|
||||
// Extract and register RocksDB batch for commit at provider level
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if let Some(batch) = writer.into_raw_rocksdb_batch() {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
}
|
||||
Ok(((), writer.into_raw_rocksdb_batch()))
|
||||
})?;
|
||||
|
||||
trace!(target: "sync::stages::transaction_lookup",
|
||||
total_hashes,
|
||||
@@ -223,36 +220,31 @@ where
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
let (range, unwind_to, _) = input.unwind_block_range_with_threshold(self.chunk_size);
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb_batch = rocksdb.batch();
|
||||
#[cfg(not(all(unix, feature = "rocksdb")))]
|
||||
let rocksdb_batch = ();
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
provider.with_rocksdb_batch(|rocksdb_batch| {
|
||||
let mut writer = EitherWriter::new_transaction_hash_numbers(provider, rocksdb_batch)?;
|
||||
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
let rev_walker =
|
||||
provider.block_body_indices_range(range.clone())?.into_iter().rev().zip(range.rev());
|
||||
let static_file_provider = provider.static_file_provider();
|
||||
let rev_walker = provider
|
||||
.block_body_indices_range(range.clone())?
|
||||
.into_iter()
|
||||
.rev()
|
||||
.zip(range.rev());
|
||||
|
||||
for (body, number) in rev_walker {
|
||||
if number <= unwind_to {
|
||||
break;
|
||||
}
|
||||
for (body, number) in rev_walker {
|
||||
if number <= unwind_to {
|
||||
break;
|
||||
}
|
||||
|
||||
// Delete all transactions that belong to this block
|
||||
for tx_id in body.tx_num_range() {
|
||||
if let Some(transaction) = static_file_provider.transaction_by_id(tx_id)? {
|
||||
// Delete all transactions that belong to this block
|
||||
for transaction in
|
||||
static_file_provider.transactions_by_tx_range(body.tx_num_range())?
|
||||
{
|
||||
writer.delete_transaction_hash_number(transaction.trie_hash())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Extract and register RocksDB batch for commit at provider level
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if let Some(batch) = writer.into_raw_rocksdb_batch() {
|
||||
provider.set_pending_rocksdb_batch(batch);
|
||||
}
|
||||
Ok(((), writer.into_raw_rocksdb_batch()))
|
||||
})?;
|
||||
|
||||
Ok(UnwindOutput {
|
||||
checkpoint: StageCheckpoint::new(unwind_to)
|
||||
|
||||
@@ -30,6 +30,10 @@ where
|
||||
let mut static_file_writer =
|
||||
provider.get_static_file_writer(*block_range.start(), StaticFileSegment::Receipts)?;
|
||||
|
||||
let mut receipts_cursor = provider
|
||||
.tx_ref()
|
||||
.cursor_read::<tables::Receipts<<Provider::Primitives as NodePrimitives>::Receipt>>()?;
|
||||
|
||||
for block in block_range {
|
||||
static_file_writer.increment_block(block)?;
|
||||
|
||||
@@ -37,10 +41,6 @@ where
|
||||
.block_body_indices(block)?
|
||||
.ok_or(ProviderError::BlockBodyIndicesNotFound(block))?;
|
||||
|
||||
let mut receipts_cursor = provider
|
||||
.tx_ref()
|
||||
.cursor_read::<tables::Receipts<<Provider::Primitives as NodePrimitives>::Receipt>>(
|
||||
)?;
|
||||
let receipts_walker = receipts_cursor.walk_range(block_body_indices.tx_num_range())?;
|
||||
|
||||
static_file_writer.append_receipts(
|
||||
|
||||
@@ -173,11 +173,10 @@ where
|
||||
/// Returns highest block numbers for all static file segments.
|
||||
pub fn copy_to_static_files(&self) -> ProviderResult<HighestStaticFiles> {
|
||||
let provider = self.provider.database_provider_ro()?;
|
||||
let stages_checkpoints = std::iter::once(StageId::Execution)
|
||||
.map(|stage| provider.get_stage_checkpoint(stage).map(|c| c.map(|c| c.block_number)))
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
let execution_checkpoint =
|
||||
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number);
|
||||
|
||||
let highest_static_files = HighestStaticFiles { receipts: stages_checkpoints[0] };
|
||||
let highest_static_files = HighestStaticFiles { receipts: execution_checkpoint };
|
||||
let targets = self.get_static_file_targets(highest_static_files)?;
|
||||
self.run(targets)?;
|
||||
|
||||
|
||||
@@ -81,6 +81,20 @@ impl StaticFileSegment {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a short string representation of the segment.
|
||||
///
|
||||
/// Linux threads have a length limit of 15 characters: <https://man7.org/linux/man-pages/man3/pthread_setname_np.3.html#DESCRIPTION>.
|
||||
pub const fn as_short_str(&self) -> &'static str {
|
||||
match self {
|
||||
Self::Headers => "headers",
|
||||
Self::Transactions => "transactions",
|
||||
Self::Receipts => "receipts",
|
||||
Self::TransactionSenders => "tx-senders",
|
||||
Self::AccountChangeSets => "account-changes",
|
||||
Self::StorageChangeSets => "storage-changes",
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an iterator over all segments.
|
||||
pub fn iter() -> impl Iterator<Item = Self> {
|
||||
// The order of segments is significant and must be maintained to ensure correctness.
|
||||
@@ -861,4 +875,15 @@ mod tests {
|
||||
assert_eq!(ser, format!("\"{expected_str}\""));
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_static_file_segment_as_short_str() {
|
||||
for segment in StaticFileSegment::iter() {
|
||||
assert!(
|
||||
segment.as_short_str().len() <= 15,
|
||||
"{segment} segment name is too long: {}",
|
||||
segment.as_short_str()
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -643,9 +643,11 @@ mod tests {
|
||||
use std::str::FromStr;
|
||||
use tempfile::TempDir;
|
||||
|
||||
/// Create database for testing
|
||||
fn create_test_db(kind: DatabaseEnvKind) -> DatabaseEnv {
|
||||
create_test_db_with_path(kind, &tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep())
|
||||
/// Create database for testing. Returns the `TempDir` to prevent cleanup until test ends.
|
||||
fn create_test_db(kind: DatabaseEnvKind) -> (TempDir, DatabaseEnv) {
|
||||
let tempdir = tempfile::TempDir::new().expect(ERROR_TEMPDIR);
|
||||
let env = create_test_db_with_path(kind, tempdir.path());
|
||||
(tempdir, env)
|
||||
}
|
||||
|
||||
/// Create database for testing with specified path
|
||||
@@ -670,13 +672,13 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_creation() {
|
||||
create_test_db(DatabaseEnvKind::RW);
|
||||
let _tempdir = create_test_db(DatabaseEnvKind::RW);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn db_drop_orphan_table() {
|
||||
let path = tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep();
|
||||
let db = create_test_db_with_path(DatabaseEnvKind::RW, &path);
|
||||
let tempdir = tempfile::TempDir::new().expect(ERROR_TEMPDIR);
|
||||
let db = create_test_db_with_path(DatabaseEnvKind::RW, tempdir.path());
|
||||
|
||||
// Create an orphan table by manually creating it
|
||||
let orphan_table_name = "OrphanTestTable";
|
||||
@@ -715,7 +717,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_manual_put_get() {
|
||||
let env = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
let value = Header::default();
|
||||
let key = 1u64;
|
||||
@@ -734,7 +736,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_dup_cursor_delete_first() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
|
||||
@@ -774,7 +776,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_walk() {
|
||||
let env = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
let value = Header::default();
|
||||
let key = 1u64;
|
||||
@@ -799,7 +801,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_walk_range() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -863,7 +865,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_walk_range_on_dup_table() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
let address0 = Address::ZERO;
|
||||
let address1 = Address::with_last_byte(1);
|
||||
@@ -923,7 +925,7 @@ mod tests {
|
||||
#[expect(clippy::reversed_empty_ranges)]
|
||||
#[test]
|
||||
fn db_cursor_walk_range_invalid() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (2, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -951,7 +953,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_walker() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -981,7 +983,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_reverse_walker() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1011,7 +1013,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_walk_back() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT (0, 0), (1, 0), (3, 0)
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1050,7 +1052,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_seek_exact_or_previous_key() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1074,7 +1076,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_insert() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1114,7 +1116,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_insert_dup() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let mut dup_cursor = tx.cursor_dup_write::<PlainStorageState>().unwrap();
|
||||
@@ -1132,7 +1134,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_delete_current_non_existent() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let key1 = Address::with_last_byte(1);
|
||||
@@ -1162,7 +1164,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_insert_wherever_cursor_is() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
// PUT
|
||||
@@ -1195,7 +1197,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_append() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1222,7 +1224,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_append_failure() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
// PUT
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
@@ -1257,7 +1259,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_upsert() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
let tx = db.tx_mut().expect(ERROR_INIT_TX);
|
||||
|
||||
let mut cursor = tx.cursor_write::<PlainAccountState>().unwrap();
|
||||
@@ -1292,7 +1294,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_cursor_dupsort_append() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
|
||||
let transition_id = 2;
|
||||
|
||||
@@ -1356,7 +1358,8 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_closure_put_get() {
|
||||
let path = TempDir::new().expect(ERROR_TEMPDIR).keep();
|
||||
let tempdir = TempDir::new().expect(ERROR_TEMPDIR);
|
||||
let path = tempdir.path();
|
||||
|
||||
let value = Account {
|
||||
nonce: 18446744073709551615,
|
||||
@@ -1367,7 +1370,7 @@ mod tests {
|
||||
.expect(ERROR_ETH_ADDRESS);
|
||||
|
||||
{
|
||||
let env = create_test_db_with_path(DatabaseEnvKind::RW, &path);
|
||||
let env = create_test_db_with_path(DatabaseEnvKind::RW, path);
|
||||
|
||||
// PUT
|
||||
let result = env.update(|tx| {
|
||||
@@ -1378,7 +1381,7 @@ mod tests {
|
||||
}
|
||||
|
||||
let env = DatabaseEnv::open(
|
||||
&path,
|
||||
path,
|
||||
DatabaseEnvKind::RO,
|
||||
DatabaseArguments::new(ClientVersion::default()),
|
||||
)
|
||||
@@ -1393,7 +1396,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_dup_sort() {
|
||||
let env = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
|
||||
let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047")
|
||||
.expect(ERROR_ETH_ADDRESS);
|
||||
|
||||
@@ -1437,7 +1440,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_walk_dup_with_not_existing_key() {
|
||||
let env = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
|
||||
let key = Address::from_str("0xa2c122be93b0074270ebee7f6b7292c7deb45047")
|
||||
.expect(ERROR_ETH_ADDRESS);
|
||||
|
||||
@@ -1465,7 +1468,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_iterate_over_all_dup_values() {
|
||||
let env = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
|
||||
let key1 = Address::from_str("0x1111111111111111111111111111111111111111")
|
||||
.expect(ERROR_ETH_ADDRESS);
|
||||
let key2 = Address::from_str("0x2222222222222222222222222222222222222222")
|
||||
@@ -1511,7 +1514,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn dup_value_with_same_subkey() {
|
||||
let env = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, env) = create_test_db(DatabaseEnvKind::RW);
|
||||
let key1 = Address::new([0x11; 20]);
|
||||
let key2 = Address::new([0x22; 20]);
|
||||
|
||||
@@ -1554,7 +1557,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn db_sharded_key() {
|
||||
let db: DatabaseEnv = create_test_db(DatabaseEnvKind::RW);
|
||||
let (_tempdir, db) = create_test_db(DatabaseEnvKind::RW);
|
||||
let real_key = address!("0xa2c122be93b0074270ebee7f6b7292c7deb45047");
|
||||
|
||||
let shards = 5;
|
||||
|
||||
@@ -133,7 +133,7 @@ pub enum Error {
|
||||
#[error("permission denied to setup database")]
|
||||
Permission,
|
||||
/// Unknown error code.
|
||||
#[error("unknown error code: {0}")]
|
||||
#[error("{}", Error::fmt_other(*.0))]
|
||||
Other(i32),
|
||||
}
|
||||
|
||||
@@ -215,6 +215,13 @@ impl Error {
|
||||
Self::Other(err_code) => *err_code,
|
||||
}
|
||||
}
|
||||
|
||||
fn fmt_other(code: i32) -> String {
|
||||
let mut s = String::with_capacity(1024);
|
||||
let desc = unsafe { ffi::mdbx_strerror_r(code, s.as_mut_ptr().cast(), 1024) };
|
||||
let desc = unsafe { std::ffi::CStr::from_ptr(desc) }.to_string_lossy();
|
||||
desc.into_owned()
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Error> for i32 {
|
||||
|
||||
@@ -97,7 +97,7 @@ impl TxnManager {
|
||||
}
|
||||
}
|
||||
};
|
||||
std::thread::Builder::new().name("mdbx-rs-txn-manager".to_string()).spawn(task).unwrap();
|
||||
std::thread::Builder::new().name("mdbx-rs-txn-mgr".to_string()).spawn(task).unwrap();
|
||||
}
|
||||
|
||||
pub(crate) fn send_message(&self, message: TxnManagerMessage) {
|
||||
|
||||
@@ -16,7 +16,7 @@ workspace = true
|
||||
reth-chainspec.workspace = true
|
||||
reth-execution-types.workspace = true
|
||||
reth-ethereum-primitives = { workspace = true, features = ["reth-codec"] }
|
||||
reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k1"] }
|
||||
reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k1", "dashmap"] }
|
||||
reth-errors.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
|
||||
@@ -31,6 +31,7 @@ reth-codecs.workspace = true
|
||||
reth-chain-state.workspace = true
|
||||
reth-node-types.workspace = true
|
||||
reth-static-file-types.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
# ethereum
|
||||
alloy-eips.workspace = true
|
||||
alloy-primitives.workspace = true
|
||||
@@ -50,7 +51,6 @@ metrics.workspace = true
|
||||
itertools.workspace = true
|
||||
notify = { workspace = true, default-features = false, features = ["macos_fsevent"] }
|
||||
parking_lot.workspace = true
|
||||
dashmap = { workspace = true, features = ["inline"] }
|
||||
strum.workspace = true
|
||||
eyre.workspace = true
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@ pub mod providers;
|
||||
pub use providers::{
|
||||
DatabaseProvider, DatabaseProviderRO, DatabaseProviderRW, HistoricalStateProvider,
|
||||
HistoricalStateProviderRef, LatestStateProvider, LatestStateProviderRef, ProviderFactory,
|
||||
PruneShardOutcome, SaveBlocksMode, StaticFileAccess, StaticFileProviderBuilder,
|
||||
PruneShardOutcome, PrunedIndices, SaveBlocksMode, StaticFileAccess, StaticFileProviderBuilder,
|
||||
StaticFileWriteCtx, StaticFileWriter,
|
||||
};
|
||||
|
||||
|
||||
@@ -745,9 +745,10 @@ mod tests {
|
||||
fn provider_factory_with_database_path() {
|
||||
let chain_spec = ChainSpecBuilder::mainnet().build();
|
||||
let (_static_dir, static_dir_path) = create_test_static_files_dir();
|
||||
let (_, rocksdb_path) = create_test_rocksdb_dir();
|
||||
let (_rocksdb_dir, rocksdb_path) = create_test_rocksdb_dir();
|
||||
let _db_tempdir = tempfile::TempDir::new().expect(ERROR_TEMPDIR);
|
||||
let factory = ProviderFactory::<MockNodeTypesWithDB<DatabaseEnv>>::new_with_database_path(
|
||||
tempfile::TempDir::new().expect(ERROR_TEMPDIR).keep(),
|
||||
_db_tempdir.path(),
|
||||
Arc::new(chain_spec),
|
||||
DatabaseArguments::new(Default::default()),
|
||||
StaticFileProvider::read_write(static_dir_path).unwrap(),
|
||||
@@ -785,8 +786,9 @@ mod tests {
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
..PruneModes::default()
|
||||
};
|
||||
let factory = create_test_provider_factory();
|
||||
let provider = factory.with_prune_modes(prune_modes).provider_rw().unwrap();
|
||||
// Keep factory alive until provider is dropped to prevent TempDatabase cleanup
|
||||
let factory = create_test_provider_factory().with_prune_modes(prune_modes);
|
||||
let provider = factory.provider_rw().unwrap();
|
||||
assert_matches!(provider.insert_block(&block.clone().try_recover().unwrap()), Ok(_));
|
||||
assert_matches!(provider.transaction_sender(0), Ok(None));
|
||||
assert_matches!(
|
||||
|
||||
@@ -64,6 +64,7 @@ use reth_storage_api::{
|
||||
StorageSettingsCache, TryIntoHistoricalStateProvider, WriteStateInput,
|
||||
};
|
||||
use reth_storage_errors::provider::{ProviderResult, StaticFileWriterError};
|
||||
use reth_tasks::spawn_scoped_os_thread;
|
||||
use reth_trie::{
|
||||
updates::{StorageTrieUpdatesSorted, TrieUpdatesSorted},
|
||||
HashedPostStateSorted, StoredNibbles,
|
||||
@@ -552,7 +553,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
|
||||
thread::scope(|s| {
|
||||
// SF writes
|
||||
let sf_handle = s.spawn(|| {
|
||||
let sf_handle = spawn_scoped_os_thread(s, "static-files", || {
|
||||
let start = Instant::now();
|
||||
sf_provider.write_blocks_data(&blocks, &tx_nums, sf_ctx)?;
|
||||
Ok::<_, ProviderError>(start.elapsed())
|
||||
@@ -561,7 +562,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
// RocksDB writes
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
let rocksdb_handle = rocksdb_ctx.storage_settings.any_in_rocksdb().then(|| {
|
||||
s.spawn(|| {
|
||||
spawn_scoped_os_thread(s, "rocksdb", || {
|
||||
let start = Instant::now();
|
||||
rocksdb_provider.write_blocks_data(&blocks, &tx_nums, rocksdb_ctx)?;
|
||||
Ok::<_, ProviderError>(start.elapsed())
|
||||
|
||||
@@ -39,8 +39,8 @@ pub use consistent::ConsistentProvider;
|
||||
pub(crate) mod rocksdb;
|
||||
|
||||
pub use rocksdb::{
|
||||
PruneShardOutcome, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider, RocksDBRawIter,
|
||||
RocksDBStats, RocksDBTableStats, RocksTx,
|
||||
PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider,
|
||||
RocksDBRawIter, RocksDBStats, RocksDBTableStats, RocksTx,
|
||||
};
|
||||
|
||||
/// Helper trait to bound [`NodeTypes`] so that combined with database they satisfy
|
||||
|
||||
@@ -6,6 +6,6 @@ mod provider;
|
||||
|
||||
pub(crate) use provider::{PendingRocksDBBatches, RocksDBWriteCtx};
|
||||
pub use provider::{
|
||||
PruneShardOutcome, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider, RocksDBRawIter,
|
||||
RocksDBStats, RocksDBTableStats, RocksTx,
|
||||
PruneShardOutcome, PrunedIndices, RocksDBBatch, RocksDBBuilder, RocksDBIter, RocksDBProvider,
|
||||
RocksDBRawIter, RocksDBStats, RocksDBTableStats, RocksTx,
|
||||
};
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -219,3 +219,14 @@ pub enum PruneShardOutcome {
|
||||
/// Shard was unchanged (no blocks <= `to_block`).
|
||||
Unchanged,
|
||||
}
|
||||
|
||||
/// Tracks pruning outcomes for batch operations (stub).
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub struct PrunedIndices {
|
||||
/// Number of shards completely deleted.
|
||||
pub deleted: usize,
|
||||
/// Number of shards that were updated (filtered but still have entries).
|
||||
pub updated: usize,
|
||||
/// Number of shards that were unchanged.
|
||||
pub unchanged: usize,
|
||||
}
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
use metrics::{Counter, Histogram};
|
||||
use parking_lot::RwLock;
|
||||
use reth_chain_state::LazyOverlay;
|
||||
use reth_db_api::DatabaseError;
|
||||
use reth_errors::{ProviderError, ProviderResult};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::dashmap::{self, DashMap};
|
||||
use reth_prune_types::PruneSegment;
|
||||
use reth_stages_types::StageId;
|
||||
use reth_storage_api::{
|
||||
@@ -22,7 +22,6 @@ use reth_trie_db::{
|
||||
ChangesetCache, DatabaseHashedCursorFactory, DatabaseHashedPostState, DatabaseTrieCursorFactory,
|
||||
};
|
||||
use std::{
|
||||
collections::{hash_map::Entry, HashMap},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
@@ -102,7 +101,7 @@ pub struct OverlayStateProviderFactory<F> {
|
||||
metrics: OverlayStateProviderMetrics,
|
||||
/// A cache which maps `db_tip -> Overlay`. If the db tip changes during usage of the factory
|
||||
/// then a new entry will get added to this, but in most cases only one entry is present.
|
||||
overlay_cache: Arc<RwLock<HashMap<BlockNumber, Overlay>>>,
|
||||
overlay_cache: Arc<DashMap<BlockNumber, Overlay>>,
|
||||
}
|
||||
|
||||
impl<F> OverlayStateProviderFactory<F> {
|
||||
@@ -419,17 +418,16 @@ where
|
||||
let db_tip_block = self.get_db_tip_block_number(provider)?;
|
||||
|
||||
// If the overlay is present in the cache then return it directly.
|
||||
if let Some(overlay) = self.overlay_cache.as_ref().read().get(&db_tip_block) {
|
||||
return Ok(overlay.clone());
|
||||
if let Some(entry) = self.overlay_cache.get(&db_tip_block) {
|
||||
return Ok(entry.value().clone());
|
||||
}
|
||||
|
||||
// If the overlay is not present then we need to calculate a new one. We grab a write lock,
|
||||
// and then check the cache again in case some other thread populated the cache since we
|
||||
// checked with the read-lock. If still not present we calculate and populate.
|
||||
// If the overlay is not present then we need to calculate a new one.
|
||||
// DashMap's entry API handles the race condition internally.
|
||||
let mut cache_miss = false;
|
||||
let overlay = match self.overlay_cache.as_ref().write().entry(db_tip_block) {
|
||||
Entry::Occupied(entry) => entry.get().clone(),
|
||||
Entry::Vacant(entry) => {
|
||||
let overlay = match self.overlay_cache.entry(db_tip_block) {
|
||||
dashmap::Entry::Occupied(entry) => entry.get().clone(),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
cache_miss = true;
|
||||
let overlay = self.calculate_overlay(provider, db_tip_block)?;
|
||||
entry.insert(overlay.clone());
|
||||
|
||||
@@ -11,7 +11,6 @@ use crate::{
|
||||
use alloy_consensus::{transaction::TransactionMeta, Header};
|
||||
use alloy_eips::{eip2718::Encodable2718, BlockHashOrNumber};
|
||||
use alloy_primitives::{b256, keccak256, Address, BlockHash, BlockNumber, TxHash, TxNumber, B256};
|
||||
use dashmap::DashMap;
|
||||
use notify::{RecommendedWatcher, RecursiveMode, Watcher};
|
||||
use parking_lot::RwLock;
|
||||
use reth_chain_state::ExecutedBlock;
|
||||
@@ -34,8 +33,8 @@ use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
use reth_nippy_jar::{NippyJar, NippyJarChecker, CONFIG_FILE_EXTENSION};
|
||||
use reth_node_types::NodePrimitives;
|
||||
use reth_primitives_traits::{
|
||||
AlloyBlockHeader as _, BlockBody as _, RecoveredBlock, SealedHeader, SignedTransaction,
|
||||
StorageEntry,
|
||||
dashmap::DashMap, AlloyBlockHeader as _, BlockBody as _, RecoveredBlock, SealedHeader,
|
||||
SignedTransaction, StorageEntry,
|
||||
};
|
||||
use reth_prune_types::PruneSegment;
|
||||
use reth_stages_types::PipelineTarget;
|
||||
@@ -48,6 +47,7 @@ use reth_storage_api::{
|
||||
StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_storage_errors::provider::{ProviderError, ProviderResult, StaticFileWriterError};
|
||||
use reth_tasks::spawn_scoped_os_thread;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
fmt::Debug,
|
||||
@@ -670,7 +670,7 @@ impl<N: NodePrimitives> StaticFileProvider<N> {
|
||||
where
|
||||
F: FnOnce(&mut StaticFileProviderRWRefMut<'_, N>) -> ProviderResult<()> + Send + 'env,
|
||||
{
|
||||
scope.spawn(move || {
|
||||
spawn_scoped_os_thread(scope, segment.as_short_str(), move || {
|
||||
let mut w = self.get_writer(first_block_number, segment)?;
|
||||
f(&mut w)?;
|
||||
w.sync_all()
|
||||
|
||||
@@ -17,7 +17,8 @@ use reth_storage_errors::provider::{ProviderError, ProviderResult};
|
||||
use std::{ops::Deref, sync::Arc};
|
||||
|
||||
/// Alias type for each specific `NippyJar`.
|
||||
type LoadedJarRef<'a> = dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
|
||||
type LoadedJarRef<'a> =
|
||||
reth_primitives_traits::dashmap::mapref::one::Ref<'a, (u64, StaticFileSegment), LoadedJar>;
|
||||
|
||||
/// Helper type to reuse an associated static file mmap handle on created cursors.
|
||||
#[derive(Debug)]
|
||||
|
||||
@@ -204,8 +204,9 @@ impl<N> std::ops::Deref for StaticFileProviderRWRefMut<'_, N> {
|
||||
/// Extends `StaticFileProvider` with writing capabilities
|
||||
pub struct StaticFileProviderRW<N> {
|
||||
/// Reference back to the provider. We need [Weak] here because [`StaticFileProviderRW`] is
|
||||
/// stored in a [`dashmap::DashMap`] inside the parent [`StaticFileProvider`].which is an
|
||||
/// [Arc]. If we were to use an [Arc] here, we would create a reference cycle.
|
||||
/// stored in a [`reth_primitives_traits::dashmap::DashMap`] inside the parent
|
||||
/// [`StaticFileProvider`].which is an [Arc]. If we were to use an [Arc] here, we would
|
||||
/// create a reference cycle.
|
||||
reader: Weak<StaticFileProviderInner<N>>,
|
||||
/// A [`NippyJarWriter`] instance.
|
||||
writer: NippyJarWriter<SegmentHeader>,
|
||||
|
||||
@@ -4,12 +4,7 @@ use crate::{
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use reth_chainspec::{ChainSpec, MAINNET};
|
||||
use reth_db::{
|
||||
test_utils::{
|
||||
create_test_rocksdb_dir, create_test_rw_db, create_test_static_files_dir, TempDatabase,
|
||||
},
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_db::{test_utils::TempDatabase, DatabaseEnv};
|
||||
use reth_errors::ProviderResult;
|
||||
use reth_ethereum_engine_primitives::EthEngineTypes;
|
||||
use reth_node_types::NodeTypesWithDBAdapter;
|
||||
@@ -55,14 +50,23 @@ pub fn create_test_provider_factory_with_chain_spec(
|
||||
pub fn create_test_provider_factory_with_node_types<N: NodeTypesForProvider>(
|
||||
chain_spec: Arc<N::ChainSpec>,
|
||||
) -> ProviderFactory<NodeTypesWithDBAdapter<N, Arc<TempDatabase<DatabaseEnv>>>> {
|
||||
let (static_dir, _) = create_test_static_files_dir();
|
||||
let (rocksdb_dir, _) = create_test_rocksdb_dir();
|
||||
let db = create_test_rw_db();
|
||||
let rocksdb_path = rocksdb_dir.keep();
|
||||
// Create a single temp directory that contains all data dirs (db, static_files, rocksdb).
|
||||
// TempDatabase will clean up the entire directory on drop.
|
||||
let datadir_path = reth_db::test_utils::tempdir_path();
|
||||
|
||||
let static_files_path = datadir_path.join("static_files");
|
||||
let rocksdb_path = datadir_path.join("rocksdb");
|
||||
|
||||
// Create static_files directory
|
||||
std::fs::create_dir_all(&static_files_path).expect("failed to create static_files dir");
|
||||
|
||||
// Create database with the datadir path so TempDatabase cleans up everything on drop
|
||||
let db = reth_db::test_utils::create_test_rw_db_with_datadir(&datadir_path);
|
||||
|
||||
ProviderFactory::new(
|
||||
db,
|
||||
chain_spec,
|
||||
StaticFileProvider::read_write(static_dir.keep()).expect("static file provider"),
|
||||
StaticFileProvider::read_write(static_files_path).expect("static file provider"),
|
||||
RocksDBBuilder::new(&rocksdb_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
|
||||
@@ -40,7 +40,7 @@ tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
|
||||
|
||||
# other
|
||||
tracing.workspace = true
|
||||
parking_lot.workspace = true
|
||||
dashmap = { workspace = true, features = ["inline"] }
|
||||
|
||||
# revm
|
||||
revm.workspace = true
|
||||
|
||||
@@ -27,13 +27,11 @@
|
||||
use alloy_consensus::{constants::KECCAK_EMPTY, BlockHeader};
|
||||
use alloy_eips::{BlockHashOrNumber, BlockNumberOrTag};
|
||||
use alloy_network::{primitives::HeaderResponse, BlockResponse};
|
||||
use alloy_primitives::{
|
||||
map::HashMap, Address, BlockHash, BlockNumber, StorageKey, TxHash, TxNumber, B256, U256,
|
||||
};
|
||||
use alloy_primitives::{Address, BlockHash, BlockNumber, StorageKey, TxHash, TxNumber, B256, U256};
|
||||
use alloy_provider::{ext::DebugApi, network::Network, Provider};
|
||||
use alloy_rpc_types::{AccountInfo, BlockId};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use parking_lot::RwLock;
|
||||
use dashmap::DashMap;
|
||||
use reth_chainspec::{ChainInfo, ChainSpecProvider};
|
||||
use reth_db_api::{
|
||||
mock::{DatabaseMock, TxMock},
|
||||
@@ -912,7 +910,7 @@ where
|
||||
/// Cached bytecode for accounts
|
||||
///
|
||||
/// Since the state provider is short-lived, we don't worry about memory leaks.
|
||||
code_store: RwLock<HashMap<B256, Bytecode>>,
|
||||
code_store: DashMap<B256, Bytecode>,
|
||||
/// Whether to use Reth-specific RPC methods for better performance
|
||||
reth_rpc_support: bool,
|
||||
}
|
||||
@@ -942,7 +940,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
|
||||
network: std::marker::PhantomData,
|
||||
chain_spec: None,
|
||||
compute_state_root: false,
|
||||
code_store: RwLock::new(HashMap::default()),
|
||||
code_store: Default::default(),
|
||||
reth_rpc_support: true,
|
||||
}
|
||||
}
|
||||
@@ -960,7 +958,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
|
||||
network: std::marker::PhantomData,
|
||||
chain_spec: Some(chain_spec),
|
||||
compute_state_root: false,
|
||||
code_store: RwLock::new(HashMap::default()),
|
||||
code_store: Default::default(),
|
||||
reth_rpc_support: true,
|
||||
}
|
||||
}
|
||||
@@ -982,7 +980,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
|
||||
network: self.network,
|
||||
chain_spec: self.chain_spec.clone(),
|
||||
compute_state_root: self.compute_state_root,
|
||||
code_store: RwLock::new(HashMap::default()),
|
||||
code_store: Default::default(),
|
||||
reth_rpc_support: self.reth_rpc_support,
|
||||
}
|
||||
}
|
||||
@@ -1038,9 +1036,7 @@ impl<P: Clone, Node: NodeTypes, N> RpcBlockchainStateProvider<P, Node, N> {
|
||||
let code_hash = account_info.code_hash();
|
||||
if code_hash != KECCAK_EMPTY {
|
||||
// Insert code into the cache
|
||||
self.code_store
|
||||
.write()
|
||||
.insert(code_hash, Bytecode::new_raw(account_info.code.clone()));
|
||||
self.code_store.insert(code_hash, Bytecode::new_raw(account_info.code.clone()));
|
||||
}
|
||||
|
||||
Ok(account_info)
|
||||
@@ -1119,7 +1115,7 @@ where
|
||||
{
|
||||
fn bytecode_by_hash(&self, code_hash: &B256) -> Result<Option<Bytecode>, ProviderError> {
|
||||
if !self.reth_rpc_support {
|
||||
return Ok(self.code_store.read().get(code_hash).cloned());
|
||||
return Ok(self.code_store.get(code_hash).map(|entry| entry.value().clone()));
|
||||
}
|
||||
|
||||
self.block_on_async(async {
|
||||
|
||||
@@ -30,6 +30,7 @@ use std::{
|
||||
Arc, OnceLock,
|
||||
},
|
||||
task::{ready, Context, Poll},
|
||||
thread,
|
||||
};
|
||||
use tokio::{
|
||||
runtime::Handle,
|
||||
@@ -48,6 +49,51 @@ pub mod pool;
|
||||
/// Global [`TaskExecutor`] instance that can be accessed from anywhere.
|
||||
static GLOBAL_EXECUTOR: OnceLock<TaskExecutor> = OnceLock::new();
|
||||
|
||||
/// Spawns an OS thread with the current tokio runtime context propagated.
|
||||
///
|
||||
/// This function captures the current tokio runtime handle (if available) and enters it
|
||||
/// in the newly spawned thread. This ensures that code running in the spawned thread can
|
||||
/// use [`Handle::current()`], [`Handle::spawn_blocking()`], and other tokio utilities that
|
||||
/// require a runtime context.
|
||||
#[track_caller]
|
||||
pub fn spawn_os_thread<F, T>(name: &str, f: F) -> thread::JoinHandle<T>
|
||||
where
|
||||
F: FnOnce() -> T + Send + 'static,
|
||||
T: Send + 'static,
|
||||
{
|
||||
let handle = Handle::try_current().ok();
|
||||
thread::Builder::new()
|
||||
.name(name.to_string())
|
||||
.spawn(move || {
|
||||
let _guard = handle.as_ref().map(Handle::enter);
|
||||
f()
|
||||
})
|
||||
.unwrap_or_else(|e| panic!("failed to spawn thread {name:?}: {e}"))
|
||||
}
|
||||
|
||||
/// Spawns a scoped OS thread with the current tokio runtime context propagated.
|
||||
///
|
||||
/// This is the scoped thread version of [`spawn_os_thread`], for use with [`std::thread::scope`].
|
||||
#[track_caller]
|
||||
pub fn spawn_scoped_os_thread<'scope, 'env, F, T>(
|
||||
scope: &'scope thread::Scope<'scope, 'env>,
|
||||
name: &str,
|
||||
f: F,
|
||||
) -> thread::ScopedJoinHandle<'scope, T>
|
||||
where
|
||||
F: FnOnce() -> T + Send + 'scope,
|
||||
T: Send + 'scope,
|
||||
{
|
||||
let handle = Handle::try_current().ok();
|
||||
thread::Builder::new()
|
||||
.name(name.to_string())
|
||||
.spawn_scoped(scope, move || {
|
||||
let _guard = handle.as_ref().map(Handle::enter);
|
||||
f()
|
||||
})
|
||||
.unwrap_or_else(|e| panic!("failed to spawn scoped thread {name:?}: {e}"))
|
||||
}
|
||||
|
||||
/// A type that can spawn tasks.
|
||||
///
|
||||
/// The main purpose of this type is to abstract over [`TaskExecutor`] so it's more convenient to
|
||||
|
||||
@@ -610,6 +610,14 @@ where
|
||||
self.pool.pending_transactions()
|
||||
}
|
||||
|
||||
fn get_pending_transaction_by_sender_and_nonce(
|
||||
&self,
|
||||
sender: Address,
|
||||
nonce: u64,
|
||||
) -> Option<Arc<ValidPoolTransaction<Self::Transaction>>> {
|
||||
self.pool.get_pending_transaction_by_sender_and_nonce(sender, nonce)
|
||||
}
|
||||
|
||||
fn pending_transactions_max(
|
||||
&self,
|
||||
max: usize,
|
||||
|
||||
@@ -1090,6 +1090,16 @@ where
|
||||
self.get_pool_data().get_transactions_by_sender(sender_id)
|
||||
}
|
||||
|
||||
/// Returns a pending transaction sent by the given sender with the given nonce.
|
||||
pub fn get_pending_transaction_by_sender_and_nonce(
|
||||
&self,
|
||||
sender: Address,
|
||||
nonce: u64,
|
||||
) -> Option<Arc<ValidPoolTransaction<T::Transaction>>> {
|
||||
let sender_id = self.get_sender_id(sender);
|
||||
self.get_pool_data().get_pending_transaction_by_sender_and_nonce(sender_id, nonce)
|
||||
}
|
||||
|
||||
/// Returns all queued transactions of the address by sender
|
||||
pub fn get_queued_transactions_by_sender(
|
||||
&self,
|
||||
|
||||
@@ -560,6 +560,18 @@ impl<T: TransactionOrdering> TxPool<T> {
|
||||
self.all_transactions.txs_iter(sender).map(|(_, tx)| Arc::clone(&tx.transaction)).collect()
|
||||
}
|
||||
|
||||
/// Returns a pending transaction sent by the given sender with the given nonce.
|
||||
pub(crate) fn get_pending_transaction_by_sender_and_nonce(
|
||||
&self,
|
||||
sender: SenderId,
|
||||
nonce: u64,
|
||||
) -> Option<Arc<ValidPoolTransaction<T::Transaction>>> {
|
||||
self.all_transactions
|
||||
.txs_iter(sender)
|
||||
.find(|(id, tx)| id.nonce == nonce && tx.subpool == SubPool::Pending)
|
||||
.map(|(_, tx)| Arc::clone(&tx.transaction))
|
||||
}
|
||||
|
||||
/// Updates only the pending fees without triggering subpool updates.
|
||||
/// Returns the previous base fee and blob fee values.
|
||||
const fn update_pending_fees_only(
|
||||
|
||||
@@ -405,6 +405,16 @@ pub trait TransactionPool: Clone + Debug + Send + Sync {
|
||||
/// Consumer: RPC
|
||||
fn pending_transactions(&self) -> Vec<Arc<ValidPoolTransaction<Self::Transaction>>>;
|
||||
|
||||
/// Returns a pending transaction if it exists and is ready for immediate execution
|
||||
/// (i.e., has the lowest nonce among the sender's pending transactions).
|
||||
fn get_pending_transaction_by_sender_and_nonce(
|
||||
&self,
|
||||
sender: Address,
|
||||
nonce: u64,
|
||||
) -> Option<Arc<ValidPoolTransaction<Self::Transaction>>> {
|
||||
self.best_transactions().find(|tx| tx.sender() == sender && tx.nonce() == nonce)
|
||||
}
|
||||
|
||||
/// Returns first `max` transactions that can be included in the next block.
|
||||
/// See <https://github.com/paradigmxyz/reth/issues/12767#issuecomment-2493223579>
|
||||
///
|
||||
|
||||
@@ -85,4 +85,6 @@ pub mod serde_bincode_compat {
|
||||
}
|
||||
|
||||
/// Re-export
|
||||
pub use alloy_trie::{nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, EMPTY_ROOT_HASH};
|
||||
pub use alloy_trie::{
|
||||
nodes::*, proof, BranchNodeCompact, HashBuilder, TrieMask, TrieMaskIter, EMPTY_ROOT_HASH,
|
||||
};
|
||||
|
||||
@@ -13,7 +13,7 @@ workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["dashmap"] }
|
||||
reth-execution-errors.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
@@ -29,7 +29,6 @@ alloy-primitives.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
# misc
|
||||
dashmap.workspace = true
|
||||
thiserror.workspace = true
|
||||
derive_more.workspace = true
|
||||
rayon.workspace = true
|
||||
|
||||
@@ -42,8 +42,8 @@ use alloy_primitives::{
|
||||
};
|
||||
use alloy_rlp::{BufMut, Encodable};
|
||||
use crossbeam_channel::{unbounded, Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use dashmap::DashMap;
|
||||
use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, StateProofError};
|
||||
use reth_primitives_traits::dashmap::{self, DashMap};
|
||||
use reth_provider::{DatabaseProviderROFactory, ProviderError, ProviderResult};
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
@@ -155,10 +155,10 @@ impl ProofWorkerHandle {
|
||||
|
||||
// Initialize availability counters at zero. Each worker will increment when it
|
||||
// successfully initializes, ensuring only healthy workers are counted.
|
||||
let storage_available_workers = Arc::new(AtomicUsize::new(0));
|
||||
let account_available_workers = Arc::new(AtomicUsize::new(0));
|
||||
let storage_available_workers = Arc::<AtomicUsize>::default();
|
||||
let account_available_workers = Arc::<AtomicUsize>::default();
|
||||
|
||||
let cached_storage_roots = Arc::new(DashMap::new());
|
||||
let cached_storage_roots = Arc::<DashMap<_, _>>::default();
|
||||
|
||||
debug!(
|
||||
target: "trie::proof_task",
|
||||
@@ -168,101 +168,112 @@ impl ProofWorkerHandle {
|
||||
"Spawning proof worker pools"
|
||||
);
|
||||
|
||||
let parent_span =
|
||||
debug_span!(target: "trie::proof_task", "storage proof workers", ?storage_worker_count)
|
||||
.entered();
|
||||
// Spawn storage workers
|
||||
for worker_id in 0..storage_worker_count {
|
||||
let span = debug_span!(target: "trie::proof_task", "storage worker", ?worker_id);
|
||||
let task_ctx_clone = task_ctx.clone();
|
||||
let work_rx_clone = storage_work_rx.clone();
|
||||
let storage_available_workers_clone = storage_available_workers.clone();
|
||||
let cached_storage_roots = cached_storage_roots.clone();
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics = ProofTaskTrieMetrics::default();
|
||||
#[cfg(feature = "metrics")]
|
||||
let cursor_metrics = ProofTaskCursorMetrics::new();
|
||||
|
||||
let _guard = span.enter();
|
||||
let worker = StorageProofWorker::new(
|
||||
task_ctx_clone,
|
||||
work_rx_clone,
|
||||
worker_id,
|
||||
storage_available_workers_clone,
|
||||
cached_storage_roots,
|
||||
#[cfg(feature = "metrics")]
|
||||
metrics,
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics,
|
||||
)
|
||||
.with_v2_proofs(v2_proofs_enabled);
|
||||
if let Err(error) = worker.run() {
|
||||
error!(
|
||||
target: "trie::proof_task",
|
||||
worker_id,
|
||||
?error,
|
||||
"Storage worker failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
drop(parent_span);
|
||||
|
||||
let parent_span =
|
||||
debug_span!(target: "trie::proof_task", "account proof workers", ?account_worker_count)
|
||||
.entered();
|
||||
// Spawn account workers
|
||||
for worker_id in 0..account_worker_count {
|
||||
let span = debug_span!(target: "trie::proof_task", "account worker", ?worker_id);
|
||||
let task_ctx_clone = task_ctx.clone();
|
||||
let work_rx_clone = account_work_rx.clone();
|
||||
let storage_work_tx_clone = storage_work_tx.clone();
|
||||
let account_available_workers_clone = account_available_workers.clone();
|
||||
let cached_storage_roots = cached_storage_roots.clone();
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics = ProofTaskTrieMetrics::default();
|
||||
#[cfg(feature = "metrics")]
|
||||
let cursor_metrics = ProofTaskCursorMetrics::new();
|
||||
|
||||
let _guard = span.enter();
|
||||
let worker = AccountProofWorker::new(
|
||||
task_ctx_clone,
|
||||
work_rx_clone,
|
||||
worker_id,
|
||||
storage_work_tx_clone,
|
||||
account_available_workers_clone,
|
||||
cached_storage_roots,
|
||||
#[cfg(feature = "metrics")]
|
||||
metrics,
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics,
|
||||
)
|
||||
.with_v2_proofs(v2_proofs_enabled);
|
||||
if let Err(error) = worker.run() {
|
||||
error!(
|
||||
target: "trie::proof_task",
|
||||
worker_id,
|
||||
?error,
|
||||
"Account worker failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
drop(parent_span);
|
||||
|
||||
Self {
|
||||
storage_work_tx,
|
||||
let this = Self {
|
||||
storage_work_tx: storage_work_tx.clone(),
|
||||
account_work_tx,
|
||||
storage_available_workers,
|
||||
account_available_workers,
|
||||
storage_available_workers: storage_available_workers.clone(),
|
||||
account_available_workers: account_available_workers.clone(),
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
v2_proofs_enabled,
|
||||
}
|
||||
};
|
||||
|
||||
// Clone for the first spawn_blocking (storage workers)
|
||||
let task_ctx_for_storage = task_ctx.clone();
|
||||
let executor_for_storage = executor.clone();
|
||||
let cached_storage_roots_for_storage = cached_storage_roots.clone();
|
||||
|
||||
executor_for_storage.clone().spawn_blocking(move || {
|
||||
let parent_span =
|
||||
debug_span!(target: "trie::proof_task", "storage proof workers", ?storage_worker_count)
|
||||
.entered();
|
||||
// Spawn storage workers
|
||||
for worker_id in 0..storage_worker_count {
|
||||
let span = debug_span!(target: "trie::proof_task", "storage worker", ?worker_id);
|
||||
let task_ctx_clone = task_ctx_for_storage.clone();
|
||||
let work_rx_clone = storage_work_rx.clone();
|
||||
let storage_available_workers_clone = storage_available_workers.clone();
|
||||
let cached_storage_roots = cached_storage_roots_for_storage.clone();
|
||||
|
||||
executor_for_storage.spawn_blocking(move || {
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics = ProofTaskTrieMetrics::default();
|
||||
#[cfg(feature = "metrics")]
|
||||
let cursor_metrics = ProofTaskCursorMetrics::new();
|
||||
|
||||
let _guard = span.enter();
|
||||
let worker = StorageProofWorker::new(
|
||||
task_ctx_clone,
|
||||
work_rx_clone,
|
||||
worker_id,
|
||||
storage_available_workers_clone,
|
||||
cached_storage_roots,
|
||||
#[cfg(feature = "metrics")]
|
||||
metrics,
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics,
|
||||
)
|
||||
.with_v2_proofs(v2_proofs_enabled);
|
||||
if let Err(error) = worker.run() {
|
||||
error!(
|
||||
target: "trie::proof_task",
|
||||
worker_id,
|
||||
?error,
|
||||
"Storage worker failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
drop(parent_span);
|
||||
});
|
||||
|
||||
executor.clone().spawn_blocking(move || {
|
||||
let parent_span =
|
||||
debug_span!(target: "trie::proof_task", "account proof workers", ?account_worker_count)
|
||||
.entered();
|
||||
// Spawn account workers
|
||||
for worker_id in 0..account_worker_count {
|
||||
let span = debug_span!(target: "trie::proof_task", "account worker", ?worker_id);
|
||||
let task_ctx_clone = task_ctx.clone();
|
||||
let work_rx_clone = account_work_rx.clone();
|
||||
let storage_work_tx_clone = storage_work_tx.clone();
|
||||
let account_available_workers_clone = account_available_workers.clone();
|
||||
let cached_storage_roots = cached_storage_roots.clone();
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
#[cfg(feature = "metrics")]
|
||||
let metrics = ProofTaskTrieMetrics::default();
|
||||
#[cfg(feature = "metrics")]
|
||||
let cursor_metrics = ProofTaskCursorMetrics::new();
|
||||
|
||||
let _guard = span.enter();
|
||||
let worker = AccountProofWorker::new(
|
||||
task_ctx_clone,
|
||||
work_rx_clone,
|
||||
worker_id,
|
||||
storage_work_tx_clone,
|
||||
account_available_workers_clone,
|
||||
cached_storage_roots,
|
||||
#[cfg(feature = "metrics")]
|
||||
metrics,
|
||||
#[cfg(feature = "metrics")]
|
||||
cursor_metrics,
|
||||
)
|
||||
.with_v2_proofs(v2_proofs_enabled);
|
||||
if let Err(error) = worker.run() {
|
||||
error!(
|
||||
target: "trie::proof_task",
|
||||
worker_id,
|
||||
?error,
|
||||
"Account worker failed"
|
||||
);
|
||||
}
|
||||
});
|
||||
}
|
||||
drop(parent_span);
|
||||
});
|
||||
|
||||
this
|
||||
}
|
||||
|
||||
/// Returns whether V2 storage proofs are enabled for this worker pool.
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::{stats::ParallelTrieTracker, storage_root_targets::StorageRootTargets
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::{BufMut, Encodable};
|
||||
use itertools::Itertools;
|
||||
use reth_execution_errors::{StateProofError, StorageRootError};
|
||||
use reth_execution_errors::{SparseTrieError, StateProofError, StorageRootError};
|
||||
use reth_provider::{DatabaseProviderROFactory, ProviderError};
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
@@ -232,6 +232,9 @@ pub enum ParallelStateRootError {
|
||||
/// Provider error.
|
||||
#[error(transparent)]
|
||||
Provider(#[from] ProviderError),
|
||||
/// Sparse trie error.
|
||||
#[error(transparent)]
|
||||
SparseTrie(#[from] SparseTrieError),
|
||||
/// Other unspecified error.
|
||||
#[error("{_0}")]
|
||||
Other(String),
|
||||
@@ -244,6 +247,7 @@ impl From<ParallelStateRootError> for ProviderError {
|
||||
ParallelStateRootError::StorageRoot(StorageRootError::Database(error)) => {
|
||||
Self::Database(error)
|
||||
}
|
||||
ParallelStateRootError::SparseTrie(error) => Self::other(error),
|
||||
ParallelStateRootError::Other(other) => Self::Database(DatabaseError::Other(other)),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,6 +18,17 @@ impl MultiProofTargetsV2 {
|
||||
pub fn is_empty(&self) -> bool {
|
||||
self.account_targets.is_empty() && self.storage_targets.is_empty()
|
||||
}
|
||||
|
||||
/// Returns the number of items that will be considered during chunking.
|
||||
pub fn chunking_length(&self) -> usize {
|
||||
self.account_targets.len() +
|
||||
self.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
|
||||
/// Returns an iterator that yields chunks of the specified size.
|
||||
pub fn chunks(self, chunk_size: usize) -> impl Iterator<Item = Self> {
|
||||
ChunkedMultiProofTargetsV2::new(self, chunk_size)
|
||||
}
|
||||
}
|
||||
|
||||
/// An iterator that yields chunks of V2 proof targets of at most `size` account and storage
|
||||
|
||||
@@ -3,9 +3,8 @@ use alloy_primitives::{map::B256Map, B256};
|
||||
use alloy_rlp::Encodable;
|
||||
use core::cell::RefCell;
|
||||
use crossbeam_channel::Receiver as CrossbeamReceiver;
|
||||
use dashmap::DashMap;
|
||||
use reth_execution_errors::trie::StateProofError;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_primitives_traits::{dashmap::DashMap, Account};
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie::{
|
||||
hashed_cursor::HashedStorageCursor,
|
||||
|
||||
@@ -128,4 +128,16 @@ impl LowerSparseSubtrie {
|
||||
Self::Blind(None) => {}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a heuristic for the in-memory size of this subtrie in bytes.
|
||||
pub(crate) fn memory_size(&self) -> usize {
|
||||
match self {
|
||||
Self::Revealed(subtrie) | Self::Blind(Some(subtrie)) => subtrie.memory_size(),
|
||||
Self::Blind(None) => 0,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) fn wipe(&mut self) {
|
||||
*self = Self::default();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ use reth_execution_errors::{SparseTrieError, SparseTrieErrorKind, SparseTrieResu
|
||||
use reth_trie_common::{
|
||||
prefix_set::{PrefixSet, PrefixSetMut},
|
||||
BranchNodeMasks, BranchNodeMasksMap, BranchNodeRef, ExtensionNodeRef, LeafNodeRef, Nibbles,
|
||||
ProofTrieNode, RlpNode, TrieNode, CHILD_INDEX_RANGE,
|
||||
ProofTrieNode, RlpNode, TrieNode,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
provider::{RevealedNode, TrieNodeProvider},
|
||||
@@ -106,7 +106,7 @@ pub struct ParallelSparseTrie {
|
||||
/// This contains the trie nodes for the upper part of the trie.
|
||||
upper_subtrie: Box<SparseSubtrie>,
|
||||
/// An array containing the subtries at the second level of the trie.
|
||||
lower_subtries: [LowerSparseSubtrie; NUM_LOWER_SUBTRIES],
|
||||
lower_subtries: Box<[LowerSparseSubtrie; NUM_LOWER_SUBTRIES]>,
|
||||
/// Set of prefixes (key paths) that have been marked as updated.
|
||||
/// This is used to track which parts of the trie need to be recalculated.
|
||||
prefix_set: PrefixSetMut,
|
||||
@@ -138,7 +138,9 @@ impl Default for ParallelSparseTrie {
|
||||
nodes: HashMap::from_iter([(Nibbles::default(), SparseNode::Empty)]),
|
||||
..Default::default()
|
||||
}),
|
||||
lower_subtries: [const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES],
|
||||
lower_subtries: Box::new(
|
||||
[const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES],
|
||||
),
|
||||
prefix_set: PrefixSetMut::default(),
|
||||
updates: None,
|
||||
branch_node_masks: BranchNodeMasksMap::default(),
|
||||
@@ -184,11 +186,12 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
|ProofTrieNode { path: path_a, .. }, ProofTrieNode { path: path_b, .. }| {
|
||||
let subtrie_type_a = SparseSubtrieType::from_path(path_a);
|
||||
let subtrie_type_b = SparseSubtrieType::from_path(path_b);
|
||||
subtrie_type_a.cmp(&subtrie_type_b).then(path_a.cmp(path_b))
|
||||
subtrie_type_a.cmp(&subtrie_type_b).then_with(|| path_a.cmp(path_b))
|
||||
},
|
||||
);
|
||||
|
||||
// Update the top-level branch node masks. This is simple and can't be done in parallel.
|
||||
self.branch_node_masks.reserve(nodes.len());
|
||||
for ProofTrieNode { path, masks, .. } in &nodes {
|
||||
if let Some(branch_masks) = masks {
|
||||
self.branch_node_masks.insert(*path, *branch_masks);
|
||||
@@ -202,9 +205,7 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
.iter()
|
||||
.position(|n| !SparseSubtrieType::path_len_is_upper(n.path.len()))
|
||||
.unwrap_or(nodes.len());
|
||||
|
||||
let upper_nodes = &nodes[..num_upper_nodes];
|
||||
let lower_nodes = &nodes[num_upper_nodes..];
|
||||
let (upper_nodes, lower_nodes) = nodes.split_at(num_upper_nodes);
|
||||
|
||||
// Reserve the capacity of the upper subtrie's `nodes` HashMap before iterating, so we don't
|
||||
// end up making many small capacity changes as we loop.
|
||||
@@ -913,6 +914,19 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
fn take_updates(&mut self) -> SparseTrieUpdates {
|
||||
match self.updates.take() {
|
||||
Some(updates) => {
|
||||
// Sync branch_node_masks with what's being committed to DB.
|
||||
// This ensures that on subsequent root() calls, the masks reflect the actual
|
||||
// DB state, which is needed for correct removal detection.
|
||||
for (path, node) in &updates.updated_nodes {
|
||||
self.branch_node_masks.insert(
|
||||
*path,
|
||||
BranchNodeMasks { tree_mask: node.tree_mask, hash_mask: node.hash_mask },
|
||||
);
|
||||
}
|
||||
for path in &updates.removed_nodes {
|
||||
self.branch_node_masks.remove(path);
|
||||
}
|
||||
|
||||
// NOTE: we need to preserve Some case
|
||||
self.updates = Some(SparseTrieUpdates::with_capacity(
|
||||
updates.updated_nodes.len(),
|
||||
@@ -926,7 +940,9 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
|
||||
fn wipe(&mut self) {
|
||||
self.upper_subtrie.wipe();
|
||||
self.lower_subtries = [const { LowerSparseSubtrie::Blind(None) }; NUM_LOWER_SUBTRIES];
|
||||
for trie in &mut *self.lower_subtries {
|
||||
trie.wipe();
|
||||
}
|
||||
self.prefix_set = PrefixSetMut::all();
|
||||
self.updates = self.updates.is_some().then(SparseTrieUpdates::wiped);
|
||||
self.subtrie_heat.clear();
|
||||
@@ -935,7 +951,7 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
fn clear(&mut self) {
|
||||
self.upper_subtrie.clear();
|
||||
self.upper_subtrie.nodes.insert(Nibbles::default(), SparseNode::Empty);
|
||||
for subtrie in &mut self.lower_subtries {
|
||||
for subtrie in &mut *self.lower_subtries {
|
||||
subtrie.clear();
|
||||
}
|
||||
self.prefix_set.clear();
|
||||
@@ -1020,7 +1036,7 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
self.upper_subtrie.shrink_nodes_to(size_per_subtrie);
|
||||
|
||||
// Shrink lower subtries (works for both revealed and blind with allocation)
|
||||
for subtrie in &mut self.lower_subtries {
|
||||
for subtrie in &mut *self.lower_subtries {
|
||||
subtrie.shrink_nodes_to(size_per_subtrie);
|
||||
}
|
||||
|
||||
@@ -1039,7 +1055,7 @@ impl SparseTrie for ParallelSparseTrie {
|
||||
self.upper_subtrie.shrink_values_to(size_per_subtrie);
|
||||
|
||||
// Shrink lower subtries (works for both revealed and blind with allocation)
|
||||
for subtrie in &mut self.lower_subtries {
|
||||
for subtrie in &mut *self.lower_subtries {
|
||||
subtrie.shrink_values_to(size_per_subtrie);
|
||||
}
|
||||
}
|
||||
@@ -1093,7 +1109,7 @@ impl SparseTrieExt for ParallelSparseTrie {
|
||||
SparseNode::Extension { key, .. } => {
|
||||
let mut child = path;
|
||||
child.extend(key);
|
||||
SmallVec::from_buf_and_len([child; 16], 1)
|
||||
SmallVec::from_slice(&[child])
|
||||
}
|
||||
SparseNode::Branch { state_mask, .. } => {
|
||||
let mut children = SmallVec::new();
|
||||
@@ -1167,7 +1183,7 @@ impl SparseTrieExt for ParallelSparseTrie {
|
||||
// Upper prune roots that are prefixes of lower subtrie root paths cause the entire
|
||||
// subtrie to be cleared (preserving allocations for reuse).
|
||||
if !roots_upper.is_empty() {
|
||||
for subtrie in &mut self.lower_subtries {
|
||||
for subtrie in &mut *self.lower_subtries {
|
||||
let should_clear = subtrie.as_revealed_ref().is_some_and(|s| {
|
||||
let search_idx = roots_upper.partition_point(|(root, _)| root <= &s.path);
|
||||
search_idx > 0 && s.path.starts_with(&roots_upper[search_idx - 1].0)
|
||||
@@ -1441,8 +1457,6 @@ impl ParallelSparseTrie {
|
||||
///
|
||||
/// Returns `None` if a lower subtrie does not exist for the given path.
|
||||
fn subtrie_for_path(&self, path: &Nibbles) -> Option<&SparseSubtrie> {
|
||||
// We can't just call `lower_subtrie_for_path` and return `upper_subtrie` if it returns
|
||||
// None, because Rust complains about double mutable borrowing `self`.
|
||||
if SparseSubtrieType::path_len_is_upper(path.len()) {
|
||||
Some(&self.upper_subtrie)
|
||||
} else {
|
||||
@@ -2073,15 +2087,13 @@ impl ParallelSparseTrie {
|
||||
// in the lower subtrie, and reveal accordingly.
|
||||
if !SparseSubtrieType::path_len_is_upper(path.len() + 1) {
|
||||
let mut stack_ptr = branch.as_ref().first_child_index();
|
||||
for idx in CHILD_INDEX_RANGE {
|
||||
if branch.state_mask.is_bit_set(idx) {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(idx);
|
||||
self.lower_subtrie_for_path_mut(&child_path)
|
||||
.expect("child_path must have a lower subtrie")
|
||||
.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
|
||||
stack_ptr += 1;
|
||||
}
|
||||
for idx in branch.state_mask.iter() {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(idx);
|
||||
self.lower_subtrie_for_path_mut(&child_path)
|
||||
.expect("child_path must have a lower subtrie")
|
||||
.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
|
||||
stack_ptr += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2118,6 +2130,50 @@ impl ParallelSparseTrie {
|
||||
self.subtrie_heat.mark_modified(index);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a heuristic for the in-memory size of this trie in bytes.
|
||||
///
|
||||
/// This is an approximation that accounts for:
|
||||
/// - The upper subtrie nodes and values
|
||||
/// - All revealed lower subtries nodes and values
|
||||
/// - The prefix set keys
|
||||
/// - The branch node masks map
|
||||
/// - Updates if retained
|
||||
/// - Update action buffers
|
||||
///
|
||||
/// Note: Heap allocations for hash maps may be larger due to load factor overhead.
|
||||
pub fn memory_size(&self) -> usize {
|
||||
let mut size = core::mem::size_of::<Self>();
|
||||
|
||||
// Upper subtrie
|
||||
size += self.upper_subtrie.memory_size();
|
||||
|
||||
// Lower subtries (both Revealed and Blind with allocation)
|
||||
for subtrie in self.lower_subtries.iter() {
|
||||
size += subtrie.memory_size();
|
||||
}
|
||||
|
||||
// Prefix set keys
|
||||
size += self.prefix_set.len() * core::mem::size_of::<Nibbles>();
|
||||
|
||||
// Branch node masks map
|
||||
size += self.branch_node_masks.len() *
|
||||
(core::mem::size_of::<Nibbles>() + core::mem::size_of::<BranchNodeMasks>());
|
||||
|
||||
// Updates if present
|
||||
if let Some(updates) = &self.updates {
|
||||
size += updates.updated_nodes.len() *
|
||||
(core::mem::size_of::<Nibbles>() + core::mem::size_of::<BranchNodeCompact>());
|
||||
size += updates.removed_nodes.len() * core::mem::size_of::<Nibbles>();
|
||||
}
|
||||
|
||||
// Update actions buffers
|
||||
for buf in &self.update_actions_buffers {
|
||||
size += buf.capacity() * core::mem::size_of::<SparseTrieUpdatesAction>();
|
||||
}
|
||||
|
||||
size
|
||||
}
|
||||
}
|
||||
|
||||
/// Bitset tracking which of the 256 lower subtries were modified in the current cycle.
|
||||
@@ -2571,19 +2627,17 @@ impl SparseSubtrie {
|
||||
self.nodes.insert(path, SparseNode::Empty);
|
||||
}
|
||||
TrieNode::Branch(branch) => {
|
||||
// For a branch node, iterate over all potential children
|
||||
// For a branch node, iterate over all children
|
||||
let mut stack_ptr = branch.as_ref().first_child_index();
|
||||
for idx in CHILD_INDEX_RANGE {
|
||||
if branch.state_mask.is_bit_set(idx) {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(idx);
|
||||
if Self::is_child_same_level(&path, &child_path) {
|
||||
// Reveal each child node or hash it has, but only if the child is on
|
||||
// the same level as the parent.
|
||||
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
|
||||
}
|
||||
stack_ptr += 1;
|
||||
for idx in branch.state_mask.iter() {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(idx);
|
||||
if Self::is_child_same_level(&path, &child_path) {
|
||||
// Reveal each child node or hash it has, but only if the child is on
|
||||
// the same level as the parent.
|
||||
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
|
||||
}
|
||||
stack_ptr += 1;
|
||||
}
|
||||
// Update the branch node entry in the nodes map, handling cases where a blinded
|
||||
// node is now replaced with a revealed node.
|
||||
@@ -2812,6 +2866,30 @@ impl SparseSubtrie {
|
||||
pub(crate) fn shrink_values_to(&mut self, size: usize) {
|
||||
self.inner.values.shrink_to(size);
|
||||
}
|
||||
|
||||
/// Returns a heuristic for the in-memory size of this subtrie in bytes.
|
||||
pub(crate) fn memory_size(&self) -> usize {
|
||||
let mut size = core::mem::size_of::<Self>();
|
||||
|
||||
// Nodes map: key (Nibbles) + value (SparseNode)
|
||||
for (path, node) in &self.nodes {
|
||||
size += core::mem::size_of::<Nibbles>();
|
||||
size += path.len(); // Nibbles heap allocation
|
||||
size += node.memory_size();
|
||||
}
|
||||
|
||||
// Values map: key (Nibbles) + value (Vec<u8>)
|
||||
for (path, value) in &self.inner.values {
|
||||
size += core::mem::size_of::<Nibbles>();
|
||||
size += path.len(); // Nibbles heap allocation
|
||||
size += core::mem::size_of::<Vec<u8>>() + value.capacity();
|
||||
}
|
||||
|
||||
// Buffers
|
||||
size += self.inner.buffers.memory_size();
|
||||
|
||||
size
|
||||
}
|
||||
}
|
||||
|
||||
/// Helper type for [`SparseSubtrie`] to mutably access only a subset of fields from the original
|
||||
@@ -2989,12 +3067,10 @@ impl SparseSubtrieInner {
|
||||
self.buffers.branch_child_buf.clear();
|
||||
// Walk children in a reverse order from `f` to `0`, so we pop the `0` first
|
||||
// from the stack and keep walking in the sorted order.
|
||||
for bit in CHILD_INDEX_RANGE.rev() {
|
||||
if state_mask.is_bit_set(bit) {
|
||||
let mut child = path;
|
||||
child.push_unchecked(bit);
|
||||
self.buffers.branch_child_buf.push(child);
|
||||
}
|
||||
for bit in state_mask.iter().rev() {
|
||||
let mut child = path;
|
||||
child.push_unchecked(bit);
|
||||
self.buffers.branch_child_buf.push(child);
|
||||
}
|
||||
|
||||
self.buffers
|
||||
@@ -3285,6 +3361,19 @@ impl SparseSubtrieBuffers {
|
||||
self.branch_value_stack_buf.clear();
|
||||
self.rlp_buf.clear();
|
||||
}
|
||||
|
||||
/// Returns a heuristic for the in-memory size of these buffers in bytes.
|
||||
const fn memory_size(&self) -> usize {
|
||||
let mut size = core::mem::size_of::<Self>();
|
||||
|
||||
size += self.path_stack.capacity() * core::mem::size_of::<RlpNodePathStackItem>();
|
||||
size += self.rlp_node_stack.capacity() * core::mem::size_of::<RlpNodeStackItem>();
|
||||
size += self.branch_child_buf.capacity() * core::mem::size_of::<Nibbles>();
|
||||
size += self.branch_value_stack_buf.capacity() * core::mem::size_of::<RlpNode>();
|
||||
size += self.rlp_buf.capacity();
|
||||
|
||||
size
|
||||
}
|
||||
}
|
||||
|
||||
/// RLP node path stack item.
|
||||
@@ -3318,7 +3407,10 @@ struct ChangedSubtrie {
|
||||
/// If the path is shorter than [`UPPER_TRIE_MAX_DEPTH`] nibbles.
|
||||
fn path_subtrie_index_unchecked(path: &Nibbles) -> usize {
|
||||
debug_assert_eq!(UPPER_TRIE_MAX_DEPTH, 2);
|
||||
path.get_byte_unchecked(0) as usize
|
||||
let idx = path.get_byte_unchecked(0) as usize;
|
||||
// SAFETY: always true.
|
||||
unsafe { core::hint::assert_unchecked(idx < NUM_LOWER_SUBTRIES) };
|
||||
idx
|
||||
}
|
||||
|
||||
/// Checks if `path` is a strict descendant of any root in a sorted slice.
|
||||
@@ -8895,4 +8987,41 @@ mod tests {
|
||||
b256!("f000000000000000000000000000000000000000000000000000000000000000");
|
||||
assert_eq!(ParallelSparseTrie::nibbles_to_padded_b256(&single), expected_single);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_memory_size() {
|
||||
// Test that memory_size returns a reasonable value for an empty trie
|
||||
let trie = ParallelSparseTrie::default();
|
||||
let empty_size = trie.memory_size();
|
||||
|
||||
// Should at least be the size of the struct itself
|
||||
assert!(empty_size >= core::mem::size_of::<ParallelSparseTrie>());
|
||||
|
||||
// Create a trie with some data
|
||||
let mut trie = ParallelSparseTrie::default();
|
||||
let nodes = vec![
|
||||
ProofTrieNode {
|
||||
path: Nibbles::from_nibbles_unchecked([0x1, 0x2]),
|
||||
node: TrieNode::Leaf(LeafNode {
|
||||
key: Nibbles::from_nibbles_unchecked([0x3, 0x4]),
|
||||
value: vec![1, 2, 3],
|
||||
}),
|
||||
masks: None,
|
||||
},
|
||||
ProofTrieNode {
|
||||
path: Nibbles::from_nibbles_unchecked([0x5, 0x6]),
|
||||
node: TrieNode::Leaf(LeafNode {
|
||||
key: Nibbles::from_nibbles_unchecked([0x7, 0x8]),
|
||||
value: vec![4, 5, 6],
|
||||
}),
|
||||
masks: None,
|
||||
},
|
||||
];
|
||||
trie.reveal_nodes(nodes).unwrap();
|
||||
|
||||
let populated_size = trie.memory_size();
|
||||
|
||||
// Populated trie should use more memory than an empty one
|
||||
assert!(populated_size > empty_size);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,11 +182,23 @@ where
|
||||
self.storage.tries.get_mut(address).and_then(|e| e.as_revealed_mut())
|
||||
}
|
||||
|
||||
/// Returns mutable reference to storage tries.
|
||||
pub const fn storage_tries_mut(&mut self) -> &mut B256Map<RevealableSparseTrie<S>> {
|
||||
&mut self.storage.tries
|
||||
}
|
||||
|
||||
/// Takes the storage trie for the provided address.
|
||||
pub fn take_storage_trie(&mut self, address: &B256) -> Option<RevealableSparseTrie<S>> {
|
||||
self.storage.tries.remove(address)
|
||||
}
|
||||
|
||||
/// Takes the storage trie for the provided address, creating a blind one if it doesn't exist.
|
||||
pub fn take_or_create_storage_trie(&mut self, address: &B256) -> RevealableSparseTrie<S> {
|
||||
self.storage.tries.remove(address).unwrap_or_else(|| {
|
||||
self.storage.cleared_tries.pop().unwrap_or_else(|| self.storage.default_trie.clone())
|
||||
})
|
||||
}
|
||||
|
||||
/// Inserts storage trie for the provided address.
|
||||
pub fn insert_storage_trie(&mut self, address: B256, storage_trie: RevealableSparseTrie<S>) {
|
||||
self.storage.tries.insert(address, storage_trie);
|
||||
|
||||
@@ -21,8 +21,7 @@ use reth_execution_errors::{SparseTrieErrorKind, SparseTrieResult};
|
||||
use reth_trie_common::{
|
||||
prefix_set::{PrefixSet, PrefixSetMut},
|
||||
BranchNodeCompact, BranchNodeMasks, BranchNodeMasksMap, BranchNodeRef, ExtensionNodeRef,
|
||||
LeafNodeRef, Nibbles, ProofTrieNode, RlpNode, TrieMask, TrieNode, CHILD_INDEX_RANGE,
|
||||
EMPTY_ROOT_HASH,
|
||||
LeafNodeRef, Nibbles, ProofTrieNode, RlpNode, TrieMask, TrieNode, EMPTY_ROOT_HASH,
|
||||
};
|
||||
use tracing::{debug, instrument, trace};
|
||||
|
||||
@@ -422,13 +421,11 @@ impl fmt::Display for SerialSparseTrie {
|
||||
SparseNode::Branch { state_mask, .. } => {
|
||||
writeln!(f, "{packed_path} -> {node:?}")?;
|
||||
|
||||
for i in CHILD_INDEX_RANGE.rev() {
|
||||
if state_mask.is_bit_set(i) {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(i);
|
||||
if let Some(child_node) = self.nodes_ref().get(&child_path) {
|
||||
stack.push((child_path, child_node, depth + 1));
|
||||
}
|
||||
for i in state_mask.iter().rev() {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(i);
|
||||
if let Some(child_node) = self.nodes_ref().get(&child_path) {
|
||||
stack.push((child_path, child_node, depth + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -503,16 +500,14 @@ impl SparseTrieTrait for SerialSparseTrie {
|
||||
self.nodes.insert(path, SparseNode::Empty);
|
||||
}
|
||||
TrieNode::Branch(branch) => {
|
||||
// For a branch node, iterate over all potential children
|
||||
// For a branch node, iterate over all children
|
||||
let mut stack_ptr = branch.as_ref().first_child_index();
|
||||
for idx in CHILD_INDEX_RANGE {
|
||||
if branch.state_mask.is_bit_set(idx) {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(idx);
|
||||
// Reveal each child node or hash it has
|
||||
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
|
||||
stack_ptr += 1;
|
||||
}
|
||||
for idx in branch.state_mask.iter() {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(idx);
|
||||
// Reveal each child node or hash it has
|
||||
self.reveal_node_or_hash(child_path, &branch.stack[stack_ptr])?;
|
||||
stack_ptr += 1;
|
||||
}
|
||||
// Update the branch node entry in the nodes map, handling cases where a blinded
|
||||
// node is now replaced with a revealed node.
|
||||
@@ -980,6 +975,19 @@ impl SparseTrieTrait for SerialSparseTrie {
|
||||
fn take_updates(&mut self) -> SparseTrieUpdates {
|
||||
match self.updates.take() {
|
||||
Some(updates) => {
|
||||
// Sync branch_node_masks with what's being committed to DB.
|
||||
// This ensures that on subsequent root() calls, the masks reflect the actual
|
||||
// DB state, which is needed for correct removal detection.
|
||||
for (path, node) in &updates.updated_nodes {
|
||||
self.branch_node_masks.insert(
|
||||
*path,
|
||||
BranchNodeMasks { tree_mask: node.tree_mask, hash_mask: node.hash_mask },
|
||||
);
|
||||
}
|
||||
for path in &updates.removed_nodes {
|
||||
self.branch_node_masks.remove(path);
|
||||
}
|
||||
|
||||
// NOTE: we need to preserve Some case
|
||||
self.updates = Some(SparseTrieUpdates::with_capacity(
|
||||
updates.updated_nodes.len(),
|
||||
@@ -1487,12 +1495,10 @@ impl SerialSparseTrie {
|
||||
} else {
|
||||
unchanged_prefix_set.insert(path);
|
||||
|
||||
for bit in CHILD_INDEX_RANGE.rev() {
|
||||
if state_mask.is_bit_set(bit) {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(bit);
|
||||
paths.push((child_path, level + 1));
|
||||
}
|
||||
for bit in state_mask.iter().rev() {
|
||||
let mut child_path = path;
|
||||
child_path.push_unchecked(bit);
|
||||
paths.push((child_path, level + 1));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1646,12 +1652,10 @@ impl SerialSparseTrie {
|
||||
buffers.branch_child_buf.clear();
|
||||
// Walk children in a reverse order from `f` to `0`, so we pop the `0` first
|
||||
// from the stack and keep walking in the sorted order.
|
||||
for bit in CHILD_INDEX_RANGE.rev() {
|
||||
if state_mask.is_bit_set(bit) {
|
||||
let mut child = path;
|
||||
child.push_unchecked(bit);
|
||||
buffers.branch_child_buf.push(child);
|
||||
}
|
||||
for bit in state_mask.iter().rev() {
|
||||
let mut child = path;
|
||||
child.push_unchecked(bit);
|
||||
buffers.branch_child_buf.push(child);
|
||||
}
|
||||
|
||||
buffers
|
||||
@@ -1971,6 +1975,16 @@ impl SparseNode {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the memory size of this node in bytes.
|
||||
pub const fn memory_size(&self) -> usize {
|
||||
match self {
|
||||
Self::Empty | Self::Hash(_) | Self::Branch { .. } => core::mem::size_of::<Self>(),
|
||||
Self::Leaf { key, .. } | Self::Extension { key, .. } => {
|
||||
core::mem::size_of::<Self>() + key.len()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A helper struct used to store information about a node that has been removed
|
||||
|
||||
@@ -57,20 +57,16 @@ impl<'a, K, V> ForwardInMemoryCursor<'a, K, V> {
|
||||
/// For small slices, linear scan has better cache locality and lower overhead.
|
||||
const BINARY_SEARCH_THRESHOLD: usize = 64;
|
||||
|
||||
impl<K, V> ForwardInMemoryCursor<'_, K, V>
|
||||
where
|
||||
K: Ord + Clone,
|
||||
V: Clone,
|
||||
{
|
||||
impl<K: Ord, V> ForwardInMemoryCursor<'_, K, V> {
|
||||
/// Returns the first entry from the current cursor position that's greater or equal to the
|
||||
/// provided key. This method advances the cursor forward.
|
||||
pub fn seek(&mut self, key: &K) -> Option<(K, V)> {
|
||||
pub fn seek(&mut self, key: &K) -> Option<&(K, V)> {
|
||||
self.advance_while(|k| k < key)
|
||||
}
|
||||
|
||||
/// Returns the first entry from the current cursor position that's greater than the provided
|
||||
/// key. This method advances the cursor forward.
|
||||
pub fn first_after(&mut self, key: &K) -> Option<(K, V)> {
|
||||
pub fn first_after(&mut self, key: &K) -> Option<&(K, V)> {
|
||||
self.advance_while(|k| k <= key)
|
||||
}
|
||||
|
||||
@@ -81,7 +77,7 @@ where
|
||||
///
|
||||
/// Returns the first entry for which `predicate` returns `false` or `None`. The cursor will
|
||||
/// point to the returned entry.
|
||||
fn advance_while(&mut self, predicate: impl Fn(&K) -> bool) -> Option<(K, V)> {
|
||||
fn advance_while(&mut self, predicate: impl Fn(&K) -> bool) -> Option<&(K, V)> {
|
||||
let remaining = self.entries.len().saturating_sub(self.idx);
|
||||
if remaining >= BINARY_SEARCH_THRESHOLD {
|
||||
let slice = &self.entries[self.idx..];
|
||||
@@ -92,7 +88,7 @@ where
|
||||
self.next();
|
||||
}
|
||||
}
|
||||
self.current().cloned()
|
||||
self.current()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -105,16 +101,16 @@ mod tests {
|
||||
let mut cursor = ForwardInMemoryCursor::new(&[(1, ()), (2, ()), (3, ()), (4, ()), (5, ())]);
|
||||
assert_eq!(cursor.current(), Some(&(1, ())));
|
||||
|
||||
assert_eq!(cursor.seek(&0), Some((1, ())));
|
||||
assert_eq!(cursor.seek(&0), Some(&(1, ())));
|
||||
assert_eq!(cursor.current(), Some(&(1, ())));
|
||||
|
||||
assert_eq!(cursor.seek(&3), Some((3, ())));
|
||||
assert_eq!(cursor.seek(&3), Some(&(3, ())));
|
||||
assert_eq!(cursor.current(), Some(&(3, ())));
|
||||
|
||||
assert_eq!(cursor.seek(&3), Some((3, ())));
|
||||
assert_eq!(cursor.seek(&3), Some(&(3, ())));
|
||||
assert_eq!(cursor.current(), Some(&(3, ())));
|
||||
|
||||
assert_eq!(cursor.seek(&4), Some((4, ())));
|
||||
assert_eq!(cursor.seek(&4), Some(&(4, ())));
|
||||
assert_eq!(cursor.current(), Some(&(4, ())));
|
||||
|
||||
assert_eq!(cursor.seek(&6), None);
|
||||
@@ -128,19 +124,19 @@ mod tests {
|
||||
let mut cursor = ForwardInMemoryCursor::new(&entries);
|
||||
|
||||
// Seek to beginning
|
||||
assert_eq!(cursor.seek(&0), Some((0, ())));
|
||||
assert_eq!(cursor.seek(&0), Some(&(0, ())));
|
||||
assert_eq!(cursor.idx, 0);
|
||||
|
||||
// Seek to middle (should use binary search)
|
||||
assert_eq!(cursor.seek(&100), Some((100, ())));
|
||||
assert_eq!(cursor.seek(&100), Some(&(100, ())));
|
||||
assert_eq!(cursor.idx, 50);
|
||||
|
||||
// Seek to non-existent key (should find next greater)
|
||||
assert_eq!(cursor.seek(&101), Some((102, ())));
|
||||
assert_eq!(cursor.seek(&101), Some(&(102, ())));
|
||||
assert_eq!(cursor.idx, 51);
|
||||
|
||||
// Seek to end
|
||||
assert_eq!(cursor.seek(&398), Some((398, ())));
|
||||
assert_eq!(cursor.seek(&398), Some(&(398, ())));
|
||||
assert_eq!(cursor.idx, 199);
|
||||
|
||||
// Seek past end
|
||||
@@ -153,16 +149,16 @@ mod tests {
|
||||
let mut cursor = ForwardInMemoryCursor::new(&entries);
|
||||
|
||||
// first_after should find strictly greater
|
||||
assert_eq!(cursor.first_after(&0), Some((2, ())));
|
||||
assert_eq!(cursor.first_after(&0), Some(&(2, ())));
|
||||
assert_eq!(cursor.idx, 1);
|
||||
|
||||
// Reset and test from beginning
|
||||
cursor.reset();
|
||||
assert_eq!(cursor.first_after(&99), Some((100, ())));
|
||||
assert_eq!(cursor.first_after(&99), Some(&(100, ())));
|
||||
|
||||
// first_after on exact match
|
||||
cursor.reset();
|
||||
assert_eq!(cursor.first_after(&100), Some((102, ())));
|
||||
assert_eq!(cursor.first_after(&100), Some(&(102, ())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -215,8 +215,8 @@ impl<C: TrieCursor> TrieCursor for InMemoryTrieCursor<'_, C> {
|
||||
self.seeked = true;
|
||||
|
||||
let entry = match (mem_entry, &self.cursor_entry) {
|
||||
(Some((mem_key, entry_inner)), _) if mem_key == key => {
|
||||
entry_inner.map(|node| (key, node))
|
||||
(Some((mem_key, entry_inner)), _) if *mem_key == key => {
|
||||
entry_inner.clone().map(|node| (key, node))
|
||||
}
|
||||
(_, Some((db_key, node))) if db_key == &key => Some((key, node.clone())),
|
||||
_ => None,
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use crate::{BranchNodeCompact, Nibbles, StoredSubNode, CHILD_INDEX_RANGE};
|
||||
use crate::{BranchNodeCompact, Nibbles, StoredSubNode};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_trie::proof::AddedRemovedKeys;
|
||||
|
||||
@@ -57,15 +57,12 @@ impl CursorSubNode {
|
||||
/// Creates a new [`CursorSubNode`] from a key and an optional node.
|
||||
pub fn new(key: Nibbles, node: Option<BranchNodeCompact>) -> Self {
|
||||
// Find the first nibble that is set in the state mask of the node.
|
||||
let position = node.as_ref().filter(|n| n.root_hash.is_none()).map_or(
|
||||
SubNodePosition::ParentBranch,
|
||||
|n| {
|
||||
let mut child_index_range = CHILD_INDEX_RANGE;
|
||||
SubNodePosition::Child(
|
||||
child_index_range.find(|i| n.state_mask.is_bit_set(*i)).unwrap(),
|
||||
)
|
||||
},
|
||||
);
|
||||
let position = node
|
||||
.as_ref()
|
||||
.filter(|n| n.root_hash.is_none())
|
||||
.map_or(SubNodePosition::ParentBranch, |n| {
|
||||
SubNodePosition::Child(n.state_mask.iter().next().unwrap())
|
||||
});
|
||||
Self::new_with_full_key(key, node, position)
|
||||
}
|
||||
|
||||
|
||||
@@ -1,8 +1,4 @@
|
||||
// Docker Bake configuration for reth and op-reth images
|
||||
// Usage:
|
||||
// docker buildx bake ethereum # Build reth
|
||||
// docker buildx bake optimism # Build op-reth
|
||||
// docker buildx bake # Build all
|
||||
// Docker Bake configuration for reth images
|
||||
|
||||
variable "REGISTRY" {
|
||||
default = "ghcr.io/paradigmxyz"
|
||||
@@ -35,23 +31,35 @@ variable "VERGEN_GIT_DIRTY" {
|
||||
|
||||
// Common settings for all targets
|
||||
group "default" {
|
||||
targets = ["ethereum", "optimism"]
|
||||
targets = [
|
||||
"ethereum-amd64",
|
||||
"ethereum-arm64",
|
||||
"optimism-amd64",
|
||||
"optimism-arm64"
|
||||
]
|
||||
}
|
||||
|
||||
group "nightly" {
|
||||
targets = ["ethereum", "ethereum-profiling", "ethereum-edge-profiling", "optimism", "optimism-profiling"]
|
||||
targets = [
|
||||
"ethereum-amd64",
|
||||
"ethereum-arm64",
|
||||
"ethereum-profiling",
|
||||
"ethereum-edge-profiling",
|
||||
"optimism-amd64",
|
||||
"optimism-arm64",
|
||||
"optimism-profiling"
|
||||
]
|
||||
}
|
||||
|
||||
// Base target with shared configuration
|
||||
target "_base" {
|
||||
dockerfile = "Dockerfile.depot"
|
||||
platforms = ["linux/amd64", "linux/arm64"]
|
||||
args = {
|
||||
BUILD_PROFILE = "${BUILD_PROFILE}"
|
||||
FEATURES = "${FEATURES}"
|
||||
VERGEN_GIT_SHA = "${VERGEN_GIT_SHA}"
|
||||
BUILD_PROFILE = "${BUILD_PROFILE}"
|
||||
FEATURES = "${FEATURES}"
|
||||
VERGEN_GIT_SHA = "${VERGEN_GIT_SHA}"
|
||||
VERGEN_GIT_DESCRIBE = "${VERGEN_GIT_DESCRIBE}"
|
||||
VERGEN_GIT_DIRTY = "${VERGEN_GIT_DIRTY}"
|
||||
VERGEN_GIT_DIRTY = "${VERGEN_GIT_DIRTY}"
|
||||
}
|
||||
secret = [
|
||||
{
|
||||
@@ -60,14 +68,42 @@ target "_base" {
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
// amd64 base with x86-64-v3 optimizations
|
||||
target "_base_amd64" {
|
||||
inherits = ["_base"]
|
||||
platforms = ["linux/amd64"]
|
||||
args = {
|
||||
# `x86-64-v3` features match the 2013 Intel Haswell architecture, excluding Intel-specific instructions;
|
||||
# see: https://en.wikipedia.org/wiki/X86-64
|
||||
#
|
||||
# `pclmulqdq` is required for rocksdb: https://github.com/rust-rocksdb/rust-rocksdb/issues/1069
|
||||
RUSTFLAGS = "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
|
||||
}
|
||||
}
|
||||
|
||||
// arm64 base
|
||||
target "_base_arm64" {
|
||||
inherits = ["_base"]
|
||||
platforms = ["linux/arm64"]
|
||||
}
|
||||
|
||||
target "_base_profiling" {
|
||||
inherits = ["_base"]
|
||||
platforms = ["linux/amd64"]
|
||||
inherits = ["_base_amd64"]
|
||||
}
|
||||
|
||||
// Ethereum (reth)
|
||||
target "ethereum" {
|
||||
inherits = ["_base"]
|
||||
target "ethereum-amd64" {
|
||||
inherits = ["_base_amd64"]
|
||||
args = {
|
||||
BINARY = "reth"
|
||||
MANIFEST_PATH = "bin/reth"
|
||||
}
|
||||
tags = ["${REGISTRY}/reth:${TAG}"]
|
||||
}
|
||||
|
||||
target "ethereum-arm64" {
|
||||
inherits = ["_base_arm64"]
|
||||
args = {
|
||||
BINARY = "reth"
|
||||
MANIFEST_PATH = "bin/reth"
|
||||
@@ -98,8 +134,17 @@ target "ethereum-edge-profiling" {
|
||||
}
|
||||
|
||||
// Optimism (op-reth)
|
||||
target "optimism" {
|
||||
inherits = ["_base"]
|
||||
target "optimism-amd64" {
|
||||
inherits = ["_base_amd64"]
|
||||
args = {
|
||||
BINARY = "op-reth"
|
||||
MANIFEST_PATH = "crates/optimism/bin"
|
||||
}
|
||||
tags = ["${REGISTRY}/op-reth:${TAG}"]
|
||||
}
|
||||
|
||||
target "optimism-arm64" {
|
||||
inherits = ["_base_arm64"]
|
||||
args = {
|
||||
BINARY = "op-reth"
|
||||
MANIFEST_PATH = "crates/optimism/bin"
|
||||
|
||||
Reference in New Issue
Block a user