mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
86 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
903c58749d | ||
|
|
d1dfce17ab | ||
|
|
abe603b94c | ||
|
|
9258bcdc7b | ||
|
|
7134ae4ec7 | ||
|
|
d9f81bc104 | ||
|
|
89be91de0e | ||
|
|
3af5a4a4e2 | ||
|
|
95f6bbe922 | ||
|
|
abab83facd | ||
|
|
9359e21f94 | ||
|
|
32d5ddfe40 | ||
|
|
d7e740f96c | ||
|
|
87bae74094 | ||
|
|
648f19fb56 | ||
|
|
e6fc5ff54b | ||
|
|
bc729671d9 | ||
|
|
eee27df27c | ||
|
|
6d02565c5e | ||
|
|
e706d76aa9 | ||
|
|
b9b7d092f6 | ||
|
|
d0fb5f31c2 | ||
|
|
9621b78586 | ||
|
|
3722071a7c | ||
|
|
6273530501 | ||
|
|
ce29101277 | ||
|
|
b1b95f9825 | ||
|
|
7f970e136a | ||
|
|
6b7cc00289 | ||
|
|
786140a99d | ||
|
|
ffcb486388 | ||
|
|
59d68f92c4 | ||
|
|
0e0271a612 | ||
|
|
df12fee965 | ||
|
|
11a4f65624 | ||
|
|
a782e1a18a | ||
|
|
2dc76f9abe | ||
|
|
65100971e5 | ||
|
|
8e21afa9cc | ||
|
|
46a9b9ad3d | ||
|
|
3f77af4f98 | ||
|
|
79cabbf89c | ||
|
|
e04afe6e0e | ||
|
|
ee224fe20f | ||
|
|
972f23745e | ||
|
|
49f60822f7 | ||
|
|
47ebc79c85 | ||
|
|
53f922927a | ||
|
|
f1f3980d29 | ||
|
|
6946f26d77 | ||
|
|
f663d1d110 | ||
|
|
f4943abf73 | ||
|
|
102a6944ba | ||
|
|
1592e51d34 | ||
|
|
4280ccf470 | ||
|
|
05ab98107c | ||
|
|
49128ed28f | ||
|
|
f74e594292 | ||
|
|
e7d4a05e36 | ||
|
|
9382a4c713 | ||
|
|
28409558f9 | ||
|
|
5ef32726db | ||
|
|
60c3bef1e8 | ||
|
|
af96eeae56 | ||
|
|
5528aae8f6 | ||
|
|
83364aa2d6 | ||
|
|
749a742bcf | ||
|
|
2970624413 | ||
|
|
7e18aa4be8 | ||
|
|
9f8c22e2c3 | ||
|
|
3d699ac9c6 | ||
|
|
9be31d504d | ||
|
|
34cc65cfe6 | ||
|
|
6e161f0fc9 | ||
|
|
63a3e18404 | ||
|
|
7d10e791b2 | ||
|
|
a9b2c1d454 | ||
|
|
9127563914 | ||
|
|
a500fb22ba | ||
|
|
e869cd4670 | ||
|
|
de69654b73 | ||
|
|
8d28c4c8f2 | ||
|
|
bfe778ab51 | ||
|
|
e523a76fb8 | ||
|
|
cd12ae58f2 | ||
|
|
370a548f34 |
20
.changelog/config.toml
Normal file
20
.changelog/config.toml
Normal file
@@ -0,0 +1,20 @@
|
||||
# Changelogs configuration for reth
|
||||
# https://github.com/wevm/changelogs
|
||||
|
||||
# How to bump packages that depend on changed packages
|
||||
dependent_bump = "patch"
|
||||
|
||||
[changelog]
|
||||
# Generate per-crate changelogs (vs single root changelog)
|
||||
format = "per-crate"
|
||||
|
||||
# Fixed groups: all always share the same version
|
||||
# reth binaries share version
|
||||
[[fixed]]
|
||||
members = ["reth", "op-reth"]
|
||||
|
||||
# Packages to ignore (internal/test-only crates)
|
||||
ignore = [
|
||||
"reth-testing-utils",
|
||||
"reth-bench",
|
||||
]
|
||||
5
.changelog/gentle-moons-cry.md
Normal file
5
.changelog/gentle-moons-cry.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-engine-tree: patch
|
||||
---
|
||||
|
||||
Reordered cache size calculations in `ExecutionCache::new` to group related operations together.
|
||||
6
.changelog/shy-tigers-dry.md
Normal file
6
.changelog/shy-tigers-dry.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth: patch
|
||||
op-reth: patch
|
||||
---
|
||||
|
||||
Added automated changelog generation infrastructure using wevm/changelogs-rs with Claude Code integration. Configured per-crate changelog format with fixed version groups for reth binaries and exclusions for internal test utilities.
|
||||
5
.changelog/vain-lakes-cry.md
Normal file
5
.changelog/vain-lakes-cry.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth: patch
|
||||
---
|
||||
|
||||
Updated Alloy dependencies from 1.5.2 to 1.6.1.
|
||||
21
.github/workflows/changelog.yml
vendored
Normal file
21
.github/workflows/changelog.yml
vendored
Normal file
@@ -0,0 +1,21 @@
|
||||
name: Changelog
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
changelog:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
with:
|
||||
fetch-depth: 0
|
||||
ref: ${{ github.head_ref }}
|
||||
- run: npm install -g @anthropic-ai/claude-code
|
||||
- uses: wevm/changelogs-rs/gen@master
|
||||
with:
|
||||
ai: 'claude -p'
|
||||
env:
|
||||
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
|
||||
6
.github/workflows/docker.yml
vendored
6
.github/workflows/docker.yml
vendored
@@ -100,5 +100,7 @@ jobs:
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
set: |
|
||||
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
optimism.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
ethereum-amd64.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
ethereum-arm64.tags=${{ steps.params.outputs.ethereum_tags }}
|
||||
optimism-amd64.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
optimism-arm64.tags=${{ steps.params.outputs.optimism_tags }}
|
||||
|
||||
2
.github/workflows/unit.yml
vendored
2
.github/workflows/unit.yml
vendored
@@ -90,7 +90,7 @@ jobs:
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo nextest run --release -p ef-tests --features "asm-keccak ef-tests"
|
||||
- run: cargo nextest run --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
|
||||
|
||||
doc:
|
||||
name: doc tests
|
||||
|
||||
11
CLAUDE.md
11
CLAUDE.md
@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
|
||||
|
||||
2. **Linting**: Run clippy with all features
|
||||
```bash
|
||||
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
|
||||
```
|
||||
|
||||
3. **Testing**: Use nextest for faster test execution
|
||||
@@ -169,12 +169,11 @@ Based on PR patterns, avoid:
|
||||
Before submitting changes, ensure:
|
||||
|
||||
1. **Format Check**: `cargo +nightly fmt --all --check`
|
||||
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
|
||||
2. **Clippy**: No warnings
|
||||
3. **Tests Pass**: All unit and integration tests
|
||||
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
|
||||
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
|
||||
|
||||
|
||||
### Opening PRs against <https://github.com/paradigmxyz/reth>
|
||||
|
||||
Label PRs appropriately, first check the available labels and then apply the relevant ones:
|
||||
@@ -349,10 +348,10 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
}
|
||||
```
|
||||
|
||||
5. **Run checks**:
|
||||
5. **Run checks** (IMPORTANT!):
|
||||
```bash
|
||||
cargo +nightly fmt --all
|
||||
cargo clippy --all-features
|
||||
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
|
||||
cargo test -p reth-discv4
|
||||
```
|
||||
|
||||
@@ -374,7 +373,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
cargo +nightly fmt --all
|
||||
|
||||
# Run lints
|
||||
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
|
||||
cargo +nightly clippy --workspace --all-features
|
||||
|
||||
# Run tests
|
||||
cargo nextest run --workspace
|
||||
|
||||
810
Cargo.lock
generated
810
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
105
Cargo.toml
105
Cargo.toml
@@ -490,42 +490,42 @@ alloy-sol-types = { version = "1.5.4", default-features = false }
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.2", default-features = false }
|
||||
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.1", default-features = false }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
|
||||
alloy-consensus = { version = "1.5.2", default-features = false }
|
||||
alloy-contract = { version = "1.5.2", default-features = false }
|
||||
alloy-eips = { version = "1.5.2", default-features = false }
|
||||
alloy-genesis = { version = "1.5.2", default-features = false }
|
||||
alloy-json-rpc = { version = "1.5.2", default-features = false }
|
||||
alloy-network = { version = "1.5.2", default-features = false }
|
||||
alloy-network-primitives = { version = "1.5.2", default-features = false }
|
||||
alloy-provider = { version = "1.5.2", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-client = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types = { version = "1.5.2", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.5.2", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.5.2", default-features = false }
|
||||
alloy-serde = { version = "1.5.2", default-features = false }
|
||||
alloy-signer = { version = "1.5.2", default-features = false }
|
||||
alloy-signer-local = { version = "1.5.2", default-features = false }
|
||||
alloy-transport = { version = "1.5.2" }
|
||||
alloy-transport-http = { version = "1.5.2", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.5.2", default-features = false }
|
||||
alloy-transport-ws = { version = "1.5.2", default-features = false }
|
||||
alloy-consensus = { version = "1.6.1", default-features = false }
|
||||
alloy-contract = { version = "1.6.1", default-features = false }
|
||||
alloy-eips = { version = "1.6.1", default-features = false }
|
||||
alloy-genesis = { version = "1.6.1", default-features = false }
|
||||
alloy-json-rpc = { version = "1.6.1", default-features = false }
|
||||
alloy-network = { version = "1.6.1", default-features = false }
|
||||
alloy-network-primitives = { version = "1.6.1", default-features = false }
|
||||
alloy-provider = { version = "1.6.1", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-client = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types = { version = "1.6.1", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.6.1", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.6.1", default-features = false }
|
||||
alloy-serde = { version = "1.6.1", default-features = false }
|
||||
alloy-signer = { version = "1.6.1", default-features = false }
|
||||
alloy-signer-local = { version = "1.6.1", default-features = false }
|
||||
alloy-transport = { version = "1.6.1" }
|
||||
alloy-transport-http = { version = "1.6.1", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.6.1", default-features = false }
|
||||
alloy-transport-ws = { version = "1.6.1", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-evm = { version = "0.27.0", default-features = false }
|
||||
alloy-op-evm = { version = "0.27.2", default-features = false }
|
||||
alloy-op-hardforks = "0.4.4"
|
||||
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
|
||||
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
|
||||
@@ -543,7 +543,7 @@ backon = { version = "1.2", default-features = false, features = ["std-blocking-
|
||||
bincode = "1.3"
|
||||
bitflags = "2.4"
|
||||
boyer-moore-magiclen = "0.2.16"
|
||||
bytes = { version = "1.5", default-features = false }
|
||||
bytes = { version = "1.11.1", default-features = false }
|
||||
brotli = "8"
|
||||
cfg-if = "1.0"
|
||||
clap = "4"
|
||||
@@ -560,9 +560,9 @@ humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
lz4 = "1.28.1"
|
||||
modular-bitfield = "0.11.2"
|
||||
modular-bitfield = "0.13.1"
|
||||
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
|
||||
nybbles = { version = "0.4.2", default-features = false }
|
||||
nybbles = { version = "0.4.8", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
parking_lot = "0.12"
|
||||
paste = "1.0"
|
||||
@@ -589,13 +589,13 @@ zstd = "0.13"
|
||||
byteorder = "1"
|
||||
fixed-cache = { version = "0.1.7", features = ["stats"] }
|
||||
moka = "0.12"
|
||||
tar-no-std = { version = "0.3.2", default-features = false }
|
||||
miniz_oxide = { version = "0.8.4", default-features = false }
|
||||
tar-no-std = { version = "0.4.2", default-features = false }
|
||||
miniz_oxide = { version = "0.9.0", default-features = false }
|
||||
chrono = "0.4.41"
|
||||
|
||||
# metrics
|
||||
metrics = "0.24.0"
|
||||
metrics-derive = "0.1"
|
||||
metrics-derive = "0.1.1"
|
||||
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
|
||||
metrics-process = "2.1.0"
|
||||
metrics-util = { default-features = false, version = "0.20.0" }
|
||||
@@ -607,7 +607,7 @@ quote = "1.0"
|
||||
# tokio
|
||||
tokio = { version = "1.44.2", default-features = false }
|
||||
tokio-stream = "0.1.11"
|
||||
tokio-tungstenite = "0.26.2"
|
||||
tokio-tungstenite = "0.28.0"
|
||||
tokio-util = { version = "0.7.4", features = ["codec"] }
|
||||
|
||||
# async
|
||||
@@ -620,7 +620,7 @@ futures-util = { version = "0.3", default-features = false }
|
||||
hyper = "1.3"
|
||||
hyper-util = "0.1.5"
|
||||
pin-project = "1.0.12"
|
||||
reqwest = { version = "0.12", default-features = false }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
|
||||
tracing-futures = "0.2"
|
||||
tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
@@ -640,7 +640,6 @@ jsonrpsee-types = "0.26.0"
|
||||
http = "1.0"
|
||||
http-body = "1.0"
|
||||
http-body-util = "0.1.2"
|
||||
jsonwebtoken = "9"
|
||||
proptest-arbitrary-interop = "0.1.0"
|
||||
|
||||
# crypto
|
||||
@@ -654,7 +653,7 @@ rand_08 = { package = "rand", version = "0.8" }
|
||||
c-kzg = "2.1.5"
|
||||
|
||||
# config
|
||||
toml = "0.8"
|
||||
toml = "0.9"
|
||||
|
||||
# rocksdb
|
||||
rocksdb = { version = "0.24" }
|
||||
@@ -673,16 +672,16 @@ assert_matches = "1.5.0"
|
||||
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
|
||||
insta = "1.41"
|
||||
proptest = "1.7"
|
||||
proptest-derive = "0.5"
|
||||
proptest-derive = "0.7"
|
||||
similar-asserts = { version = "1.5.0", features = ["serde"] }
|
||||
tempfile = "3.20"
|
||||
test-fuzz = "7"
|
||||
rstest = "0.24.0"
|
||||
rstest = "0.26.1"
|
||||
test-case = "3"
|
||||
|
||||
# ssz encoding
|
||||
ethereum_ssz = "0.9.0"
|
||||
ethereum_ssz_derive = "0.9.0"
|
||||
ethereum_ssz = "0.10.1"
|
||||
ethereum_ssz_derive = "0.10.1"
|
||||
|
||||
# allocators
|
||||
jemalloc_pprof = { version = "0.8", default-features = false }
|
||||
@@ -694,14 +693,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
|
||||
aes = "0.8.1"
|
||||
ahash = "0.8"
|
||||
anyhow = "1.0"
|
||||
bindgen = { version = "0.71", default-features = false }
|
||||
block-padding = "0.3.2"
|
||||
bindgen = { version = "0.72", default-features = false }
|
||||
block-padding = "0.3"
|
||||
cc = "1.2.15"
|
||||
cipher = "0.4.3"
|
||||
comfy-table = "7.0"
|
||||
concat-kdf = "0.1.0"
|
||||
crossbeam-channel = "0.5.13"
|
||||
crossterm = "0.28.0"
|
||||
crossterm = "0.29.0"
|
||||
csv = "1.3.0"
|
||||
ctrlc = "3.4"
|
||||
ctr = "0.9.2"
|
||||
@@ -714,7 +713,7 @@ hmac = "0.12.1"
|
||||
human_bytes = "0.4.1"
|
||||
indexmap = "2"
|
||||
interprocess = "2.2.0"
|
||||
lz4_flex = { version = "0.11", default-features = false }
|
||||
lz4_flex = { version = "0.12", default-features = false }
|
||||
memmap2 = "0.9.4"
|
||||
mev-share-sse = { version = "0.5.0", default-features = false }
|
||||
num-traits = "0.2.15"
|
||||
@@ -722,15 +721,15 @@ page_size = "0.6.0"
|
||||
parity-scale-codec = "3.2.1"
|
||||
plain_hasher = "0.2"
|
||||
pretty_assertions = "1.4"
|
||||
ratatui = { version = "0.29", default-features = false }
|
||||
ringbuffer = "0.15.0"
|
||||
ratatui = { version = "0.30", default-features = false }
|
||||
ringbuffer = "0.16.0"
|
||||
rmp-serde = "1.3"
|
||||
roaring = "0.10.2"
|
||||
roaring = "0.11.3"
|
||||
rolling-file = "0.2.0"
|
||||
sha3 = "0.10.5"
|
||||
snap = "1.1.1"
|
||||
socket2 = { version = "0.5", default-features = false }
|
||||
sysinfo = { version = "0.33", default-features = false }
|
||||
socket2 = { version = "0.6", default-features = false }
|
||||
sysinfo = { version = "0.38", default-features = false }
|
||||
tracing-journald = "0.3"
|
||||
tracing-logfmt = "=0.3.5"
|
||||
tracing-samply = "0.1"
|
||||
|
||||
@@ -56,7 +56,7 @@ ctrlc.workspace = true
|
||||
shlex.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { version = "0.29", features = ["signal", "process"] }
|
||||
nix = { version = "0.31", features = ["signal", "process"] }
|
||||
|
||||
[features]
|
||||
default = ["jemalloc"]
|
||||
|
||||
@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
|
||||
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
|
||||
|
||||
# reqwest
|
||||
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
|
||||
# tower
|
||||
tower.workspace = true
|
||||
|
||||
@@ -572,13 +572,22 @@ impl Command {
|
||||
|
||||
for i in 0..self.count {
|
||||
// Get initial batch of transactions for this payload
|
||||
let mut result = tx_buffer
|
||||
.take_batch()
|
||||
.await
|
||||
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
|
||||
let Some(mut result) = tx_buffer.take_batch().await else {
|
||||
info!(
|
||||
payloads_built = i,
|
||||
payloads_requested = self.count,
|
||||
"Transaction source exhausted, stopping"
|
||||
);
|
||||
break;
|
||||
};
|
||||
|
||||
if result.transactions.is_empty() {
|
||||
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
|
||||
info!(
|
||||
payloads_built = i,
|
||||
payloads_requested = self.count,
|
||||
"No more transactions available, stopping"
|
||||
);
|
||||
break;
|
||||
}
|
||||
|
||||
// Build with retry - may need to request more transactions
|
||||
|
||||
@@ -541,7 +541,7 @@ impl<H: BlockHeader> ChainSpec<H> {
|
||||
}
|
||||
}
|
||||
|
||||
bf_params.first().map(|(_, params)| *params).unwrap_or(BaseFeeParams::ethereum())
|
||||
bf_params.first().map(|(_, params)| *params).unwrap_or_else(BaseFeeParams::ethereum)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -133,4 +133,4 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb"]
|
||||
edge = ["reth-db-common/edge", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]
|
||||
|
||||
@@ -121,14 +121,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
let genesis_block_number = self.chain.genesis().number.unwrap_or_default();
|
||||
let (db, sfp) = match access {
|
||||
AccessRights::RW => (
|
||||
Arc::new(init_db(db_path, self.db.database_args())?),
|
||||
init_db(db_path, self.db.database_args())?,
|
||||
StaticFileProviderBuilder::read_write(sf_path)
|
||||
.with_metrics()
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?,
|
||||
),
|
||||
AccessRights::RO | AccessRights::RoInconsistent => {
|
||||
(Arc::new(open_db_read_only(&db_path, self.db.database_args())?), {
|
||||
(open_db_read_only(&db_path, self.db.database_args())?, {
|
||||
let provider = StaticFileProviderBuilder::read_only(sf_path)
|
||||
.with_metrics()
|
||||
.with_genesis_block_number(genesis_block_number)
|
||||
.build()?;
|
||||
provider.watch_directory();
|
||||
@@ -160,16 +162,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
fn create_provider_factory<N: CliNodeTypes>(
|
||||
&self,
|
||||
config: &Config,
|
||||
db: Arc<DatabaseEnv>,
|
||||
db: DatabaseEnv,
|
||||
static_file_provider: StaticFileProvider<N::Primitives>,
|
||||
rocksdb_provider: RocksDBProvider,
|
||||
access: AccessRights,
|
||||
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>>
|
||||
) -> eyre::Result<ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>>
|
||||
where
|
||||
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
|
||||
{
|
||||
let prune_modes = config.prune.segments.clone();
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::new(
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
|
||||
db,
|
||||
self.chain.clone(),
|
||||
static_file_provider,
|
||||
@@ -200,7 +202,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
let (_tip_tx, tip_rx) = watch::channel(B256::ZERO);
|
||||
|
||||
// Builds and executes an unwind-only pipeline
|
||||
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>::builder()
|
||||
let mut pipeline = Pipeline::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::builder()
|
||||
.add_stages(DefaultStages::new(
|
||||
factory.clone(),
|
||||
tip_rx,
|
||||
@@ -229,7 +231,7 @@ pub struct Environment<N: NodeTypes> {
|
||||
/// Configuration for reth node
|
||||
pub config: Config,
|
||||
/// Provider factory.
|
||||
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
pub provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
/// Datadir path.
|
||||
pub data_dir: ChainPath<DataDirPath>,
|
||||
}
|
||||
@@ -261,8 +263,8 @@ impl AccessRights {
|
||||
/// Helper alias to satisfy `FullNodeTypes` bound on [`Node`] trait generic.
|
||||
type FullTypesAdapter<T> = FullNodeTypesAdapter<
|
||||
T,
|
||||
Arc<DatabaseEnv>,
|
||||
BlockchainProvider<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
|
||||
DatabaseEnv,
|
||||
BlockchainProvider<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
|
||||
>;
|
||||
|
||||
/// Helper trait with a common set of requirements for the
|
||||
|
||||
@@ -17,7 +17,6 @@ use reth_provider::{providers::ProviderNodeTypes, DBProvider, StaticFileProvider
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use std::{
|
||||
hash::{BuildHasher, Hasher},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
@@ -90,7 +89,7 @@ impl Command {
|
||||
/// Execute `db checksum` command
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("This command should be run without the node running!");
|
||||
|
||||
@@ -117,7 +116,7 @@ fn checksum_hasher() -> impl Hasher {
|
||||
}
|
||||
|
||||
fn checksum_static_file<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
segment: StaticFileSegment,
|
||||
start_block: Option<u64>,
|
||||
end_block: Option<u64>,
|
||||
|
||||
@@ -9,7 +9,7 @@ use reth_db_api::table::Table;
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use std::{hash::Hasher, sync::Arc, time::Instant};
|
||||
use std::{hash::Hasher, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// RocksDB tables that can be checksummed.
|
||||
@@ -36,7 +36,7 @@ impl RocksDbTable {
|
||||
|
||||
/// Computes a checksum for a RocksDB table.
|
||||
pub fn checksum_rocksdb<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
table: RocksDbTable,
|
||||
limit: Option<usize>,
|
||||
) -> eyre::Result<()> {
|
||||
|
||||
@@ -16,7 +16,6 @@ use std::{
|
||||
hash::Hash,
|
||||
io::Write,
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
@@ -56,7 +55,7 @@ impl Command {
|
||||
/// then written to a file in the output directory.
|
||||
pub fn execute<T: NodeTypes>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<T, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<T, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
warn!("Make sure the node is not running when running `reth db diff`!");
|
||||
// open second db
|
||||
|
||||
@@ -7,7 +7,7 @@ use reth_db::{transaction::DbTx, DatabaseEnv};
|
||||
use reth_db_api::{database::Database, table::Table, RawValue, TableViewer, Tables};
|
||||
use reth_db_common::{DbTool, ListFilter};
|
||||
use reth_node_builder::{NodeTypes, NodeTypesWithDBAdapter};
|
||||
use std::{cell::RefCell, sync::Arc};
|
||||
use std::cell::RefCell;
|
||||
use tracing::error;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
@@ -55,7 +55,7 @@ impl Command {
|
||||
/// Execute `db list` command
|
||||
pub fn execute<N: NodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
self.table.view(&ListTableViewer { tool, args: &self })
|
||||
}
|
||||
@@ -89,7 +89,7 @@ impl Command {
|
||||
}
|
||||
|
||||
struct ListTableViewer<'a, N: NodeTypes> {
|
||||
tool: &'a DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &'a DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
args: &'a Command,
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ use reth_db_common::DbTool;
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_provider::providers::ProviderNodeTypes;
|
||||
use reth_storage_api::{BlockNumReader, StateProvider, StorageSettingsCache};
|
||||
use reth_tasks::spawn_scoped_os_thread;
|
||||
use std::{
|
||||
collections::BTreeSet,
|
||||
thread,
|
||||
@@ -230,7 +231,7 @@ impl Command {
|
||||
thread::scope(|s| {
|
||||
let handles: Vec<_> = (0..num_threads)
|
||||
.map(|thread_id| {
|
||||
s.spawn(move || {
|
||||
spawn_scoped_os_thread(s, "db-state-worker", move || {
|
||||
loop {
|
||||
// Get next chunk to process
|
||||
let chunk_idx = {
|
||||
|
||||
@@ -16,7 +16,7 @@ use reth_provider::{
|
||||
RocksDBProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::SegmentRangeInclusive;
|
||||
use std::{sync::Arc, time::Duration};
|
||||
use std::time::Duration;
|
||||
|
||||
#[derive(Parser, Debug)]
|
||||
/// The arguments for the `reth db stats` command
|
||||
@@ -48,7 +48,7 @@ impl Command {
|
||||
pub fn execute<N: CliNodeTypes<ChainSpec: EthereumHardforks>>(
|
||||
self,
|
||||
data_dir: ChainPath<DataDirPath>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, Arc<DatabaseEnv>>>,
|
||||
tool: &DbTool<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()> {
|
||||
if self.checksum {
|
||||
let checksum_report = self.checksum_report(tool)?;
|
||||
@@ -72,7 +72,7 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn db_stats_table<N: NodeTypesWithDB<DB = Arc<DatabaseEnv>>>(
|
||||
fn db_stats_table<N: NodeTypesWithDB<DB = DatabaseEnv>>(
|
||||
&self,
|
||||
tool: &DbTool<N>,
|
||||
) -> eyre::Result<ComfyTable> {
|
||||
|
||||
@@ -227,8 +227,9 @@ where
|
||||
|
||||
// Handle errors
|
||||
if let Err(err) = res {
|
||||
error!("{:?}", err)
|
||||
error!("{err}");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -241,6 +242,7 @@ fn event_loop<B: Backend, F, T: Table>(
|
||||
) -> io::Result<()>
|
||||
where
|
||||
F: FnMut(usize, usize) -> Vec<TableRow<T>>,
|
||||
io::Error: From<B::Error>,
|
||||
{
|
||||
let mut last_tick = Instant::now();
|
||||
let mut running = true;
|
||||
|
||||
@@ -2,7 +2,7 @@ use futures::Future;
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_db::DatabaseEnv;
|
||||
use reth_node_builder::{NodeBuilder, WithLaunchContext};
|
||||
use std::{fmt, sync::Arc};
|
||||
use std::fmt;
|
||||
|
||||
/// A trait for launching a reth node with custom configuration strategies.
|
||||
///
|
||||
@@ -30,7 +30,7 @@ where
|
||||
/// * `builder_args` - Extension arguments for configuration
|
||||
fn entrypoint(
|
||||
self,
|
||||
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
builder_args: Ext,
|
||||
) -> impl Future<Output = eyre::Result<()>>;
|
||||
}
|
||||
@@ -58,7 +58,7 @@ impl<F> FnLauncher<F> {
|
||||
where
|
||||
C: ChainSpecParser,
|
||||
F: AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
{
|
||||
@@ -77,13 +77,13 @@ where
|
||||
C: ChainSpecParser,
|
||||
Ext: clap::Args + fmt::Debug,
|
||||
F: AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
{
|
||||
fn entrypoint(
|
||||
self,
|
||||
builder: WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
builder: WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
builder_args: Ext,
|
||||
) -> impl Future<Output = eyre::Result<()>> {
|
||||
(self.func)(builder, builder_args)
|
||||
|
||||
@@ -206,7 +206,7 @@ where
|
||||
let db_path = data_dir.db();
|
||||
|
||||
tracing::info!(target: "reth::cli", path = ?db_path, "Opening database");
|
||||
let database = Arc::new(init_db(db_path.clone(), self.db.database_args())?.with_metrics());
|
||||
let database = init_db(db_path.clone(), self.db.database_args())?.with_metrics();
|
||||
|
||||
if with_unused_ports {
|
||||
node_config = node_config.with_unused_ports();
|
||||
|
||||
@@ -1,9 +1,10 @@
|
||||
//! Command that runs pruning without any limits.
|
||||
//! Command that runs pruning.
|
||||
use crate::common::{AccessRights, CliNodeTypes, EnvironmentArgs};
|
||||
use clap::Parser;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_node_builder::common::metrics_hooks;
|
||||
use reth_node_core::{args::MetricArgs, version::version_metadata};
|
||||
use reth_node_metrics::{
|
||||
@@ -11,12 +12,14 @@ use reth_node_metrics::{
|
||||
server::{MetricServer, MetricServerConfig},
|
||||
version::VersionInfo,
|
||||
};
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_prune::PrunerBuilder;
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
/// Prunes according to the configuration without any limits
|
||||
/// Prunes according to the configuration
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct PruneCommand<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
@@ -50,7 +53,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
build_profile: version_metadata().build_profile_name.as_ref(),
|
||||
},
|
||||
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
|
||||
ctx.task_executor,
|
||||
ctx.task_executor.clone(),
|
||||
metrics_hooks(&provider_factory),
|
||||
data_dir.pprof_dumps(),
|
||||
);
|
||||
@@ -69,13 +72,66 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
|
||||
// Delete data which has been copied to static files.
|
||||
if let Some(prune_tip) = lowest_static_file_height {
|
||||
info!(target: "reth::cli", ?prune_tip, ?config, "Pruning data from database...");
|
||||
// Run the pruner according to the configuration, and don't enforce any limits on it
|
||||
let mut pruner = PrunerBuilder::new(config)
|
||||
.delete_limit(usize::MAX)
|
||||
.build_with_provider_factory(provider_factory);
|
||||
|
||||
pruner.run(prune_tip)?;
|
||||
info!(target: "reth::cli", "Pruned data from database");
|
||||
// Set up cancellation token for graceful shutdown on Ctrl+C
|
||||
let cancellation = CancellationToken::new();
|
||||
let cancellation_clone = cancellation.clone();
|
||||
ctx.task_executor.spawn_critical("prune-ctrl-c", async move {
|
||||
tokio::signal::ctrl_c().await.expect("failed to listen for ctrl-c");
|
||||
cancellation_clone.cancel();
|
||||
});
|
||||
|
||||
// Use batched pruning with a limit to bound memory, running in a loop until complete.
|
||||
//
|
||||
// A limit of 20_000_000 results in a max memory usage of ~5G.
|
||||
const DELETE_LIMIT: usize = 20_000_000;
|
||||
let mut pruner = PrunerBuilder::new(config)
|
||||
.delete_limit(DELETE_LIMIT)
|
||||
.build_with_provider_factory(provider_factory.clone());
|
||||
|
||||
let mut total_pruned = 0usize;
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
info!(target: "reth::cli", total_pruned, "Pruning interrupted by user");
|
||||
break;
|
||||
}
|
||||
|
||||
let output = pruner.run(prune_tip)?;
|
||||
let batch_pruned: usize = output.segments.iter().map(|(_, seg)| seg.pruned).sum();
|
||||
total_pruned = total_pruned.saturating_add(batch_pruned);
|
||||
|
||||
// Check if all segments are finished (not just the overall progress,
|
||||
// since the pruner sets overall progress from the last segment only)
|
||||
let all_segments_finished =
|
||||
output.segments.iter().all(|(_, seg)| seg.progress.is_finished());
|
||||
|
||||
if all_segments_finished {
|
||||
info!(target: "reth::cli", total_pruned, "Pruned data from database");
|
||||
break;
|
||||
}
|
||||
|
||||
if batch_pruned == 0 {
|
||||
return Err(eyre::eyre!(
|
||||
"pruner made no progress but reported more data remaining; \
|
||||
aborting to prevent infinite loop"
|
||||
));
|
||||
}
|
||||
|
||||
info!(
|
||||
target: "reth::cli",
|
||||
batch_pruned,
|
||||
total_pruned,
|
||||
"Pruning batch complete, continuing..."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Flush and compact RocksDB to reclaim disk space after pruning
|
||||
#[cfg(all(unix, feature = "edge"))]
|
||||
{
|
||||
info!(target: "reth::cli", "Flushing and compacting RocksDB...");
|
||||
provider_factory.rocksdb_provider().flush_and_compact()?;
|
||||
info!(target: "reth::cli", "RocksDB compaction complete");
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -26,7 +26,7 @@ pub(crate) async fn dump_execution_stage<N, E, C>(
|
||||
consensus: C,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
|
||||
N: ProviderNodeTypes<DB = DatabaseEnv>,
|
||||
E: ConfigureEvm<Primitives = N::Primitives> + 'static,
|
||||
C: FullConsensus<E::Primitives> + 'static,
|
||||
{
|
||||
@@ -39,7 +39,7 @@ where
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -10,10 +10,9 @@ use reth_provider::{
|
||||
DatabaseProviderFactory, ProviderFactory,
|
||||
};
|
||||
use reth_stages::{stages::AccountHashingStage, Stage, StageCheckpoint, UnwindInput};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
db_tool: &DbTool<N>,
|
||||
from: BlockNumber,
|
||||
to: BlockNumber,
|
||||
@@ -36,7 +35,7 @@ pub(crate) async fn dump_hashing_account_stage<N: ProviderNodeTypes<DB = Arc<Dat
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -9,10 +9,9 @@ use reth_provider::{
|
||||
DatabaseProviderFactory, ProviderFactory,
|
||||
};
|
||||
use reth_stages::{stages::StorageHashingStage, Stage, StageCheckpoint, UnwindInput};
|
||||
use std::sync::Arc;
|
||||
use tracing::info;
|
||||
|
||||
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
db_tool: &DbTool<N>,
|
||||
from: u64,
|
||||
to: u64,
|
||||
@@ -26,7 +25,7 @@ pub(crate) async fn dump_hashing_storage_stage<N: ProviderNodeTypes<DB = Arc<Dat
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -34,7 +34,7 @@ pub(crate) async fn dump_merkle_stage<N>(
|
||||
consensus: impl FullConsensus<N::Primitives> + 'static,
|
||||
) -> Result<()>
|
||||
where
|
||||
N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>,
|
||||
N: ProviderNodeTypes<DB = DatabaseEnv>,
|
||||
{
|
||||
let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
|
||||
|
||||
@@ -59,7 +59,7 @@ where
|
||||
if should_run {
|
||||
dry_run(
|
||||
ProviderFactory::<N>::new(
|
||||
Arc::new(output_db),
|
||||
output_db,
|
||||
db_tool.chain(),
|
||||
StaticFileProvider::read_write(output_datadir.static_files())?,
|
||||
RocksDBProvider::builder(output_datadir.rocksdb()).build()?,
|
||||
|
||||
@@ -158,7 +158,7 @@ enum Subcommands {
|
||||
|
||||
impl Subcommands {
|
||||
/// Returns the block to unwind to. The returned block will stay in database.
|
||||
fn unwind_target<N: ProviderNodeTypes<DB = Arc<DatabaseEnv>>>(
|
||||
fn unwind_target<N: ProviderNodeTypes<DB = DatabaseEnv>>(
|
||||
&self,
|
||||
factory: ProviderFactory<N>,
|
||||
) -> eyre::Result<u64> {
|
||||
|
||||
@@ -83,22 +83,7 @@ impl CliRunner {
|
||||
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
|
||||
}
|
||||
|
||||
// `drop(tokio_runtime)` would block the current thread until its pools
|
||||
// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
|
||||
// it on a separate thread and wait for up to 5 seconds for this operation to
|
||||
// complete.
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-runtime-shutdown".to_string())
|
||||
.spawn(move || {
|
||||
drop(tokio_runtime);
|
||||
let _ = tx.send(());
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
|
||||
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
|
||||
});
|
||||
tokio_shutdown(tokio_runtime, true);
|
||||
|
||||
command_res
|
||||
}
|
||||
@@ -137,19 +122,7 @@ impl CliRunner {
|
||||
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
|
||||
}
|
||||
|
||||
// Shutdown the runtime on a separate thread
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-runtime-shutdown".to_string())
|
||||
.spawn(move || {
|
||||
drop(tokio_runtime);
|
||||
let _ = tx.send(());
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
|
||||
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
|
||||
});
|
||||
tokio_shutdown(tokio_runtime, true);
|
||||
|
||||
command_res
|
||||
}
|
||||
@@ -179,13 +152,7 @@ impl CliRunner {
|
||||
tokio_runtime
|
||||
.block_on(run_until_ctrl_c(async move { fut.await.expect("Failed to join task") }))?;
|
||||
|
||||
// drop the tokio runtime on a separate thread because drop blocks until its pools
|
||||
// (including blocking pool) are shutdown. In other words `drop(tokio_runtime)` would block
|
||||
// the current thread but we want to exit right away.
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-runtime-shutdown".to_string())
|
||||
.spawn(move || drop(tokio_runtime))
|
||||
.unwrap();
|
||||
tokio_shutdown(tokio_runtime, false);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -321,3 +288,27 @@ where
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Shut down the given Tokio runtime, and wait for it if `wait` is set.
|
||||
///
|
||||
/// `drop(tokio_runtime)` would block the current thread until its pools
|
||||
/// (including blocking pool) are shutdown. Since we want to exit as soon as possible, drop
|
||||
/// it on a separate thread and wait for up to 5 seconds for this operation to
|
||||
/// complete.
|
||||
fn tokio_shutdown(rt: tokio::runtime::Runtime, wait: bool) {
|
||||
// Shutdown the runtime on a separate thread
|
||||
let (tx, rx) = mpsc::channel();
|
||||
std::thread::Builder::new()
|
||||
.name("tokio-shutdown".to_string())
|
||||
.spawn(move || {
|
||||
drop(rt);
|
||||
let _ = tx.send(());
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
if wait {
|
||||
let _ = rx.recv_timeout(Duration::from_secs(5)).inspect_err(|err| {
|
||||
debug!(target: "reth::cli", %err, "tokio runtime shutdown timed out");
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -29,7 +29,7 @@ auto_impl.workspace = true
|
||||
derive_more.workspace = true
|
||||
futures.workspace = true
|
||||
eyre.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls"] }
|
||||
reqwest.workspace = true
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
tokio = { workspace = true, features = ["time"] }
|
||||
serde_json.workspace = true
|
||||
|
||||
@@ -95,7 +95,7 @@ where
|
||||
let block_hash = payload.block_hash();
|
||||
let block_number = payload.block_number();
|
||||
|
||||
previous_block_hashes.push(block_hash);
|
||||
previous_block_hashes.enqueue(block_hash);
|
||||
|
||||
// Send new events to execution client
|
||||
let _ = self.engine_handle.new_payload(payload).await;
|
||||
@@ -160,7 +160,7 @@ mod tests {
|
||||
|
||||
// Push hashes 0..65
|
||||
for i in 0..65u8 {
|
||||
buffer.push(B256::with_last_byte(i));
|
||||
buffer.enqueue(B256::with_last_byte(i));
|
||||
}
|
||||
|
||||
// offset=0 should return the most recent (64)
|
||||
@@ -181,7 +181,7 @@ mod tests {
|
||||
let mut buffer: AllocRingBuffer<B256> = AllocRingBuffer::new(65);
|
||||
|
||||
// With only 1 entry, only offset=0 works
|
||||
buffer.push(B256::with_last_byte(1));
|
||||
buffer.enqueue(B256::with_last_byte(1));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 0), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 1), None);
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), None);
|
||||
@@ -189,7 +189,7 @@ mod tests {
|
||||
|
||||
// With 33 entries, offset=32 works but offset=64 doesn't
|
||||
for i in 2..=33u8 {
|
||||
buffer.push(B256::with_last_byte(i));
|
||||
buffer.enqueue(B256::with_last_byte(i));
|
||||
}
|
||||
assert_eq!(get_hash_at_offset(&buffer, 32), Some(B256::with_last_byte(1)));
|
||||
assert_eq!(get_hash_at_offset(&buffer, 64), None);
|
||||
|
||||
@@ -114,22 +114,22 @@ pub async fn setup_engine_with_chain_import(
|
||||
|
||||
// Initialize the database using init_db (same as CLI import command)
|
||||
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
|
||||
let db_env = reth_db::init_db(&db_path, db_args)?;
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, db_args)?;
|
||||
|
||||
// Create a provider factory with the initialized database (use regular DB, not
|
||||
// TempDatabase) We need to specify the node types properly for the adapter
|
||||
let provider_factory = ProviderFactory::<
|
||||
NodeTypesWithDBAdapter<EthereumNode, Arc<DatabaseEnv>>,
|
||||
>::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(static_files_path.clone())?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)?;
|
||||
let provider_factory =
|
||||
ProviderFactory::<NodeTypesWithDBAdapter<EthereumNode, DatabaseEnv>>::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
reth_provider::providers::StaticFileProvider::read_write(
|
||||
static_files_path.clone(),
|
||||
)?,
|
||||
reth_provider::providers::RocksDBProvider::builder(rocksdb_dir_path)
|
||||
.with_default_tables()
|
||||
.build()
|
||||
.unwrap(),
|
||||
)?;
|
||||
|
||||
// Initialize genesis if needed
|
||||
reth_db_common::init::init_genesis(&provider_factory)?;
|
||||
@@ -320,11 +320,10 @@ mod tests {
|
||||
// Import the chain
|
||||
{
|
||||
let db_args = reth_node_core::args::DatabaseArgs::default().database_args();
|
||||
let db_env = reth_db::init_db(&db_path, db_args).unwrap();
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, db_args).unwrap();
|
||||
|
||||
let provider_factory: ProviderFactory<
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
|
||||
> = ProviderFactory::new(
|
||||
db.clone(),
|
||||
chain_spec.clone(),
|
||||
@@ -385,11 +384,10 @@ mod tests {
|
||||
|
||||
// Now reopen the database and verify checkpoints are still there
|
||||
{
|
||||
let db_env = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
|
||||
let db = Arc::new(db_env);
|
||||
let db = reth_db::init_db(&db_path, DatabaseArguments::default()).unwrap();
|
||||
|
||||
let provider_factory: ProviderFactory<
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, Arc<DatabaseEnv>>,
|
||||
NodeTypesWithDBAdapter<reth_node_ethereum::EthereumNode, DatabaseEnv>,
|
||||
> = ProviderFactory::new(
|
||||
db,
|
||||
chain_spec.clone(),
|
||||
|
||||
@@ -528,8 +528,12 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for the number of storage proof worker threads.
|
||||
pub fn with_storage_worker_count(mut self, storage_worker_count: usize) -> Self {
|
||||
self.storage_worker_count = storage_worker_count.max(MIN_WORKER_COUNT);
|
||||
///
|
||||
/// No-op if it's [`None`].
|
||||
pub fn with_storage_worker_count_opt(mut self, storage_worker_count: Option<usize>) -> Self {
|
||||
if let Some(count) = storage_worker_count {
|
||||
self.storage_worker_count = count.max(MIN_WORKER_COUNT);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
@@ -539,8 +543,12 @@ impl TreeConfig {
|
||||
}
|
||||
|
||||
/// Setter for the number of account proof worker threads.
|
||||
pub fn with_account_worker_count(mut self, account_worker_count: usize) -> Self {
|
||||
self.account_worker_count = account_worker_count.max(MIN_WORKER_COUNT);
|
||||
///
|
||||
/// No-op if it's [`None`].
|
||||
pub fn with_account_worker_count_opt(mut self, account_worker_count: Option<usize>) -> Self {
|
||||
if let Some(count) = account_worker_count {
|
||||
self.account_worker_count = count.max(MIN_WORKER_COUNT);
|
||||
}
|
||||
self
|
||||
}
|
||||
|
||||
|
||||
@@ -17,7 +17,6 @@ reth-engine-tree.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-stages-api.workspace = true
|
||||
|
||||
@@ -14,7 +14,6 @@ pub use reth_engine_tree::{
|
||||
chain::{ChainEvent, ChainOrchestrator},
|
||||
engine::EngineApiEvent,
|
||||
};
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_network_p2p::BlockClient;
|
||||
use reth_node_types::{BlockTy, NodeTypes};
|
||||
@@ -97,7 +96,7 @@ where
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<EthPrimitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
|
||||
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
|
||||
|
||||
|
||||
@@ -23,7 +23,7 @@ reth-evm = { workspace = true, features = ["metrics"] }
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-payload-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["rayon", "dashmap"] }
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
@@ -62,7 +62,6 @@ metrics.workspace = true
|
||||
reth-metrics = { workspace = true, features = ["common"] }
|
||||
|
||||
# misc
|
||||
dashmap.workspace = true
|
||||
schnellru.workspace = true
|
||||
rayon.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
@@ -20,7 +20,7 @@ pub(crate) struct PersistenceMetrics {
|
||||
/// How long it took for blocks to be saved
|
||||
pub(crate) save_blocks_duration_seconds: Histogram,
|
||||
/// How many blocks we persist at once.
|
||||
pub(crate) save_blocks_block_count: Histogram,
|
||||
pub(crate) save_blocks_batch_size: Histogram,
|
||||
/// How long it took for blocks to be pruned
|
||||
pub(crate) prune_before_duration_seconds: Histogram,
|
||||
}
|
||||
|
||||
@@ -11,8 +11,13 @@ use reth_provider::{
|
||||
};
|
||||
use reth_prune::{PrunerError, PrunerOutput, PrunerWithFactory};
|
||||
use reth_stages_api::{MetricEvent, MetricEventsSender};
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use std::{
|
||||
sync::mpsc::{Receiver, SendError, Sender},
|
||||
sync::{
|
||||
mpsc::{Receiver, SendError, Sender},
|
||||
Arc,
|
||||
},
|
||||
thread::JoinHandle,
|
||||
time::Instant,
|
||||
};
|
||||
use thiserror::Error;
|
||||
@@ -40,6 +45,12 @@ where
|
||||
metrics: PersistenceMetrics,
|
||||
/// Sender for sync metrics - we only submit sync metrics for persisted blocks
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
/// Pending finalized block number to be committed with the next block save.
|
||||
/// This avoids triggering a separate fsync for each finalized block update.
|
||||
pending_finalized_block: Option<u64>,
|
||||
/// Pending safe block number to be committed with the next block save.
|
||||
/// This avoids triggering a separate fsync for each safe block update.
|
||||
pending_safe_block: Option<u64>,
|
||||
}
|
||||
|
||||
impl<N> PersistenceService<N>
|
||||
@@ -53,7 +64,15 @@ where
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
) -> Self {
|
||||
Self { provider, incoming, pruner, metrics: PersistenceMetrics::default(), sync_metrics_tx }
|
||||
Self {
|
||||
provider,
|
||||
incoming,
|
||||
pruner,
|
||||
metrics: PersistenceMetrics::default(),
|
||||
sync_metrics_tx,
|
||||
pending_finalized_block: None,
|
||||
pending_safe_block: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Prunes block data before the given block number according to the configured prune
|
||||
@@ -106,14 +125,10 @@ where
|
||||
}
|
||||
}
|
||||
PersistenceAction::SaveFinalizedBlock(finalized_block) => {
|
||||
let provider = self.provider.database_provider_rw()?;
|
||||
provider.save_finalized_block_number(finalized_block)?;
|
||||
provider.commit()?;
|
||||
self.pending_finalized_block = Some(finalized_block);
|
||||
}
|
||||
PersistenceAction::SaveSafeBlock(safe_block) => {
|
||||
let provider = self.provider.database_provider_rw()?;
|
||||
provider.save_safe_block_number(safe_block)?;
|
||||
provider.commit()?;
|
||||
self.pending_safe_block = Some(safe_block);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -138,26 +153,39 @@ where
|
||||
}
|
||||
|
||||
fn on_save_blocks(
|
||||
&self,
|
||||
&mut self,
|
||||
blocks: Vec<ExecutedBlock<N::Primitives>>,
|
||||
) -> Result<Option<BlockNumHash>, PersistenceError> {
|
||||
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
|
||||
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
|
||||
let block_count = blocks.len();
|
||||
|
||||
// Take any pending finalized/safe block updates to commit together
|
||||
let pending_finalized = self.pending_finalized_block.take();
|
||||
let pending_safe = self.pending_safe_block.take();
|
||||
|
||||
debug!(target: "engine::persistence", ?block_count, first=?first_block, last=?last_block, "Saving range of blocks");
|
||||
|
||||
let start_time = Instant::now();
|
||||
|
||||
if last_block.is_some() {
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
|
||||
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
|
||||
|
||||
// Commit pending finalized/safe block updates in the same transaction
|
||||
if let Some(finalized) = pending_finalized {
|
||||
provider_rw.save_finalized_block_number(finalized)?;
|
||||
}
|
||||
if let Some(safe) = pending_safe {
|
||||
provider_rw.save_safe_block_number(safe)?;
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
self.metrics.save_blocks_block_count.record(block_count as f64);
|
||||
self.metrics.save_blocks_batch_size.record(block_count as f64);
|
||||
self.metrics.save_blocks_duration_seconds.record(start_time.elapsed());
|
||||
|
||||
Ok(last_block)
|
||||
@@ -204,15 +232,25 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
|
||||
pub struct PersistenceHandle<N: NodePrimitives = EthPrimitives> {
|
||||
/// The channel used to communicate with the persistence service
|
||||
sender: Sender<PersistenceAction<N>>,
|
||||
/// Guard that joins the service thread when all handles are dropped.
|
||||
/// Uses `Arc` so the handle remains `Clone`.
|
||||
_service_guard: Arc<ServiceGuard>,
|
||||
}
|
||||
|
||||
impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
/// Create a new [`PersistenceHandle`] from a [`Sender<PersistenceAction>`].
|
||||
pub const fn new(sender: Sender<PersistenceAction<T>>) -> Self {
|
||||
Self { sender }
|
||||
///
|
||||
/// This is intended for testing purposes where you want to mock the persistence service.
|
||||
/// For production use, prefer [`spawn_service`](Self::spawn_service).
|
||||
pub fn new(sender: Sender<PersistenceAction<T>>) -> Self {
|
||||
Self { sender, _service_guard: Arc::new(ServiceGuard(None)) }
|
||||
}
|
||||
|
||||
/// Create a new [`PersistenceHandle`], and spawn the persistence service.
|
||||
///
|
||||
/// The returned handle can be cloned and shared. When all clones are dropped, the service
|
||||
/// thread will be joined, ensuring graceful shutdown before resources (like `RocksDB`) are
|
||||
/// released.
|
||||
pub fn spawn_service<N>(
|
||||
provider_factory: ProviderFactory<N>,
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
@@ -224,22 +262,19 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
// create the initial channels
|
||||
let (db_service_tx, db_service_rx) = std::sync::mpsc::channel();
|
||||
|
||||
// construct persistence handle
|
||||
let persistence_handle = PersistenceHandle::new(db_service_tx);
|
||||
|
||||
// spawn the persistence service
|
||||
let db_service =
|
||||
PersistenceService::new(provider_factory, db_service_rx, pruner, sync_metrics_tx);
|
||||
std::thread::Builder::new()
|
||||
.name("Persistence Service".to_string())
|
||||
.spawn(|| {
|
||||
if let Err(err) = db_service.run() {
|
||||
error!(target: "engine::persistence", ?err, "Persistence service failed");
|
||||
}
|
||||
})
|
||||
.unwrap();
|
||||
let join_handle = spawn_os_thread("persistence", || {
|
||||
if let Err(err) = db_service.run() {
|
||||
error!(target: "engine::persistence", ?err, "Persistence service failed");
|
||||
}
|
||||
});
|
||||
|
||||
persistence_handle
|
||||
PersistenceHandle {
|
||||
sender: db_service_tx,
|
||||
_service_guard: Arc::new(ServiceGuard(Some(join_handle))),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sends a specific [`PersistenceAction`] in the contained channel. The caller is responsible
|
||||
@@ -267,7 +302,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
|
||||
}
|
||||
|
||||
/// Persists the finalized block number on disk.
|
||||
/// Queues the finalized block number to be persisted on disk.
|
||||
///
|
||||
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
|
||||
/// call to avoid triggering a separate fsync for each update.
|
||||
pub fn save_finalized_block_number(
|
||||
&self,
|
||||
finalized_block: u64,
|
||||
@@ -275,7 +313,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
self.send_action(PersistenceAction::SaveFinalizedBlock(finalized_block))
|
||||
}
|
||||
|
||||
/// Persists the safe block number on disk.
|
||||
/// Queues the safe block number to be persisted on disk.
|
||||
///
|
||||
/// The update is deferred and will be committed together with the next [`Self::save_blocks`]
|
||||
/// call to avoid triggering a separate fsync for each update.
|
||||
pub fn save_safe_block_number(
|
||||
&self,
|
||||
safe_block: u64,
|
||||
@@ -297,6 +338,27 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that joins the persistence service thread when dropped.
|
||||
///
|
||||
/// This ensures graceful shutdown - the service thread completes before resources like
|
||||
/// `RocksDB` are released. Stored in an `Arc` inside [`PersistenceHandle`] so the handle
|
||||
/// can be cloned while sharing the same guard.
|
||||
struct ServiceGuard(Option<JoinHandle<()>>);
|
||||
|
||||
impl std::fmt::Debug for ServiceGuard {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_tuple("ServiceGuard").field(&self.0.as_ref().map(|_| "...")).finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ServiceGuard {
|
||||
fn drop(&mut self) {
|
||||
if let Some(join_handle) = self.0.take() {
|
||||
let _ = join_handle.join();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -323,12 +385,12 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_empty() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let blocks = vec![];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let hash = rx.recv().unwrap();
|
||||
assert_eq!(hash, None);
|
||||
@@ -337,7 +399,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_single_block() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
let block_number = 0;
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let executed =
|
||||
@@ -347,7 +409,7 @@ mod tests {
|
||||
let blocks = vec![executed];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx
|
||||
.recv_timeout(std::time::Duration::from_secs(10))
|
||||
@@ -360,14 +422,14 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_multiple_blocks() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
let blocks = test_block_builder.get_executed_blocks(0..5).collect::<Vec<_>>();
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
|
||||
assert_eq!(last_hash, actual_hash);
|
||||
}
|
||||
@@ -375,7 +437,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_save_blocks_multiple_calls() {
|
||||
reth_tracing::init_test_tracing();
|
||||
let persistence_handle = default_persistence_handle();
|
||||
let handle = default_persistence_handle();
|
||||
|
||||
let ranges = [0..1, 1..2, 2..4, 4..5];
|
||||
let mut test_block_builder = TestBlockBuilder::eth();
|
||||
@@ -384,7 +446,7 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
persistence_handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
|
||||
let BlockNumHash { hash: actual_hash, number: _ } = rx.recv().unwrap().unwrap();
|
||||
assert_eq!(last_hash, actual_hash);
|
||||
|
||||
@@ -534,9 +534,9 @@ impl ExecutionCache {
|
||||
|
||||
/// Build an [`ExecutionCache`] struct, so that execution caches can be easily cloned.
|
||||
pub fn new(total_cache_size: usize) -> Self {
|
||||
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
|
||||
let storage_cache_size = (total_cache_size * 8888) / 10000; // 88.88% of total
|
||||
let account_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
|
||||
let code_cache_size = (total_cache_size * 556) / 10000; // 5.56% of total
|
||||
|
||||
let code_capacity = Self::bytes_to_entries(code_cache_size, CODE_CACHE_ENTRY_SIZE);
|
||||
let storage_capacity = Self::bytes_to_entries(storage_cache_size, STORAGE_CACHE_ENTRY_SIZE);
|
||||
|
||||
@@ -37,6 +37,7 @@ use reth_provider::{
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages_api::ControlFlow;
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use revm::state::EvmState;
|
||||
use state::TreeState;
|
||||
@@ -61,7 +62,6 @@ mod persistence_state;
|
||||
pub mod precompile_cache;
|
||||
#[cfg(test)]
|
||||
mod tests;
|
||||
#[expect(unused)]
|
||||
mod trie_updates;
|
||||
|
||||
use crate::tree::error::AdvancePersistenceError;
|
||||
@@ -432,7 +432,7 @@ where
|
||||
changeset_cache,
|
||||
);
|
||||
let incoming = task.incoming_tx.clone();
|
||||
std::thread::Builder::new().name("Engine Task".to_string()).spawn(|| task.run()).unwrap();
|
||||
spawn_os_thread("engine", || task.run());
|
||||
(incoming, outgoing)
|
||||
}
|
||||
|
||||
@@ -2613,19 +2613,27 @@ where
|
||||
let block_num_hash = block_id.block;
|
||||
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
|
||||
|
||||
match self.sealed_header_by_hash(block_num_hash.hash) {
|
||||
Err(err) => {
|
||||
let block = convert_to_block(self, input)?;
|
||||
return Err(InsertBlockError::new(block, err.into()).into());
|
||||
// Check if block already exists - first in memory, then DB only if it could be persisted
|
||||
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
|
||||
// Only query DB if block could be persisted (number <= last persisted block).
|
||||
// New blocks from CL always have number > last persisted, so skip DB lookup for them.
|
||||
if block_num_hash.number <= self.persistence_state.last_persisted_block.number {
|
||||
match self.provider.sealed_header_by_hash(block_num_hash.hash) {
|
||||
Err(err) => {
|
||||
let block = convert_to_block(self, input)?;
|
||||
return Err(InsertBlockError::new(block, err.into()).into());
|
||||
}
|
||||
Ok(Some(_)) => {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
Ok(None) => {}
|
||||
}
|
||||
Ok(Some(_)) => {
|
||||
// We now assume that we already have this block in the tree. However, we need to
|
||||
// run the conversion to ensure that the block hash is valid.
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid))
|
||||
}
|
||||
_ => {}
|
||||
};
|
||||
}
|
||||
|
||||
// Ensure that the parent state is available.
|
||||
match self.state_provider_builder(block_id.parent) {
|
||||
|
||||
@@ -39,7 +39,7 @@ use reth_trie_parallel::{
|
||||
proof_task::{ProofTaskCtx, ProofWorkerHandle},
|
||||
root::ParallelStateRootError,
|
||||
};
|
||||
use reth_trie_sparse::{ClearedSparseStateTrie, RevealableSparseTrie, SparseStateTrie};
|
||||
use reth_trie_sparse::{RevealableSparseTrie, SparseStateTrie};
|
||||
use reth_trie_sparse_parallel::{ParallelSparseTrie, ParallelismThresholds};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
@@ -235,8 +235,7 @@ where
|
||||
+ 'static,
|
||||
{
|
||||
// start preparing transactions immediately
|
||||
let (prewarm_rx, execution_rx, transaction_count_hint) =
|
||||
self.spawn_tx_iterator(transactions);
|
||||
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
|
||||
|
||||
let span = Span::current();
|
||||
let (to_sparse_trie, sparse_trie_rx) = channel();
|
||||
@@ -260,7 +259,6 @@ where
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
None, // Don't send proof targets when BAL is present
|
||||
Some(bal),
|
||||
@@ -271,7 +269,6 @@ where
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
transaction_count_hint,
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
None,
|
||||
@@ -355,10 +352,10 @@ where
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx, size_hint) = self.spawn_tx_iterator(transactions);
|
||||
let (prewarm_rx, execution_rx) = self.spawn_tx_iterator(transactions);
|
||||
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
|
||||
let prewarm_handle =
|
||||
self.spawn_caching_with(env, prewarm_rx, size_hint, provider_builder, None, bal, false);
|
||||
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
|
||||
PayloadHandle {
|
||||
to_multi_proof: None,
|
||||
prewarm_handle,
|
||||
@@ -376,19 +373,15 @@ where
|
||||
) -> (
|
||||
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
|
||||
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
|
||||
usize,
|
||||
) {
|
||||
let (transactions, convert) = transactions.into();
|
||||
let transactions = transactions.into_par_iter();
|
||||
let transaction_count_hint = transactions.len();
|
||||
|
||||
let (ooo_tx, ooo_rx) = mpsc::channel();
|
||||
let (prewarm_tx, prewarm_rx) = mpsc::channel();
|
||||
let (execute_tx, execute_rx) = mpsc::channel();
|
||||
|
||||
// Spawn a task that `convert`s all transactions in parallel and sends them out-of-order.
|
||||
self.executor.spawn_blocking(move || {
|
||||
transactions.enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
rayon::spawn(move || {
|
||||
let (transactions, convert) = transactions.into();
|
||||
transactions.into_par_iter().enumerate().for_each_with(ooo_tx, |ooo_tx, (idx, tx)| {
|
||||
let tx = convert(tx);
|
||||
let tx = tx.map(|tx| {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
@@ -424,16 +417,14 @@ where
|
||||
}
|
||||
});
|
||||
|
||||
(prewarm_rx, execute_rx, transaction_count_hint)
|
||||
(prewarm_rx, execute_rx)
|
||||
}
|
||||
|
||||
/// Spawn prewarming optionally wired to the multiproof task for target updates.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
mut transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
|
||||
transaction_count_hint: usize,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
@@ -468,7 +459,6 @@ where
|
||||
self.execution_cache.clone(),
|
||||
prewarm_ctx,
|
||||
to_multi_proof,
|
||||
transaction_count_hint,
|
||||
self.prewarm_max_concurrency,
|
||||
);
|
||||
|
||||
@@ -527,6 +517,8 @@ where
|
||||
let disable_sparse_trie_as_cache = !config.enable_sparse_trie_as_cache();
|
||||
let prune_depth = self.sparse_trie_prune_depth;
|
||||
let max_storage_tries = self.sparse_trie_max_storage_tries;
|
||||
let chunk_size =
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
@@ -562,11 +554,12 @@ where
|
||||
sparse_state_trie,
|
||||
))
|
||||
} else {
|
||||
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_cleared_trie(
|
||||
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
|
||||
from_multi_proof,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
ClearedSparseStateTrie::from_state_trie(sparse_state_trie),
|
||||
sparse_state_trie,
|
||||
chunk_size,
|
||||
))
|
||||
};
|
||||
|
||||
@@ -961,6 +954,10 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
/// Used for sparse trie continuation: if the preserved trie's anchor matches this,
|
||||
/// the trie can be reused directly.
|
||||
pub parent_state_root: B256,
|
||||
/// Number of transactions in the block.
|
||||
/// Used to determine parallel worker count for prewarming.
|
||||
/// A value of 0 indicates the count is unknown.
|
||||
pub transaction_count: usize,
|
||||
}
|
||||
|
||||
impl<Evm: ConfigureEvm> Default for ExecutionEnv<Evm>
|
||||
@@ -973,6 +970,7 @@ where
|
||||
hash: Default::default(),
|
||||
parent_hash: Default::default(),
|
||||
parent_state_root: Default::default(),
|
||||
transaction_count: 0,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,7 @@ use reth_trie_parallel::{
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
},
|
||||
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
};
|
||||
use revm_primitives::map::{hash_map, B256Map};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
@@ -63,7 +63,7 @@ const PREFETCH_MAX_BATCH_MESSAGES: usize = 16;
|
||||
|
||||
/// The default max targets, for limiting the number of account and storage proof targets to be
|
||||
/// fetched by a single worker. If exceeded, chunking is forced regardless of worker availability.
|
||||
const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
pub(crate) const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
|
||||
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
|
||||
/// state.
|
||||
@@ -311,11 +311,7 @@ impl VersionedMultiProofTargets {
|
||||
fn chunking_length(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.chunking_length(),
|
||||
Self::V2(targets) => {
|
||||
// For V2, count accounts + storage slots
|
||||
targets.account_targets.len() +
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
Self::V2(targets) => targets.chunking_length(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -367,9 +363,7 @@ impl VersionedMultiProofTargets {
|
||||
Self::Legacy(targets) => {
|
||||
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
|
||||
}
|
||||
Self::V2(targets) => Box::new(targets.chunks(chunk_size).map(Self::V2)),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1494,7 +1488,7 @@ fn get_proof_targets(
|
||||
/// Dispatches work items as a single unit or in chunks based on target size and worker
|
||||
/// availability.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
fn dispatch_with_chunking<T, I>(
|
||||
pub(crate) fn dispatch_with_chunking<T, I>(
|
||||
items: T,
|
||||
chunking_len: usize,
|
||||
chunk_size: Option<usize>,
|
||||
|
||||
@@ -84,8 +84,6 @@ where
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
/// How many transactions should be executed in parallel
|
||||
max_concurrency: usize,
|
||||
/// The number of transactions to be processed
|
||||
transaction_count_hint: usize,
|
||||
/// Sender to emit evm state outcome messages, if any.
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
/// Receiver for events produced by tx execution
|
||||
@@ -106,7 +104,6 @@ where
|
||||
execution_cache: PayloadExecutionCache,
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
transaction_count_hint: usize,
|
||||
max_concurrency: usize,
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
@@ -114,7 +111,7 @@ where
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
max_concurrency,
|
||||
transaction_count_hint,
|
||||
transaction_count = ctx.env.transaction_count,
|
||||
"Initialized prewarm task"
|
||||
);
|
||||
|
||||
@@ -124,7 +121,6 @@ where
|
||||
execution_cache,
|
||||
ctx,
|
||||
max_concurrency,
|
||||
transaction_count_hint,
|
||||
to_multi_proof,
|
||||
actions_rx,
|
||||
parent_span: Span::current(),
|
||||
@@ -148,7 +144,6 @@ where
|
||||
let executor = self.executor.clone();
|
||||
let ctx = self.ctx.clone();
|
||||
let max_concurrency = self.max_concurrency;
|
||||
let transaction_count_hint = self.transaction_count_hint;
|
||||
let span = Span::current();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
@@ -156,13 +151,14 @@ where
|
||||
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
|
||||
// When transaction_count_hint is 0, it means the count is unknown. In this case, spawn
|
||||
// When transaction_count is 0, it means the count is unknown. In this case, spawn
|
||||
// max workers to handle potentially many transactions in parallel rather
|
||||
// than bottlenecking on a single worker.
|
||||
let workers_needed = if transaction_count_hint == 0 {
|
||||
let transaction_count = ctx.env.transaction_count;
|
||||
let workers_needed = if transaction_count == 0 {
|
||||
max_concurrency
|
||||
} else {
|
||||
transaction_count_hint.min(max_concurrency)
|
||||
transaction_count.min(max_concurrency)
|
||||
};
|
||||
|
||||
// Spawn workers
|
||||
|
||||
@@ -1,18 +1,21 @@
|
||||
//! Sparse Trie task related functionality.
|
||||
|
||||
use crate::tree::{
|
||||
multiproof::{evm_state_to_hashed_post_state, MultiProofMessage, VersionedMultiProofTargets},
|
||||
multiproof::{
|
||||
dispatch_with_chunking, evm_state_to_hashed_post_state, MultiProofMessage,
|
||||
VersionedMultiProofTargets, DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
},
|
||||
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::Decodable;
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
use reth_errors::ProviderError;
|
||||
use reth_primitives_traits::Account;
|
||||
use rayon::iter::{IntoParallelRefMutIterator, ParallelBridge, ParallelIterator};
|
||||
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
proof_v2::Target, updates::TrieUpdates, HashedPostState, Nibbles, TrieAccount, EMPTY_ROOT_HASH,
|
||||
proof_v2::Target, updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, Nibbles,
|
||||
TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use reth_trie_parallel::{
|
||||
proof_task::{
|
||||
@@ -23,10 +26,9 @@ use reth_trie_parallel::{
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
ClearedSparseStateTrie, LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie,
|
||||
SparseTrieExt,
|
||||
LeafUpdate, SerialSparseTrie, SparseStateTrie, SparseTrie, SparseTrieExt,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use smallvec::SmallVec;
|
||||
@@ -34,7 +36,7 @@ use std::{
|
||||
sync::mpsc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{debug, debug_span, instrument, trace};
|
||||
use tracing::{debug, debug_span, error, instrument, trace};
|
||||
|
||||
#[expect(clippy::large_enum_variant)]
|
||||
pub(super) enum SpawnedSparseTrieTask<BPF, A, S>
|
||||
@@ -203,6 +205,9 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Maximum number of pending/prewarm updates that we accumulate in memory before actually applying.
|
||||
const MAX_PENDING_UPDATES: usize = 100;
|
||||
|
||||
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
|
||||
pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie> {
|
||||
/// Sender for proof results.
|
||||
@@ -215,6 +220,15 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
|
||||
trie: SparseStateTrie<A, S>,
|
||||
/// Handle to the proof worker pools (storage and account).
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
|
||||
/// The size of proof targets chunk to spawn in one calculation.
|
||||
/// If None, chunking is disabled and all targets are processed in a single proof.
|
||||
chunk_size: Option<usize>,
|
||||
/// If this number is exceeded and chunking is enabled, then this will override whether or not
|
||||
/// there are any active workers and force chunking across workers. This is to prevent tasks
|
||||
/// which are very long from hitting a single worker.
|
||||
max_targets_for_chunking: usize,
|
||||
|
||||
/// Account trie updates.
|
||||
account_updates: B256Map<LeafUpdate>,
|
||||
/// Storage trie updates. hashed address -> slot -> update.
|
||||
@@ -239,6 +253,16 @@ pub(super) struct SparseTrieCacheTask<A = SerialSparseTrie, S = SerialSparseTrie
|
||||
/// Cache of storage proof targets that have already been fetched/requested from the proof
|
||||
/// workers. account -> slot -> lowest `min_len` requested.
|
||||
fetched_storage_targets: B256Map<B256Map<u8>>,
|
||||
/// Reusable buffer for RLP encoding of accounts.
|
||||
account_rlp_buf: Vec<u8>,
|
||||
/// Whether the last state update has been received.
|
||||
finished_state_updates: bool,
|
||||
/// Pending targets to be dispatched to the proof workers.
|
||||
pending_targets: MultiProofTargetsV2,
|
||||
/// Number of pending execution/prewarming updates received but not yet passed to
|
||||
/// `update_leaves`.
|
||||
pending_updates: usize,
|
||||
|
||||
/// Metrics for the sparse trie.
|
||||
metrics: MultiProofTaskMetrics,
|
||||
}
|
||||
@@ -248,12 +272,13 @@ where
|
||||
A: SparseTrieExt + Default,
|
||||
S: SparseTrieExt + Default + Clone,
|
||||
{
|
||||
/// Creates a new sparse trie, pre-populating with a [`ClearedSparseStateTrie`].
|
||||
pub(super) fn new_with_cleared_trie(
|
||||
/// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`].
|
||||
pub(super) fn new_with_trie(
|
||||
updates: CrossbeamReceiver<MultiProofMessage>,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
sparse_state_trie: ClearedSparseStateTrie<A, S>,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
chunk_size: Option<usize>,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
Self {
|
||||
@@ -261,12 +286,18 @@ where
|
||||
proof_result_rx,
|
||||
updates,
|
||||
proof_worker_handle,
|
||||
trie: sparse_state_trie.into_inner(),
|
||||
trie,
|
||||
chunk_size,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
account_updates: Default::default(),
|
||||
storage_updates: Default::default(),
|
||||
pending_account_updates: Default::default(),
|
||||
fetched_account_targets: Default::default(),
|
||||
fetched_storage_targets: Default::default(),
|
||||
account_rlp_buf: Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE),
|
||||
finished_state_updates: Default::default(),
|
||||
pending_targets: Default::default(),
|
||||
pending_updates: Default::default(),
|
||||
metrics,
|
||||
}
|
||||
}
|
||||
@@ -314,15 +345,8 @@ where
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
let now = Instant::now();
|
||||
|
||||
let mut finished_state_updates = false;
|
||||
loop {
|
||||
crossbeam_channel::select_biased! {
|
||||
recv(self.proof_result_rx) -> message => {
|
||||
let Ok(result) = message else {
|
||||
unreachable!("we own the sender half")
|
||||
};
|
||||
self.on_proof_result(result)?;
|
||||
},
|
||||
recv(self.updates) -> message => {
|
||||
let update = match message {
|
||||
Ok(m) => m,
|
||||
@@ -331,27 +355,48 @@ where
|
||||
}
|
||||
};
|
||||
|
||||
match update {
|
||||
MultiProofMessage::PrefetchProofs(targets) => {
|
||||
self.on_prewarm_targets(targets);
|
||||
}
|
||||
MultiProofMessage::StateUpdate(_, state) => {
|
||||
self.on_state_update(state);
|
||||
}
|
||||
MultiProofMessage::EmptyProof { sequence_number: _, state } => {
|
||||
self.on_hashed_state_update(state);
|
||||
}
|
||||
MultiProofMessage::BlockAccessList(_) => todo!(),
|
||||
MultiProofMessage::FinishedStateUpdates => {
|
||||
finished_state_updates = true;
|
||||
}
|
||||
}
|
||||
self.on_multiproof_message(update);
|
||||
self.pending_updates += 1;
|
||||
}
|
||||
recv(self.proof_result_rx) -> message => {
|
||||
let Ok(result) = message else {
|
||||
unreachable!("we own the sender half")
|
||||
};
|
||||
let ProofResult::V2(mut result) = result.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
|
||||
while let Ok(next) = self.proof_result_rx.try_recv() {
|
||||
let ProofResult::V2(res) = next.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
result.extend(res);
|
||||
}
|
||||
|
||||
self.on_proof_result(result)?;
|
||||
},
|
||||
}
|
||||
|
||||
self.process_updates()?;
|
||||
if self.updates.is_empty() && self.proof_result_rx.is_empty() {
|
||||
// If we don't have any pending messages, we can spend some time on computing
|
||||
// storage roots and promoting account updates.
|
||||
self.dispatch_pending_targets();
|
||||
self.promote_pending_account_updates()?;
|
||||
self.dispatch_pending_targets();
|
||||
} else if self.updates.is_empty() || self.pending_updates > MAX_PENDING_UPDATES {
|
||||
// If we don't have any pending updates OR we've accumulated a lot already, apply
|
||||
// them to the trie,
|
||||
self.process_leaf_updates()?;
|
||||
self.dispatch_pending_targets();
|
||||
} else if self.updates.is_empty() ||
|
||||
self.pending_targets.chunking_length() > self.chunk_size.unwrap_or_default()
|
||||
{
|
||||
// Make sure to dispatch targets if we don't have any updates or if we've
|
||||
// accumulated a lot of them.
|
||||
self.dispatch_pending_targets();
|
||||
}
|
||||
|
||||
if finished_state_updates &&
|
||||
if self.finished_state_updates &&
|
||||
self.account_updates.is_empty() &&
|
||||
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
|
||||
{
|
||||
@@ -359,11 +404,6 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
// Process any remaining pending account updates.
|
||||
if !self.pending_account_updates.is_empty() {
|
||||
self.process_updates()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::root", "All proofs processed, ending calculation");
|
||||
|
||||
let start = Instant::now();
|
||||
@@ -379,6 +419,22 @@ where
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
}
|
||||
|
||||
/// Processes a [`MultiProofMessage`].
|
||||
fn on_multiproof_message(&mut self, message: MultiProofMessage) {
|
||||
match message {
|
||||
MultiProofMessage::PrefetchProofs(targets) => self.on_prewarm_targets(targets),
|
||||
MultiProofMessage::StateUpdate(_, state) => self.on_state_update(state),
|
||||
MultiProofMessage::EmptyProof { .. } => unreachable!(),
|
||||
MultiProofMessage::BlockAccessList(_) => todo!(),
|
||||
MultiProofMessage::FinishedStateUpdates => self.finished_state_updates = true,
|
||||
}
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
|
||||
let VersionedMultiProofTargets::V2(targets) = targets else {
|
||||
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
|
||||
@@ -414,11 +470,7 @@ where
|
||||
)]
|
||||
fn on_state_update(&mut self, update: EvmState) {
|
||||
let hashed_state_update = evm_state_to_hashed_post_state(update);
|
||||
self.on_hashed_state_update(hashed_state_update)
|
||||
}
|
||||
|
||||
/// Processes a hashed state update and encodes all state changes as trie updates.
|
||||
fn on_hashed_state_update(&mut self, hashed_state_update: HashedPostState) {
|
||||
for (address, storage) in hashed_state_update.storages {
|
||||
for (slot, value) in storage.storage {
|
||||
let encoded = if value.is_zero() {
|
||||
@@ -456,150 +508,242 @@ where
|
||||
|
||||
fn on_proof_result(
|
||||
&mut self,
|
||||
result: ProofResultMessage,
|
||||
result: DecodedMultiProofV2,
|
||||
) -> Result<(), ParallelStateRootError> {
|
||||
let ProofResult::V2(result) = result.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
|
||||
self.trie.reveal_decoded_multiproof_v2(result).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not reveal multiproof: {e:?}"))
|
||||
})
|
||||
}
|
||||
|
||||
/// Applies updates to the sparse trie and dispatches requested multiproof targets.
|
||||
fn process_updates(&mut self) -> Result<(), ProviderError> {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
/// Applies all account and storage leaf updates to corresponding tries and collects any new
|
||||
/// multiproof targets.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn process_leaf_updates(&mut self) -> SparseTrieResult<()> {
|
||||
self.pending_updates = 0;
|
||||
|
||||
for (addr, updates) in &mut self.storage_updates {
|
||||
let trie = self.trie.get_or_create_storage_trie_mut(*addr);
|
||||
let fetched_storage = self.fetched_storage_targets.entry(*addr).or_default();
|
||||
// Start with processing all storage updates in parallel.
|
||||
let storage_results = self
|
||||
.storage_updates
|
||||
.iter_mut()
|
||||
.map(|(address, updates)| {
|
||||
let trie = self.trie.take_or_create_storage_trie(address);
|
||||
let fetched = self.fetched_storage_targets.remove(address).unwrap_or_default();
|
||||
|
||||
trie.update_leaves(updates, |path, min_len| match fetched_storage.entry(path) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(*addr)
|
||||
.or_default()
|
||||
.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(*addr)
|
||||
.or_default()
|
||||
.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
(address, updates, fetched, trie)
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
.par_bridge()
|
||||
.map(|(address, updates, mut fetched, mut trie)| {
|
||||
let mut targets = Vec::new();
|
||||
|
||||
// If all storage updates were processed, we can now compute the new storage root.
|
||||
if updates.is_empty() {
|
||||
let storage_root =
|
||||
trie.root().expect("updates are drained, trie should be revealed by now");
|
||||
|
||||
// If there is a pending account update for this address with known info, we can
|
||||
// encode it into proper update right away.
|
||||
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
|
||||
entry.get().is_some()
|
||||
{
|
||||
let account = entry.remove().expect("just checked, should be Some");
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) &&
|
||||
storage_root == EMPTY_ROOT_HASH
|
||||
{
|
||||
Vec::new()
|
||||
} else {
|
||||
// TODO: optimize allocation
|
||||
alloy_rlp::encode(
|
||||
account.unwrap_or_default().into_trie_account(storage_root),
|
||||
)
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Now handle pending account updates that can be upgraded to a proper update.
|
||||
self.pending_account_updates.retain(|addr, account| {
|
||||
// If account has pending storage updates, it is still pending.
|
||||
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Get the current account state either from the trie or from latest account update.
|
||||
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
|
||||
Some(encoded).filter(|encoded| !encoded.is_empty())
|
||||
} else if !self.account_updates.contains_key(addr) {
|
||||
self.trie.get_account_value(addr)
|
||||
} else {
|
||||
// Needs to be revealed first
|
||||
return true;
|
||||
};
|
||||
|
||||
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
|
||||
|
||||
let (account, storage_root) = if let Some(account) = account.take() {
|
||||
// If account is Some(_) here it means it didn't have any storage updates
|
||||
// and we can fetch the storage root directly from the account trie.
|
||||
//
|
||||
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
|
||||
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
|
||||
|
||||
(account, storage_root)
|
||||
} else {
|
||||
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
|
||||
};
|
||||
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
|
||||
Vec::new()
|
||||
} else {
|
||||
let account = account.unwrap_or_default().into_trie_account(storage_root);
|
||||
|
||||
// TODO: optimize allocation
|
||||
alloy_rlp::encode(account)
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
|
||||
false
|
||||
});
|
||||
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.trie
|
||||
.trie_mut()
|
||||
.update_leaves(&mut self.account_updates, |target, min_len| {
|
||||
match self.fetched_account_targets.entry(target) {
|
||||
trie.update_leaves(updates, |path, min_len| match fetched.entry(path) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
targets.account_targets.push(Target::new(target).with_min_len(min_len));
|
||||
targets.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
targets.account_targets.push(Target::new(target).with_min_len(min_len));
|
||||
targets.push(Target::new(path).with_min_len(min_len));
|
||||
}
|
||||
})?;
|
||||
|
||||
SparseTrieResult::Ok((address, targets, fetched, trie))
|
||||
})
|
||||
.collect::<Result<Vec<_>, _>>()?;
|
||||
|
||||
for (address, targets, fetched, trie) in storage_results {
|
||||
self.fetched_storage_targets.insert(*address, fetched);
|
||||
self.trie.insert_storage_trie(*address, trie);
|
||||
|
||||
if !targets.is_empty() {
|
||||
self.pending_targets.storage_targets.entry(*address).or_default().extend(targets);
|
||||
}
|
||||
}
|
||||
|
||||
// Process account trie updates and fill the account targets.
|
||||
self.process_account_leaf_updates()?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Invokes `update_leaves` for the accounts trie and collects any new targets.
|
||||
///
|
||||
/// Returns whether any updates were drained (applied to the trie).
|
||||
fn process_account_leaf_updates(&mut self) -> SparseTrieResult<bool> {
|
||||
let updates_len_before = self.account_updates.len();
|
||||
|
||||
self.trie.trie_mut().update_leaves(
|
||||
&mut self.account_updates,
|
||||
|target, min_len| match self.fetched_account_targets.entry(target) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
if min_len < *entry.get() {
|
||||
entry.insert(min_len);
|
||||
self.pending_targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
}
|
||||
})
|
||||
.map_err(ProviderError::other)?;
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(min_len);
|
||||
self.pending_targets
|
||||
.account_targets
|
||||
.push(Target::new(target).with_min_len(min_len));
|
||||
}
|
||||
},
|
||||
)?;
|
||||
|
||||
if !targets.is_empty() {
|
||||
self.proof_worker_handle.dispatch_account_multiproof(AccountMultiproofInput::V2 {
|
||||
targets,
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
0,
|
||||
HashedPostState::default(),
|
||||
Instant::now(),
|
||||
),
|
||||
})?;
|
||||
Ok(self.account_updates.len() < updates_len_before)
|
||||
}
|
||||
|
||||
/// Iterates through all storage tries for which all updates were processed, computes their
|
||||
/// storage roots, and promotes corresponding pending account updates into proper leaf updates
|
||||
/// for accounts trie.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn promote_pending_account_updates(&mut self) -> SparseTrieResult<()> {
|
||||
self.process_leaf_updates()?;
|
||||
|
||||
if self.pending_account_updates.is_empty() {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let roots = self
|
||||
.trie
|
||||
.storage_tries_mut()
|
||||
.par_iter_mut()
|
||||
.filter(|(address, _)| {
|
||||
self.storage_updates.get(*address).is_some_and(|updates| updates.is_empty())
|
||||
})
|
||||
.map(|(address, trie)| {
|
||||
let root =
|
||||
trie.root().expect("updates are drained, trie should be revealed by now");
|
||||
|
||||
(address, root)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
for (addr, storage_root) in roots {
|
||||
// If the storage root is known and we have a pending update for this account, encode it
|
||||
// into a proper update.
|
||||
if let Entry::Occupied(entry) = self.pending_account_updates.entry(*addr) &&
|
||||
entry.get().is_some()
|
||||
{
|
||||
let account = entry.remove().expect("just checked, should be Some");
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) &&
|
||||
storage_root == EMPTY_ROOT_HASH
|
||||
{
|
||||
Vec::new()
|
||||
} else {
|
||||
self.account_rlp_buf.clear();
|
||||
account
|
||||
.unwrap_or_default()
|
||||
.into_trie_account(storage_root)
|
||||
.encode(&mut self.account_rlp_buf);
|
||||
self.account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
}
|
||||
}
|
||||
|
||||
loop {
|
||||
// Now handle pending account updates that can be upgraded to a proper update.
|
||||
let account_rlp_buf = &mut self.account_rlp_buf;
|
||||
self.pending_account_updates.retain(|addr, account| {
|
||||
// If account has pending storage updates, it is still pending.
|
||||
if self.storage_updates.get(addr).is_some_and(|updates| !updates.is_empty()) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// Get the current account state either from the trie or from latest account update.
|
||||
let trie_account = if let Some(LeafUpdate::Changed(encoded)) = self.account_updates.get(addr) {
|
||||
Some(encoded).filter(|encoded| !encoded.is_empty())
|
||||
} else if !self.account_updates.contains_key(addr) {
|
||||
self.trie.get_account_value(addr)
|
||||
} else {
|
||||
// Needs to be revealed first
|
||||
return true;
|
||||
};
|
||||
|
||||
let trie_account = trie_account.map(|value| TrieAccount::decode(&mut &value[..]).expect("invalid account RLP"));
|
||||
|
||||
let (account, storage_root) = if let Some(account) = account.take() {
|
||||
// If account is Some(_) here it means it didn't have any storage updates
|
||||
// and we can fetch the storage root directly from the account trie.
|
||||
//
|
||||
// If it did have storage updates, we would've had processed it above when iterating over storage tries.
|
||||
let storage_root = trie_account.map(|account| account.storage_root).unwrap_or(EMPTY_ROOT_HASH);
|
||||
|
||||
(account, storage_root)
|
||||
} else {
|
||||
(trie_account.map(Into::into), self.trie.storage_root(addr).expect("account had storage updates that were applied to its trie, storage root must be revealed by now"))
|
||||
};
|
||||
|
||||
let encoded = if account.is_none_or(|account| account.is_empty()) && storage_root == EMPTY_ROOT_HASH {
|
||||
Vec::new()
|
||||
} else {
|
||||
account_rlp_buf.clear();
|
||||
account.unwrap_or_default().into_trie_account(storage_root).encode(account_rlp_buf);
|
||||
account_rlp_buf.clone()
|
||||
};
|
||||
self.account_updates.insert(*addr, LeafUpdate::Changed(encoded));
|
||||
|
||||
false
|
||||
});
|
||||
|
||||
// Only exit when no new updates are processed.
|
||||
//
|
||||
// We need to keep iterating if any updates are being drained because that might
|
||||
// indicate that more pending account updates can be promoted.
|
||||
if !self.process_account_leaf_updates()? {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn dispatch_pending_targets(&mut self) {
|
||||
if !self.pending_targets.is_empty() {
|
||||
let chunking_length = self.pending_targets.chunking_length();
|
||||
dispatch_with_chunking(
|
||||
std::mem::take(&mut self.pending_targets),
|
||||
chunking_length,
|
||||
self.chunk_size,
|
||||
self.max_targets_for_chunking,
|
||||
self.proof_worker_handle.available_account_workers(),
|
||||
self.proof_worker_handle.available_storage_workers(),
|
||||
MultiProofTargetsV2::chunks,
|
||||
|proof_targets| {
|
||||
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(
|
||||
AccountMultiproofInput::V2 {
|
||||
targets: proof_targets,
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
0,
|
||||
HashedPostState::default(),
|
||||
Instant::now(),
|
||||
),
|
||||
},
|
||||
) {
|
||||
error!("failed to dispatch account multiproof: {e:?}");
|
||||
}
|
||||
},
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Outcome of the state root computation, including the state root itself with
|
||||
@@ -651,7 +795,7 @@ where
|
||||
.storages
|
||||
.into_iter()
|
||||
.map(|(address, storage)| (address, storage, trie.take_storage_trie(&address)))
|
||||
.par_bridge()
|
||||
.par_bridge_buffered()
|
||||
.map(|(address, storage, storage_trie)| {
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie", ?address)
|
||||
|
||||
@@ -407,6 +407,7 @@ where
|
||||
hash: input.hash(),
|
||||
parent_hash: input.parent_hash(),
|
||||
parent_state_root: parent_block.state_root(),
|
||||
transaction_count: input.transaction_count(),
|
||||
};
|
||||
|
||||
// Plan the strategy used for state root computation.
|
||||
@@ -519,6 +520,14 @@ where
|
||||
info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished");
|
||||
// we double check the state root here for good measure
|
||||
if state_root == block.header().state_root() {
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
self.compare_trie_updates_with_serial(
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
trie_updates.clone(),
|
||||
);
|
||||
}
|
||||
maybe_state_root = Some((state_root, trie_updates, elapsed))
|
||||
} else {
|
||||
warn!(
|
||||
@@ -894,6 +903,62 @@ where
|
||||
.root_with_updates()?)
|
||||
}
|
||||
|
||||
/// Compares trie updates from the state root task with serial state root computation.
|
||||
///
|
||||
/// This is used for debugging and validating the correctness of the parallel state root
|
||||
/// task implementation. When enabled via `--engine.state-root-task-compare-updates`, this
|
||||
/// method runs a separate serial state root computation and compares the resulting trie
|
||||
/// updates.
|
||||
fn compare_trie_updates_with_serial(
|
||||
&self,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &HashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match self.compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
Ok((serial_root, serial_trie_updates)) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
?serial_root,
|
||||
"Serial state root computation finished for comparison"
|
||||
);
|
||||
|
||||
// Get a database provider to use as trie cursor factory
|
||||
match overlay_factory.database_provider_ro() {
|
||||
Ok(provider) => {
|
||||
if let Err(err) = super::trie_updates::compare_trie_updates(
|
||||
&provider,
|
||||
task_trie_updates,
|
||||
serial_trie_updates,
|
||||
) {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Error comparing trie updates"
|
||||
);
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to get database provider for trie update comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to compute serial state root for comparison"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the block after execution.
|
||||
///
|
||||
/// This performs:
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
//! Contains a precompile cache backed by `schnellru::LruMap` (LRU by length).
|
||||
|
||||
use alloy_primitives::Bytes;
|
||||
use dashmap::DashMap;
|
||||
use moka::policy::EvictionPolicy;
|
||||
use reth_evm::precompiles::{DynPrecompile, Precompile, PrecompileInput};
|
||||
use reth_primitives_traits::dashmap::DashMap;
|
||||
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
|
||||
use revm_primitives::Address;
|
||||
use std::{hash::Hash, sync::Arc};
|
||||
|
||||
@@ -28,6 +28,7 @@ use reth_ethereum_primitives::{Block, EthPrimitives};
|
||||
use reth_evm_ethereum::MockEvmConfig;
|
||||
use reth_primitives_traits::Block as _;
|
||||
use reth_provider::test_utils::MockEthProvider;
|
||||
use reth_tasks::spawn_os_thread;
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
str::FromStr,
|
||||
@@ -538,10 +539,7 @@ async fn test_tree_persist_blocks() {
|
||||
.get_executed_blocks(1..tree_config.persistence_threshold() + 2)
|
||||
.collect();
|
||||
let test_harness = TestHarness::new(chain_spec).with_blocks(blocks.clone());
|
||||
std::thread::Builder::new()
|
||||
.name("Engine Task".to_string())
|
||||
.spawn(|| test_harness.tree.run())
|
||||
.unwrap();
|
||||
spawn_os_thread("engine", || test_harness.tree.run());
|
||||
|
||||
// send a message to the tree to enter the main loop.
|
||||
test_harness.to_tree_tx.send(FromEngine::DownloadedBlocks(vec![])).unwrap();
|
||||
@@ -1989,10 +1987,7 @@ mod forkchoice_updated_tests {
|
||||
let action_rx = test_harness.action_rx;
|
||||
|
||||
// Spawn tree in background thread
|
||||
std::thread::Builder::new()
|
||||
.name("Engine Task".to_string())
|
||||
.spawn(|| test_harness.tree.run())
|
||||
.unwrap();
|
||||
spawn_os_thread("engine", || test_harness.tree.run());
|
||||
|
||||
// Send terminate request
|
||||
to_tree_tx
|
||||
|
||||
@@ -98,7 +98,7 @@ impl StorageTrieUpdatesDiff {
|
||||
|
||||
/// Compares the trie updates from state root task, regular state root calculation and database,
|
||||
/// and logs the differences if there's any.
|
||||
pub(super) fn compare_trie_updates(
|
||||
pub(crate) fn compare_trie_updates(
|
||||
trie_cursor_factory: impl TrieCursorFactory,
|
||||
task: TrieUpdates,
|
||||
regular: TrieUpdates,
|
||||
@@ -186,7 +186,8 @@ fn compare_storage_trie_updates<C: TrieCursor>(
|
||||
task: &mut StorageTrieUpdates,
|
||||
regular: &mut StorageTrieUpdates,
|
||||
) -> Result<StorageTrieUpdatesDiff, DatabaseError> {
|
||||
let database_not_exists = trie_cursor()?.next()?.is_none();
|
||||
// Check if the storage trie exists by seeking to the first entry
|
||||
let database_not_exists = trie_cursor()?.seek(Nibbles::default())?.is_none();
|
||||
let mut diff = StorageTrieUpdatesDiff {
|
||||
// If the deletion is a no-op, meaning that the entry is not in the
|
||||
// database, do not add it to the diff.
|
||||
|
||||
@@ -20,8 +20,6 @@ reth-era.workspace = true
|
||||
# http
|
||||
bytes.workspace = true
|
||||
reqwest.workspace = true
|
||||
reqwest.default-features = false
|
||||
reqwest.features = ["stream", "rustls-tls-native-roots"]
|
||||
|
||||
# async
|
||||
tokio.workspace = true
|
||||
|
||||
@@ -86,7 +86,7 @@ where
|
||||
mut self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> Result<()>,
|
||||
) -> Result<()>
|
||||
@@ -132,7 +132,7 @@ pub(crate) fn run_commands_with<C, Ext, Rpc, N, SubCmd>(
|
||||
runner: CliRunner,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> Result<()>,
|
||||
) -> Result<()>
|
||||
|
||||
@@ -131,7 +131,7 @@ impl<
|
||||
/// ````
|
||||
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
C: ChainSpecParser<ChainSpec = ChainSpec>,
|
||||
{
|
||||
@@ -148,7 +148,7 @@ impl<
|
||||
self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
) -> eyre::Result<()>
|
||||
@@ -180,7 +180,7 @@ impl<
|
||||
/// ```
|
||||
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
C: ChainSpecParser<ChainSpec = ChainSpec>,
|
||||
{
|
||||
@@ -196,7 +196,7 @@ impl<
|
||||
runner: CliRunner,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
launcher: impl AsyncFnOnce(
|
||||
WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>,
|
||||
WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>,
|
||||
Ext,
|
||||
) -> eyre::Result<()>,
|
||||
) -> eyre::Result<()>
|
||||
|
||||
@@ -119,10 +119,9 @@ impl EthereumNode {
|
||||
/// use reth_db::open_db_read_only;
|
||||
/// use reth_node_ethereum::EthereumNode;
|
||||
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
|
||||
/// use std::sync::Arc;
|
||||
///
|
||||
/// let factory = EthereumNode::provider_factory_builder()
|
||||
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
|
||||
/// .db(open_db_read_only("db", Default::default()).unwrap())
|
||||
/// .chainspec(ChainSpecBuilder::mainnet().build().into())
|
||||
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
|
||||
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())
|
||||
|
||||
@@ -100,10 +100,12 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
|
||||
ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(),
|
||||
);
|
||||
let genesis_hash = chain_spec.genesis_hash();
|
||||
let node_config = NodeConfig::test()
|
||||
.with_chain(chain_spec)
|
||||
.with_unused_ports()
|
||||
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
|
||||
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
|
||||
RpcServerArgs::default()
|
||||
.with_unused_ports()
|
||||
.with_http()
|
||||
.with_force_blob_sidecar_upcasting(),
|
||||
);
|
||||
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
|
||||
.testing_node(exec.clone())
|
||||
.node(EthereumNode::default())
|
||||
@@ -125,7 +127,7 @@ async fn can_send_legacy_sidecar_post_activation() -> eyre::Result<()> {
|
||||
let blob_tx_hash = node.rpc.inject_tx(blob_tx).await?;
|
||||
// fetch it from rpc
|
||||
let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?;
|
||||
// assert that sidecar was converted to eip7594
|
||||
// assert that sidecar was converted to eip7594 (force upcasting is enabled)
|
||||
assert!(envelope.as_eip4844().unwrap().tx().sidecar().unwrap().is_eip7594());
|
||||
// validate sidecar
|
||||
TransactionTestContext::validate_sidecar(envelope);
|
||||
@@ -161,10 +163,12 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> {
|
||||
.build(),
|
||||
);
|
||||
let genesis_hash = chain_spec.genesis_hash();
|
||||
let node_config = NodeConfig::test()
|
||||
.with_chain(chain_spec)
|
||||
.with_unused_ports()
|
||||
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
|
||||
let node_config = NodeConfig::test().with_chain(chain_spec).with_unused_ports().with_rpc(
|
||||
RpcServerArgs::default()
|
||||
.with_unused_ports()
|
||||
.with_http()
|
||||
.with_force_blob_sidecar_upcasting(),
|
||||
);
|
||||
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config.clone())
|
||||
.testing_node(exec.clone())
|
||||
.node(EthereumNode::default())
|
||||
|
||||
@@ -511,9 +511,8 @@ mod compact {
|
||||
total_length += flags.len() + buffer.len();
|
||||
buf.put_slice(&flags);
|
||||
if zstd {
|
||||
reth_zstd_compressors::RECEIPT_COMPRESSOR.with(|compressor| {
|
||||
let compressed =
|
||||
compressor.borrow_mut().compress(&buffer).expect("Failed to compress.");
|
||||
reth_zstd_compressors::with_receipt_compressor(|compressor| {
|
||||
let compressed = compressor.compress(&buffer).expect("Failed to compress.");
|
||||
buf.put(compressed.as_slice());
|
||||
});
|
||||
} else {
|
||||
@@ -525,8 +524,7 @@ mod compact {
|
||||
fn from_compact(buf: &[u8], _len: usize) -> (Self, &[u8]) {
|
||||
let (flags, mut buf) = ReceiptFlags::from(buf);
|
||||
if flags.__zstd() != 0 {
|
||||
reth_zstd_compressors::RECEIPT_DECOMPRESSOR.with(|decompressor| {
|
||||
let decompressor = &mut decompressor.borrow_mut();
|
||||
reth_zstd_compressors::with_receipt_decompressor(|decompressor| {
|
||||
let decompressed = decompressor.decompress(buf);
|
||||
let original_buf = buf;
|
||||
let mut buf: &[u8] = decompressed;
|
||||
|
||||
@@ -577,19 +577,11 @@ impl reth_codecs::Compact for TransactionSigned {
|
||||
|
||||
let tx_bits = if zstd_bit {
|
||||
let mut tmp = Vec::with_capacity(256);
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
|
||||
let mut compressor = compressor.borrow_mut();
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
})
|
||||
} else {
|
||||
let mut compressor = reth_zstd_compressors::create_tx_compressor();
|
||||
reth_zstd_compressors::with_tx_compressor(|compressor| {
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
}
|
||||
})
|
||||
} else {
|
||||
self.transaction.to_compact(buf) as u8
|
||||
};
|
||||
@@ -611,26 +603,13 @@ impl reth_codecs::Compact for TransactionSigned {
|
||||
|
||||
let zstd_bit = bitflags >> 3;
|
||||
let (transaction, buf) = if zstd_bit != 0 {
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
|
||||
let mut decompressor = decompressor.borrow_mut();
|
||||
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) =
|
||||
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
|
||||
|
||||
(transaction, buf)
|
||||
})
|
||||
} else {
|
||||
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
|
||||
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) =
|
||||
Transaction::from_compact(decompressor.decompress(buf), transaction_type);
|
||||
|
||||
(transaction, buf)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
let transaction_type = bitflags >> 1;
|
||||
Transaction::from_compact(buf, transaction_type)
|
||||
|
||||
@@ -36,7 +36,6 @@ rayon = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-ethereum-forks.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["std"]
|
||||
@@ -47,7 +46,6 @@ std = [
|
||||
"alloy-primitives/std",
|
||||
"alloy-consensus/std",
|
||||
"revm/std",
|
||||
"reth-ethereum-forks/std",
|
||||
"alloy-evm/std",
|
||||
"reth-execution-errors/std",
|
||||
"reth-execution-types/std",
|
||||
|
||||
@@ -41,6 +41,7 @@ metrics.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
reth-tracing.workspace = true
|
||||
alloy-primitives = { workspace = true, features = ["rand"] }
|
||||
tokio = { workspace = true, features = ["rt-multi-thread"] }
|
||||
secp256k1 = { workspace = true, features = ["std", "rand"] }
|
||||
rand_08.workspace = true
|
||||
|
||||
@@ -30,12 +30,12 @@ tokio-stream.workspace = true
|
||||
hickory-resolver = { workspace = true, features = ["tokio"] }
|
||||
|
||||
# misc
|
||||
dashmap = { workspace = true, features = ["inline"] }
|
||||
data-encoding.workspace = true
|
||||
linked_hash_set.workspace = true
|
||||
schnellru.workspace = true
|
||||
thiserror.workspace = true
|
||||
tracing.workspace = true
|
||||
parking_lot.workspace = true
|
||||
serde = { workspace = true, optional = true }
|
||||
serde_with = { workspace = true, optional = true }
|
||||
|
||||
@@ -56,9 +56,9 @@ serde = [
|
||||
"alloy-primitives/serde",
|
||||
"enr/serde",
|
||||
"linked_hash_set/serde",
|
||||
"parking_lot/serde",
|
||||
"rand/serde",
|
||||
"secp256k1/serde",
|
||||
"hickory-resolver/serde",
|
||||
"reth-ethereum-forks/serde",
|
||||
"dashmap/serde",
|
||||
]
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
//! Perform DNS lookups
|
||||
|
||||
use dashmap::DashMap;
|
||||
use hickory_resolver::name_server::ConnectionProvider;
|
||||
pub use hickory_resolver::{ResolveError, TokioResolver};
|
||||
use parking_lot::RwLock;
|
||||
use std::{collections::HashMap, future::Future};
|
||||
use std::future::Future;
|
||||
use tracing::trace;
|
||||
|
||||
/// A type that can lookup DNS entries
|
||||
@@ -72,25 +72,25 @@ impl Resolver for DnsResolver {
|
||||
|
||||
/// A [Resolver] that uses an in memory map to lookup entries
|
||||
#[derive(Debug, Default)]
|
||||
pub struct MapResolver(RwLock<HashMap<String, String>>);
|
||||
pub struct MapResolver(DashMap<String, String>);
|
||||
|
||||
// === impl MapResolver ===
|
||||
|
||||
impl MapResolver {
|
||||
/// Inserts a key-value pair into the map.
|
||||
pub fn insert(&self, k: String, v: String) -> Option<String> {
|
||||
self.0.write().insert(k, v)
|
||||
self.0.insert(k, v)
|
||||
}
|
||||
|
||||
/// Returns the value corresponding to the key
|
||||
pub fn get(&self, k: &str) -> Option<String> {
|
||||
self.0.read().get(k).cloned()
|
||||
self.0.get(k).map(|entry| entry.value().clone())
|
||||
}
|
||||
|
||||
/// Removes a key from the map, returning the value at the key if the key was previously in the
|
||||
/// map.
|
||||
pub fn remove(&self, k: &str) -> Option<String> {
|
||||
self.0.write().remove(k)
|
||||
self.0.remove(k).map(|(_, v)| v)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -251,6 +251,8 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
|
||||
}
|
||||
|
||||
/// Creates a preconfigured node for testing purposes with a specific datadir.
|
||||
///
|
||||
/// The entire `datadir` will be cleaned up when the node is dropped.
|
||||
#[cfg(feature = "test-utils")]
|
||||
pub fn testing_node_with_datadir(
|
||||
mut self,
|
||||
@@ -268,7 +270,7 @@ impl<DB, ChainSpec: EthChainSpec> NodeBuilder<DB, ChainSpec> {
|
||||
let data_dir =
|
||||
path.unwrap_or_chain_default(self.config.chain.chain(), self.config.datadir.clone());
|
||||
|
||||
let db = reth_db::test_utils::create_test_rw_db_with_path(data_dir.db());
|
||||
let db = reth_db::test_utils::create_test_rw_db_with_datadir(data_dir.data_dir());
|
||||
|
||||
WithLaunchContext { builder: self.with_database(db), task_executor }
|
||||
}
|
||||
|
||||
@@ -236,7 +236,7 @@ impl LaunchContext {
|
||||
.map_or(0, |num| num.get().saturating_sub(reserved_cpu_cores).max(1));
|
||||
if let Err(err) = ThreadPoolBuilder::new()
|
||||
.num_threads(num_threads)
|
||||
.thread_name(|i| format!("reth-rayon-{i}"))
|
||||
.thread_name(|i| format!("rayon-{i}"))
|
||||
.build_global()
|
||||
{
|
||||
warn!(%err, "Failed to build global thread pool")
|
||||
|
||||
@@ -218,9 +218,9 @@ impl<Node: FullNodeComponents, AddOns: NodeAddOns<Node>> DerefMut for FullNode<N
|
||||
}
|
||||
|
||||
/// Helper type alias to define [`FullNode`] for a given [`Node`].
|
||||
pub type FullNodeFor<N, DB = Arc<DatabaseEnv>> =
|
||||
pub type FullNodeFor<N, DB = DatabaseEnv> =
|
||||
FullNode<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;
|
||||
|
||||
/// Helper type alias to define [`NodeHandle`] for a given [`Node`].
|
||||
pub type NodeHandleFor<N, DB = Arc<DatabaseEnv>> =
|
||||
pub type NodeHandleFor<N, DB = DatabaseEnv> =
|
||||
NodeHandle<NodeAdapter<RethFullAdapter<DB, N>>, <N as Node<RethFullAdapter<DB, N>>>::AddOns>;
|
||||
|
||||
@@ -1192,6 +1192,7 @@ impl<'a, N: FullNodeComponents<Types: NodeTypes<ChainSpec: Hardforks + EthereumH
|
||||
.pending_block_kind(self.config.pending_block_kind)
|
||||
.raw_tx_forwarder(self.config.raw_tx_forwarder)
|
||||
.evm_memory_limit(self.config.rpc_evm_memory_limit)
|
||||
.force_blob_sidecar_upcasting(self.config.force_blob_sidecar_upcasting)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,7 +1,10 @@
|
||||
//! clap [Args](clap::Args) for engine purposes
|
||||
|
||||
use clap::{builder::Resettable, Args};
|
||||
use reth_engine_primitives::{TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE};
|
||||
use reth_engine_primitives::{
|
||||
TreeConfig, DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE, DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
};
|
||||
use std::sync::OnceLock;
|
||||
|
||||
use crate::node_config::{
|
||||
@@ -38,6 +41,8 @@ pub struct DefaultEngineValues {
|
||||
disable_proof_v2: bool,
|
||||
cache_metrics_disabled: bool,
|
||||
enable_sparse_trie_as_cache: bool,
|
||||
sparse_trie_prune_depth: usize,
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
}
|
||||
|
||||
impl DefaultEngineValues {
|
||||
@@ -179,6 +184,18 @@ impl DefaultEngineValues {
|
||||
self.enable_sparse_trie_as_cache = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the sparse trie prune depth by default
|
||||
pub const fn with_sparse_trie_prune_depth(mut self, v: usize) -> Self {
|
||||
self.sparse_trie_prune_depth = v;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set the maximum number of storage tries to retain after sparse trie pruning by default
|
||||
pub const fn with_sparse_trie_max_storage_tries(mut self, v: usize) -> Self {
|
||||
self.sparse_trie_max_storage_tries = v;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for DefaultEngineValues {
|
||||
@@ -205,6 +222,8 @@ impl Default for DefaultEngineValues {
|
||||
disable_proof_v2: false,
|
||||
cache_metrics_disabled: false,
|
||||
enable_sparse_trie_as_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -336,6 +355,14 @@ pub struct EngineArgs {
|
||||
/// Enable sparse trie as cache.
|
||||
#[arg(long = "engine.enable-sparse-trie-as-cache", default_value_t = DefaultEngineValues::get_global().enable_sparse_trie_as_cache, conflicts_with = "disable_proof_v2")]
|
||||
pub enable_sparse_trie_as_cache: bool,
|
||||
|
||||
/// Sparse trie prune depth.
|
||||
#[arg(long = "engine.sparse-trie-prune-depth", default_value_t = DefaultEngineValues::get_global().sparse_trie_prune_depth, requires = "enable_sparse_trie_as_cache")]
|
||||
pub sparse_trie_prune_depth: usize,
|
||||
|
||||
/// Maximum number of storage tries to retain after sparse trie pruning.
|
||||
#[arg(long = "engine.sparse-trie-max-storage-tries", default_value_t = DefaultEngineValues::get_global().sparse_trie_max_storage_tries, requires = "enable_sparse_trie_as_cache")]
|
||||
pub sparse_trie_max_storage_tries: usize,
|
||||
}
|
||||
|
||||
#[allow(deprecated)]
|
||||
@@ -363,6 +390,8 @@ impl Default for EngineArgs {
|
||||
disable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
enable_sparse_trie_as_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
} = DefaultEngineValues::get_global().clone();
|
||||
Self {
|
||||
persistence_threshold,
|
||||
@@ -390,6 +419,8 @@ impl Default for EngineArgs {
|
||||
disable_proof_v2,
|
||||
cache_metrics_disabled,
|
||||
enable_sparse_trie_as_cache,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -397,7 +428,7 @@ impl Default for EngineArgs {
|
||||
impl EngineArgs {
|
||||
/// Creates a [`TreeConfig`] from the engine arguments.
|
||||
pub fn tree_config(&self) -> TreeConfig {
|
||||
let mut config = TreeConfig::default()
|
||||
TreeConfig::default()
|
||||
.with_persistence_threshold(self.persistence_threshold)
|
||||
.with_memory_block_buffer_target(self.memory_block_buffer_target)
|
||||
.with_legacy_state_root(self.legacy_state_root_task_enabled)
|
||||
@@ -414,21 +445,14 @@ impl EngineArgs {
|
||||
.with_always_process_payload_attributes_on_canonical_head(
|
||||
self.always_process_payload_attributes_on_canonical_head,
|
||||
)
|
||||
.with_unwind_canonical_header(self.allow_unwind_canonical_header);
|
||||
|
||||
if let Some(count) = self.storage_worker_count {
|
||||
config = config.with_storage_worker_count(count);
|
||||
}
|
||||
|
||||
if let Some(count) = self.account_worker_count {
|
||||
config = config.with_account_worker_count(count);
|
||||
}
|
||||
|
||||
config = config.with_disable_proof_v2(self.disable_proof_v2);
|
||||
config = config.without_cache_metrics(self.cache_metrics_disabled);
|
||||
config = config.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache);
|
||||
|
||||
config
|
||||
.with_unwind_canonical_header(self.allow_unwind_canonical_header)
|
||||
.with_storage_worker_count_opt(self.storage_worker_count)
|
||||
.with_account_worker_count_opt(self.account_worker_count)
|
||||
.with_disable_proof_v2(self.disable_proof_v2)
|
||||
.without_cache_metrics(self.cache_metrics_disabled)
|
||||
.with_enable_sparse_trie_as_cache(self.enable_sparse_trie_as_cache)
|
||||
.with_sparse_trie_prune_depth(self.sparse_trie_prune_depth)
|
||||
.with_sparse_trie_max_storage_tries(self.sparse_trie_max_storage_tries)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -479,7 +503,9 @@ mod tests {
|
||||
account_worker_count: Some(8),
|
||||
disable_proof_v2: false,
|
||||
cache_metrics_disabled: true,
|
||||
enable_sparse_trie_as_cache: false,
|
||||
enable_sparse_trie_as_cache: true,
|
||||
sparse_trie_prune_depth: 10,
|
||||
sparse_trie_max_storage_tries: 100,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<EngineArgs>::parse_from([
|
||||
@@ -510,6 +536,11 @@ mod tests {
|
||||
"--engine.account-worker-count",
|
||||
"8",
|
||||
"--engine.disable-cache-metrics",
|
||||
"--engine.enable-sparse-trie-as-cache",
|
||||
"--engine.sparse-trie-prune-depth",
|
||||
"10",
|
||||
"--engine.sparse-trie-max-storage-tries",
|
||||
"100",
|
||||
])
|
||||
.args;
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ use clap::{builder::RangedU64ValueParser, Args};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_config::config::PruneConfig;
|
||||
use reth_prune_types::{
|
||||
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_DISTANCE, MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
};
|
||||
use std::{collections::BTreeMap, ops::Not, sync::OnceLock};
|
||||
|
||||
@@ -81,7 +81,7 @@ impl Default for DefaultPruningValues {
|
||||
minimal_prune_modes: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Distance(MINIMUM_DISTANCE)),
|
||||
account_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
|
||||
storage_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
|
||||
bodies_history: Some(PruneMode::Distance(MINIMUM_UNWIND_SAFE_DISTANCE)),
|
||||
|
||||
@@ -647,6 +647,14 @@ pub struct RpcServerArgs {
|
||||
/// transactions from the same sender will also be skipped.
|
||||
#[arg(long = "testing.skip-invalid-transactions", default_value_t = true)]
|
||||
pub testing_skip_invalid_transactions: bool,
|
||||
|
||||
/// Force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
|
||||
///
|
||||
/// When enabled, blob transactions submitted via `eth_sendRawTransaction` with EIP-4844
|
||||
/// sidecars will be automatically converted to EIP-7594 format if the next block is Osaka.
|
||||
/// By default this is disabled, meaning transactions are submitted as-is.
|
||||
#[arg(long = "rpc.force-blob-sidecar-upcasting", default_value_t = false)]
|
||||
pub rpc_force_blob_sidecar_upcasting: bool,
|
||||
}
|
||||
|
||||
impl RpcServerArgs {
|
||||
@@ -768,6 +776,12 @@ impl RpcServerArgs {
|
||||
self.rpc_send_raw_transaction_sync_timeout = timeout;
|
||||
self
|
||||
}
|
||||
|
||||
/// Enables forced blob sidecar upcasting from EIP-4844 to EIP-7594 format.
|
||||
pub const fn with_force_blob_sidecar_upcasting(mut self) -> Self {
|
||||
self.rpc_force_blob_sidecar_upcasting = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl Default for RpcServerArgs {
|
||||
@@ -860,6 +874,7 @@ impl Default for RpcServerArgs {
|
||||
gas_price_oracle,
|
||||
rpc_send_raw_transaction_sync_timeout,
|
||||
testing_skip_invalid_transactions: true,
|
||||
rpc_force_blob_sidecar_upcasting: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1036,6 +1051,7 @@ mod tests {
|
||||
},
|
||||
rpc_send_raw_transaction_sync_timeout: std::time::Duration::from_secs(30),
|
||||
testing_skip_invalid_transactions: true,
|
||||
rpc_force_blob_sidecar_upcasting: false,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<RpcServerArgs>::parse_from([
|
||||
|
||||
@@ -21,7 +21,7 @@ alloy-primitives.workspace = true
|
||||
alloy-consensus.workspace = true
|
||||
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
tokio-tungstenite.workspace = true
|
||||
futures-util.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
|
||||
|
||||
@@ -37,7 +37,7 @@ tempfile = { workspace = true, optional = true }
|
||||
tikv-jemalloc-ctl = { workspace = true, optional = true, features = ["stats"] }
|
||||
|
||||
[target.'cfg(target_os = "linux")'.dependencies]
|
||||
procfs = "0.17.0"
|
||||
procfs = "0.18.0"
|
||||
|
||||
[dev-dependencies]
|
||||
reqwest.workspace = true
|
||||
|
||||
@@ -37,7 +37,7 @@ pub use commands::{import::ImportOpCommand, import_receipts::ImportReceiptsOpCom
|
||||
use reth_optimism_chainspec::OpChainSpec;
|
||||
use reth_rpc_server_types::{DefaultRpcModuleValidator, RpcModuleValidator};
|
||||
|
||||
use std::{ffi::OsString, fmt, marker::PhantomData, sync::Arc};
|
||||
use std::{ffi::OsString, fmt, marker::PhantomData};
|
||||
|
||||
use chainspec::OpChainSpecParser;
|
||||
use clap::Parser;
|
||||
@@ -121,7 +121,7 @@ where
|
||||
/// [`NodeCommand`](reth_cli_commands::node::NodeCommand).
|
||||
pub fn run<L, Fut>(self, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
{
|
||||
self.with_runner(CliRunner::try_default_runtime()?, launcher)
|
||||
@@ -130,7 +130,7 @@ where
|
||||
/// Execute the configured cli command with the provided [`CliRunner`].
|
||||
pub fn with_runner<L, Fut>(self, runner: CliRunner, launcher: L) -> eyre::Result<()>
|
||||
where
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<Arc<DatabaseEnv>, C::ChainSpec>>, Ext) -> Fut,
|
||||
L: FnOnce(WithLaunchContext<NodeBuilder<DatabaseEnv, C::ChainSpec>>, Ext) -> Fut,
|
||||
Fut: Future<Output = eyre::Result<()>>,
|
||||
{
|
||||
let mut this = self.configure();
|
||||
|
||||
@@ -38,7 +38,7 @@ op-alloy-rpc-types-engine = { workspace = true, features = ["k256"] }
|
||||
|
||||
# io
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
tokio-tungstenite.workspace = true
|
||||
serde_json.workspace = true
|
||||
url.workspace = true
|
||||
futures-util.workspace = true
|
||||
|
||||
@@ -101,7 +101,7 @@ impl<T: SignedTransaction> SequenceManager<T> {
|
||||
// Bundle completed sequence with its decoded transactions and push to cache
|
||||
// Ring buffer automatically evicts oldest entry when full
|
||||
let txs = std::mem::take(&mut self.pending_transactions);
|
||||
self.completed_cache.push((completed, txs));
|
||||
self.completed_cache.enqueue((completed, txs));
|
||||
|
||||
// ensure cache is wiped on new flashblock
|
||||
let _ = self.pending.take_cached_reads();
|
||||
|
||||
@@ -2,6 +2,7 @@ use futures_util::stream::StreamExt;
|
||||
use reth_optimism_flashblocks::WsFlashBlockStream;
|
||||
|
||||
#[tokio::test]
|
||||
#[ignore = "depends on external service availability"]
|
||||
async fn test_streaming_flashblocks_from_remote_source_is_successful() {
|
||||
let items = 3;
|
||||
let ws_url = "wss://sepolia.flashblocks.base.org/ws".parse().unwrap();
|
||||
|
||||
@@ -219,10 +219,9 @@ impl OpNode {
|
||||
/// use reth_optimism_chainspec::OpChainSpecBuilder;
|
||||
/// use reth_optimism_node::OpNode;
|
||||
/// use reth_provider::providers::{RocksDBProvider, StaticFileProvider};
|
||||
/// use std::sync::Arc;
|
||||
///
|
||||
/// let factory = OpNode::provider_factory_builder()
|
||||
/// .db(Arc::new(open_db_read_only("db", Default::default()).unwrap()))
|
||||
/// .db(open_db_read_only("db", Default::default()).unwrap())
|
||||
/// .chainspec(OpChainSpecBuilder::base_mainnet().build().into())
|
||||
/// .static_file(StaticFileProvider::read_only("db/static_files", false).unwrap())
|
||||
/// .rocksdb_provider(RocksDBProvider::builder("db/rocksdb").build().unwrap())
|
||||
|
||||
@@ -435,19 +435,11 @@ impl reth_codecs::Compact for OpTransactionSigned {
|
||||
|
||||
let tx_bits = if zstd_bit {
|
||||
let mut tmp = Vec::with_capacity(256);
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_COMPRESSOR.with(|compressor| {
|
||||
let mut compressor = compressor.borrow_mut();
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
})
|
||||
} else {
|
||||
let mut compressor = reth_zstd_compressors::create_tx_compressor();
|
||||
reth_zstd_compressors::with_tx_compressor(|compressor| {
|
||||
let tx_bits = self.transaction.to_compact(&mut tmp);
|
||||
buf.put_slice(&compressor.compress(&tmp).expect("Failed to compress"));
|
||||
tx_bits as u8
|
||||
}
|
||||
})
|
||||
} else {
|
||||
self.transaction.to_compact(buf) as u8
|
||||
};
|
||||
@@ -469,29 +461,15 @@ impl reth_codecs::Compact for OpTransactionSigned {
|
||||
|
||||
let zstd_bit = bitflags >> 3;
|
||||
let (transaction, buf) = if zstd_bit != 0 {
|
||||
if cfg!(feature = "std") {
|
||||
reth_zstd_compressors::TRANSACTION_DECOMPRESSOR.with(|decompressor| {
|
||||
let mut decompressor = decompressor.borrow_mut();
|
||||
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) = OpTypedTransaction::from_compact(
|
||||
decompressor.decompress(buf),
|
||||
transaction_type,
|
||||
);
|
||||
|
||||
(transaction, buf)
|
||||
})
|
||||
} else {
|
||||
let mut decompressor = reth_zstd_compressors::create_tx_decompressor();
|
||||
reth_zstd_compressors::with_tx_decompressor(|decompressor| {
|
||||
// TODO: enforce that zstd is only present at a "top" level type
|
||||
let transaction_type = (bitflags & 0b110) >> 1;
|
||||
let (transaction, _) = OpTypedTransaction::from_compact(
|
||||
decompressor.decompress(buf),
|
||||
transaction_type,
|
||||
);
|
||||
|
||||
(transaction, buf)
|
||||
}
|
||||
})
|
||||
} else {
|
||||
let transaction_type = bitflags >> 1;
|
||||
OpTypedTransaction::from_compact(buf, transaction_type)
|
||||
|
||||
@@ -61,7 +61,7 @@ op-revm.workspace = true
|
||||
tokio.workspace = true
|
||||
futures.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
async-trait.workspace = true
|
||||
tower.workspace = true
|
||||
|
||||
|
||||
@@ -36,6 +36,7 @@ secp256k1 = { workspace = true, features = ["recovery"], optional = true }
|
||||
auto_impl.workspace = true
|
||||
byteorder = { workspace = true, optional = true }
|
||||
bytes.workspace = true
|
||||
dashmap = { workspace = true, features = ["inline"], optional = true }
|
||||
derive_more.workspace = true
|
||||
once_cell.workspace = true
|
||||
serde_with = { workspace = true, optional = true }
|
||||
@@ -116,6 +117,7 @@ arbitrary = [
|
||||
"alloy-trie/arbitrary",
|
||||
"reth-chainspec/arbitrary",
|
||||
"alloy-rpc-types-eth?/arbitrary",
|
||||
"dashmap?/arbitrary",
|
||||
]
|
||||
serde-bincode-compat = [
|
||||
"serde",
|
||||
@@ -144,6 +146,7 @@ serde = [
|
||||
"revm-state/serde",
|
||||
"rand_08/serde",
|
||||
"alloy-rpc-types-eth?/serde",
|
||||
"dashmap?/serde",
|
||||
]
|
||||
reth-codec = [
|
||||
"dep:reth-codecs",
|
||||
@@ -157,4 +160,5 @@ op = [
|
||||
rayon = [
|
||||
"dep:rayon",
|
||||
]
|
||||
dashmap = ["dep:dashmap"]
|
||||
rpc-compat = ["alloy-rpc-types-eth"]
|
||||
|
||||
@@ -188,6 +188,12 @@ pub mod serde_bincode_compat;
|
||||
pub mod size;
|
||||
pub use size::InMemorySize;
|
||||
|
||||
/// Rayon utilities
|
||||
#[cfg(feature = "rayon")]
|
||||
pub mod rayon;
|
||||
#[cfg(feature = "rayon")]
|
||||
pub use rayon::ParallelBridgeBuffered;
|
||||
|
||||
/// Node traits
|
||||
pub mod node;
|
||||
pub use node::{BlockTy, BodyTy, HeaderTy, NodePrimitives, ReceiptTy, TxTy};
|
||||
@@ -239,3 +245,12 @@ pub mod test_utils {
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
pub use crate::{block::TestBlock, header::test_utils::TestHeader};
|
||||
}
|
||||
|
||||
/// Re-exports of `dashmap` types with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
|
||||
#[cfg(feature = "dashmap")]
|
||||
pub mod dashmap {
|
||||
pub use ::dashmap::{mapref, DashSet, Entry};
|
||||
/// Re-export of `DashMap` with [`alloy_primitives::map::DefaultHashBuilder`] as the hasher.
|
||||
pub type DashMap<K, V, S = alloy_primitives::map::DefaultHashBuilder> =
|
||||
::dashmap::DashMap<K, V, S>;
|
||||
}
|
||||
|
||||
32
crates/primitives-traits/src/rayon.rs
Normal file
32
crates/primitives-traits/src/rayon.rs
Normal file
@@ -0,0 +1,32 @@
|
||||
//! Rayon parallel iterator utilities.
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use rayon::iter::IntoParallelIterator;
|
||||
|
||||
/// Extension trait for iterators to convert them to parallel iterators via collection.
|
||||
///
|
||||
/// This is an alternative to [`rayon::iter::ParallelBridge`] that first collects the iterator
|
||||
/// into a `Vec`, then calls [`IntoParallelIterator`] on it. This avoids the mutex contention
|
||||
/// that can occur with `par_bridge` when either the iterator's `next()` method is fast or the
|
||||
/// parallel tasks are fast, as `par_bridge` wraps the iterator in a mutex.
|
||||
///
|
||||
/// # When to use
|
||||
///
|
||||
/// Use `par_bridge_buffered` instead of `par_bridge` when:
|
||||
/// - The iterator produces items quickly
|
||||
/// - The parallel work per item is relatively light
|
||||
/// - The total number of items is known to be reasonable for memory
|
||||
///
|
||||
/// Stick with `par_bridge` when:
|
||||
/// - The iterator is slow (e.g., I/O bound) and you want to overlap iteration with processing
|
||||
/// - Memory is constrained and you cannot afford to collect all items upfront
|
||||
pub trait ParallelBridgeBuffered: Iterator<Item: Send> + Sized {
|
||||
/// Collects this iterator into a `Vec` and returns a parallel iterator over it.
|
||||
///
|
||||
/// See [this trait's documentation](ParallelBridgeBuffered) for more details.
|
||||
fn par_bridge_buffered(self) -> rayon::vec::IntoIter<Self::Item> {
|
||||
self.collect::<Vec<_>>().into_par_iter()
|
||||
}
|
||||
}
|
||||
|
||||
impl<I: Iterator<Item: Send>> ParallelBridgeBuffered for I {}
|
||||
@@ -42,11 +42,15 @@ rayon.workspace = true
|
||||
tokio.workspace = true
|
||||
rustc-hash.workspace = true
|
||||
|
||||
[features]
|
||||
rocksdb = ["reth-provider/rocksdb"]
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
reth-db = { workspace = true, features = ["test-utils"] }
|
||||
reth-stages = { workspace = true, features = ["test-utils"] }
|
||||
reth-stages = { workspace = true, features = ["test-utils", "rocksdb"] }
|
||||
reth-primitives-traits = { workspace = true, features = ["arbitrary"] }
|
||||
reth-storage-api.workspace = true
|
||||
reth-testing-utils.workspace = true
|
||||
reth-tracing.workspace = true
|
||||
|
||||
|
||||
@@ -7,10 +7,10 @@ use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
|
||||
DatabaseProviderFactory, NodePrimitivesProvider, PruneCheckpointReader, PruneCheckpointWriter,
|
||||
StageCheckpointReader, StaticFileProviderFactory, StorageSettingsCache,
|
||||
RocksDBProviderFactory, StageCheckpointReader, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::watch;
|
||||
|
||||
@@ -85,6 +85,7 @@ impl PrunerBuilder {
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory
|
||||
+ StaticFileProviderFactory<
|
||||
Primitives: NodePrimitives<SignedTx: Value, Receipt: Value, BlockHeader: Value>,
|
||||
>,
|
||||
@@ -121,7 +122,8 @@ impl PrunerBuilder {
|
||||
+ StorageSettingsCache
|
||||
+ StageCheckpointReader
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
let segments = SegmentSet::<Provider>::from_components(static_file_provider, self.segments);
|
||||
|
||||
|
||||
@@ -25,6 +25,10 @@ pub use user::{
|
||||
///
|
||||
/// This is a generic helper function used by both receipts and bodies pruning
|
||||
/// when data is stored in static files.
|
||||
///
|
||||
/// The checkpoint block number is set to the highest block in the actually deleted files,
|
||||
/// not `input.to_block`, since `to_block` might refer to a block in the middle of an
|
||||
/// undeleted file.
|
||||
pub(crate) fn prune_static_files<Provider>(
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
@@ -36,6 +40,51 @@ where
|
||||
let deleted_headers =
|
||||
provider.static_file_provider().delete_segment_below_block(segment, input.to_block + 1)?;
|
||||
|
||||
if deleted_headers.is_empty() {
|
||||
return Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned: 0,
|
||||
checkpoint: input
|
||||
.previous_checkpoint
|
||||
.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
})
|
||||
}
|
||||
|
||||
let tx_ranges = deleted_headers.iter().filter_map(|header| header.tx_range());
|
||||
|
||||
let pruned = tx_ranges.clone().map(|range| range.len()).sum::<u64>() as usize;
|
||||
|
||||
// The highest block number in the deleted files is the actual checkpoint.
|
||||
let checkpoint_block = deleted_headers
|
||||
.iter()
|
||||
.filter_map(|header| header.block_range())
|
||||
.map(|range| range.end())
|
||||
.max();
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: checkpoint_block,
|
||||
tx_number: tx_ranges.map(|range| range.end()).max(),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Deletes ALL static file jars for a given segment.
|
||||
///
|
||||
/// This is used for `PruneMode::Full` where all data should be removed, including the highest jar.
|
||||
/// Unlike [`prune_static_files`], this does not preserve the most recent jar.
|
||||
pub(crate) fn delete_static_files_segment<Provider>(
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
segment: StaticFileSegment,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: StaticFileProviderFactory,
|
||||
{
|
||||
let deleted_headers = provider.static_file_provider().delete_segment(segment)?;
|
||||
|
||||
if deleted_headers.is_empty() {
|
||||
return Ok(SegmentOutput::done())
|
||||
}
|
||||
|
||||
@@ -7,10 +7,11 @@ use reth_db_api::{table::Value, transaction::DbTxMut};
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::StaticFileProvider, BlockReader, ChainStateBlockReader, DBProvider,
|
||||
PruneCheckpointReader, PruneCheckpointWriter, StaticFileProviderFactory, StorageSettingsCache,
|
||||
PruneCheckpointReader, PruneCheckpointWriter, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader};
|
||||
use reth_storage_api::{ChangeSetReader, StorageChangeSetReader, StorageSettingsCache};
|
||||
|
||||
/// Collection of [`Segment`]. Thread-safe, allocated on the heap.
|
||||
#[derive(Debug)]
|
||||
@@ -55,7 +56,8 @@ where
|
||||
+ ChainStateBlockReader
|
||||
+ StorageSettingsCache
|
||||
+ ChangeSetReader
|
||||
+ StorageChangeSetReader,
|
||||
+ StorageChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
/// Creates a [`SegmentSet`] from an existing components, such as [`StaticFileProvider`] and
|
||||
/// [`PruneModes`].
|
||||
@@ -74,8 +76,11 @@ where
|
||||
} = prune_modes;
|
||||
|
||||
Self::default()
|
||||
// Bodies - run first since file deletion is fast
|
||||
.segment_opt(bodies_history.map(Bodies::new))
|
||||
// Transaction lookup must run before bodies because it needs to read transaction
|
||||
// data from static files before bodies deletes them.
|
||||
.segment_opt(transaction_lookup.map(TransactionLookup::new))
|
||||
// Bodies
|
||||
.segment_opt(bodies_history.map(|mode| Bodies::new(mode, transaction_lookup)))
|
||||
// Account history
|
||||
.segment_opt(account_history.map(AccountHistory::new))
|
||||
// Storage history
|
||||
@@ -87,8 +92,6 @@ where
|
||||
(!receipts_log_filter.is_empty())
|
||||
.then(|| ReceiptsByLogs::new(receipts_log_filter.clone())),
|
||||
)
|
||||
// Transaction lookup
|
||||
.segment_opt(transaction_lookup.map(TransactionLookup::new))
|
||||
// Sender recovery
|
||||
.segment_opt(sender_recovery.map(SenderRecovery::new))
|
||||
}
|
||||
|
||||
@@ -10,20 +10,20 @@ use alloy_primitives::BlockNumber;
|
||||
use reth_db_api::{models::ShardedKey, tables, transaction::DbTxMut};
|
||||
use reth_provider::{
|
||||
changeset_walker::StaticFileAccountChangesetWalker, DBProvider, EitherWriter,
|
||||
StaticFileProviderFactory, StorageSettingsCache,
|
||||
RocksDBProviderFactory, StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::{
|
||||
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::ChangeSetReader;
|
||||
use reth_storage_api::{ChangeSetReader, StorageSettingsCache};
|
||||
use rustc_hash::FxHashMap;
|
||||
use tracing::{instrument, trace};
|
||||
|
||||
/// Number of account history tables to prune in one step.
|
||||
///
|
||||
/// Account History consists of two tables: [`tables::AccountChangeSets`] and
|
||||
/// [`tables::AccountsHistory`]. We want to prune them to the same block number.
|
||||
/// Account History consists of two tables: [`tables::AccountChangeSets`] (either in database or
|
||||
/// static files) and [`tables::AccountsHistory`]. We want to prune them to the same block number.
|
||||
const ACCOUNT_HISTORY_TABLES_TO_PRUNE: usize = 2;
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -42,7 +42,8 @@ where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageSettingsCache
|
||||
+ ChangeSetReader,
|
||||
+ ChangeSetReader
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::AccountHistory
|
||||
@@ -67,7 +68,13 @@ where
|
||||
};
|
||||
let range_end = *range.end();
|
||||
|
||||
// Check where account changesets are stored
|
||||
// Check where account history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().account_history_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
// Check where account changesets are stored (MDBX path)
|
||||
if EitherWriter::account_changesets_destination(provider).is_static_file() {
|
||||
self.prune_static_files(provider, input, range, range_end)
|
||||
} else {
|
||||
@@ -94,6 +101,8 @@ impl AccountHistory {
|
||||
input.limiter
|
||||
};
|
||||
|
||||
// The limiter may already be exhausted from a previous segment in the same prune run.
|
||||
// Early exit avoids unnecessary iteration when no budget remains.
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
@@ -101,11 +110,14 @@ impl AccountHistory {
|
||||
))
|
||||
}
|
||||
|
||||
// The size of this map it's limited by `prune_delete_limit * blocks_since_last_run /
|
||||
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with the current defaults it's usually `3500 * 5 /
|
||||
// 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total size should be up to
|
||||
// ~0.25MB + some hashmap overhead. `blocks_since_last_run` is additionally limited by the
|
||||
// `max_reorg_depth`, so no OOM is expected here.
|
||||
// Deleted account changeset keys (account addresses) with the highest block number deleted
|
||||
// for that key.
|
||||
//
|
||||
// The size of this map is limited by `prune_delete_limit * blocks_since_last_run /
|
||||
// ACCOUNT_HISTORY_TABLES_TO_PRUNE`, and with current default it's usually `3500 * 5
|
||||
// / 2`, so 8750 entries. Each entry is `160 bit + 64 bit`, so the total
|
||||
// size should be up to ~0.25MB + some hashmap overhead. `blocks_since_last_run` is
|
||||
// additionally limited by the `max_reorg_depth`, so no OOM is expected here.
|
||||
let mut highest_deleted_accounts = FxHashMap::default();
|
||||
let mut last_changeset_pruned_block = None;
|
||||
let mut pruned_changesets = 0;
|
||||
@@ -124,8 +136,8 @@ impl AccountHistory {
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
|
||||
// Delete static file jars below the pruned block
|
||||
if let Some(last_block) = last_changeset_pruned_block {
|
||||
// Delete static file jars only when fully processed
|
||||
if done && let Some(last_block) = last_changeset_pruned_block {
|
||||
provider
|
||||
.static_file_provider()
|
||||
.delete_segment_below_block(StaticFileSegment::AccountChangeSets, last_block + 1)?;
|
||||
@@ -210,6 +222,105 @@ impl AccountHistory {
|
||||
)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Prunes account history when indices are stored in `RocksDB`.
|
||||
///
|
||||
/// Reads account changesets from static files and prunes the corresponding
|
||||
/// `RocksDB` history shards.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn prune_rocksdb<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
range: std::ops::RangeInclusive<BlockNumber>,
|
||||
range_end: BlockNumber,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: DBProvider + StaticFileProviderFactory + ChangeSetReader + RocksDBProviderFactory,
|
||||
{
|
||||
// Unlike MDBX path, we don't divide the limit by 2 because RocksDB path only prunes
|
||||
// history shards (no separate changeset table to delete from). The changesets are in
|
||||
// static files which are deleted separately.
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
))
|
||||
}
|
||||
|
||||
let mut highest_deleted_accounts = FxHashMap::default();
|
||||
let mut last_changeset_pruned_block = None;
|
||||
let mut changesets_processed = 0usize;
|
||||
let mut done = true;
|
||||
|
||||
// Walk account changesets from static files using a streaming iterator.
|
||||
// For each changeset, track the highest block number seen for each address
|
||||
// to determine which history shard entries need pruning.
|
||||
let walker = StaticFileAccountChangesetWalker::new(provider, range);
|
||||
for result in walker {
|
||||
if limiter.is_limit_reached() {
|
||||
done = false;
|
||||
break;
|
||||
}
|
||||
let (block_number, changeset) = result?;
|
||||
highest_deleted_accounts.insert(changeset.address, block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
changesets_processed += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned account changesets from static files");
|
||||
|
||||
let last_changeset_pruned_block = last_changeset_pruned_block
|
||||
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
|
||||
.unwrap_or(range_end);
|
||||
|
||||
// Prune RocksDB history shards for affected accounts
|
||||
let mut deleted_shards = 0usize;
|
||||
let mut updated_shards = 0usize;
|
||||
|
||||
// Sort by address for better RocksDB cache locality
|
||||
let mut sorted_accounts: Vec<_> = highest_deleted_accounts.into_iter().collect();
|
||||
sorted_accounts.sort_unstable_by_key(|(addr, _)| *addr);
|
||||
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
let targets: Vec<_> = sorted_accounts
|
||||
.iter()
|
||||
.map(|(addr, highest)| (*addr, (*highest).min(last_changeset_pruned_block)))
|
||||
.collect();
|
||||
|
||||
let outcomes = batch.prune_account_history_batch(&targets)?;
|
||||
deleted_shards = outcomes.deleted;
|
||||
updated_shards = outcomes.updated;
|
||||
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned account history (RocksDB indices)");
|
||||
|
||||
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
|
||||
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
|
||||
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
|
||||
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
|
||||
// is idempotent (re-pruning already-pruned shards is a no-op).
|
||||
if done {
|
||||
provider.static_file_provider().delete_segment_below_block(
|
||||
StaticFileSegment::AccountChangeSets,
|
||||
last_changeset_pruned_block + 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
let progress = limiter.progress(done);
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress,
|
||||
pruned: changesets_processed + deleted_shards + updated_shards,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_changeset_pruned_block),
|
||||
tx_number: None,
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -539,4 +650,272 @@ mod tests {
|
||||
test_prune(998, 2, (PruneProgress::Finished, 1000));
|
||||
test_prune(1400, 3, (PruneProgress::Finished, 804));
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
#[test]
|
||||
fn prune_rocksdb_path() {
|
||||
use reth_db_api::models::ShardedKey;
|
||||
use reth_provider::{RocksDBProviderFactory, StaticFileProviderFactory};
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=100,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
|
||||
|
||||
let (changesets, _) = random_changeset_range(
|
||||
&mut rng,
|
||||
blocks.iter(),
|
||||
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
|
||||
0..0,
|
||||
0..0,
|
||||
);
|
||||
|
||||
db.insert_changesets_to_static_files(changesets.clone(), None)
|
||||
.expect("insert changesets to static files");
|
||||
|
||||
let mut account_blocks: BTreeMap<_, Vec<u64>> = BTreeMap::new();
|
||||
for (block, changeset) in changesets.iter().enumerate() {
|
||||
for (address, _, _) in changeset {
|
||||
account_blocks.entry(*address).or_default().push(block as u64);
|
||||
}
|
||||
}
|
||||
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let mut batch = rocksdb.batch();
|
||||
for (address, block_numbers) in &account_blocks {
|
||||
let shard = BlockNumberList::new_pre_sorted(block_numbers.iter().copied());
|
||||
batch
|
||||
.put::<tables::AccountsHistory>(ShardedKey::new(*address, u64::MAX), &shard)
|
||||
.unwrap();
|
||||
}
|
||||
batch.commit().unwrap();
|
||||
|
||||
for (address, expected_blocks) in &account_blocks {
|
||||
let shards = rocksdb.account_history_shards(*address).unwrap();
|
||||
assert_eq!(shards.len(), 1);
|
||||
assert_eq!(shards[0].1.iter().collect::<Vec<_>>(), *expected_blocks);
|
||||
}
|
||||
|
||||
let to_block: BlockNumber = 50;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let input =
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_account_changesets_in_static_files(true)
|
||||
.with_account_history_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
assert_matches!(
|
||||
result,
|
||||
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
|
||||
if pruned > 0
|
||||
);
|
||||
|
||||
for (address, original_blocks) in &account_blocks {
|
||||
let shards = rocksdb.account_history_shards(*address).unwrap();
|
||||
|
||||
let expected_blocks: Vec<u64> =
|
||||
original_blocks.iter().copied().filter(|b| *b > to_block).collect();
|
||||
|
||||
if expected_blocks.is_empty() {
|
||||
assert!(
|
||||
shards.is_empty(),
|
||||
"Expected no shards for address {address:?} after pruning"
|
||||
);
|
||||
} else {
|
||||
assert_eq!(shards.len(), 1, "Expected 1 shard for address {address:?}");
|
||||
assert_eq!(
|
||||
shards[0].1.iter().collect::<Vec<_>>(),
|
||||
expected_blocks,
|
||||
"Shard blocks mismatch for address {address:?}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
let static_file_provider = db.factory.static_file_provider();
|
||||
let highest_block = static_file_provider.get_highest_static_file_block(
|
||||
reth_static_file_types::StaticFileSegment::AccountChangeSets,
|
||||
);
|
||||
if let Some(block) = highest_block {
|
||||
assert!(
|
||||
block > to_block,
|
||||
"Static files should only contain blocks above to_block ({to_block}), got {block}"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Tests that when a limiter stops mid-block (with multiple changes for the same block),
|
||||
/// the checkpoint is set to `block_number - 1` to avoid dangling index entries.
|
||||
#[test]
|
||||
fn prune_partial_progress_mid_block() {
|
||||
use alloy_primitives::{Address, U256};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_testing_utils::generators::ChangeSet;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
// Create blocks 0..=10
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=10,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
// Create specific changesets where block 5 has 4 account changes
|
||||
let addr1 = Address::with_last_byte(1);
|
||||
let addr2 = Address::with_last_byte(2);
|
||||
let addr3 = Address::with_last_byte(3);
|
||||
let addr4 = Address::with_last_byte(4);
|
||||
let addr5 = Address::with_last_byte(5);
|
||||
|
||||
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
|
||||
|
||||
// Build changesets: blocks 0-4 have 1 change each, block 5 has 4 changes, block 6 has 1
|
||||
let changesets: Vec<ChangeSet> = vec![
|
||||
vec![(addr1, account, vec![])], // block 0
|
||||
vec![(addr1, account, vec![])], // block 1
|
||||
vec![(addr1, account, vec![])], // block 2
|
||||
vec![(addr1, account, vec![])], // block 3
|
||||
vec![(addr1, account, vec![])], // block 4
|
||||
// block 5: 4 different account changes (sorted by address for consistency)
|
||||
vec![
|
||||
(addr1, account, vec![]),
|
||||
(addr2, account, vec![]),
|
||||
(addr3, account, vec![]),
|
||||
(addr4, account, vec![]),
|
||||
],
|
||||
vec![(addr5, account, vec![])], // block 6
|
||||
];
|
||||
|
||||
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
|
||||
db.insert_history(changesets.clone(), None).expect("insert history");
|
||||
|
||||
// Total changesets: 5 (blocks 0-4) + 4 (block 5) + 1 (block 6) = 10
|
||||
assert_eq!(
|
||||
db.table::<tables::AccountChangeSets>().unwrap().len(),
|
||||
changesets.iter().flatten().count()
|
||||
);
|
||||
|
||||
let prune_mode = PruneMode::Before(10);
|
||||
|
||||
// Set limiter to stop after 7 entries (mid-block 5: 5 from blocks 0-4, then 2 of 4 from
|
||||
// block 5). Due to ACCOUNT_HISTORY_TABLES_TO_PRUNE=2, actual limit is 7/2=3
|
||||
// changesets. So we'll process blocks 0, 1, 2 (3 changesets), stopping before block
|
||||
// 3. Actually, let's use a higher limit to reach block 5. With limit=14, we get 7
|
||||
// changeset slots. Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so
|
||||
// we stop mid-block 5.
|
||||
let deleted_entries_limit = 14; // 14/2 = 7 changeset entries before limit
|
||||
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
|
||||
|
||||
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
|
||||
let segment = AccountHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
|
||||
|
||||
// Save checkpoint and commit
|
||||
segment
|
||||
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
|
||||
.unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
|
||||
let checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
assert_eq!(
|
||||
checkpoint.block_number,
|
||||
Some(4),
|
||||
"Checkpoint should be block 4 (block before incomplete block 5)"
|
||||
);
|
||||
|
||||
// Verify remaining changesets (block 5 and 6 should still have entries)
|
||||
let remaining_changesets = db.table::<tables::AccountChangeSets>().unwrap();
|
||||
// After pruning blocks 0-4, remaining should be block 5 (4 entries) + block 6 (1 entry) = 5
|
||||
// But since we stopped mid-block 5, some of block 5 might be pruned
|
||||
// However, checkpoint is 4, so on re-run we should re-process from block 5
|
||||
assert!(
|
||||
!remaining_changesets.is_empty(),
|
||||
"Should have remaining changesets for blocks 5-6"
|
||||
);
|
||||
|
||||
// Verify no dangling history indices for blocks that weren't fully pruned
|
||||
// The indices for block 5 should still reference blocks <= 5 appropriately
|
||||
let history = db.table::<tables::AccountsHistory>().unwrap();
|
||||
for (key, _blocks) in &history {
|
||||
// All blocks in the history should be > checkpoint block number
|
||||
// OR the shard's highest_block_number should be > checkpoint
|
||||
assert!(
|
||||
key.highest_block_number > 4,
|
||||
"Found stale history shard with highest_block_number {} <= checkpoint 4",
|
||||
key.highest_block_number
|
||||
);
|
||||
}
|
||||
|
||||
// Run prune again to complete - should finish processing block 5 and 6
|
||||
let input2 = PruneInput {
|
||||
previous_checkpoint: Some(checkpoint),
|
||||
to_block: 10,
|
||||
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_account_changesets_in_static_files(false),
|
||||
);
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
|
||||
segment
|
||||
.save_checkpoint(
|
||||
&provider2,
|
||||
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
.unwrap();
|
||||
provider2.commit().expect("commit");
|
||||
|
||||
// Verify final checkpoint
|
||||
let final_checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
// Should now be at block 6 (the last block with changesets)
|
||||
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
|
||||
|
||||
// All changesets should be pruned
|
||||
let final_changesets = db.table::<tables::AccountChangeSets>().unwrap();
|
||||
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,9 +2,14 @@ use crate::{
|
||||
segments::{self, PruneInput, Segment},
|
||||
PrunerError,
|
||||
};
|
||||
use reth_provider::{BlockReader, StaticFileProviderFactory};
|
||||
use reth_prune_types::{PruneMode, PrunePurpose, PruneSegment, SegmentOutput};
|
||||
use alloy_primitives::BlockNumber;
|
||||
use reth_provider::{BlockReader, PruneCheckpointReader, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneInterruptReason, PruneMode, PrunePurpose, PruneSegment, SegmentOutput,
|
||||
SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use tracing::debug;
|
||||
|
||||
/// Segment responsible for pruning transactions in static files.
|
||||
///
|
||||
@@ -12,18 +17,79 @@ use reth_static_file_types::StaticFileSegment;
|
||||
#[derive(Debug)]
|
||||
pub struct Bodies {
|
||||
mode: PruneMode,
|
||||
/// Transaction lookup prune mode. Used to determine if we need to wait for tx lookup pruning
|
||||
/// before deleting transaction bodies.
|
||||
tx_lookup_mode: Option<PruneMode>,
|
||||
}
|
||||
|
||||
impl Bodies {
|
||||
/// Creates a new [`Bodies`] segment with the given prune mode.
|
||||
pub const fn new(mode: PruneMode) -> Self {
|
||||
Self { mode }
|
||||
/// Creates a new [`Bodies`] segment with the given prune mode and optional transaction lookup
|
||||
/// prune mode for coordination.
|
||||
pub const fn new(mode: PruneMode, tx_lookup_mode: Option<PruneMode>) -> Self {
|
||||
Self { mode, tx_lookup_mode }
|
||||
}
|
||||
|
||||
/// Returns the next best block that bodies can prune up to considering the transaction lookup
|
||||
/// pruning configuration (if any) and progress.
|
||||
///
|
||||
/// Returns `None` if there's no block available to prune (e.g., waiting on `tx_lookup`).
|
||||
fn next_bodies_prune_target<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: &PruneInput,
|
||||
) -> Result<Option<BlockNumber>, PrunerError>
|
||||
where
|
||||
Provider: PruneCheckpointReader,
|
||||
{
|
||||
let Some(tx_lookup_mode) = self.tx_lookup_mode else { return Ok(Some(input.to_block)) };
|
||||
|
||||
let tx_lookup_checkpoint = provider
|
||||
.get_prune_checkpoint(PruneSegment::TransactionLookup)?
|
||||
.and_then(|cp| cp.block_number);
|
||||
|
||||
// Determine the safe prune target, if any.
|
||||
// tx_lookup's next_pruned_block tells us what block it will prune next.
|
||||
// - None: tx_lookup will never prune more blocks (e.g. Before(N) reached its target), so
|
||||
// bodies can prune freely
|
||||
// - Some(next) > to_block: tx_lookup is ahead of our target, so we're safe to prune
|
||||
// to_block
|
||||
// - Some(next) <= to_block: tx_lookup still needs to prune blocks we want to delete, so we
|
||||
// must wait and only prune up to (next - 1) to preserve tx data it needs
|
||||
let to_block = match tx_lookup_mode.next_pruned_block(tx_lookup_checkpoint) {
|
||||
None => Some(input.to_block),
|
||||
Some(tx_lookup_next) if tx_lookup_next > input.to_block => Some(input.to_block),
|
||||
Some(tx_lookup_next) => {
|
||||
// We can only prune bodies up to the block BEFORE tx_lookup's next target.
|
||||
// tx_lookup_next is the next block tx_lookup will prune, meaning it still needs
|
||||
// to read transactions from that block. We must preserve those transactions,
|
||||
// so bodies can only safely delete up to (tx_lookup_next - 1).
|
||||
let Some(safe) = tx_lookup_next.checked_sub(1) else {
|
||||
return Ok(None);
|
||||
};
|
||||
|
||||
if input.previous_checkpoint.is_some_and(|cp| cp.block_number.unwrap_or(0) >= safe)
|
||||
{
|
||||
// we have pruned what we can
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
debug!(
|
||||
target: "pruner",
|
||||
to_block = input.to_block,
|
||||
safe,
|
||||
"Bodies pruning limited by tx_lookup progress"
|
||||
);
|
||||
Some(safe)
|
||||
}
|
||||
};
|
||||
|
||||
Ok(to_block)
|
||||
}
|
||||
}
|
||||
|
||||
impl<Provider> Segment<Provider> for Bodies
|
||||
where
|
||||
Provider: StaticFileProviderFactory + BlockReader,
|
||||
Provider: StaticFileProviderFactory + BlockReader + PruneCheckpointReader,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::Bodies
|
||||
@@ -38,7 +104,20 @@ where
|
||||
}
|
||||
|
||||
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
|
||||
segments::prune_static_files(provider, input, StaticFileSegment::Transactions)
|
||||
let Some(to_block) = self.next_bodies_prune_target(provider, &input)? else {
|
||||
debug!(
|
||||
to_block = input.to_block,
|
||||
"Transaction lookup still has work to be done up to target block"
|
||||
);
|
||||
return Ok(SegmentOutput::not_done(
|
||||
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
|
||||
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
));
|
||||
};
|
||||
|
||||
// Use the coordinated to_block instead of input.to_block
|
||||
let adjusted_input = PruneInput { to_block, ..input };
|
||||
segments::prune_static_files(provider, adjusted_input, StaticFileSegment::Transactions)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,7 +129,8 @@ mod tests {
|
||||
use reth_exex_types::FinishedExExHeight;
|
||||
use reth_provider::{
|
||||
test_utils::{create_test_provider_factory, MockNodeTypesWithDB},
|
||||
ProviderFactory, StaticFileWriter,
|
||||
DBProvider, DatabaseProviderFactory, ProviderFactory, PruneCheckpointWriter,
|
||||
StaticFileWriter,
|
||||
};
|
||||
use reth_prune_types::{PruneMode, PruneProgress, PruneSegment};
|
||||
use reth_static_file_types::{
|
||||
@@ -93,19 +173,83 @@ mod tests {
|
||||
static_file_provider.initialize_index().expect("initialize index");
|
||||
}
|
||||
|
||||
struct PruneTestCase {
|
||||
prune_mode: PruneMode,
|
||||
struct TestCase {
|
||||
tx_lookup_mode: Option<PruneMode>,
|
||||
tx_lookup_checkpoint_block: Option<BlockNumber>,
|
||||
bodies_mode: PruneMode,
|
||||
expected_pruned: usize,
|
||||
expected_lowest_block: Option<BlockNumber>,
|
||||
expected_progress: PruneProgress,
|
||||
}
|
||||
|
||||
impl TestCase {
|
||||
fn new() -> Self {
|
||||
Self {
|
||||
tx_lookup_mode: None,
|
||||
tx_lookup_checkpoint_block: None,
|
||||
bodies_mode: PruneMode::Full,
|
||||
expected_pruned: 0,
|
||||
expected_lowest_block: None,
|
||||
expected_progress: PruneProgress::Finished,
|
||||
}
|
||||
}
|
||||
|
||||
fn with_bodies_mode(mut self, mode: PruneMode) -> Self {
|
||||
self.bodies_mode = mode;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_expected_pruned(mut self, pruned: usize) -> Self {
|
||||
self.expected_pruned = pruned;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_expected_progress(mut self, progress: PruneProgress) -> Self {
|
||||
self.expected_progress = progress;
|
||||
self
|
||||
}
|
||||
|
||||
fn with_lowest_block(mut self, block: BlockNumber) -> Self {
|
||||
self.expected_lowest_block = Some(block);
|
||||
self
|
||||
}
|
||||
|
||||
fn with_tx_lookup(mut self, mode: PruneMode, checkpoint: Option<BlockNumber>) -> Self {
|
||||
self.tx_lookup_mode = Some(mode);
|
||||
self.tx_lookup_checkpoint_block = checkpoint;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
fn run_prune_test(
|
||||
factory: &ProviderFactory<MockNodeTypesWithDB>,
|
||||
finished_exex_height_rx: &tokio::sync::watch::Receiver<FinishedExExHeight>,
|
||||
test_case: PruneTestCase,
|
||||
test_case: TestCase,
|
||||
tip: BlockNumber,
|
||||
) {
|
||||
let bodies = Bodies::new(test_case.prune_mode);
|
||||
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
// Capture highest block before pruning
|
||||
let static_provider = factory.static_file_provider();
|
||||
let highest_before =
|
||||
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions);
|
||||
|
||||
// Set up tx_lookup checkpoint if provided
|
||||
if let Some(checkpoint_block) = test_case.tx_lookup_checkpoint_block {
|
||||
let provider = factory.database_provider_rw().unwrap();
|
||||
provider
|
||||
.save_prune_checkpoint(
|
||||
PruneSegment::TransactionLookup,
|
||||
reth_prune_types::PruneCheckpoint {
|
||||
block_number: Some(checkpoint_block),
|
||||
tx_number: None,
|
||||
prune_mode: test_case.tx_lookup_mode.unwrap(),
|
||||
},
|
||||
)
|
||||
.unwrap();
|
||||
provider.commit().unwrap();
|
||||
}
|
||||
|
||||
let bodies = Bodies::new(test_case.bodies_mode, test_case.tx_lookup_mode);
|
||||
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
|
||||
|
||||
let mut pruner = Pruner::new_with_factory(
|
||||
@@ -114,27 +258,29 @@ mod tests {
|
||||
5,
|
||||
10000,
|
||||
None,
|
||||
finished_exex_height_rx.clone(),
|
||||
finished_exex_height_rx,
|
||||
);
|
||||
|
||||
let result = pruner.run(tip).expect("pruner run");
|
||||
|
||||
assert_eq!(result.progress, PruneProgress::Finished);
|
||||
assert_eq!(result.progress, test_case.expected_progress);
|
||||
assert_eq!(result.segments.len(), 1);
|
||||
|
||||
let (segment, output) = &result.segments[0];
|
||||
assert_eq!(*segment, PruneSegment::Bodies);
|
||||
assert_eq!(output.pruned, test_case.expected_pruned);
|
||||
|
||||
let static_provider = factory.static_file_provider();
|
||||
assert_eq!(
|
||||
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
|
||||
test_case.expected_lowest_block
|
||||
);
|
||||
assert_eq!(
|
||||
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
|
||||
Some(tip)
|
||||
);
|
||||
if let Some(expected_lowest) = test_case.expected_lowest_block {
|
||||
let static_provider = factory.static_file_provider();
|
||||
assert_eq!(
|
||||
static_provider.get_lowest_range_end(StaticFileSegment::Transactions),
|
||||
Some(expected_lowest)
|
||||
);
|
||||
assert_eq!(
|
||||
static_provider.get_highest_static_file_block(StaticFileSegment::Transactions),
|
||||
highest_before
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -143,50 +289,82 @@ mod tests {
|
||||
let tip = 2_499_999;
|
||||
setup_static_file_jars(&factory, tip);
|
||||
|
||||
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
let (_, _finished_exex_height_rx) =
|
||||
tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
let test_cases = vec![
|
||||
// Test 1: PruneMode::Before(750_000) → deletes jar 1 (0-499_999)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(750_000),
|
||||
expected_pruned: 1000,
|
||||
expected_lowest_block: Some(999_999),
|
||||
},
|
||||
// Test 2: PruneMode::Before(850_000) → no deletion (jar 2: 500_000-999_999 contains
|
||||
// Test 1: PruneMode::Before(750_000) → deletes jar 0 (0-499_999)
|
||||
// Checkpoint 499_999 != target 749_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(750_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(999_999),
|
||||
// Test 2: PruneMode::Before(850_000) → no deletion (jar 1: 500_000-999_999 contains
|
||||
// target)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(850_000),
|
||||
expected_pruned: 0,
|
||||
expected_lowest_block: Some(999_999),
|
||||
},
|
||||
// Test 3: PruneMode::Before(1_599_999) → deletes jar 2 (500_000-999_999) and jar 3
|
||||
// (1_000_000-1_499_999)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(1_599_999),
|
||||
expected_pruned: 2000,
|
||||
expected_lowest_block: Some(1_999_999),
|
||||
},
|
||||
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 4
|
||||
// (1_500_000-1_999_999)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Distance(500_000),
|
||||
expected_pruned: 1000,
|
||||
expected_lowest_block: Some(2_499_999),
|
||||
},
|
||||
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 5: 2_000_000-2_499_999
|
||||
TestCase::new().with_bodies_mode(PruneMode::Before(850_000)).with_lowest_block(999_999),
|
||||
// Test 3: PruneMode::Before(1_599_999) → deletes jars 0 and 1 (0-999_999)
|
||||
// Checkpoint 999_999 != target 1_599_998 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(1_599_999))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_999_999),
|
||||
// Test 4: PruneMode::Distance(500_000) with tip=2_499_999 → deletes jar 3
|
||||
// (1_500_000-1_999_999) Checkpoint 1_999_999 == target 1_999_999 ->
|
||||
// Finished
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Distance(500_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(2_499_999),
|
||||
// Test 5: PruneMode::Before(2_300_000) → no deletion (jar 4: 2_000_000-2_499_999
|
||||
// contains target)
|
||||
PruneTestCase {
|
||||
prune_mode: PruneMode::Before(2_300_000),
|
||||
expected_pruned: 0,
|
||||
expected_lowest_block: Some(2_499_999),
|
||||
},
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(2_300_000))
|
||||
.with_lowest_block(2_499_999),
|
||||
];
|
||||
|
||||
for test_case in test_cases {
|
||||
run_prune_test(&factory, &finished_exex_height_rx, test_case, tip);
|
||||
run_prune_test(&factory, test_case, tip);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn checkpoint_reflects_deleted_files_not_target() {
|
||||
// Test that checkpoint is set to the highest deleted block, not to_block.
|
||||
// When to_block falls in the middle of an undeleted file, checkpoint should reflect
|
||||
// what was actually deleted.
|
||||
let factory = create_test_provider_factory();
|
||||
let tip = 1_499_999;
|
||||
setup_static_file_jars(&factory, tip);
|
||||
|
||||
// Use PruneMode::Before(900_000) which targets 899_999.
|
||||
// This should delete jar 0 (0-499_999) since it's entirely below the target.
|
||||
// Jar 1 (500_000-999_999) contains the target, so it won't be deleted.
|
||||
// Checkpoint should be 499_999 (end of jar 0), not 899_999 (to_block).
|
||||
let bodies = Bodies::new(PruneMode::Before(900_000), None);
|
||||
let segments: Vec<Box<dyn Segment<_>>> = vec![Box::new(bodies)];
|
||||
|
||||
let (_, finished_exex_height_rx) = tokio::sync::watch::channel(FinishedExExHeight::NoExExs);
|
||||
|
||||
let mut pruner =
|
||||
Pruner::new_with_factory(factory, segments, 5, 10000, None, finished_exex_height_rx);
|
||||
|
||||
let result = pruner.run(tip).expect("pruner run");
|
||||
|
||||
assert_eq!(result.progress, PruneProgress::Finished);
|
||||
assert_eq!(result.segments.len(), 1);
|
||||
|
||||
let (segment, output) = &result.segments[0];
|
||||
assert_eq!(*segment, PruneSegment::Bodies);
|
||||
|
||||
// Verify checkpoint is set to the end of deleted jar (499_999), not to_block (899_999)
|
||||
let checkpoint_block = output.checkpoint.as_ref().and_then(|cp| cp.block_number);
|
||||
assert_eq!(
|
||||
checkpoint_block,
|
||||
Some(499_999),
|
||||
"Checkpoint should be 499_999 (end of deleted jar 0), not 899_999 (to_block)"
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn min_block_updated_on_sync() {
|
||||
// Regression test: update_index must update min_block to prevent stale values
|
||||
@@ -302,4 +480,100 @@ mod tests {
|
||||
assert_eq!(deleted.len(), expected_deleted);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn bodies_with_tx_lookup_coordination() {
|
||||
// Test that bodies pruning correctly coordinates with tx lookup pruning
|
||||
// Using tip = 1_523_000 creates 4 static file jars:
|
||||
// - Jar 0: blocks 0-499_999, txs 0-999
|
||||
// - Jar 1: blocks 500_000-999_999, txs 1000-1999
|
||||
// - Jar 2: blocks 1_000_000-1_499_999, txs 2000-2999
|
||||
// - Jar 3: blocks 1_500_000-1_523_000, txs 3000-3999
|
||||
let tip = 1_523_000;
|
||||
|
||||
let test_cases = vec![
|
||||
// Scenario 1: tx_lookup disabled, bodies can prune freely (deletes jar 0)
|
||||
// Checkpoint is 499_999 (end of jar 0), target is 599_999, so HasMoreData
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(999_999),
|
||||
// Scenario 2: tx_lookup enabled but not run yet, bodies cannot prune
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), None)
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_expected_progress(PruneProgress::HasMoreData(
|
||||
PruneInterruptReason::WaitingOnSegment(PruneSegment::TransactionLookup),
|
||||
))
|
||||
.with_lowest_block(499_999), // No jars deleted, jar 0 ends at 499_999
|
||||
// Scenario 3: tx_lookup caught up to its target, bodies can prune freely
|
||||
// Deletes jar 0, checkpoint is 499_999, target is 599_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_expected_pruned(1000)
|
||||
.with_lowest_block(999_999),
|
||||
// Scenario 4: tx_lookup behind its target, bodies limited to tx_lookup checkpoint
|
||||
// tx_lookup should prune up to 599_999, but checkpoint is only at 250_000
|
||||
// bodies wants to prune up to 599_999, but limited to 250_000
|
||||
// No jars deleted because jar 0 (0-499_999) ends beyond 250_000
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), Some(250_000))
|
||||
.with_bodies_mode(PruneMode::Before(600_000))
|
||||
.with_lowest_block(499_999), // No jars deleted
|
||||
// Scenario 5: Both use Distance, tx_lookup caught up
|
||||
// With tip=1_523_000, Distance(500_000) targets block 1_023_000
|
||||
// Deletes jars 0 and 1, checkpoint is 999_999, target is 1_023_000 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Distance(500_000), Some(1_023_000))
|
||||
.with_bodies_mode(PruneMode::Distance(500_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_499_999),
|
||||
// Scenario 6: Both use Distance, tx_lookup less aggressive (bigger distance) than
|
||||
// bodies With tip=1_523_000:
|
||||
// - tx_lookup: Distance(1_000_000) targets block 523_000, checkpoint at 523_000
|
||||
// - bodies: Distance(500_000) targets block 1_023_000
|
||||
// Bodies can prune up to what tx_lookup has finished (523_000), deleting jar 0
|
||||
// Checkpoint is 499_999, target is 1_023_000 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Distance(1_000_000), Some(523_000))
|
||||
.with_bodies_mode(PruneMode::Distance(500_000))
|
||||
.with_expected_pruned(1000) // Jar 0 deleted
|
||||
.with_lowest_block(999_999), // Jar 0 (0-499_999) deleted
|
||||
// Scenario 7: tx_lookup more aggressive than bodies (deletes jar 0 and 1)
|
||||
// tx_lookup: Before(1_100_000) -> prune up to 1_099_999
|
||||
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
|
||||
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(1_100_000), Some(1_099_999))
|
||||
.with_bodies_mode(PruneMode::Before(1_100_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
|
||||
// Scenario 8: tx_lookup has lower target than bodies, but is done
|
||||
// tx_lookup: Before(600_000) -> prune up to 599_999 (checkpoint at 599_999, DONE)
|
||||
// bodies: Before(1_100_000) -> wants to prune up to 1_099_999
|
||||
// Since tx_lookup is done (next_pruned_block returns None), bodies can prune freely
|
||||
// Checkpoint is 999_999, target is 1_099_999 -> HasMoreData
|
||||
TestCase::new()
|
||||
.with_tx_lookup(PruneMode::Before(600_000), Some(599_999))
|
||||
.with_bodies_mode(PruneMode::Before(1_100_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
|
||||
// Scenario 9: Perfect alignment - checkpoint equals target
|
||||
// bodies: Before(1_000_000) -> targets 999_999
|
||||
// Deletes jars 0 and 1 (0-999_999), checkpoint is 999_999 which equals target ->
|
||||
// Finished
|
||||
TestCase::new()
|
||||
.with_bodies_mode(PruneMode::Before(1_000_000))
|
||||
.with_expected_pruned(2000)
|
||||
.with_expected_progress(PruneProgress::Finished)
|
||||
.with_lowest_block(1_499_999), // Jars 0 and 1 deleted
|
||||
];
|
||||
|
||||
for test_case in test_cases {
|
||||
let factory = create_test_provider_factory();
|
||||
setup_static_file_jars(&factory, tip);
|
||||
run_prune_test(&factory, test_case, tip);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,6 +49,16 @@ where
|
||||
fn prune(&self, provider: &Provider, input: PruneInput) -> Result<SegmentOutput, PrunerError> {
|
||||
if EitherWriterDestination::senders(provider).is_static_file() {
|
||||
debug!(target: "pruner", "Pruning transaction senders from static files.");
|
||||
|
||||
if self.mode.is_full() {
|
||||
debug!(target: "pruner", "PruneMode::Full: deleting all transaction senders static files.");
|
||||
return segments::delete_static_files_segment(
|
||||
provider,
|
||||
input,
|
||||
StaticFileSegment::TransactionSenders,
|
||||
)
|
||||
}
|
||||
|
||||
return segments::prune_static_files(
|
||||
provider,
|
||||
input,
|
||||
|
||||
@@ -12,7 +12,7 @@ use reth_db_api::{
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_provider::{DBProvider, EitherWriter, StaticFileProviderFactory};
|
||||
use reth_provider::{DBProvider, EitherWriter, RocksDBProviderFactory, StaticFileProviderFactory};
|
||||
use reth_prune_types::{
|
||||
PruneMode, PrunePurpose, PruneSegment, SegmentOutput, SegmentOutputCheckpoint,
|
||||
};
|
||||
@@ -43,7 +43,8 @@ where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageChangeSetReader
|
||||
+ StorageSettingsCache,
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::StorageHistory
|
||||
@@ -68,6 +69,13 @@ where
|
||||
};
|
||||
let range_end = *range.end();
|
||||
|
||||
// Check where storage history indices are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().storages_history_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, range, range_end);
|
||||
}
|
||||
|
||||
// Check where storage changesets are stored (MDBX path)
|
||||
if EitherWriter::storage_changesets_destination(provider).is_static_file() {
|
||||
self.prune_static_files(provider, input, range, range_end)
|
||||
} else {
|
||||
@@ -94,6 +102,8 @@ impl StorageHistory {
|
||||
input.limiter
|
||||
};
|
||||
|
||||
// The limiter may already be exhausted from a previous segment in the same prune run.
|
||||
// Early exit avoids unnecessary iteration when no budget remains.
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
@@ -126,8 +136,8 @@ impl StorageHistory {
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
|
||||
// Delete static file jars below the pruned block
|
||||
if let Some(last_block) = last_changeset_pruned_block {
|
||||
// Delete static file jars only when fully processed
|
||||
if done && let Some(last_block) = last_changeset_pruned_block {
|
||||
provider
|
||||
.static_file_provider()
|
||||
.delete_segment_below_block(StaticFileSegment::StorageChangeSets, last_block + 1)?;
|
||||
@@ -216,6 +226,108 @@ impl StorageHistory {
|
||||
)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
/// Prunes storage history when indices are stored in `RocksDB`.
|
||||
///
|
||||
/// Reads storage changesets from static files and prunes the corresponding
|
||||
/// `RocksDB` history shards.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn prune_rocksdb<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
range: std::ops::RangeInclusive<BlockNumber>,
|
||||
range_end: BlockNumber,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: DBProvider + StaticFileProviderFactory + RocksDBProviderFactory,
|
||||
{
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
if limiter.is_limit_reached() {
|
||||
return Ok(SegmentOutput::not_done(
|
||||
limiter.interrupt_reason(),
|
||||
input.previous_checkpoint.map(SegmentOutputCheckpoint::from_prune_checkpoint),
|
||||
))
|
||||
}
|
||||
|
||||
let mut highest_deleted_storages: FxHashMap<_, _> = FxHashMap::default();
|
||||
let mut last_changeset_pruned_block = None;
|
||||
let mut changesets_processed = 0usize;
|
||||
let mut done = true;
|
||||
|
||||
// Walk storage changesets from static files using a streaming iterator.
|
||||
// For each changeset, track the highest block number seen for each (address, storage_key)
|
||||
// pair to determine which history shard entries need pruning.
|
||||
let walker = provider.static_file_provider().walk_storage_changeset_range(range);
|
||||
for result in walker {
|
||||
if limiter.is_limit_reached() {
|
||||
done = false;
|
||||
break;
|
||||
}
|
||||
let (block_address, entry) = result?;
|
||||
let block_number = block_address.block_number();
|
||||
let address = block_address.address();
|
||||
highest_deleted_storages.insert((address, entry.key), block_number);
|
||||
last_changeset_pruned_block = Some(block_number);
|
||||
changesets_processed += 1;
|
||||
limiter.increment_deleted_entries_count();
|
||||
}
|
||||
|
||||
trace!(target: "pruner", processed = %changesets_processed, %done, "Scanned storage changesets from static files");
|
||||
|
||||
let last_changeset_pruned_block = last_changeset_pruned_block
|
||||
.map(|block_number| if done { block_number } else { block_number.saturating_sub(1) })
|
||||
.unwrap_or(range_end);
|
||||
|
||||
// Prune RocksDB history shards for affected storage slots
|
||||
let mut deleted_shards = 0usize;
|
||||
let mut updated_shards = 0usize;
|
||||
|
||||
// Sort by (address, storage_key) for better RocksDB cache locality
|
||||
let mut sorted_storages: Vec<_> = highest_deleted_storages.into_iter().collect();
|
||||
sorted_storages.sort_unstable_by_key(|((addr, key), _)| (*addr, *key));
|
||||
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
let targets: Vec<_> = sorted_storages
|
||||
.iter()
|
||||
.map(|((addr, key), highest)| {
|
||||
((*addr, *key), (*highest).min(last_changeset_pruned_block))
|
||||
})
|
||||
.collect();
|
||||
|
||||
let outcomes = batch.prune_storage_history_batch(&targets)?;
|
||||
deleted_shards = outcomes.deleted;
|
||||
updated_shards = outcomes.updated;
|
||||
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
|
||||
trace!(target: "pruner", deleted = deleted_shards, updated = updated_shards, %done, "Pruned storage history (RocksDB indices)");
|
||||
|
||||
// Delete static file jars only when fully processed. During provider.commit(), RocksDB
|
||||
// batch is committed before the MDBX checkpoint. If crash occurs after RocksDB commit
|
||||
// but before MDBX commit, on restart the pruner checkpoint indicates data needs
|
||||
// re-pruning, but the RocksDB shards are already pruned - this is safe because pruning
|
||||
// is idempotent (re-pruning already-pruned shards is a no-op).
|
||||
if done {
|
||||
provider.static_file_provider().delete_segment_below_block(
|
||||
StaticFileSegment::StorageChangeSets,
|
||||
last_changeset_pruned_block + 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
let progress = limiter.progress(done);
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress,
|
||||
pruned: changesets_processed + deleted_shards + updated_shards,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_changeset_pruned_block),
|
||||
tx_number: None,
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
@@ -553,4 +665,270 @@ mod tests {
|
||||
test_prune(998, 2, (PruneProgress::Finished, 500));
|
||||
test_prune(1200, 3, (PruneProgress::Finished, 202));
|
||||
}
|
||||
|
||||
/// Tests that when a limiter stops mid-block (with multiple storage changes for the same
|
||||
/// block), the checkpoint is set to `block_number - 1` to avoid dangling index entries.
|
||||
#[test]
|
||||
fn prune_partial_progress_mid_block() {
|
||||
use alloy_primitives::{Address, U256};
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_testing_utils::generators::ChangeSet;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
// Create blocks 0..=10
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=10,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
// Create specific changesets where block 5 has 4 storage changes
|
||||
let addr1 = Address::with_last_byte(1);
|
||||
let addr2 = Address::with_last_byte(2);
|
||||
|
||||
let account = Account { nonce: 1, balance: U256::from(100), bytecode_hash: None };
|
||||
|
||||
// Create storage entries
|
||||
let storage_entry = |key: u8| reth_primitives_traits::StorageEntry {
|
||||
key: B256::with_last_byte(key),
|
||||
value: U256::from(100),
|
||||
};
|
||||
|
||||
// Build changesets: blocks 0-4 have 1 storage change each, block 5 has 4 changes, block 6
|
||||
// has 1. Entries within each account must be sorted by key.
|
||||
let changesets: Vec<ChangeSet> = vec![
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 0
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 1
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 2
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 3
|
||||
vec![(addr1, account, vec![storage_entry(1)])], // block 4
|
||||
// block 5: 4 different storage changes (2 addresses, each with 2 storage slots)
|
||||
// Sorted by address, then by storage key within each address
|
||||
vec![
|
||||
(addr1, account, vec![storage_entry(1), storage_entry(2)]),
|
||||
(addr2, account, vec![storage_entry(1), storage_entry(2)]),
|
||||
],
|
||||
vec![(addr1, account, vec![storage_entry(3)])], // block 6
|
||||
];
|
||||
|
||||
db.insert_changesets(changesets.clone(), None).expect("insert changesets");
|
||||
db.insert_history(changesets.clone(), None).expect("insert history");
|
||||
|
||||
// Total storage changesets
|
||||
let total_storage_entries: usize =
|
||||
changesets.iter().flat_map(|c| c.iter()).map(|(_, _, entries)| entries.len()).sum();
|
||||
assert_eq!(db.table::<tables::StorageChangeSets>().unwrap().len(), total_storage_entries);
|
||||
|
||||
let prune_mode = PruneMode::Before(10);
|
||||
|
||||
// Set limiter to stop mid-block 5
|
||||
// With STORAGE_HISTORY_TABLES_TO_PRUNE=2, limit=14 gives us 7 storage entries before limit
|
||||
// Blocks 0-4 use 5 slots, leaving 2 for block 5 (which has 4), so we stop mid-block 5
|
||||
let deleted_entries_limit = 14; // 14/2 = 7 storage entries before limit
|
||||
let limiter = PruneLimiter::default().set_deleted_entries_limit(deleted_entries_limit);
|
||||
|
||||
let input = PruneInput { previous_checkpoint: None, to_block: 10, limiter };
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
|
||||
// Should report that there's more data
|
||||
assert!(!result.progress.is_finished(), "Expected HasMoreData since we stopped mid-block");
|
||||
|
||||
// Save checkpoint and commit
|
||||
segment
|
||||
.save_checkpoint(&provider, result.checkpoint.unwrap().as_prune_checkpoint(prune_mode))
|
||||
.unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
// Verify checkpoint is set to block 4 (not 5), since block 5 is incomplete
|
||||
let checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
assert_eq!(
|
||||
checkpoint.block_number,
|
||||
Some(4),
|
||||
"Checkpoint should be block 4 (block before incomplete block 5)"
|
||||
);
|
||||
|
||||
// Verify remaining changesets
|
||||
let remaining_changesets = db.table::<tables::StorageChangeSets>().unwrap();
|
||||
assert!(
|
||||
!remaining_changesets.is_empty(),
|
||||
"Should have remaining changesets for blocks 5-6"
|
||||
);
|
||||
|
||||
// Verify no dangling history indices for blocks that weren't fully pruned
|
||||
let history = db.table::<tables::StoragesHistory>().unwrap();
|
||||
for (key, _blocks) in &history {
|
||||
assert!(
|
||||
key.sharded_key.highest_block_number > 4,
|
||||
"Found stale history shard with highest_block_number {} <= checkpoint 4",
|
||||
key.sharded_key.highest_block_number
|
||||
);
|
||||
}
|
||||
|
||||
// Run prune again to complete - should finish processing block 5 and 6
|
||||
let input2 = PruneInput {
|
||||
previous_checkpoint: Some(checkpoint),
|
||||
to_block: 10,
|
||||
limiter: PruneLimiter::default().set_deleted_entries_limit(100), // high limit
|
||||
};
|
||||
|
||||
let provider2 = db.factory.database_provider_rw().unwrap();
|
||||
provider2.set_storage_settings_cache(
|
||||
StorageSettings::default().with_storage_changesets_in_static_files(false),
|
||||
);
|
||||
let result2 = segment.prune(&provider2, input2).unwrap();
|
||||
|
||||
assert!(result2.progress.is_finished(), "Second run should complete");
|
||||
|
||||
segment
|
||||
.save_checkpoint(
|
||||
&provider2,
|
||||
result2.checkpoint.unwrap().as_prune_checkpoint(prune_mode),
|
||||
)
|
||||
.unwrap();
|
||||
provider2.commit().expect("commit");
|
||||
|
||||
// Verify final checkpoint
|
||||
let final_checkpoint = db
|
||||
.factory
|
||||
.provider()
|
||||
.unwrap()
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)
|
||||
.unwrap()
|
||||
.expect("checkpoint should exist");
|
||||
|
||||
// Should now be at block 6 (the last block with changesets)
|
||||
assert_eq!(final_checkpoint.block_number, Some(6), "Final checkpoint should be at block 6");
|
||||
|
||||
// All changesets should be pruned
|
||||
let final_changesets = db.table::<tables::StorageChangeSets>().unwrap();
|
||||
assert!(final_changesets.is_empty(), "All changesets up to block 10 should be pruned");
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
#[test]
|
||||
fn prune_rocksdb() {
|
||||
use reth_db_api::models::storage_sharded_key::StorageShardedKey;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettings;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
0..=100,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 0..1, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Database(None)).expect("insert blocks");
|
||||
|
||||
let accounts = random_eoa_accounts(&mut rng, 2).into_iter().collect::<BTreeMap<_, _>>();
|
||||
|
||||
let (changesets, _) = random_changeset_range(
|
||||
&mut rng,
|
||||
blocks.iter(),
|
||||
accounts.into_iter().map(|(addr, acc)| (addr, (acc, Vec::new()))),
|
||||
1..2,
|
||||
1..2,
|
||||
);
|
||||
|
||||
db.insert_changesets_to_static_files(changesets.clone(), None)
|
||||
.expect("insert changesets to static files");
|
||||
|
||||
let mut storage_indices: BTreeMap<(alloy_primitives::Address, B256), Vec<u64>> =
|
||||
BTreeMap::new();
|
||||
for (block, changeset) in changesets.iter().enumerate() {
|
||||
for (address, _, storage_entries) in changeset {
|
||||
for entry in storage_entries {
|
||||
storage_indices.entry((*address, entry.key)).or_default().push(block as u64);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let mut batch = rocksdb.batch();
|
||||
for ((address, storage_key), block_numbers) in &storage_indices {
|
||||
let shard = BlockNumberList::new_pre_sorted(block_numbers.clone());
|
||||
batch
|
||||
.put::<tables::StoragesHistory>(
|
||||
StorageShardedKey::last(*address, *storage_key),
|
||||
&shard,
|
||||
)
|
||||
.expect("insert storage history shard");
|
||||
}
|
||||
batch.commit().expect("commit rocksdb batch");
|
||||
}
|
||||
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for (address, storage_key) in storage_indices.keys() {
|
||||
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
|
||||
assert!(!shards.is_empty(), "RocksDB should contain storage history before prune");
|
||||
}
|
||||
}
|
||||
|
||||
let to_block = 50u64;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let input =
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = StorageHistory::new(prune_mode);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
provider.set_storage_settings_cache(
|
||||
StorageSettings::default()
|
||||
.with_storage_changesets_in_static_files(true)
|
||||
.with_storages_history_in_rocksdb(true),
|
||||
);
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
assert_matches!(
|
||||
result,
|
||||
SegmentOutput { progress: PruneProgress::Finished, checkpoint: Some(_), .. }
|
||||
);
|
||||
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for ((address, storage_key), block_numbers) in &storage_indices {
|
||||
let shards = rocksdb.storage_history_shards(*address, *storage_key).unwrap();
|
||||
|
||||
let remaining_blocks: Vec<u64> =
|
||||
block_numbers.iter().copied().filter(|&b| b > to_block).collect();
|
||||
|
||||
if remaining_blocks.is_empty() {
|
||||
assert!(
|
||||
shards.is_empty(),
|
||||
"Shard for {:?}/{:?} should be deleted when all blocks pruned",
|
||||
address,
|
||||
storage_key
|
||||
);
|
||||
} else {
|
||||
assert!(!shards.is_empty(), "Shard should exist with remaining blocks");
|
||||
let actual_blocks: Vec<u64> =
|
||||
shards.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
assert_eq!(
|
||||
actual_blocks, remaining_blocks,
|
||||
"RocksDB shard should only contain blocks > {}",
|
||||
to_block
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,11 +6,15 @@ use crate::{
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use rayon::prelude::*;
|
||||
use reth_db_api::{tables, transaction::DbTxMut};
|
||||
use reth_provider::{BlockReader, DBProvider, PruneCheckpointReader, StaticFileProviderFactory};
|
||||
use reth_provider::{
|
||||
BlockReader, DBProvider, PruneCheckpointReader, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_prune_types::{
|
||||
PruneCheckpoint, PruneMode, PruneProgress, PrunePurpose, PruneSegment, SegmentOutputCheckpoint,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
use tracing::{debug, instrument, trace};
|
||||
|
||||
#[derive(Debug)]
|
||||
@@ -29,7 +33,9 @@ where
|
||||
Provider: DBProvider<Tx: DbTxMut>
|
||||
+ BlockReader<Transaction: Encodable2718>
|
||||
+ PruneCheckpointReader
|
||||
+ StaticFileProviderFactory,
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageSettingsCache
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
fn segment(&self) -> PruneSegment {
|
||||
PruneSegment::TransactionLookup
|
||||
@@ -83,6 +89,12 @@ where
|
||||
}
|
||||
.into_inner();
|
||||
|
||||
// Check where transaction hash numbers are stored
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
if provider.cached_storage_settings().transaction_hash_numbers_in_rocksdb {
|
||||
return self.prune_rocksdb(provider, input, start, end);
|
||||
}
|
||||
|
||||
// For PruneMode::Full, clear the entire table in one operation
|
||||
if self.mode.is_full() {
|
||||
let pruned = provider.tx_ref().clear_table::<tables::TransactionHashNumbers>()?;
|
||||
@@ -174,6 +186,106 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl TransactionLookup {
|
||||
/// Prunes transaction lookup when indices are stored in `RocksDB`.
|
||||
///
|
||||
/// Reads transactions from static files and deletes corresponding entries
|
||||
/// from the `RocksDB` `TransactionHashNumbers` table.
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
fn prune_rocksdb<Provider>(
|
||||
&self,
|
||||
provider: &Provider,
|
||||
input: PruneInput,
|
||||
start: alloy_primitives::TxNumber,
|
||||
end: alloy_primitives::TxNumber,
|
||||
) -> Result<SegmentOutput, PrunerError>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ BlockReader<Transaction: Encodable2718>
|
||||
+ StaticFileProviderFactory
|
||||
+ RocksDBProviderFactory,
|
||||
{
|
||||
// For PruneMode::Full, clear the entire RocksDB table in one operation
|
||||
if self.mode.is_full() {
|
||||
let rocksdb = provider.rocksdb_provider();
|
||||
rocksdb.clear::<tables::TransactionHashNumbers>()?;
|
||||
trace!(target: "pruner", "Cleared transaction lookup table (RocksDB)");
|
||||
|
||||
let last_pruned_block = provider
|
||||
.block_by_transaction_id(end)?
|
||||
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?;
|
||||
|
||||
return Ok(SegmentOutput {
|
||||
progress: PruneProgress::Finished,
|
||||
pruned: 0, // RocksDB clear doesn't return count
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: Some(last_pruned_block),
|
||||
tx_number: Some(end),
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
let tx_range_end = input
|
||||
.limiter
|
||||
.deleted_entries_limit_left()
|
||||
.map(|left| start.saturating_add(left as u64).saturating_sub(1))
|
||||
.map_or(end, |limited| limited.min(end));
|
||||
let tx_range = start..=tx_range_end;
|
||||
|
||||
// Retrieve transactions in the range and calculate their hashes in parallel
|
||||
let hashes: Vec<_> = provider
|
||||
.transactions_by_tx_range(tx_range.clone())?
|
||||
.into_par_iter()
|
||||
.map(|transaction| transaction.trie_hash())
|
||||
.collect();
|
||||
|
||||
// Number of transactions retrieved from the database should match the tx range count
|
||||
let tx_count = tx_range.count();
|
||||
if hashes.len() != tx_count {
|
||||
return Err(PrunerError::InconsistentData(
|
||||
"Unexpected number of transaction hashes retrieved by transaction number range",
|
||||
))
|
||||
}
|
||||
|
||||
let mut limiter = input.limiter;
|
||||
|
||||
// Delete transaction hash -> number mappings from RocksDB
|
||||
let mut deleted = 0usize;
|
||||
provider.with_rocksdb_batch(|mut batch| {
|
||||
for hash in &hashes {
|
||||
if limiter.is_limit_reached() {
|
||||
break;
|
||||
}
|
||||
batch.delete::<tables::TransactionHashNumbers>(*hash)?;
|
||||
limiter.increment_deleted_entries_count();
|
||||
deleted += 1;
|
||||
}
|
||||
Ok(((), Some(batch.into_inner())))
|
||||
})?;
|
||||
|
||||
let done = deleted == hashes.len() && tx_range_end == end;
|
||||
trace!(target: "pruner", %deleted, %done, "Pruned transaction lookup (RocksDB)");
|
||||
|
||||
let last_pruned_transaction = if deleted > 0 { start + deleted as u64 - 1 } else { start };
|
||||
|
||||
let last_pruned_block = provider
|
||||
.block_by_transaction_id(last_pruned_transaction)?
|
||||
.ok_or(PrunerError::InconsistentData("Block for transaction is not found"))?
|
||||
.checked_sub(if done { 0 } else { 1 });
|
||||
|
||||
let progress = limiter.progress(done);
|
||||
|
||||
Ok(SegmentOutput {
|
||||
progress,
|
||||
pruned: deleted,
|
||||
checkpoint: Some(SegmentOutputCheckpoint {
|
||||
block_number: last_pruned_block,
|
||||
tx_number: Some(last_pruned_transaction),
|
||||
}),
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use crate::segments::{PruneInput, PruneLimiter, Segment, SegmentOutput, TransactionLookup};
|
||||
@@ -319,4 +431,100 @@ mod tests {
|
||||
test_prune(6, (PruneProgress::Finished, 2));
|
||||
test_prune(10, (PruneProgress::Finished, 8));
|
||||
}
|
||||
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
#[test]
|
||||
fn prune_rocksdb() {
|
||||
use reth_db_api::models::StorageSettings;
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
use reth_storage_api::StorageSettingsCache;
|
||||
|
||||
let db = TestStageDB::default();
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let blocks = random_block_range(
|
||||
&mut rng,
|
||||
1..=10,
|
||||
BlockRangeParams { parent: Some(B256::ZERO), tx_count: 2..3, ..Default::default() },
|
||||
);
|
||||
db.insert_blocks(blocks.iter(), StorageKind::Static).expect("insert blocks");
|
||||
|
||||
// Collect transaction hashes and their tx numbers
|
||||
let mut tx_hash_numbers = Vec::new();
|
||||
for block in &blocks {
|
||||
tx_hash_numbers.reserve_exact(block.transaction_count());
|
||||
for transaction in &block.body().transactions {
|
||||
tx_hash_numbers.push((*transaction.tx_hash(), tx_hash_numbers.len() as u64));
|
||||
}
|
||||
}
|
||||
let tx_hash_numbers_len = tx_hash_numbers.len();
|
||||
|
||||
// Insert into RocksDB instead of MDBX
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let mut batch = rocksdb.batch();
|
||||
for (hash, tx_num) in &tx_hash_numbers {
|
||||
batch.put::<tables::TransactionHashNumbers>(*hash, tx_num).unwrap();
|
||||
}
|
||||
batch.commit().expect("commit rocksdb batch");
|
||||
}
|
||||
|
||||
// Verify RocksDB has all entries
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for (hash, expected_tx_num) in &tx_hash_numbers {
|
||||
let actual = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
|
||||
assert_eq!(actual, Some(*expected_tx_num));
|
||||
}
|
||||
}
|
||||
|
||||
let to_block: BlockNumber = 6;
|
||||
let prune_mode = PruneMode::Before(to_block);
|
||||
let input =
|
||||
PruneInput { previous_checkpoint: None, to_block, limiter: PruneLimiter::default() };
|
||||
let segment = TransactionLookup::new(prune_mode);
|
||||
|
||||
// Enable RocksDB storage for transaction hash numbers
|
||||
db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_hash_numbers_in_rocksdb(true),
|
||||
);
|
||||
|
||||
let provider = db.factory.database_provider_rw().unwrap();
|
||||
let result = segment.prune(&provider, input).unwrap();
|
||||
provider.commit().expect("commit");
|
||||
|
||||
assert_matches!(
|
||||
result,
|
||||
SegmentOutput { progress: PruneProgress::Finished, pruned, checkpoint: Some(_) }
|
||||
if pruned > 0
|
||||
);
|
||||
|
||||
// Calculate expected: blocks 1-6 should have their tx hashes pruned
|
||||
let txs_up_to_block_6: usize = blocks.iter().take(6).map(|b| b.transaction_count()).sum();
|
||||
|
||||
// Verify RocksDB entries: first `txs_up_to_block_6` should be gone
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
for (i, (hash, _)) in tx_hash_numbers.iter().enumerate() {
|
||||
let entry = rocksdb.get::<tables::TransactionHashNumbers>(*hash).unwrap();
|
||||
if i < txs_up_to_block_6 {
|
||||
assert!(entry.is_none(), "Entry {} (hash {:?}) should be pruned", i, hash);
|
||||
} else {
|
||||
assert!(entry.is_some(), "Entry {} (hash {:?}) should still exist", i, hash);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify remaining count
|
||||
{
|
||||
let rocksdb = db.factory.rocksdb_provider();
|
||||
let remaining: Vec<_> =
|
||||
rocksdb.iter::<tables::TransactionHashNumbers>().unwrap().collect();
|
||||
assert_eq!(
|
||||
remaining.len(),
|
||||
tx_hash_numbers_len - txs_up_to_block_6,
|
||||
"Remaining RocksDB entries should match expected"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,6 +84,38 @@ impl PruneMode {
|
||||
pub const fn is_distance(&self) -> bool {
|
||||
matches!(self, Self::Distance(_))
|
||||
}
|
||||
|
||||
/// Returns the next block number that will EVENTUALLY be pruned after the given checkpoint. It
|
||||
/// should not be used to find if there are blocks to be pruned right now. For that, use
|
||||
/// [`Self::prune_target_block`].
|
||||
///
|
||||
/// This is independent of the current tip and indicates what block is next in the pruning
|
||||
/// sequence according to this mode's configuration. Returns `None` if no more blocks will
|
||||
/// be pruned (i.e., the mode has reached its target).
|
||||
///
|
||||
/// # Examples
|
||||
///
|
||||
/// - `Before(10)` with checkpoint at block 5 returns `Some(6)`
|
||||
/// - `Before(10)` with checkpoint at block 9 returns `None` (done)
|
||||
/// - `Distance(100)` with checkpoint at block 1000 returns `Some(1001)` (always has more)
|
||||
/// - `Full` always returns the next block after checkpoint
|
||||
pub const fn next_pruned_block(&self, checkpoint: Option<BlockNumber>) -> Option<BlockNumber> {
|
||||
let next = match checkpoint {
|
||||
Some(c) => c + 1,
|
||||
None => 0,
|
||||
};
|
||||
|
||||
match self {
|
||||
Self::Before(n) => {
|
||||
if next < *n {
|
||||
Some(next)
|
||||
} else {
|
||||
None
|
||||
}
|
||||
}
|
||||
Self::Distance(_) | Self::Full => Some(next),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -93,7 +93,7 @@ impl SegmentOutput {
|
||||
Self { progress: PruneProgress::Finished, pruned: 0, checkpoint: None }
|
||||
}
|
||||
|
||||
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and `checkpoint = None`.
|
||||
/// Returns a [`SegmentOutput`] with `done = false`, `pruned = 0` and the given checkpoint.
|
||||
/// Use when pruning is needed but cannot be done.
|
||||
pub const fn not_done(
|
||||
reason: PruneInterruptReason,
|
||||
@@ -142,6 +142,8 @@ pub enum PruneInterruptReason {
|
||||
Timeout,
|
||||
/// Limit on the number of deleted entries (rows in the database) per prune run was reached.
|
||||
DeletedEntriesLimitReached,
|
||||
/// Waiting for another segment to finish pruning before this segment can proceed.
|
||||
WaitingOnSegment(PruneSegment),
|
||||
/// Unknown reason for stopping prune run.
|
||||
Unknown,
|
||||
}
|
||||
|
||||
@@ -54,7 +54,7 @@ pub struct PruneModes {
|
||||
pub transaction_lookup: Option<PruneMode>,
|
||||
/// Receipts pruning configuration. This setting overrides `receipts_log_filter`
|
||||
/// and offers improved performance.
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
|
||||
pub receipts: Option<PruneMode>,
|
||||
/// Account History pruning configuration.
|
||||
#[cfg_attr(
|
||||
@@ -75,7 +75,7 @@ pub struct PruneModes {
|
||||
)]
|
||||
pub storage_history: Option<PruneMode>,
|
||||
/// Bodies History pruning configuration.
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none"))]
|
||||
pub bodies_history: Option<PruneMode>,
|
||||
/// Receipts pruning configuration by retaining only those receipts that contain logs emitted
|
||||
/// by the specified addresses, discarding others. This setting is overridden by `receipts`.
|
||||
@@ -112,7 +112,13 @@ impl PruneModes {
|
||||
///
|
||||
/// Returns `true` if any migration was performed.
|
||||
pub const fn migrate(&mut self) -> bool {
|
||||
false
|
||||
match &self.receipts {
|
||||
Some(PruneMode::Full | PruneMode::Distance(0..MINIMUM_DISTANCE)) => {
|
||||
self.receipts = Some(PruneMode::Distance(MINIMUM_DISTANCE));
|
||||
true
|
||||
}
|
||||
_ => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns an error if we can't unwind to the targeted block because the target block is
|
||||
|
||||
@@ -2,7 +2,7 @@ use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{BlockId, BlockNumberOrTag};
|
||||
use alloy_genesis::ChainConfig;
|
||||
use alloy_json_rpc::RpcObject;
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use alloy_primitives::{Address, Bytes, B256, U64};
|
||||
use alloy_rpc_types_debug::ExecutionWitness;
|
||||
use alloy_rpc_types_eth::{Bundle, StateContext};
|
||||
use alloy_rpc_types_trace::geth::{
|
||||
@@ -325,7 +325,7 @@ pub trait DebugApi<TxReq: RpcObject> {
|
||||
/// Sets the current head of the local chain by block number. Note, this is a destructive action
|
||||
/// and may severely damage your chain. Use with extreme caution.
|
||||
#[method(name = "setHead")]
|
||||
async fn debug_set_head(&self, number: u64) -> RpcResult<()>;
|
||||
async fn debug_set_head(&self, number: U64) -> RpcResult<()>;
|
||||
|
||||
/// Sets the rate of mutex profiling.
|
||||
#[method(name = "setMutexProfileFraction")]
|
||||
|
||||
@@ -107,6 +107,7 @@ impl RethRpcServerConfig for RpcServerArgs {
|
||||
.pending_block_kind(self.rpc_pending_block)
|
||||
.raw_tx_forwarder(self.rpc_forwarder.clone())
|
||||
.rpc_evm_memory_limit(self.rpc_evm_memory_limit)
|
||||
.force_blob_sidecar_upcasting(self.rpc_force_blob_sidecar_upcasting)
|
||||
}
|
||||
|
||||
fn flashbots_config(&self) -> ValidationApiConfig {
|
||||
|
||||
@@ -394,6 +394,17 @@ pub trait EthApi<
|
||||
address: Address,
|
||||
block: BlockId,
|
||||
) -> RpcResult<alloy_rpc_types_eth::AccountInfo>;
|
||||
|
||||
/// Returns the EIP-7928 block access list for a block by hash.
|
||||
#[method(name = "getBlockAccessListByBlockHash")]
|
||||
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>>;
|
||||
|
||||
/// Returns the EIP-7928 block access list for a block by number.
|
||||
#[method(name = "getBlockAccessListByBlockNumber")]
|
||||
async fn block_access_list_by_block_number(
|
||||
&self,
|
||||
number: BlockNumberOrTag,
|
||||
) -> RpcResult<Option<Bytes>>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -881,4 +892,19 @@ where
|
||||
trace!(target: "rpc::eth", "Serving eth_getAccountInfo");
|
||||
Ok(EthState::get_account_info(self, address, block).await?)
|
||||
}
|
||||
|
||||
/// Handler for: `eth_getBlockAccessListByBlockHash`
|
||||
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Bytes>> {
|
||||
trace!(target: "rpc::eth", ?hash, "Serving eth_getBlockAccessListByBlockHash");
|
||||
Err(internal_rpc_err("unimplemented"))
|
||||
}
|
||||
|
||||
/// Handler for: `eth_getBlockAccessListByBlockNumber`
|
||||
async fn block_access_list_by_block_number(
|
||||
&self,
|
||||
number: BlockNumberOrTag,
|
||||
) -> RpcResult<Option<Bytes>> {
|
||||
trace!(target: "rpc::eth", ?number, "Serving eth_getBlockAccessListByBlockNumber");
|
||||
Err(internal_rpc_err("unimplemented"))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -49,7 +49,7 @@ jsonrpsee-types.workspace = true
|
||||
futures.workspace = true
|
||||
tokio.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
reqwest = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
reqwest.workspace = true
|
||||
|
||||
# metrics
|
||||
metrics.workspace = true
|
||||
|
||||
@@ -107,6 +107,11 @@ pub struct EthConfig {
|
||||
pub send_raw_transaction_sync_timeout: Duration,
|
||||
/// Maximum memory the EVM can allocate per RPC request.
|
||||
pub rpc_evm_memory_limit: u64,
|
||||
/// Whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format when Osaka is active.
|
||||
///
|
||||
/// This is disabled by default, allowing blob transactions with EIP-4844 sidecars to be
|
||||
/// submitted without automatic conversion.
|
||||
pub force_blob_sidecar_upcasting: bool,
|
||||
}
|
||||
|
||||
impl EthConfig {
|
||||
@@ -140,6 +145,7 @@ impl Default for EthConfig {
|
||||
raw_tx_forwarder: ForwardConfig::default(),
|
||||
send_raw_transaction_sync_timeout: RPC_DEFAULT_SEND_RAW_TX_SYNC_TIMEOUT_SECS,
|
||||
rpc_evm_memory_limit: (1 << 32) - 1,
|
||||
force_blob_sidecar_upcasting: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -242,6 +248,12 @@ impl EthConfig {
|
||||
self.rpc_evm_memory_limit = memory_limit;
|
||||
self
|
||||
}
|
||||
|
||||
/// Configures whether to force upcasting EIP-4844 blob sidecars to EIP-7594 format.
|
||||
pub const fn force_blob_sidecar_upcasting(mut self, force: bool) -> Self {
|
||||
self.force_blob_sidecar_upcasting = force;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Config for the filter
|
||||
|
||||
@@ -71,10 +71,6 @@ revm-primitives = { workspace = true, features = ["serde"] }
|
||||
|
||||
# rpc
|
||||
jsonrpsee.workspace = true
|
||||
http.workspace = true
|
||||
http-body.workspace = true
|
||||
hyper.workspace = true
|
||||
jsonwebtoken.workspace = true
|
||||
serde_json.workspace = true
|
||||
jsonrpsee-types.workspace = true
|
||||
|
||||
@@ -82,7 +78,6 @@ jsonrpsee-types.workspace = true
|
||||
async-trait.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio-stream.workspace = true
|
||||
tower.workspace = true
|
||||
pin-project.workspace = true
|
||||
parking_lot.workspace = true
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@ use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip2718::Encodable2718, BlockId, BlockNumberOrTag};
|
||||
use alloy_evm::env::BlockEnvironment;
|
||||
use alloy_genesis::ChainConfig;
|
||||
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256};
|
||||
use alloy_primitives::{hex::decode, uint, Address, Bytes, B256, U64};
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use alloy_rpc_types::BlockTransactionsKind;
|
||||
use alloy_rpc_types_debug::ExecutionWitness;
|
||||
@@ -998,7 +998,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn debug_set_head(&self, _number: u64) -> RpcResult<()> {
|
||||
async fn debug_set_head(&self, _number: U64) -> RpcResult<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user