mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
68 Commits
devnet4
...
proofs-v2-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c757e0d021 | ||
|
|
733a6d6d2b | ||
|
|
fbd9db4030 | ||
|
|
4413de3050 | ||
|
|
6283d7fd25 | ||
|
|
e55361ce17 | ||
|
|
458608939e | ||
|
|
7446f8ce30 | ||
|
|
1f5c1153b3 | ||
|
|
1820055009 | ||
|
|
45f0adf3e0 | ||
|
|
c5d1f6acee | ||
|
|
66cd979f32 | ||
|
|
a34bb9e11a | ||
|
|
9b37e519d0 | ||
|
|
6a7b2c16f2 | ||
|
|
a3ec14eae2 | ||
|
|
9fd60b879d | ||
|
|
5eb8b2d34b | ||
|
|
6d27c3ac35 | ||
|
|
2675ee0868 | ||
|
|
9879ea6b58 | ||
|
|
93da40d858 | ||
|
|
28f796372b | ||
|
|
044da64e44 | ||
|
|
a3527f231d | ||
|
|
83ef02389f | ||
|
|
7aa06ae27e | ||
|
|
118f7e8ae7 | ||
|
|
9a6c0ebb10 | ||
|
|
bc2742a23a | ||
|
|
caead766ca | ||
|
|
f4751f0cb2 | ||
|
|
cafebffe01 | ||
|
|
f90dc7a973 | ||
|
|
fcf9c8a2b9 | ||
|
|
74d839e32b | ||
|
|
c8517db1fc | ||
|
|
289b8f0e11 | ||
|
|
a36ece08f2 | ||
|
|
047788d2e0 | ||
|
|
a84862f365 | ||
|
|
a428d4ced0 | ||
|
|
0d783bda69 | ||
|
|
100045483f | ||
|
|
fb270dd0fa | ||
|
|
16df1397c0 | ||
|
|
78121b4ff0 | ||
|
|
0356817ec4 | ||
|
|
da7bb168f4 | ||
|
|
f810209806 | ||
|
|
799019b959 | ||
|
|
8a3b50a93a | ||
|
|
3df17c3281 | ||
|
|
b279eb3ca6 | ||
|
|
b9104f31de | ||
|
|
947035f0c5 | ||
|
|
da58733487 | ||
|
|
c1d5ef3a79 | ||
|
|
d1937867e1 | ||
|
|
cad4fa46ed | ||
|
|
a592181b20 | ||
|
|
447e113f5b | ||
|
|
4c88b0f52b | ||
|
|
045f352c3d | ||
|
|
73729d267f | ||
|
|
4d1a14409b | ||
|
|
24e998547c |
@@ -12,7 +12,7 @@ workflows:
|
||||
# Check that `A` activates the features of `B`.
|
||||
"propagate-feature",
|
||||
# These are the features to check:
|
||||
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,js-tracer,portable,keccak-cache-global",
|
||||
"--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global",
|
||||
# Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
|
||||
"--left-side-feature-missing=ignore",
|
||||
# Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.
|
||||
|
||||
@@ -18,7 +18,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
|
||||
6. **Pipeline (`crates/stages/`)**: Staged sync architecture for blockchain synchronization
|
||||
7. **Trie (`crates/trie/`)**: Merkle Patricia Trie implementation with parallel state root computation
|
||||
8. **Node Builder (`crates/node/`)**: High-level node orchestration and configuration
|
||||
9 **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated)
|
||||
9. **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated)
|
||||
|
||||
### Key Design Principles
|
||||
|
||||
|
||||
@@ -51,9 +51,7 @@ elsewhere.
|
||||
<!-- - **Asking in the support Telegram:** The [Foundry Support Telegram][support-tg] is a fast and easy way to ask questions. -->
|
||||
<!-- - **Opening a discussion:** This repository comes with a discussions board where you can also ask for help. Click the "Discussions" tab at the top. -->
|
||||
|
||||
If you have reviewed existing documentation and still have questions, or you are having problems, you can get help by *
|
||||
*opening a discussion**. This repository comes with a discussions board where you can also ask for help. Click the "
|
||||
Discussions" tab at the top.
|
||||
If you have reviewed existing documentation and still have questions, or you are having problems, you can get help by **opening a discussion**. This repository comes with a discussions board where you can also ask for help. Click the "Discussions" tab at the top.
|
||||
|
||||
As Reth is still in heavy development, the documentation can be a bit scattered. The [Reth Docs][reth-docs] is our
|
||||
current best-effort attempt at keeping up-to-date information.
|
||||
|
||||
476
Cargo.lock
generated
476
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
65
Cargo.toml
65
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "1.9.3"
|
||||
version = "1.10.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -485,7 +485,7 @@ revm-inspectors = "0.33.2"
|
||||
|
||||
# eth
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-dyn-abi = "1.4.1"
|
||||
alloy-dyn-abi = "1.4.3"
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.1.0", default-features = false }
|
||||
alloy-evm = { version = "0.25.1", default-features = false }
|
||||
@@ -497,33 +497,33 @@ alloy-trie = { version = "0.9.1", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
|
||||
alloy-consensus = { version = "1.2.1", default-features = false }
|
||||
alloy-contract = { version = "1.2.1", default-features = false }
|
||||
alloy-eips = { version = "1.2.1", default-features = false }
|
||||
alloy-genesis = { version = "1.2.1", default-features = false }
|
||||
alloy-json-rpc = { version = "1.2.1", default-features = false }
|
||||
alloy-network = { version = "1.2.1", default-features = false }
|
||||
alloy-network-primitives = { version = "1.2.1", default-features = false }
|
||||
alloy-provider = { version = "1.2.1", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-client = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types = { version = "1.2.1", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.2.1", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.2.1", default-features = false }
|
||||
alloy-serde = { version = "1.2.1", default-features = false }
|
||||
alloy-signer = { version = "1.2.1", default-features = false }
|
||||
alloy-signer-local = { version = "1.2.1", default-features = false }
|
||||
alloy-transport = { version = "1.2.1" }
|
||||
alloy-transport-http = { version = "1.2.1", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.2.1", default-features = false }
|
||||
alloy-transport-ws = { version = "1.2.1", default-features = false }
|
||||
alloy-consensus = { version = "1.4.3", default-features = false }
|
||||
alloy-contract = { version = "1.4.3", default-features = false }
|
||||
alloy-eips = { version = "1.4.3", default-features = false }
|
||||
alloy-genesis = { version = "1.4.3", default-features = false }
|
||||
alloy-json-rpc = { version = "1.4.3", default-features = false }
|
||||
alloy-network = { version = "1.4.3", default-features = false }
|
||||
alloy-network-primitives = { version = "1.4.3", default-features = false }
|
||||
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-client = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
|
||||
alloy-serde = { version = "1.4.3", default-features = false }
|
||||
alloy-signer = { version = "1.4.3", default-features = false }
|
||||
alloy-signer-local = { version = "1.4.3", default-features = false }
|
||||
alloy-transport = { version = "1.4.3" }
|
||||
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.4.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.4.3", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-evm = { version = "0.25.0", default-features = false }
|
||||
@@ -555,6 +555,7 @@ dirs-next = "2.0.0"
|
||||
dyn-clone = "1.0.17"
|
||||
eyre = "0.6"
|
||||
fdlimit = "0.3.0"
|
||||
fixed-map = { version = "0.9", default-features = false }
|
||||
humantime = "2.1"
|
||||
humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
@@ -596,9 +597,9 @@ chrono = "0.4.41"
|
||||
# metrics
|
||||
metrics = "0.24.0"
|
||||
metrics-derive = "0.1"
|
||||
metrics-exporter-prometheus = { version = "0.16.0", default-features = false }
|
||||
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
|
||||
metrics-process = "2.1.0"
|
||||
metrics-util = { default-features = false, version = "0.19.0" }
|
||||
metrics-util = { default-features = false, version = "0.20.0" }
|
||||
|
||||
# proc-macros
|
||||
proc-macro2 = "1.0"
|
||||
@@ -664,6 +665,7 @@ opentelemetry_sdk = "0.31"
|
||||
opentelemetry = "0.31"
|
||||
opentelemetry-otlp = "0.31"
|
||||
opentelemetry-semantic-conventions = "0.31"
|
||||
opentelemetry-appender-tracing = "0.31"
|
||||
tracing-opentelemetry = "0.32"
|
||||
|
||||
# misc-testing
|
||||
@@ -734,6 +736,7 @@ tracing-journald = "0.3"
|
||||
tracing-logfmt = "0.3.3"
|
||||
tracing-samply = "0.1"
|
||||
tracing-subscriber = { version = "0.3", default-features = false }
|
||||
tracing-tracy = "0.11"
|
||||
triehash = "0.8"
|
||||
typenum = "1.15.0"
|
||||
vergen = "9.0.4"
|
||||
|
||||
8
Makefile
8
Makefile
@@ -283,11 +283,11 @@ docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker im
|
||||
|
||||
# Create a cross-arch Docker image with the given tags and push it
|
||||
define docker_build_push
|
||||
$(MAKE) build-x86_64-unknown-linux-gnu
|
||||
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/amd64
|
||||
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
|
||||
|
||||
$(MAKE) build-aarch64-unknown-linux-gnu
|
||||
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/arm64
|
||||
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
|
||||
|
||||
@@ -357,11 +357,11 @@ op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker imag
|
||||
|
||||
# Create a cross-arch Docker image with the given tags and push it
|
||||
define op_docker_build_push
|
||||
$(MAKE) op-build-x86_64-unknown-linux-gnu
|
||||
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/amd64
|
||||
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
|
||||
|
||||
$(MAKE) op-build-aarch64-unknown-linux-gnu
|
||||
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
|
||||
mkdir -p $(BIN_DIR)/arm64
|
||||
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
|
||||
|
||||
|
||||
@@ -44,7 +44,7 @@ More historical context below:
|
||||
- We released 1.0 "production-ready" stable Reth in June 2024.
|
||||
- Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf).
|
||||
- Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon.
|
||||
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024,the last beta release.
|
||||
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024, the last beta release.
|
||||
- We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4, 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives.
|
||||
- We shipped iterative improvements until the last alpha release on February 28, 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21).
|
||||
- We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) on June 20, 2023.
|
||||
|
||||
@@ -71,7 +71,11 @@ jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
]
|
||||
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
|
||||
tracy = [
|
||||
"reth-node-core/tracy",
|
||||
"reth-tracing/tracy",
|
||||
]
|
||||
|
||||
min-error-logs = [
|
||||
"tracing/release_max_level_error",
|
||||
|
||||
50
bin/reth-bench-compare/README.md
Normal file
50
bin/reth-bench-compare/README.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# reth-bench-compare
|
||||
|
||||
Compare reth performance between two git references.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
reth-bench-compare \
|
||||
--baseline-ref main \
|
||||
--feature-ref my-feature \
|
||||
--blocks 100 \
|
||||
--wait-for-persistence
|
||||
```
|
||||
|
||||
## Arguments
|
||||
|
||||
| Argument | Description | Default | Required |
|
||||
|----------|-------------|---------|----------|
|
||||
| `--baseline-ref <REF>` | Git reference for baseline | - | Yes |
|
||||
| `--feature-ref <REF>` | Git reference to compare | - | Yes |
|
||||
| `--blocks <N>` | Number of blocks to benchmark | `100` | No |
|
||||
| `--chain <CHAIN>` | Chain to benchmark | `mainnet` | No |
|
||||
| `--datadir <PATH>` | Data directory path | OS-specific | No |
|
||||
| `--rpc-url <URL>` | RPC endpoint for block data | Chain default | No |
|
||||
| `--output-dir <PATH>` | Output directory | `./reth-bench-compare` | No |
|
||||
| `--wait-for-persistence` | Wait for block persistence | `false` | No |
|
||||
| `--persistence-threshold <N>` | Wait after every N+1 blocks | `2` | No |
|
||||
| `--wait-time <DURATION>` | Fixed delay (legacy) | - | No |
|
||||
| `--warmup-blocks <N>` | Cache warmup blocks | Same as `--blocks` | No |
|
||||
| `--draw` | Generate charts (needs Python/uv) | `false` | No |
|
||||
| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
|
||||
| `-vvvv` | Debug logging | Info | No |
|
||||
| `--features <FEATURES>` | Rust features for both builds | `jemalloc,asm-keccak` | No |
|
||||
| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
|
||||
| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
|
||||
| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |
|
||||
| `--baseline-rustflags <FLAGS>` | RUSTFLAGS for baseline only | Inherits `--rustflags` | No |
|
||||
| `--feature-rustflags <FLAGS>` | RUSTFLAGS for feature only | Inherits `--rustflags` | No |
|
||||
| `--baseline-args <ARGS>` | Extra args for baseline node | - | No |
|
||||
| `--feature-args <ARGS>` | Extra args for feature node | - | No |
|
||||
| `--metrics-port <PORT>` | Metrics endpoint port | `5005` | No |
|
||||
| `--sudo` | Run with elevated privileges | `false` | No |
|
||||
|
||||
## Output
|
||||
|
||||
Results in `./reth-bench-compare/results/<timestamp>/`:
|
||||
- `comparison_report.json` - Metrics comparison
|
||||
- `per_block_comparison.csv` - Per-block statistics
|
||||
- `baseline/` and `feature/` - Individual run results
|
||||
- `latency_comparison.png` - Chart (if `--draw` used)
|
||||
@@ -18,6 +18,8 @@ pub(crate) struct BenchmarkRunner {
|
||||
rpc_url: String,
|
||||
jwt_secret: String,
|
||||
wait_time: Option<String>,
|
||||
wait_for_persistence: bool,
|
||||
persistence_threshold: Option<u64>,
|
||||
warmup_blocks: u64,
|
||||
}
|
||||
|
||||
@@ -28,6 +30,8 @@ impl BenchmarkRunner {
|
||||
rpc_url: args.get_rpc_url(),
|
||||
jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(),
|
||||
wait_time: args.wait_time.clone(),
|
||||
wait_for_persistence: args.wait_for_persistence,
|
||||
persistence_threshold: args.persistence_threshold,
|
||||
warmup_blocks: args.get_warmup_blocks(),
|
||||
}
|
||||
}
|
||||
@@ -182,9 +186,16 @@ impl BenchmarkRunner {
|
||||
&output_dir.to_string_lossy(),
|
||||
]);
|
||||
|
||||
// If wait_time is provided, use wait-time mode; otherwise uses persistence-based flow
|
||||
// Configure wait mode: wait-time takes precedence over persistence-based flow
|
||||
if let Some(ref wait_time) = self.wait_time {
|
||||
cmd.args(["--wait-time", wait_time]);
|
||||
} else if self.wait_for_persistence {
|
||||
cmd.arg("--wait-for-persistence");
|
||||
|
||||
// Add persistence threshold if specified
|
||||
if let Some(threshold) = self.persistence_threshold {
|
||||
cmd.args(["--persistence-threshold", &threshold.to_string()]);
|
||||
}
|
||||
}
|
||||
|
||||
cmd.env("RUST_LOG_STYLE", "never")
|
||||
|
||||
@@ -121,6 +121,22 @@ pub(crate) struct Args {
|
||||
#[arg(long, value_name = "DURATION", hide = true)]
|
||||
pub wait_time: Option<String>,
|
||||
|
||||
/// Wait for blocks to be persisted before sending the next batch (passed to reth-bench).
|
||||
///
|
||||
/// When enabled, waits for every Nth block to be persisted using the
|
||||
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
|
||||
/// doesn't outpace persistence.
|
||||
#[arg(long)]
|
||||
pub wait_for_persistence: bool,
|
||||
|
||||
/// Engine persistence threshold (passed to reth-bench).
|
||||
///
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
|
||||
/// matches the engine's default persistence threshold (2), so waits occur
|
||||
/// at blocks 3, 6, 9, etc.
|
||||
#[arg(long, value_name = "PERSISTENCE_THRESHOLD")]
|
||||
pub persistence_threshold: Option<u64>,
|
||||
|
||||
/// Number of blocks to run for cache warmup after clearing caches.
|
||||
/// If not specified, defaults to the same as --blocks
|
||||
#[arg(long, value_name = "N")]
|
||||
@@ -131,6 +147,11 @@ pub(crate) struct Args {
|
||||
#[arg(long)]
|
||||
pub no_clear_cache: bool,
|
||||
|
||||
/// Skip waiting for the node to sync before starting benchmarks.
|
||||
/// When enabled, assumes the node is already synced and skips the initial tip check.
|
||||
#[arg(long)]
|
||||
pub skip_wait_syncing: bool,
|
||||
|
||||
#[command(flatten)]
|
||||
pub logs: LogArgs,
|
||||
|
||||
@@ -562,7 +583,11 @@ async fn run_warmup_phase(
|
||||
node_manager.start_node(&binary_path, warmup_ref, "warmup", &additional_args).await?;
|
||||
|
||||
// Wait for node to be ready and get its current tip
|
||||
let current_tip = node_manager.wait_for_node_ready_and_get_tip().await?;
|
||||
let current_tip = if args.skip_wait_syncing {
|
||||
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
|
||||
} else {
|
||||
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
|
||||
};
|
||||
info!("Warmup node is ready at tip: {}", current_tip);
|
||||
|
||||
// Clear filesystem caches before warmup run only (unless disabled)
|
||||
@@ -616,7 +641,11 @@ async fn run_benchmark_workflow(
|
||||
let (mut node_process, _) = node_manager
|
||||
.start_node(&binary_path, &args.baseline_ref, "baseline", &additional_args)
|
||||
.await?;
|
||||
let starting_tip = node_manager.wait_for_node_ready_and_get_tip().await?;
|
||||
let starting_tip = if args.skip_wait_syncing {
|
||||
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
|
||||
} else {
|
||||
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
|
||||
};
|
||||
info!("Node starting tip: {}", starting_tip);
|
||||
node_manager.stop_node(&mut node_process).await?;
|
||||
|
||||
@@ -683,7 +712,11 @@ async fn run_benchmark_workflow(
|
||||
node_manager.start_node(&binary_path, git_ref, ref_type, &additional_args).await?;
|
||||
|
||||
// Wait for node to be ready and get its current tip (wherever it is)
|
||||
let current_tip = node_manager.wait_for_node_ready_and_get_tip().await?;
|
||||
let current_tip = if args.skip_wait_syncing {
|
||||
node_manager.wait_for_rpc_and_get_tip(&mut node_process).await?
|
||||
} else {
|
||||
node_manager.wait_for_node_ready_and_get_tip(&mut node_process).await?
|
||||
};
|
||||
info!("Node is ready at tip: {}", current_tip);
|
||||
|
||||
// Calculate benchmark range
|
||||
|
||||
@@ -367,8 +367,13 @@ impl NodeManager {
|
||||
Ok((child, reth_command))
|
||||
}
|
||||
|
||||
/// Wait for the node to be ready and return its current tip
|
||||
pub(crate) async fn wait_for_node_ready_and_get_tip(&self) -> Result<u64> {
|
||||
/// Wait for the node to be ready and return its current tip.
|
||||
///
|
||||
/// Fails early if the node process exits before becoming ready.
|
||||
pub(crate) async fn wait_for_node_ready_and_get_tip(
|
||||
&self,
|
||||
child: &mut tokio::process::Child,
|
||||
) -> Result<u64> {
|
||||
info!("Waiting for node to be ready and synced...");
|
||||
|
||||
let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync
|
||||
@@ -391,6 +396,11 @@ impl NodeManager {
|
||||
start_time.elapsed()
|
||||
);
|
||||
|
||||
// Check if the node process has exited.
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Err(eyre!("Node process exited unexpectedly with {status}"));
|
||||
}
|
||||
|
||||
// First check if RPC is up and node is not syncing
|
||||
match provider.syncing().await {
|
||||
Ok(sync_result) => {
|
||||
@@ -448,6 +458,76 @@ impl NodeManager {
|
||||
.wrap_err("Timed out waiting for node to be ready and synced")?
|
||||
}
|
||||
|
||||
/// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
|
||||
///
|
||||
/// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
|
||||
/// the node is still syncing.
|
||||
pub(crate) async fn wait_for_rpc_and_get_tip(
|
||||
&self,
|
||||
child: &mut tokio::process::Child,
|
||||
) -> Result<u64> {
|
||||
info!("Waiting for node RPC to be ready (skipping sync wait)...");
|
||||
|
||||
let max_wait = Duration::from_secs(60);
|
||||
let check_interval = Duration::from_secs(2);
|
||||
let rpc_url = "http://localhost:8545";
|
||||
|
||||
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
|
||||
let provider = ProviderBuilder::new().connect_http(url);
|
||||
|
||||
let start_time = tokio::time::Instant::now();
|
||||
let mut iteration = 0;
|
||||
|
||||
timeout(max_wait, async {
|
||||
loop {
|
||||
iteration += 1;
|
||||
debug!(
|
||||
"RPC readiness check iteration {} (elapsed: {:?})",
|
||||
iteration,
|
||||
start_time.elapsed()
|
||||
);
|
||||
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Err(eyre!("Node process exited unexpectedly with {status}"));
|
||||
}
|
||||
|
||||
match provider.get_block_number().await {
|
||||
Ok(tip) => {
|
||||
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
|
||||
|
||||
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
|
||||
let ws_connect = WsConnect::new(&ws_url);
|
||||
|
||||
match RpcClient::connect_pubsub(ws_connect).await {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
"Node RPC is ready at block: {} (took {:?}, {} iterations)",
|
||||
tip,
|
||||
start_time.elapsed(),
|
||||
iteration
|
||||
);
|
||||
return Ok(tip);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
|
||||
iteration, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
|
||||
}
|
||||
}
|
||||
|
||||
sleep(check_interval).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.wrap_err("Timed out waiting for node RPC to be ready")?
|
||||
}
|
||||
|
||||
/// Stop the reth node gracefully
|
||||
pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
|
||||
let pid = child.id().ok_or_eyre("Child process ID should be available")?;
|
||||
|
||||
@@ -85,7 +85,11 @@ jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
]
|
||||
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
|
||||
tracy = [
|
||||
"reth-node-core/tracy",
|
||||
"reth-tracing/tracy",
|
||||
]
|
||||
|
||||
min-error-logs = [
|
||||
"tracing/release_max_level_error",
|
||||
|
||||
@@ -81,12 +81,16 @@ backon.workspace = true
|
||||
tempfile.workspace = true
|
||||
|
||||
[features]
|
||||
default = ["jemalloc", "otlp", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
default = ["jemalloc", "otlp", "otlp-logs", "reth-revm/portable", "js-tracer", "keccak-cache-global", "asm-keccak"]
|
||||
|
||||
otlp = [
|
||||
"reth-ethereum-cli/otlp",
|
||||
"reth-node-core/otlp",
|
||||
]
|
||||
otlp-logs = [
|
||||
"reth-ethereum-cli/otlp-logs",
|
||||
"reth-node-core/otlp-logs",
|
||||
]
|
||||
js-tracer = [
|
||||
"reth-node-builder/js-tracer",
|
||||
"reth-node-ethereum/js-tracer",
|
||||
@@ -131,6 +135,11 @@ jemalloc-unprefixed = [
|
||||
tracy-allocator = [
|
||||
"reth-cli-util/tracy-allocator",
|
||||
"reth-ethereum-cli/tracy-allocator",
|
||||
"tracy",
|
||||
]
|
||||
tracy = [
|
||||
"reth-ethereum-cli/tracy",
|
||||
"reth-node-core/tracy",
|
||||
]
|
||||
|
||||
# Because jemalloc is default and preferred over snmalloc when both features are
|
||||
@@ -171,7 +180,7 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
edge = ["reth-ethereum-cli/edge"]
|
||||
edge = ["reth-ethereum-cli/edge", "reth-node-core/edge"]
|
||||
|
||||
[[bin]]
|
||||
name = "reth"
|
||||
|
||||
@@ -37,12 +37,19 @@ pub struct ComputedTrieData {
|
||||
|
||||
/// Trie input bundled with its anchor hash.
|
||||
///
|
||||
/// This is used to store the trie input and anchor hash for a block together.
|
||||
/// The `trie_input` contains the **cumulative** overlay of all in-memory ancestor blocks,
|
||||
/// not just this block's changes. Child blocks reuse the parent's overlay in O(1) by
|
||||
/// cloning the Arc-wrapped data.
|
||||
///
|
||||
/// The `anchor_hash` is metadata indicating which persisted base state this overlay
|
||||
/// sits on top of. It is CRITICAL for overlay reuse decisions: an overlay built on top
|
||||
/// of Anchor A cannot be reused for a block anchored to Anchor B, as it would result
|
||||
/// in an incorrect state.
|
||||
#[derive(Clone, Debug)]
|
||||
pub struct AnchoredTrieInput {
|
||||
/// The persisted ancestor hash this trie input is anchored to.
|
||||
pub anchor_hash: B256,
|
||||
/// Trie input constructed from in-memory overlays.
|
||||
/// Cumulative trie input overlay from all in-memory ancestors.
|
||||
pub trie_input: Arc<TrieInputSorted>,
|
||||
}
|
||||
|
||||
@@ -139,8 +146,9 @@ impl DeferredTrieData {
|
||||
///
|
||||
/// # Process
|
||||
/// 1. Sort the current block's hashed state and trie updates
|
||||
/// 2. Merge ancestor overlays (oldest -> newest, so later state takes precedence)
|
||||
/// 3. Extend the merged overlay with this block's sorted data
|
||||
/// 2. Reuse parent's cached overlay if available (O(1) - the common case)
|
||||
/// 3. Otherwise, rebuild overlay from ancestors (rare fallback)
|
||||
/// 4. Extend the overlay with this block's sorted data
|
||||
///
|
||||
/// Used by both the async background task and the synchronous fallback path.
|
||||
///
|
||||
@@ -148,7 +156,7 @@ impl DeferredTrieData {
|
||||
/// * `hashed_state` - Unsorted hashed post-state (account/storage changes) from execution
|
||||
/// * `trie_updates` - Unsorted trie node updates from state root computation
|
||||
/// * `anchor_hash` - The persisted ancestor hash this trie input is anchored to
|
||||
/// * `ancestors` - Deferred trie data from ancestor blocks for merging
|
||||
/// * `ancestors` - Deferred trie data from ancestor blocks for merging (oldest -> newest)
|
||||
pub fn sort_and_build_trie_input(
|
||||
hashed_state: Arc<HashedPostState>,
|
||||
trie_updates: Arc<TrieUpdates>,
|
||||
@@ -164,9 +172,71 @@ impl DeferredTrieData {
|
||||
Err(arc) => arc.clone_into_sorted(),
|
||||
};
|
||||
|
||||
// Build overlay by merging ancestors oldest-to-newest, then this block's data last.
|
||||
// Later entries take precedence, so this block's state overwrites any ancestor conflicts.
|
||||
// Reuse parent's overlay if available and anchors match.
|
||||
// We can only reuse the parent's overlay if it was built on top of the same
|
||||
// persisted anchor. If the anchor has changed (e.g., due to persistence),
|
||||
// the parent's overlay is relative to an old state and cannot be used.
|
||||
let overlay = if let Some(parent) = ancestors.last() {
|
||||
let parent_data = parent.wait_cloned();
|
||||
|
||||
match &parent_data.anchored_trie_input {
|
||||
// Case 1: Parent has cached overlay AND anchors match.
|
||||
Some(AnchoredTrieInput { anchor_hash: parent_anchor, trie_input })
|
||||
if *parent_anchor == anchor_hash =>
|
||||
{
|
||||
// O(1): Reuse parent's overlay, extend with current block's data.
|
||||
let mut overlay = TrieInputSorted::new(
|
||||
Arc::clone(&trie_input.nodes),
|
||||
Arc::clone(&trie_input.state),
|
||||
Default::default(), // prefix_sets are per-block, not cumulative
|
||||
);
|
||||
// Only trigger COW clone if there's actually data to add.
|
||||
if !sorted_hashed_state.is_empty() {
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(&sorted_hashed_state);
|
||||
}
|
||||
if !sorted_trie_updates.is_empty() {
|
||||
Arc::make_mut(&mut overlay.nodes).extend_ref(&sorted_trie_updates);
|
||||
}
|
||||
overlay
|
||||
}
|
||||
// Case 2: Parent exists but anchor mismatch or no cached overlay.
|
||||
// We must rebuild from the ancestors list (which only contains unpersisted blocks).
|
||||
_ => Self::merge_ancestors_into_overlay(
|
||||
ancestors,
|
||||
&sorted_hashed_state,
|
||||
&sorted_trie_updates,
|
||||
),
|
||||
}
|
||||
} else {
|
||||
// Case 3: No in-memory ancestors (first block after persisted anchor).
|
||||
// Build overlay with just this block's data.
|
||||
Self::merge_ancestors_into_overlay(&[], &sorted_hashed_state, &sorted_trie_updates)
|
||||
};
|
||||
|
||||
ComputedTrieData::with_trie_input(
|
||||
Arc::new(sorted_hashed_state),
|
||||
Arc::new(sorted_trie_updates),
|
||||
anchor_hash,
|
||||
Arc::new(overlay),
|
||||
)
|
||||
}
|
||||
|
||||
/// Merge all ancestors and current block's data into a single overlay.
|
||||
///
|
||||
/// This is a rare fallback path, only used when no ancestor has a cached
|
||||
/// `anchored_trie_input` (e.g., blocks created via alternative constructors).
|
||||
/// In normal operation, the parent always has a cached overlay and this
|
||||
/// function is never called.
|
||||
///
|
||||
/// Iterates ancestors oldest -> newest, then extends with current block's data,
|
||||
/// so later state takes precedence.
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
sorted_trie_updates: &TrieUpdatesSorted,
|
||||
) -> TrieInputSorted {
|
||||
let mut overlay = TrieInputSorted::default();
|
||||
|
||||
let state_mut = Arc::make_mut(&mut overlay.state);
|
||||
let nodes_mut = Arc::make_mut(&mut overlay.nodes);
|
||||
|
||||
@@ -176,15 +246,11 @@ impl DeferredTrieData {
|
||||
nodes_mut.extend_ref(ancestor_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
state_mut.extend_ref(&sorted_hashed_state);
|
||||
nodes_mut.extend_ref(&sorted_trie_updates);
|
||||
// Extend with current block's sorted data last (takes precedence)
|
||||
state_mut.extend_ref(sorted_hashed_state);
|
||||
nodes_mut.extend_ref(sorted_trie_updates);
|
||||
|
||||
ComputedTrieData::with_trie_input(
|
||||
Arc::new(sorted_hashed_state),
|
||||
Arc::new(sorted_trie_updates),
|
||||
anchor_hash,
|
||||
Arc::new(overlay),
|
||||
)
|
||||
overlay
|
||||
}
|
||||
|
||||
/// Returns trie data, computing synchronously if the async task hasn't completed.
|
||||
@@ -441,4 +507,365 @@ mod tests {
|
||||
let (_, account) = &overlay_state[0];
|
||||
assert_eq!(account.unwrap().nonce, 2);
|
||||
}
|
||||
|
||||
/// Helper to create a ready block with anchored trie input containing specific state.
|
||||
fn ready_block_with_state(
|
||||
anchor_hash: B256,
|
||||
accounts: Vec<(B256, Option<Account>)>,
|
||||
) -> DeferredTrieData {
|
||||
let hashed_state = Arc::new(HashedPostStateSorted::new(accounts, B256Map::default()));
|
||||
let trie_updates = Arc::default();
|
||||
let mut overlay = TrieInputSorted::default();
|
||||
Arc::make_mut(&mut overlay.state).extend_ref(hashed_state.as_ref());
|
||||
|
||||
DeferredTrieData::ready(ComputedTrieData {
|
||||
hashed_state,
|
||||
trie_updates,
|
||||
anchored_trie_input: Some(AnchoredTrieInput {
|
||||
anchor_hash,
|
||||
trie_input: Arc::new(overlay),
|
||||
}),
|
||||
})
|
||||
}
|
||||
|
||||
/// Verifies that first block after anchor (no ancestors) creates empty base overlay.
|
||||
#[test]
|
||||
fn first_block_after_anchor_creates_empty_base() {
|
||||
let anchor = B256::with_last_byte(1);
|
||||
let key = B256::with_last_byte(42);
|
||||
let account = Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None };
|
||||
|
||||
// First block after anchor - no ancestors
|
||||
let first_block = DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default().with_accounts([(key, Some(account))])),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
vec![], // No ancestors
|
||||
);
|
||||
|
||||
let result = first_block.wait_cloned();
|
||||
|
||||
// Should have overlay with just this block's data
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.anchor_hash, anchor);
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
|
||||
let (found_key, found_account) = &overlay.trie_input.state.accounts[0];
|
||||
assert_eq!(*found_key, key);
|
||||
assert_eq!(found_account.unwrap().nonce, 1);
|
||||
}
|
||||
|
||||
/// Verifies that parent's overlay is reused regardless of anchor.
|
||||
#[test]
|
||||
fn reuses_parent_overlay() {
|
||||
let anchor = B256::with_last_byte(1);
|
||||
let key = B256::with_last_byte(42);
|
||||
let account = Account { nonce: 100, balance: U256::ZERO, bytecode_hash: None };
|
||||
|
||||
// Create parent with anchored trie input
|
||||
let parent = ready_block_with_state(anchor, vec![(key, Some(account))]);
|
||||
|
||||
// Create child - should reuse parent's overlay
|
||||
let child = DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
vec![parent],
|
||||
);
|
||||
|
||||
let result = child.wait_cloned();
|
||||
|
||||
// Verify parent's account is in the overlay
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.anchor_hash, anchor);
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
|
||||
let (found_key, found_account) = &overlay.trie_input.state.accounts[0];
|
||||
assert_eq!(*found_key, key);
|
||||
assert_eq!(found_account.unwrap().nonce, 100);
|
||||
}
|
||||
|
||||
/// Verifies that parent's overlay is NOT reused when anchor changes (after persist).
|
||||
/// The overlay data is dependent on the anchor, so it must be rebuilt from the
|
||||
/// remaining ancestors.
|
||||
#[test]
|
||||
fn rebuilds_overlay_when_anchor_changes() {
|
||||
let old_anchor = B256::with_last_byte(1);
|
||||
let new_anchor = B256::with_last_byte(2);
|
||||
let key = B256::with_last_byte(42);
|
||||
let account = Account { nonce: 50, balance: U256::ZERO, bytecode_hash: None };
|
||||
|
||||
// Create parent with OLD anchor
|
||||
let parent = ready_block_with_state(old_anchor, vec![(key, Some(account))]);
|
||||
|
||||
// Create child with NEW anchor (simulates after persist)
|
||||
// Should NOT reuse parent's overlay because anchor changed
|
||||
let child = DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
new_anchor,
|
||||
vec![parent],
|
||||
);
|
||||
|
||||
let result = child.wait_cloned();
|
||||
|
||||
// Verify result uses new anchor
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.anchor_hash, new_anchor);
|
||||
|
||||
// Crucially, since we provided `parent` in ancestors but it has a different anchor,
|
||||
// the code falls back to `merge_ancestors_into_overlay`.
|
||||
// `merge_ancestors_into_overlay` reads `parent.hashed_state` (which has the account).
|
||||
// So the account IS present, but it was obtained via REBUILD, not REUSE.
|
||||
// We can check `DEFERRED_TRIE_METRICS` if we want to be sure, but functionally:
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
|
||||
let (found_key, found_account) = &overlay.trie_input.state.accounts[0];
|
||||
assert_eq!(*found_key, key);
|
||||
assert_eq!(found_account.unwrap().nonce, 50);
|
||||
}
|
||||
|
||||
/// Verifies that parent without `anchored_trie_input` triggers rebuild path.
|
||||
#[test]
|
||||
fn rebuilds_when_parent_has_no_anchored_input() {
|
||||
let anchor = B256::with_last_byte(1);
|
||||
let key = B256::with_last_byte(42);
|
||||
let account = Account { nonce: 25, balance: U256::ZERO, bytecode_hash: None };
|
||||
|
||||
// Create parent WITHOUT anchored trie input (e.g., from without_trie_input constructor)
|
||||
let parent_state =
|
||||
HashedPostStateSorted::new(vec![(key, Some(account))], B256Map::default());
|
||||
let parent = DeferredTrieData::ready(ComputedTrieData {
|
||||
hashed_state: Arc::new(parent_state),
|
||||
trie_updates: Arc::default(),
|
||||
anchored_trie_input: None, // No anchored input
|
||||
});
|
||||
|
||||
// Create child - should rebuild from parent's hashed_state
|
||||
let child = DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
vec![parent],
|
||||
);
|
||||
|
||||
let result = child.wait_cloned();
|
||||
|
||||
// Verify overlay is built and contains parent's data
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.anchor_hash, anchor);
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), 1);
|
||||
}
|
||||
|
||||
/// Verifies that a chain of blocks with matching anchors builds correct cumulative overlay.
|
||||
#[test]
|
||||
fn chain_of_blocks_builds_cumulative_overlay() {
|
||||
let anchor = B256::with_last_byte(1);
|
||||
let key1 = B256::with_last_byte(1);
|
||||
let key2 = B256::with_last_byte(2);
|
||||
let key3 = B256::with_last_byte(3);
|
||||
|
||||
// Block 1: sets account at key1
|
||||
let block1 = ready_block_with_state(
|
||||
anchor,
|
||||
vec![(key1, Some(Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None }))],
|
||||
);
|
||||
|
||||
// Block 2: adds account at key2, ancestor is block1
|
||||
let block2_hashed = HashedPostState::default().with_accounts([(
|
||||
key2,
|
||||
Some(Account { nonce: 2, balance: U256::ZERO, bytecode_hash: None }),
|
||||
)]);
|
||||
let block2 = DeferredTrieData::pending(
|
||||
Arc::new(block2_hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
vec![block1.clone()],
|
||||
);
|
||||
// Compute block2's trie data
|
||||
let block2_computed = block2.wait_cloned();
|
||||
let block2_ready = DeferredTrieData::ready(block2_computed);
|
||||
|
||||
// Block 3: adds account at key3, ancestor is block2 (which includes block1)
|
||||
let block3_hashed = HashedPostState::default().with_accounts([(
|
||||
key3,
|
||||
Some(Account { nonce: 3, balance: U256::ZERO, bytecode_hash: None }),
|
||||
)]);
|
||||
let block3 = DeferredTrieData::pending(
|
||||
Arc::new(block3_hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
vec![block1, block2_ready],
|
||||
);
|
||||
|
||||
let result = block3.wait_cloned();
|
||||
|
||||
// Verify all three accounts are in the cumulative overlay
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), 3);
|
||||
|
||||
// Accounts should be sorted by key (B256 ordering)
|
||||
let accounts = &overlay.trie_input.state.accounts;
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key1 && a.unwrap().nonce == 1));
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key2 && a.unwrap().nonce == 2));
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key3 && a.unwrap().nonce == 3));
|
||||
}
|
||||
|
||||
/// Verifies that child block's state overwrites parent's state for the same key.
|
||||
#[test]
|
||||
fn child_state_overwrites_parent() {
|
||||
let anchor = B256::with_last_byte(1);
|
||||
let key = B256::with_last_byte(42);
|
||||
|
||||
// Parent sets nonce to 10
|
||||
let parent = ready_block_with_state(
|
||||
anchor,
|
||||
vec![(key, Some(Account { nonce: 10, balance: U256::ZERO, bytecode_hash: None }))],
|
||||
);
|
||||
|
||||
// Child overwrites nonce to 99
|
||||
let child_hashed = HashedPostState::default().with_accounts([(
|
||||
key,
|
||||
Some(Account { nonce: 99, balance: U256::ZERO, bytecode_hash: None }),
|
||||
)]);
|
||||
let child = DeferredTrieData::pending(
|
||||
Arc::new(child_hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
vec![parent],
|
||||
);
|
||||
|
||||
let result = child.wait_cloned();
|
||||
|
||||
// Verify child's value wins (extend_ref uses later value)
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
// Note: extend_ref may result in duplicate keys; check the last occurrence
|
||||
let accounts = &overlay.trie_input.state.accounts;
|
||||
let last_account = accounts.iter().rfind(|(k, _)| *k == key).unwrap();
|
||||
assert_eq!(last_account.1.unwrap().nonce, 99);
|
||||
}
|
||||
|
||||
/// Stress test: verify O(N) behavior by building a chain of many blocks.
|
||||
/// This test ensures the fix doesn't regress - previously this would be O(N²).
|
||||
#[test]
|
||||
fn long_chain_builds_in_linear_time() {
|
||||
let anchor = B256::with_last_byte(1);
|
||||
let num_blocks = 50; // Enough to notice O(N²) vs O(N) difference
|
||||
|
||||
let mut ancestors: Vec<DeferredTrieData> = Vec::new();
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
for i in 0..num_blocks {
|
||||
let key = B256::with_last_byte(i as u8);
|
||||
let account = Account { nonce: i as u64, balance: U256::ZERO, bytecode_hash: None };
|
||||
let hashed = HashedPostState::default().with_accounts([(key, Some(account))]);
|
||||
|
||||
let block = DeferredTrieData::pending(
|
||||
Arc::new(hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
ancestors.clone(),
|
||||
);
|
||||
|
||||
// Compute and add to ancestors for next iteration
|
||||
let computed = block.wait_cloned();
|
||||
ancestors.push(DeferredTrieData::ready(computed));
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
// With O(N) fix, 50 blocks should complete quickly (< 1 second)
|
||||
// With O(N²), this would take significantly longer
|
||||
assert!(
|
||||
elapsed < Duration::from_secs(2),
|
||||
"Chain of {num_blocks} blocks took {:?}, possible O(N²) regression",
|
||||
elapsed
|
||||
);
|
||||
|
||||
// Verify final overlay has all accounts
|
||||
let final_result = ancestors.last().unwrap().wait_cloned();
|
||||
let overlay = final_result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), num_blocks);
|
||||
}
|
||||
|
||||
/// Verifies that a multi-ancestor overlay is rebuilt when anchor changes.
|
||||
/// This simulates the "persist prefix then keep building" scenario where:
|
||||
/// 1. A chain of blocks is built with anchor A
|
||||
/// 2. Some blocks are persisted, changing anchor to B
|
||||
/// 3. New blocks must rebuild the overlay from the remaining ancestors
|
||||
#[test]
|
||||
fn multi_ancestor_overlay_rebuilt_after_anchor_change() {
|
||||
let old_anchor = B256::with_last_byte(1);
|
||||
let new_anchor = B256::with_last_byte(2);
|
||||
let key1 = B256::with_last_byte(1);
|
||||
let key2 = B256::with_last_byte(2);
|
||||
let key3 = B256::with_last_byte(3);
|
||||
let key4 = B256::with_last_byte(4);
|
||||
|
||||
// Build a chain of 3 blocks with old_anchor
|
||||
let block1 = ready_block_with_state(
|
||||
old_anchor,
|
||||
vec![(key1, Some(Account { nonce: 1, balance: U256::ZERO, bytecode_hash: None }))],
|
||||
);
|
||||
|
||||
let block2_hashed = HashedPostState::default().with_accounts([(
|
||||
key2,
|
||||
Some(Account { nonce: 2, balance: U256::ZERO, bytecode_hash: None }),
|
||||
)]);
|
||||
let block2 = DeferredTrieData::pending(
|
||||
Arc::new(block2_hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
old_anchor,
|
||||
vec![block1.clone()],
|
||||
);
|
||||
let block2_ready = DeferredTrieData::ready(block2.wait_cloned());
|
||||
|
||||
let block3_hashed = HashedPostState::default().with_accounts([(
|
||||
key3,
|
||||
Some(Account { nonce: 3, balance: U256::ZERO, bytecode_hash: None }),
|
||||
)]);
|
||||
let block3 = DeferredTrieData::pending(
|
||||
Arc::new(block3_hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
old_anchor,
|
||||
vec![block1.clone(), block2_ready.clone()],
|
||||
);
|
||||
let block3_ready = DeferredTrieData::ready(block3.wait_cloned());
|
||||
|
||||
// Verify block3's overlay has all 3 accounts with old_anchor
|
||||
let block3_overlay = block3_ready.wait_cloned().anchored_trie_input.unwrap();
|
||||
assert_eq!(block3_overlay.anchor_hash, old_anchor);
|
||||
assert_eq!(block3_overlay.trie_input.state.accounts.len(), 3);
|
||||
|
||||
// Now simulate persist: create block4 with NEW anchor but same ancestors.
|
||||
// To verify correct rebuilding, we must provide ALL unpersisted ancestors.
|
||||
// If we only provided block3, the rebuild would only see block3's state.
|
||||
// We pass block1, block2, block3 to simulate that they are all still in memory
|
||||
// but the anchor check forces a rebuild (e.g. artificial anchor change).
|
||||
let block4_hashed = HashedPostState::default().with_accounts([(
|
||||
key4,
|
||||
Some(Account { nonce: 4, balance: U256::ZERO, bytecode_hash: None }),
|
||||
)]);
|
||||
let block4 = DeferredTrieData::pending(
|
||||
Arc::new(block4_hashed),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
new_anchor, // Different anchor - simulates post-persist
|
||||
vec![block1, block2_ready, block3_ready],
|
||||
);
|
||||
|
||||
let result = block4.wait_cloned();
|
||||
|
||||
// Verify:
|
||||
// 1. New anchor is used in result
|
||||
assert_eq!(result.anchor_hash(), Some(new_anchor));
|
||||
|
||||
// 2. All 4 accounts are in the overlay (rebuilt from ancestors + extended)
|
||||
let overlay = result.anchored_trie_input.as_ref().unwrap();
|
||||
assert_eq!(overlay.trie_input.state.accounts.len(), 4);
|
||||
|
||||
// 3. All accounts have correct values
|
||||
let accounts = &overlay.trie_input.state.accounts;
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key1 && a.unwrap().nonce == 1));
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key2 && a.unwrap().nonce == 2));
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key3 && a.unwrap().nonce == 3));
|
||||
assert!(accounts.iter().any(|(k, a)| *k == key4 && a.unwrap().nonce == 4));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -83,6 +83,7 @@ backon.workspace = true
|
||||
secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] }
|
||||
tokio-stream.workspace = true
|
||||
reqwest.workspace = true
|
||||
url.workspace = true
|
||||
metrics.workspace = true
|
||||
|
||||
# io
|
||||
|
||||
@@ -29,7 +29,7 @@ impl Command {
|
||||
let static_file_provider = tool.provider_factory.static_file_provider();
|
||||
let static_files = iter_static_files(static_file_provider.directory())?;
|
||||
|
||||
if let Some(segment_static_files) = static_files.get(&segment) {
|
||||
if let Some(segment_static_files) = static_files.get(segment) {
|
||||
for (block_range, _) in segment_static_files {
|
||||
static_file_provider.delete_jar(segment, block_range.start())?;
|
||||
}
|
||||
|
||||
@@ -100,7 +100,7 @@ impl<N: NodeTypes> TableViewer<()> for ListTableViewer<'_, N> {
|
||||
tx.disable_long_read_transaction_safety();
|
||||
|
||||
let table_db = tx.inner.open_db(Some(self.args.table.name())).wrap_err("Could not open db.")?;
|
||||
let stats = tx.inner.db_stat(&table_db).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
|
||||
let stats = tx.inner.db_stat(table_db.dbi()).wrap_err(format!("Could not find table: {}", self.args.table.name()))?;
|
||||
let total_entries = stats.entries();
|
||||
let final_entry_idx = total_entries.saturating_sub(1);
|
||||
if self.args.skip > final_entry_idx {
|
||||
|
||||
@@ -88,7 +88,7 @@ impl Command {
|
||||
|
||||
let stats = tx
|
||||
.inner
|
||||
.db_stat(&table_db)
|
||||
.db_stat(table_db.dbi())
|
||||
.wrap_err(format!("Could not find table: {db_table}"))?;
|
||||
|
||||
// Defaults to 16KB right now but we should
|
||||
@@ -129,7 +129,8 @@ impl Command {
|
||||
table.add_row(row);
|
||||
|
||||
let freelist = tx.inner.env().freelist()?;
|
||||
let pagesize = tx.inner.db_stat(&mdbx::Database::freelist_db())?.page_size() as usize;
|
||||
let pagesize =
|
||||
tx.inner.db_stat(mdbx::Database::freelist_db().dbi())?.page_size() as usize;
|
||||
let freelist_size = freelist * pagesize;
|
||||
|
||||
let mut row = Row::new();
|
||||
|
||||
@@ -16,6 +16,7 @@ use std::{
|
||||
use tar::Archive;
|
||||
use tokio::task;
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
use zstd::stream::read::Decoder as ZstdDecoder;
|
||||
|
||||
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
|
||||
@@ -85,6 +86,9 @@ impl DownloadDefaults {
|
||||
"\nIf no URL is provided, the latest mainnet archive snapshot\nwill be proposed for download from ",
|
||||
);
|
||||
help.push_str(self.default_base_url.as_ref());
|
||||
help.push_str(
|
||||
".\n\nLocal file:// URLs are also supported for extracting snapshots from disk.",
|
||||
);
|
||||
help
|
||||
}
|
||||
|
||||
@@ -170,12 +174,14 @@ struct DownloadProgress {
|
||||
downloaded: u64,
|
||||
total_size: u64,
|
||||
last_displayed: Instant,
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
impl DownloadProgress {
|
||||
/// Creates new progress tracker with given total size
|
||||
fn new(total_size: u64) -> Self {
|
||||
Self { downloaded: 0, total_size, last_displayed: Instant::now() }
|
||||
let now = Instant::now();
|
||||
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
|
||||
}
|
||||
|
||||
/// Converts bytes to human readable format (B, KB, MB, GB)
|
||||
@@ -191,6 +197,18 @@ impl DownloadProgress {
|
||||
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
|
||||
}
|
||||
|
||||
/// Format duration as human readable string
|
||||
fn format_duration(duration: Duration) -> String {
|
||||
let secs = duration.as_secs();
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m {}s", secs / 60, secs % 60)
|
||||
} else {
|
||||
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates progress bar
|
||||
fn update(&mut self, chunk_size: u64) -> Result<()> {
|
||||
self.downloaded += chunk_size;
|
||||
@@ -201,8 +219,24 @@ impl DownloadProgress {
|
||||
let formatted_total = Self::format_size(self.total_size);
|
||||
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
|
||||
|
||||
// Calculate ETA based on current speed
|
||||
let elapsed = self.started_at.elapsed();
|
||||
let eta = if self.downloaded > 0 {
|
||||
let remaining = self.total_size.saturating_sub(self.downloaded);
|
||||
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
|
||||
if speed > 0.0 {
|
||||
Duration::from_secs_f64(remaining as f64 / speed)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
}
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let eta_str = Self::format_duration(eta);
|
||||
|
||||
// Pad with spaces to clear any previous longer line
|
||||
print!(
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total})",
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
|
||||
);
|
||||
io::stdout().flush()?;
|
||||
self.last_displayed = Instant::now();
|
||||
@@ -246,29 +280,30 @@ enum CompressionFormat {
|
||||
impl CompressionFormat {
|
||||
/// Detect compression format from file extension
|
||||
fn from_url(url: &str) -> Result<Self> {
|
||||
if url.ends_with(EXTENSION_TAR_LZ4) {
|
||||
let path =
|
||||
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
|
||||
|
||||
if path.ends_with(EXTENSION_TAR_LZ4) {
|
||||
Ok(Self::Lz4)
|
||||
} else if url.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
Ok(Self::Zstd)
|
||||
} else {
|
||||
Err(eyre::eyre!("Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}", url))
|
||||
Err(eyre::eyre!(
|
||||
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
|
||||
path
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
let client = reqwest::blocking::Client::builder().build()?;
|
||||
let response = client.get(url).send()?.error_for_status()?;
|
||||
|
||||
let total_size = response.content_length().ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Server did not provide Content-Length header. This is required for snapshot downloads"
|
||||
)
|
||||
})?;
|
||||
|
||||
let progress_reader = ProgressReader::new(response, total_size);
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
/// Extracts a compressed tar archive to the target directory with progress tracking.
|
||||
fn extract_archive<R: Read>(
|
||||
reader: R,
|
||||
total_size: u64,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
) -> Result<()> {
|
||||
let progress_reader = ProgressReader::new(reader, total_size);
|
||||
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
@@ -285,6 +320,45 @@ fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extracts a snapshot from a local file.
|
||||
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let total_size = file.metadata()?.len();
|
||||
extract_archive(file, total_size, format, target_dir)
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL, uncompressing it in a streaming fashion.
|
||||
fn download_and_extract(url: &str, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let client = reqwest::blocking::Client::builder().build()?;
|
||||
let response = client.get(url).send()?.error_for_status()?;
|
||||
|
||||
let total_size = response.content_length().ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"Server did not provide Content-Length header. This is required for snapshot downloads"
|
||||
)
|
||||
})?;
|
||||
|
||||
extract_archive(response, total_size, format, target_dir)
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
///
|
||||
/// Supports both `file://` URLs for local files and HTTP(S) URLs for remote downloads.
|
||||
fn blocking_download_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
|
||||
if let Ok(parsed_url) = Url::parse(url) &&
|
||||
parsed_url.scheme() == "file"
|
||||
{
|
||||
let file_path = parsed_url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
|
||||
extract_from_file(&file_path, format, target_dir)
|
||||
} else {
|
||||
download_and_extract(url, format, target_dir)
|
||||
}
|
||||
}
|
||||
|
||||
async fn stream_and_extract(url: &str, target_dir: &Path) -> Result<()> {
|
||||
let target_dir = target_dir.to_path_buf();
|
||||
let url = url.to_string();
|
||||
@@ -343,6 +417,7 @@ mod tests {
|
||||
assert!(help.contains("Available snapshot sources:"));
|
||||
assert!(help.contains("merkle.io"));
|
||||
assert!(help.contains("publicnode.com"));
|
||||
assert!(help.contains("file://"));
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -367,4 +442,25 @@ mod tests {
|
||||
assert_eq!(defaults.available_snapshots.len(), 4); // 2 defaults + 2 added
|
||||
assert_eq!(defaults.long_help, Some("Custom help for snapshots".to_string()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_detection() {
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,6 @@ use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_cli_util::get_secret_key;
|
||||
use reth_config::config::{HashingConfig, SenderRecoveryConfig, TransactionLookupConfig};
|
||||
use reth_db_api::database_metrics::DatabaseMetrics;
|
||||
use reth_downloaders::{
|
||||
bodies::bodies::BodiesDownloaderBuilder,
|
||||
headers::reverse_headers::ReverseHeadersDownloaderBuilder,
|
||||
@@ -19,19 +18,19 @@ use reth_downloaders::{
|
||||
use reth_exex::ExExManagerHandle;
|
||||
use reth_network::BlockDownloaderProvider;
|
||||
use reth_network_p2p::HeadersClient;
|
||||
use reth_node_builder::common::metrics_hooks;
|
||||
use reth_node_core::{
|
||||
args::{NetworkArgs, StageEnum},
|
||||
version::version_metadata,
|
||||
};
|
||||
use reth_node_metrics::{
|
||||
chain::ChainSpecInfo,
|
||||
hooks::Hooks,
|
||||
server::{MetricServer, MetricServerConfig},
|
||||
version::VersionInfo,
|
||||
};
|
||||
use reth_provider::{
|
||||
ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
|
||||
StageCheckpointWriter, StaticFileProviderFactory,
|
||||
StageCheckpointWriter,
|
||||
};
|
||||
use reth_stages::{
|
||||
stages::{
|
||||
@@ -139,20 +138,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
},
|
||||
ChainSpecInfo { name: provider_factory.chain_spec().chain().to_string() },
|
||||
ctx.task_executor,
|
||||
Hooks::builder()
|
||||
.with_hook({
|
||||
let db = provider_factory.db_ref().clone();
|
||||
move || db.report_metrics()
|
||||
})
|
||||
.with_hook({
|
||||
let sfp = provider_factory.static_file_provider();
|
||||
move || {
|
||||
if let Err(error) = sfp.report_metrics() {
|
||||
error!(%error, "Failed to report metrics from static file provider");
|
||||
}
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
metrics_hooks(&provider_factory),
|
||||
data_dir.pprof_dumps(),
|
||||
);
|
||||
|
||||
|
||||
@@ -18,8 +18,8 @@ use tracing::{debug, error, trace};
|
||||
///
|
||||
/// Provides utilities for running a cli command to completion.
|
||||
#[derive(Debug)]
|
||||
#[non_exhaustive]
|
||||
pub struct CliRunner {
|
||||
config: CliRunnerConfig,
|
||||
tokio_runtime: tokio::runtime::Runtime,
|
||||
}
|
||||
|
||||
@@ -29,12 +29,18 @@ impl CliRunner {
|
||||
///
|
||||
/// The default tokio runtime is multi-threaded, with both I/O and time drivers enabled.
|
||||
pub fn try_default_runtime() -> Result<Self, std::io::Error> {
|
||||
Ok(Self { tokio_runtime: tokio_runtime()? })
|
||||
Ok(Self { config: CliRunnerConfig::default(), tokio_runtime: tokio_runtime()? })
|
||||
}
|
||||
|
||||
/// Create a new [`CliRunner`] from a provided tokio [`Runtime`](tokio::runtime::Runtime).
|
||||
pub const fn from_runtime(tokio_runtime: tokio::runtime::Runtime) -> Self {
|
||||
Self { tokio_runtime }
|
||||
Self { config: CliRunnerConfig::new(), tokio_runtime }
|
||||
}
|
||||
|
||||
/// Sets the [`CliRunnerConfig`] for this runner.
|
||||
pub const fn with_config(mut self, config: CliRunnerConfig) -> Self {
|
||||
self.config = config;
|
||||
self
|
||||
}
|
||||
|
||||
/// Executes an async block on the runtime and blocks until completion.
|
||||
@@ -74,7 +80,7 @@ impl CliRunner {
|
||||
// after the command has finished or exit signal was received we shutdown the task
|
||||
// manager which fires the shutdown signal to all tasks spawned via the task
|
||||
// executor and awaiting on tasks spawned with graceful shutdown
|
||||
task_manager.graceful_shutdown_with_timeout(Duration::from_secs(5));
|
||||
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
|
||||
}
|
||||
|
||||
// `drop(tokio_runtime)` would block the current thread until its pools
|
||||
@@ -128,7 +134,7 @@ impl CliRunner {
|
||||
error!(target: "reth::cli", "shutting down due to error");
|
||||
} else {
|
||||
debug!(target: "reth::cli", "shutting down gracefully");
|
||||
task_manager.graceful_shutdown_with_timeout(Duration::from_secs(5));
|
||||
task_manager.graceful_shutdown_with_timeout(self.config.graceful_shutdown_timeout);
|
||||
}
|
||||
|
||||
// Shutdown the runtime on a separate thread
|
||||
@@ -211,6 +217,38 @@ pub struct CliContext {
|
||||
pub task_executor: TaskExecutor,
|
||||
}
|
||||
|
||||
/// Default timeout for graceful shutdown of tasks.
|
||||
const DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT: Duration = Duration::from_secs(5);
|
||||
|
||||
/// Configuration for [`CliRunner`].
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct CliRunnerConfig {
|
||||
/// Timeout for graceful shutdown of tasks.
|
||||
///
|
||||
/// After the command completes, this is the maximum time to wait for spawned tasks
|
||||
/// to finish before forcefully terminating them.
|
||||
pub graceful_shutdown_timeout: Duration,
|
||||
}
|
||||
|
||||
impl Default for CliRunnerConfig {
|
||||
fn default() -> Self {
|
||||
Self::new()
|
||||
}
|
||||
}
|
||||
|
||||
impl CliRunnerConfig {
|
||||
/// Creates a new config with default values.
|
||||
pub const fn new() -> Self {
|
||||
Self { graceful_shutdown_timeout: DEFAULT_GRACEFUL_SHUTDOWN_TIMEOUT }
|
||||
}
|
||||
|
||||
/// Sets the graceful shutdown timeout.
|
||||
pub const fn with_graceful_shutdown_timeout(mut self, timeout: Duration) -> Self {
|
||||
self.graceful_shutdown_timeout = timeout;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new default tokio multi-thread [Runtime](tokio::runtime::Runtime) with all features
|
||||
/// enabled
|
||||
pub fn tokio_runtime() -> Result<tokio::runtime::Runtime, std::io::Error> {
|
||||
|
||||
@@ -26,7 +26,8 @@ rand_08.workspace = true
|
||||
thiserror.workspace = true
|
||||
serde.workspace = true
|
||||
|
||||
tracy-client = { workspace = true, optional = true, features = ["demangle"] }
|
||||
tracy-client = { workspace = true, optional = true }
|
||||
reth-tracing = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
rand.workspace = true
|
||||
@@ -46,7 +47,7 @@ jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
|
||||
jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"]
|
||||
|
||||
# Wraps the selected allocator in the tracy profiling allocator
|
||||
tracy-allocator = ["dep:tracy-client"]
|
||||
tracy-allocator = ["dep:tracy-client", "dep:reth-tracing"]
|
||||
|
||||
snmalloc = ["dep:snmalloc-rs"]
|
||||
|
||||
|
||||
@@ -25,7 +25,6 @@ cfg_if::cfg_if! {
|
||||
cfg_if::cfg_if! {
|
||||
if #[cfg(feature = "tracy-allocator")] {
|
||||
type AllocatorWrapper = tracy_client::ProfiledAllocator<AllocatorInner>;
|
||||
tracy_client::register_demangler!();
|
||||
const fn new_allocator_wrapper() -> AllocatorWrapper {
|
||||
AllocatorWrapper::new(AllocatorInner {}, 100)
|
||||
}
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
|
||||
#[cfg(feature = "tracy-allocator")]
|
||||
use reth_tracing as _;
|
||||
|
||||
pub mod allocator;
|
||||
pub mod cancellation;
|
||||
|
||||
|
||||
@@ -2,9 +2,8 @@
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_stages_types::ExecutionStageThresholds;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
path::{Path, PathBuf},
|
||||
time::Duration,
|
||||
};
|
||||
@@ -473,8 +472,8 @@ impl StaticFilesConfig {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Converts the blocks per file configuration into a [`HashMap`] per segment.
|
||||
pub fn as_blocks_per_file_map(&self) -> HashMap<StaticFileSegment, u64> {
|
||||
/// Converts the blocks per file configuration into a [`StaticFileMap`].
|
||||
pub fn as_blocks_per_file_map(&self) -> StaticFileMap<u64> {
|
||||
let BlocksPerFileConfig {
|
||||
headers,
|
||||
transactions,
|
||||
@@ -483,7 +482,7 @@ impl StaticFilesConfig {
|
||||
account_change_sets,
|
||||
} = self.blocks_per_file;
|
||||
|
||||
let mut map = HashMap::new();
|
||||
let mut map = StaticFileMap::default();
|
||||
// Iterating over all possible segments allows us to do an exhaustive match here,
|
||||
// to not forget to configure new segments in the future.
|
||||
for segment in StaticFileSegment::iter() {
|
||||
@@ -1079,18 +1078,6 @@ transaction_lookup = 'full'
|
||||
receipts = { distance = 16384 }
|
||||
#";
|
||||
let _conf: Config = toml::from_str(s).unwrap();
|
||||
|
||||
let s = r"#
|
||||
[prune]
|
||||
block_interval = 5
|
||||
|
||||
[prune.segments]
|
||||
sender_recovery = { distance = 16384 }
|
||||
transaction_lookup = 'full'
|
||||
receipts = 'full'
|
||||
#";
|
||||
let err = toml::from_str::<Config>(s).unwrap_err().to_string();
|
||||
assert!(err.contains("invalid value: string \"full\""), "{}", err);
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -2,11 +2,11 @@ use crate::{network::NetworkTestContext, payload::PayloadTestContext, rpc::RpcTe
|
||||
use alloy_consensus::{transaction::TxHashRef, BlockHeader};
|
||||
use alloy_eips::BlockId;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bytes, Sealable, B256};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV5, ForkchoiceState};
|
||||
use alloy_rpc_types_eth::BlockNumberOrTag;
|
||||
use eyre::Ok;
|
||||
use futures_util::Future;
|
||||
use jsonrpsee::http_client::HttpClient;
|
||||
use jsonrpsee::{core::client::ClientT, http_client::HttpClient};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_network_api::test_utils::PeersHandleProvider;
|
||||
use reth_node_api::{
|
||||
@@ -20,6 +20,7 @@ use reth_provider::{
|
||||
BlockReader, BlockReaderIdExt, CanonStateNotificationStream, CanonStateSubscriptions,
|
||||
HeaderProvider, StageCheckpointReader,
|
||||
};
|
||||
use reth_rpc_api::TestingBuildBlockRequestV1;
|
||||
use reth_rpc_builder::auth::AuthServerHandle;
|
||||
use reth_rpc_eth_api::helpers::{EthApiSpec, EthTransactions, TraceExt};
|
||||
use reth_stages_types::StageId;
|
||||
@@ -319,4 +320,20 @@ where
|
||||
|
||||
Ok(crate::testsuite::NodeClient::new_with_beacon_engine(rpc, auth, url, beacon_handle))
|
||||
}
|
||||
|
||||
/// Calls the `testing_buildBlockV1` RPC on this node.
|
||||
///
|
||||
/// This endpoint builds a block using the provided parent, payload attributes, and
|
||||
/// transactions. Requires the `Testing` RPC module to be enabled.
|
||||
pub async fn testing_build_block_v1(
|
||||
&self,
|
||||
request: TestingBuildBlockRequestV1,
|
||||
) -> eyre::Result<ExecutionPayloadEnvelopeV5> {
|
||||
let client =
|
||||
self.rpc_client().ok_or_else(|| eyre::eyre!("HTTP RPC client not available"))?;
|
||||
|
||||
let res: ExecutionPayloadEnvelopeV5 =
|
||||
client.request("testing_buildBlockV1", [request]).await?;
|
||||
eyre::Ok(res)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -289,7 +289,8 @@ where
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
|
||||
to_multi_proof.clone(),
|
||||
from_multi_proof,
|
||||
);
|
||||
)
|
||||
.with_v2_proofs_enabled(v2_proofs_enabled);
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
@@ -491,38 +492,40 @@ where
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
{
|
||||
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration if
|
||||
// there's none to reuse.
|
||||
let cleared_sparse_trie = Arc::clone(&self.sparse_state_trie);
|
||||
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
|
||||
let default_trie = SparseTrie::blind_from(if self.disable_parallel_sparse_trie {
|
||||
ConfiguredSparseTrie::Serial(Default::default())
|
||||
} else {
|
||||
ConfiguredSparseTrie::Parallel(Box::new(
|
||||
ParallelSparseTrie::default()
|
||||
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
|
||||
))
|
||||
});
|
||||
ClearedSparseStateTrie::from_state_trie(
|
||||
SparseStateTrie::new()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true),
|
||||
)
|
||||
});
|
||||
|
||||
let task =
|
||||
SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
|
||||
sparse_trie_rx,
|
||||
proof_worker_handle,
|
||||
self.trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
);
|
||||
|
||||
let disable_parallel_sparse_trie = self.disable_parallel_sparse_trie;
|
||||
let trie_metrics = self.trie_metrics.clone();
|
||||
let span = Span::current();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
|
||||
// Reuse a stored SparseStateTrie, or create a new one using the desired configuration
|
||||
// if there's none to reuse.
|
||||
let sparse_state_trie = cleared_sparse_trie.lock().take().unwrap_or_else(|| {
|
||||
let default_trie = SparseTrie::blind_from(if disable_parallel_sparse_trie {
|
||||
ConfiguredSparseTrie::Serial(Default::default())
|
||||
} else {
|
||||
ConfiguredSparseTrie::Parallel(Box::new(
|
||||
ParallelSparseTrie::default()
|
||||
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
|
||||
))
|
||||
});
|
||||
ClearedSparseStateTrie::from_state_trie(
|
||||
SparseStateTrie::new()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true),
|
||||
)
|
||||
});
|
||||
|
||||
let task = SparseTrieTask::<_, ConfiguredSparseTrie, ConfiguredSparseTrie>::new_with_cleared_trie(
|
||||
sparse_trie_rx,
|
||||
proof_worker_handle,
|
||||
trie_metrics,
|
||||
sparse_state_trie,
|
||||
);
|
||||
|
||||
let (result, trie) = task.run();
|
||||
// Send state root computation result
|
||||
let _ = state_root_tx.send(result);
|
||||
|
||||
@@ -11,15 +11,18 @@ use reth_metrics::Metrics;
|
||||
use reth_provider::AccountReader;
|
||||
use reth_revm::state::EvmState;
|
||||
use reth_trie::{
|
||||
added_removed_keys::MultiAddedRemovedKeys, DecodedMultiProof, HashedPostState, HashedStorage,
|
||||
MultiProofTargets,
|
||||
added_removed_keys::MultiAddedRemovedKeys, proof_v2, DecodedMultiProof, HashedPostState,
|
||||
HashedStorage, MultiProofTargets,
|
||||
};
|
||||
use reth_trie_parallel::{
|
||||
proof::ParallelProof,
|
||||
proof_task::{
|
||||
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
},
|
||||
targets_v2::{ChunkedMultiProofTargetsV2, MultiProofTargetsV2},
|
||||
};
|
||||
use revm_primitives::map::{hash_map, B256Map};
|
||||
use std::{collections::BTreeMap, sync::Arc, time::Instant};
|
||||
use tracing::{debug, error, instrument, trace};
|
||||
|
||||
@@ -62,12 +65,12 @@ const DEFAULT_MAX_TARGETS_FOR_CHUNKING: usize = 300;
|
||||
|
||||
/// A trie update that can be applied to sparse trie alongside the proofs for touched parts of the
|
||||
/// state.
|
||||
#[derive(Default, Debug)]
|
||||
#[derive(Debug)]
|
||||
pub struct SparseTrieUpdate {
|
||||
/// The state update that was used to calculate the proof
|
||||
pub(crate) state: HashedPostState,
|
||||
/// The calculated multiproof
|
||||
pub(crate) multiproof: DecodedMultiProof,
|
||||
pub(crate) multiproof: ProofResult,
|
||||
}
|
||||
|
||||
impl SparseTrieUpdate {
|
||||
@@ -79,7 +82,10 @@ impl SparseTrieUpdate {
|
||||
/// Construct update from multiproof.
|
||||
#[cfg(test)]
|
||||
pub(super) fn from_multiproof(multiproof: reth_trie::MultiProof) -> alloy_rlp::Result<Self> {
|
||||
Ok(Self { multiproof: multiproof.try_into()?, ..Default::default() })
|
||||
Ok(Self {
|
||||
state: HashedPostState::default(),
|
||||
multiproof: ProofResult::Legacy(multiproof.try_into()?),
|
||||
})
|
||||
}
|
||||
|
||||
/// Extend update with contents of the other.
|
||||
@@ -93,7 +99,7 @@ impl SparseTrieUpdate {
|
||||
#[derive(Debug)]
|
||||
pub(super) enum MultiProofMessage {
|
||||
/// Prefetch proof targets
|
||||
PrefetchProofs(MultiProofTargets),
|
||||
PrefetchProofs(VersionedMultiProofTargets),
|
||||
/// New state update from transaction execution with its source
|
||||
StateUpdate(Source, EvmState),
|
||||
/// State update that can be applied to the sparse trie without any new proofs.
|
||||
@@ -217,12 +223,191 @@ pub(crate) fn evm_state_to_hashed_post_state(update: EvmState) -> HashedPostStat
|
||||
hashed_state
|
||||
}
|
||||
|
||||
/// Extends a `MultiProofTargets` with the contents of a `VersionedMultiProofTargets`,
|
||||
/// regardless of which variant the latter is.
|
||||
fn extend_multiproof_targets(dest: &mut MultiProofTargets, src: &VersionedMultiProofTargets) {
|
||||
match src {
|
||||
VersionedMultiProofTargets::Legacy(targets) => {
|
||||
dest.extend_ref(targets);
|
||||
}
|
||||
VersionedMultiProofTargets::V2(targets) => {
|
||||
// Add all account targets
|
||||
for target in &targets.account_targets {
|
||||
dest.entry(target.key()).or_default();
|
||||
}
|
||||
|
||||
// Add all storage targets
|
||||
for (hashed_address, slots) in &targets.storage_targets {
|
||||
let slot_set = dest.entry(*hashed_address).or_default();
|
||||
for slot in slots {
|
||||
slot_set.insert(slot.key());
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A set of multiproof targets which can be either in the legacy or V2 representations.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum VersionedMultiProofTargets {
|
||||
/// Legacy targets
|
||||
Legacy(MultiProofTargets),
|
||||
/// V2 targets
|
||||
V2(MultiProofTargetsV2),
|
||||
}
|
||||
|
||||
impl VersionedMultiProofTargets {
|
||||
/// Returns true if ther are no account or storage targets.
|
||||
fn is_empty(&self) -> bool {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.is_empty(),
|
||||
Self::V2(targets) => targets.is_empty(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of account targets in the multiproof target
|
||||
fn account_targets_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.len(),
|
||||
Self::V2(targets) => targets.account_targets.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of storage targets in the multiproof target
|
||||
fn storage_targets_len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum::<usize>(),
|
||||
Self::V2(targets) => {
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of accounts in the multiproof targets.
|
||||
fn len(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.len(),
|
||||
Self::V2(targets) => targets.account_targets.len(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total storage slot count across all accounts.
|
||||
fn storage_count(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.values().map(|slots| slots.len()).sum(),
|
||||
Self::V2(targets) => targets.storage_targets.values().map(|slots| slots.len()).sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of items that will be considered during chunking.
|
||||
fn chunking_length(&self) -> usize {
|
||||
match self {
|
||||
Self::Legacy(targets) => targets.chunking_length(),
|
||||
Self::V2(targets) => {
|
||||
// For V2, count accounts + storage slots
|
||||
targets.account_targets.len() +
|
||||
targets.storage_targets.values().map(|slots| slots.len()).sum::<usize>()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Retains the targets representing the difference with another `MultiProofTargets`.
|
||||
/// Removes all targets that are already present in `other`.
|
||||
fn retain_difference(&mut self, other: &MultiProofTargets) {
|
||||
match self {
|
||||
Self::Legacy(targets) => {
|
||||
targets.retain_difference(other);
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
// Remove account targets that exist in other
|
||||
targets.account_targets.retain(|target| !other.contains_key(&target.key()));
|
||||
|
||||
// For each account in storage_targets, remove slots that exist in other
|
||||
targets.storage_targets.retain(|hashed_address, slots| {
|
||||
if let Some(other_slots) = other.get(hashed_address) {
|
||||
slots.retain(|slot| !other_slots.contains(&slot.key()));
|
||||
!slots.is_empty()
|
||||
} else {
|
||||
true
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extends this `VersionedMultiProofTargets` with the contents of another.
|
||||
///
|
||||
/// Panics if the variants do not match.
|
||||
fn extend(&mut self, other: Self) {
|
||||
match (self, other) {
|
||||
(Self::Legacy(dest), Self::Legacy(src)) => {
|
||||
dest.extend(src);
|
||||
}
|
||||
(Self::V2(dest), Self::V2(src)) => {
|
||||
dest.account_targets.extend(src.account_targets);
|
||||
for (addr, slots) in src.storage_targets {
|
||||
dest.storage_targets.entry(addr).or_default().extend(slots);
|
||||
}
|
||||
}
|
||||
_ => panic!("Cannot extend VersionedMultiProofTargets with mismatched variants"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Chunks this `VersionedMultiProofTargets` into smaller chunks of the given size.
|
||||
fn chunks(self, chunk_size: usize) -> Box<dyn Iterator<Item = Self>> {
|
||||
match self {
|
||||
Self::Legacy(targets) => {
|
||||
Box::new(MultiProofTargets::chunks(targets, chunk_size).map(Self::Legacy))
|
||||
}
|
||||
Self::V2(targets) => {
|
||||
Box::new(ChunkedMultiProofTargetsV2::new(targets, chunk_size).map(Self::V2))
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns v2 proof targets from the given EVM state update.
|
||||
///
|
||||
/// This generates account-level v2 targets from touched accounts.
|
||||
/// For state update targets, `min_len` is set to 0 to request full proofs.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn evm_state_to_v2_targets(update: &EvmState) -> Vec<reth_trie::proof_v2::Target> {
|
||||
let mut targets = Vec::with_capacity(update.len());
|
||||
for (address, account) in update {
|
||||
if account.is_touched() && !account.is_selfdestructed() {
|
||||
targets.push(reth_trie::proof_v2::Target::new(keccak256(address)));
|
||||
}
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
/// Returns v2 storage proof targets for a specific account from the EVM state.
|
||||
///
|
||||
/// For state update targets, `min_len` is set to 0 to request full proofs.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) fn evm_state_to_v2_storage_targets(
|
||||
update: &EvmState,
|
||||
address: alloy_primitives::Address,
|
||||
) -> Vec<reth_trie::proof_v2::Target> {
|
||||
let Some(account) = update.get(&address) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut targets = Vec::with_capacity(account.storage.len());
|
||||
for (key, slot) in &account.storage {
|
||||
if slot.is_changed() {
|
||||
targets.push(reth_trie::proof_v2::Target::new(keccak256(B256::from(*key))));
|
||||
}
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
/// Input parameters for dispatching a multiproof calculation.
|
||||
#[derive(Debug)]
|
||||
struct MultiproofInput {
|
||||
source: Option<Source>,
|
||||
hashed_state_update: HashedPostState,
|
||||
proof_targets: MultiProofTargets,
|
||||
proof_targets: VersionedMultiProofTargets,
|
||||
proof_sequence_number: u64,
|
||||
state_root_message_sender: CrossbeamSender<MultiProofMessage>,
|
||||
multi_added_removed_keys: Option<Arc<MultiAddedRemovedKeys>>,
|
||||
@@ -257,8 +442,6 @@ pub struct MultiproofManager {
|
||||
proof_result_tx: CrossbeamSender<ProofResultMessage>,
|
||||
/// Metrics
|
||||
metrics: MultiProofTaskMetrics,
|
||||
/// Whether to use V2 storage proofs
|
||||
v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl MultiproofManager {
|
||||
@@ -272,9 +455,7 @@ impl MultiproofManager {
|
||||
metrics.max_storage_workers.set(proof_worker_handle.total_storage_workers() as f64);
|
||||
metrics.max_account_workers.set(proof_worker_handle.total_account_workers() as f64);
|
||||
|
||||
let v2_proofs_enabled = proof_worker_handle.v2_proofs_enabled();
|
||||
|
||||
Self { metrics, proof_worker_handle, proof_result_tx, v2_proofs_enabled }
|
||||
Self { metrics, proof_worker_handle, proof_result_tx }
|
||||
}
|
||||
|
||||
/// Dispatches a new multiproof calculation to worker pools.
|
||||
@@ -319,41 +500,48 @@ impl MultiproofManager {
|
||||
multi_added_removed_keys,
|
||||
} = multiproof_input;
|
||||
|
||||
let account_targets = proof_targets.len();
|
||||
let storage_targets = proof_targets.values().map(|slots| slots.len()).sum::<usize>();
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
proof_sequence_number,
|
||||
?proof_targets,
|
||||
account_targets,
|
||||
storage_targets,
|
||||
account_targets = proof_targets.account_targets_len(),
|
||||
storage_targets = proof_targets.storage_targets_len(),
|
||||
?source,
|
||||
"Dispatching multiproof to workers"
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Extend prefix sets with targets
|
||||
let frozen_prefix_sets =
|
||||
ParallelProof::extend_prefix_sets_with_targets(&Default::default(), &proof_targets);
|
||||
// Workers will send ProofResultMessage directly to proof_result_rx
|
||||
let proof_result_sender = ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
proof_sequence_number,
|
||||
hashed_state_update,
|
||||
start,
|
||||
);
|
||||
|
||||
// Dispatch account multiproof to worker pool with result sender
|
||||
let input = AccountMultiproofInput {
|
||||
targets: proof_targets,
|
||||
prefix_sets: frozen_prefix_sets,
|
||||
collect_branch_node_masks: true,
|
||||
multi_added_removed_keys,
|
||||
// Workers will send ProofResultMessage directly to proof_result_rx
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
proof_sequence_number,
|
||||
hashed_state_update,
|
||||
start,
|
||||
),
|
||||
v2_proofs_enabled: self.v2_proofs_enabled,
|
||||
let input = match proof_targets {
|
||||
VersionedMultiProofTargets::Legacy(proof_targets) => {
|
||||
// Extend prefix sets with targets
|
||||
let frozen_prefix_sets = ParallelProof::extend_prefix_sets_with_targets(
|
||||
&Default::default(),
|
||||
&proof_targets,
|
||||
);
|
||||
|
||||
AccountMultiproofInput::Legacy {
|
||||
targets: proof_targets,
|
||||
prefix_sets: frozen_prefix_sets,
|
||||
collect_branch_node_masks: true,
|
||||
multi_added_removed_keys,
|
||||
proof_result_sender,
|
||||
}
|
||||
}
|
||||
VersionedMultiProofTargets::V2(proof_targets) => {
|
||||
AccountMultiproofInput::V2 { targets: proof_targets, proof_result_sender }
|
||||
}
|
||||
};
|
||||
|
||||
// Dispatch account multiproof to worker pool with result sender
|
||||
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(input) {
|
||||
error!(target: "engine::tree::payload_processor::multiproof", ?e, "Failed to dispatch account multiproof");
|
||||
return;
|
||||
@@ -555,6 +743,9 @@ pub(super) struct MultiProofTask {
|
||||
/// there are any active workers and force chunking across workers. This is to prevent tasks
|
||||
/// which are very long from hitting a single worker.
|
||||
max_targets_for_chunking: usize,
|
||||
/// Whether or not V2 proof calculation is enabled. If enabled then [`MultiProofTargetsV2`]
|
||||
/// will be produced by state updates.
|
||||
v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl MultiProofTask {
|
||||
@@ -586,9 +777,16 @@ impl MultiProofTask {
|
||||
),
|
||||
metrics,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
v2_proofs_enabled: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Enables V2 proof target generation on state updates.
|
||||
pub(super) fn with_v2_proofs_enabled(mut self, v2_proofs_enabled: bool) -> Self {
|
||||
self.v2_proofs_enabled = v2_proofs_enabled;
|
||||
self
|
||||
}
|
||||
|
||||
/// Handles request for proof prefetch.
|
||||
///
|
||||
/// Returns how many multiproof tasks were dispatched for the prefetch request.
|
||||
@@ -596,25 +794,29 @@ impl MultiProofTask {
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::multiproof",
|
||||
skip_all,
|
||||
fields(accounts = targets.len(), chunks = 0)
|
||||
fields(accounts = targets.account_targets_len(), chunks = 0)
|
||||
)]
|
||||
fn on_prefetch_proof(&mut self, mut targets: MultiProofTargets) -> u64 {
|
||||
fn on_prefetch_proof(&mut self, mut targets: VersionedMultiProofTargets) -> u64 {
|
||||
// Remove already fetched proof targets to avoid redundant work.
|
||||
targets.retain_difference(&self.fetched_proof_targets);
|
||||
self.fetched_proof_targets.extend_ref(&targets);
|
||||
extend_multiproof_targets(&mut self.fetched_proof_targets, &targets);
|
||||
|
||||
// Make sure all target accounts have an `AddedRemovedKeySet` in the
|
||||
// For Legacy multiproofs, make sure all target accounts have an `AddedRemovedKeySet` in the
|
||||
// [`MultiAddedRemovedKeys`]. Even if there are not any known removed keys for the account,
|
||||
// we still want to optimistically fetch extension children for the leaf addition case.
|
||||
self.multi_added_removed_keys.touch_accounts(targets.keys().copied());
|
||||
|
||||
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
|
||||
let multi_added_removed_keys = Arc::new(self.multi_added_removed_keys.clone());
|
||||
// V2 multiproofs don't need this.
|
||||
let multi_added_removed_keys =
|
||||
if let VersionedMultiProofTargets::Legacy(legacy_targets) = &targets {
|
||||
self.multi_added_removed_keys.touch_accounts(legacy_targets.keys().copied());
|
||||
Some(Arc::new(self.multi_added_removed_keys.clone()))
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
self.metrics.prefetch_proof_targets_accounts_histogram.record(targets.len() as f64);
|
||||
self.metrics
|
||||
.prefetch_proof_targets_storages_histogram
|
||||
.record(targets.values().map(|slots| slots.len()).sum::<usize>() as f64);
|
||||
.record(targets.storage_count() as f64);
|
||||
|
||||
let chunking_len = targets.chunking_length();
|
||||
let available_account_workers =
|
||||
@@ -628,7 +830,7 @@ impl MultiProofTask {
|
||||
self.max_targets_for_chunking,
|
||||
available_account_workers,
|
||||
available_storage_workers,
|
||||
MultiProofTargets::chunks,
|
||||
VersionedMultiProofTargets::chunks,
|
||||
|proof_targets| {
|
||||
self.multiproof_manager.dispatch(MultiproofInput {
|
||||
source: None,
|
||||
@@ -636,7 +838,7 @@ impl MultiProofTask {
|
||||
proof_targets,
|
||||
proof_sequence_number: self.proof_sequencer.next_sequence(),
|
||||
state_root_message_sender: self.tx.clone(),
|
||||
multi_added_removed_keys: Some(multi_added_removed_keys.clone()),
|
||||
multi_added_removed_keys: multi_added_removed_keys.clone(),
|
||||
});
|
||||
},
|
||||
);
|
||||
@@ -645,22 +847,16 @@ impl MultiProofTask {
|
||||
num_chunks as u64
|
||||
}
|
||||
|
||||
// Returns true if all state updates finished and all proofs processed.
|
||||
fn is_done(
|
||||
&self,
|
||||
proofs_processed: u64,
|
||||
state_update_proofs_requested: u64,
|
||||
prefetch_proofs_requested: u64,
|
||||
updates_finished: bool,
|
||||
) -> bool {
|
||||
let all_proofs_processed =
|
||||
proofs_processed >= state_update_proofs_requested + prefetch_proofs_requested;
|
||||
/// Returns true if all state updates finished and all pending proofs processed.
|
||||
fn is_done(&self, metrics: &MultiproofBatchMetrics, ctx: &MultiproofBatchCtx) -> bool {
|
||||
let all_proofs_processed = metrics.all_proofs_processed();
|
||||
let no_pending = !self.proof_sequencer.has_pending();
|
||||
let updates_finished = ctx.updates_finished();
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
proofs_processed,
|
||||
state_update_proofs_requested,
|
||||
prefetch_proofs_requested,
|
||||
proofs_processed = metrics.proofs_processed,
|
||||
state_update_proofs_requested = metrics.state_update_proofs_requested,
|
||||
prefetch_proofs_requested = metrics.prefetch_proofs_requested,
|
||||
no_pending,
|
||||
updates_finished,
|
||||
"Checking end condition"
|
||||
@@ -711,7 +907,33 @@ impl MultiProofTask {
|
||||
}
|
||||
|
||||
// Clone+Arc MultiAddedRemovedKeys for sharing with the dispatched multiproof tasks
|
||||
let multi_added_removed_keys = Arc::new(self.multi_added_removed_keys.clone());
|
||||
let multi_added_removed_keys = Arc::new(MultiAddedRemovedKeys {
|
||||
account: self.multi_added_removed_keys.account.clone(),
|
||||
storages: {
|
||||
let mut storages = B256Map::with_capacity_and_hasher(
|
||||
not_fetched_state_update.storages.len(),
|
||||
Default::default(),
|
||||
);
|
||||
|
||||
for account in not_fetched_state_update
|
||||
.storages
|
||||
.keys()
|
||||
.chain(not_fetched_state_update.accounts.keys())
|
||||
{
|
||||
if let hash_map::Entry::Vacant(entry) = storages.entry(*account) {
|
||||
entry.insert(
|
||||
self.multi_added_removed_keys
|
||||
.storages
|
||||
.get(account)
|
||||
.cloned()
|
||||
.unwrap_or_default(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
storages
|
||||
},
|
||||
});
|
||||
|
||||
let chunking_len = not_fetched_state_update.chunking_length();
|
||||
let mut spawned_proof_targets = MultiProofTargets::default();
|
||||
@@ -719,6 +941,7 @@ impl MultiProofTask {
|
||||
self.multiproof_manager.proof_worker_handle.available_account_workers();
|
||||
let available_storage_workers =
|
||||
self.multiproof_manager.proof_worker_handle.available_storage_workers();
|
||||
|
||||
let num_chunks = dispatch_with_chunking(
|
||||
not_fetched_state_update,
|
||||
chunking_len,
|
||||
@@ -732,8 +955,9 @@ impl MultiProofTask {
|
||||
&hashed_state_update,
|
||||
&self.fetched_proof_targets,
|
||||
&multi_added_removed_keys,
|
||||
self.v2_proofs_enabled,
|
||||
);
|
||||
spawned_proof_targets.extend_ref(&proof_targets);
|
||||
extend_multiproof_targets(&mut spawned_proof_targets, &proof_targets);
|
||||
|
||||
self.multiproof_manager.dispatch(MultiproofInput {
|
||||
source: Some(source),
|
||||
@@ -833,7 +1057,14 @@ impl MultiProofTask {
|
||||
batch_metrics.proofs_processed += 1;
|
||||
if let Some(combined_update) = self.on_proof(
|
||||
sequence_number,
|
||||
SparseTrieUpdate { state, multiproof: Default::default() },
|
||||
SparseTrieUpdate {
|
||||
state,
|
||||
multiproof: ProofResult::Legacy(DecodedMultiProof {
|
||||
account_subtree: Default::default(),
|
||||
branch_node_masks: Default::default(),
|
||||
storages: Default::default(),
|
||||
}),
|
||||
},
|
||||
) {
|
||||
let _ = self.to_sparse_trie.send(combined_update);
|
||||
}
|
||||
@@ -860,8 +1091,7 @@ impl MultiProofTask {
|
||||
}
|
||||
|
||||
let account_targets = merged_targets.len();
|
||||
let storage_targets =
|
||||
merged_targets.values().map(|slots| slots.len()).sum::<usize>();
|
||||
let storage_targets = merged_targets.storage_count();
|
||||
batch_metrics.prefetch_proofs_requested += self.on_prefetch_proof(merged_targets);
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
@@ -933,12 +1163,7 @@ impl MultiProofTask {
|
||||
ctx.updates_finished_time = Some(Instant::now());
|
||||
|
||||
// Check if we're done (might need to wait for proofs to complete)
|
||||
if self.is_done(
|
||||
batch_metrics.proofs_processed,
|
||||
batch_metrics.state_update_proofs_requested,
|
||||
batch_metrics.prefetch_proofs_requested,
|
||||
ctx.updates_finished(),
|
||||
) {
|
||||
if self.is_done(batch_metrics, ctx) {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
"BAL processed and all proofs complete, ending calculation"
|
||||
@@ -953,12 +1178,7 @@ impl MultiProofTask {
|
||||
|
||||
ctx.updates_finished_time = Some(Instant::now());
|
||||
|
||||
if self.is_done(
|
||||
batch_metrics.proofs_processed,
|
||||
batch_metrics.state_update_proofs_requested,
|
||||
batch_metrics.prefetch_proofs_requested,
|
||||
ctx.updates_finished(),
|
||||
) {
|
||||
if self.is_done(batch_metrics, ctx) {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
"State updates finished and all proofs processed, ending calculation"
|
||||
@@ -975,17 +1195,19 @@ impl MultiProofTask {
|
||||
|
||||
if let Some(combined_update) = self.on_proof(
|
||||
sequence_number,
|
||||
SparseTrieUpdate { state, multiproof: Default::default() },
|
||||
SparseTrieUpdate {
|
||||
state,
|
||||
multiproof: ProofResult::Legacy(DecodedMultiProof {
|
||||
account_subtree: Default::default(),
|
||||
branch_node_masks: Default::default(),
|
||||
storages: Default::default(),
|
||||
}),
|
||||
},
|
||||
) {
|
||||
let _ = self.to_sparse_trie.send(combined_update);
|
||||
}
|
||||
|
||||
if self.is_done(
|
||||
batch_metrics.proofs_processed,
|
||||
batch_metrics.state_update_proofs_requested,
|
||||
batch_metrics.prefetch_proofs_requested,
|
||||
ctx.updates_finished(),
|
||||
) {
|
||||
if self.is_done(batch_metrics, ctx) {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
"State updates finished and all proofs processed, ending calculation"
|
||||
@@ -1083,7 +1305,7 @@ impl MultiProofTask {
|
||||
|
||||
let update = SparseTrieUpdate {
|
||||
state: proof_result.state,
|
||||
multiproof: proof_result_data.proof,
|
||||
multiproof: proof_result_data,
|
||||
};
|
||||
|
||||
if let Some(combined_update) =
|
||||
@@ -1098,12 +1320,7 @@ impl MultiProofTask {
|
||||
}
|
||||
}
|
||||
|
||||
if self.is_done(
|
||||
batch_metrics.proofs_processed,
|
||||
batch_metrics.state_update_proofs_requested,
|
||||
batch_metrics.prefetch_proofs_requested,
|
||||
ctx.updates_finished(),
|
||||
) {
|
||||
if self.is_done(&batch_metrics, &ctx) {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor::multiproof",
|
||||
"State updates finished and all proofs processed, ending calculation"
|
||||
@@ -1178,7 +1395,7 @@ struct MultiproofBatchCtx {
|
||||
/// received.
|
||||
updates_finished_time: Option<Instant>,
|
||||
/// Reusable buffer for accumulating prefetch targets during batching.
|
||||
accumulated_prefetch_targets: Vec<MultiProofTargets>,
|
||||
accumulated_prefetch_targets: Vec<VersionedMultiProofTargets>,
|
||||
}
|
||||
|
||||
impl MultiproofBatchCtx {
|
||||
@@ -1210,6 +1427,13 @@ struct MultiproofBatchMetrics {
|
||||
prefetch_proofs_requested: u64,
|
||||
}
|
||||
|
||||
impl MultiproofBatchMetrics {
|
||||
/// Returns `true` if all requested proofs have been processed.
|
||||
const fn all_proofs_processed(&self) -> bool {
|
||||
self.proofs_processed >= self.state_update_proofs_requested + self.prefetch_proofs_requested
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns accounts only with those storages that were not already fetched, and
|
||||
/// if there are no such storages and the account itself was already fetched, the
|
||||
/// account shouldn't be included.
|
||||
@@ -1217,40 +1441,77 @@ fn get_proof_targets(
|
||||
state_update: &HashedPostState,
|
||||
fetched_proof_targets: &MultiProofTargets,
|
||||
multi_added_removed_keys: &MultiAddedRemovedKeys,
|
||||
) -> MultiProofTargets {
|
||||
let mut targets = MultiProofTargets::default();
|
||||
v2_enabled: bool,
|
||||
) -> VersionedMultiProofTargets {
|
||||
if v2_enabled {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(hashed_address) {
|
||||
targets.insert(*hashed_address, HashSet::default());
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for &hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(&hashed_address) {
|
||||
targets.account_targets.push(hashed_address.into());
|
||||
}
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.account_targets.push(Into::<proof_v2::Target>::into(*hashed_address));
|
||||
continue
|
||||
}
|
||||
|
||||
let changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| !fetched.is_some_and(|f| f.contains(*slot)))
|
||||
.map(|slot| Into::<proof_v2::Target>::into(*slot))
|
||||
.collect::<Vec<_>>();
|
||||
|
||||
if !changed_slots.is_empty() {
|
||||
targets.account_targets.push((*hashed_address).into());
|
||||
targets.storage_targets.insert(*hashed_address, changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
VersionedMultiProofTargets::V2(targets)
|
||||
} else {
|
||||
let mut targets = MultiProofTargets::default();
|
||||
|
||||
// first collect all new accounts (not previously fetched)
|
||||
for hashed_address in state_update.accounts.keys() {
|
||||
if !fetched_proof_targets.contains_key(hashed_address) {
|
||||
targets.insert(*hashed_address, HashSet::default());
|
||||
}
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
|
||||
let mut changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| {
|
||||
!fetched.is_some_and(|f| f.contains(*slot)) ||
|
||||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.entry(*hashed_address).or_default();
|
||||
}
|
||||
|
||||
if changed_slots.peek().is_some() {
|
||||
targets.entry(*hashed_address).or_default().extend(changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
VersionedMultiProofTargets::Legacy(targets)
|
||||
}
|
||||
|
||||
// then process storage slots for all accounts in the state update
|
||||
for (hashed_address, storage) in &state_update.storages {
|
||||
let fetched = fetched_proof_targets.get(hashed_address);
|
||||
let storage_added_removed_keys = multi_added_removed_keys.get_storage(hashed_address);
|
||||
let mut changed_slots = storage
|
||||
.storage
|
||||
.keys()
|
||||
.filter(|slot| {
|
||||
!fetched.is_some_and(|f| f.contains(*slot)) ||
|
||||
storage_added_removed_keys.is_some_and(|k| k.is_removed(slot))
|
||||
})
|
||||
.peekable();
|
||||
|
||||
// If the storage is wiped, we still need to fetch the account proof.
|
||||
if storage.wiped && fetched.is_none() {
|
||||
targets.entry(*hashed_address).or_default();
|
||||
}
|
||||
|
||||
if changed_slots.peek().is_some() {
|
||||
targets.entry(*hashed_address).or_default().extend(changed_slots);
|
||||
}
|
||||
}
|
||||
|
||||
targets
|
||||
}
|
||||
|
||||
/// Dispatches work items as a single unit or in chunks based on target size and worker
|
||||
|
||||
@@ -16,7 +16,7 @@ use crate::tree::{
|
||||
payload_processor::{
|
||||
bal::{total_slots, BALSlotIter},
|
||||
executor::WorkloadExecutor,
|
||||
multiproof::MultiProofMessage,
|
||||
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
|
||||
ExecutionCache as PayloadExecutionCache,
|
||||
},
|
||||
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
|
||||
@@ -243,7 +243,9 @@ where
|
||||
}
|
||||
|
||||
if let Some((proof_targets, to_multi_proof)) = targets.zip(self.to_multi_proof.as_ref()) {
|
||||
let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(proof_targets));
|
||||
let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(
|
||||
VersionedMultiProofTargets::Legacy(proof_targets),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -798,6 +800,46 @@ fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargets, usize)
|
||||
(targets, storage_targets)
|
||||
}
|
||||
|
||||
/// Returns v2 proof targets based on the given state.
|
||||
///
|
||||
/// This generates account targets from touched accounts. Storage targets are handled
|
||||
/// separately per-account by the proof workers.
|
||||
///
|
||||
/// For prefetch targets, `min_len` is set to 0 to request full proofs.
|
||||
#[allow(dead_code)]
|
||||
fn proof_v2_targets_from_state(state: &EvmState) -> Vec<reth_trie::proof_v2::Target> {
|
||||
let mut targets = Vec::with_capacity(state.len());
|
||||
for (addr, account) in state {
|
||||
if !account.is_touched() || account.is_selfdestructed() {
|
||||
continue
|
||||
}
|
||||
targets.push(reth_trie::proof_v2::Target::new(keccak256(addr)));
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
/// Returns v2 storage proof targets for a specific account based on the given state.
|
||||
///
|
||||
/// For prefetch targets, `min_len` is set to 0 to request full proofs.
|
||||
#[allow(dead_code)]
|
||||
fn proof_v2_storage_targets_from_state(
|
||||
state: &EvmState,
|
||||
address: alloy_primitives::Address,
|
||||
) -> Vec<reth_trie::proof_v2::Target> {
|
||||
let Some(account) = state.get(&address) else {
|
||||
return Vec::new();
|
||||
};
|
||||
|
||||
let mut targets = Vec::with_capacity(account.storage.len());
|
||||
for (key, slot) in &account.storage {
|
||||
if !slot.is_changed() {
|
||||
continue
|
||||
}
|
||||
targets.push(reth_trie::proof_v2::Target::new(keccak256(B256::new(key.to_be_bytes()))));
|
||||
}
|
||||
targets
|
||||
}
|
||||
|
||||
/// The events the pre-warm task can handle.
|
||||
///
|
||||
/// Generic over `R` (receipt type) to allow sharing `Arc<ExecutionOutcome<R>>` with the main
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::tree::payload_processor::multiproof::{MultiProofTaskMetrics, SparseTr
|
||||
use alloy_primitives::B256;
|
||||
use rayon::iter::{ParallelBridge, ParallelIterator};
|
||||
use reth_trie::{updates::TrieUpdates, Nibbles};
|
||||
use reth_trie_parallel::root::ParallelStateRootError;
|
||||
use reth_trie_parallel::{proof_task::ProofResult, root::ParallelStateRootError};
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
@@ -97,8 +97,8 @@ where
|
||||
debug!(
|
||||
target: "engine::root",
|
||||
num_updates,
|
||||
account_proofs = update.multiproof.account_subtree.len(),
|
||||
storage_proofs = update.multiproof.storages.len(),
|
||||
account_proofs = update.multiproof.account_proofs_len(),
|
||||
storage_proofs = update.multiproof.storage_proofs_len(),
|
||||
"Updating sparse trie"
|
||||
);
|
||||
|
||||
@@ -121,8 +121,9 @@ where
|
||||
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
|
||||
})?;
|
||||
|
||||
self.metrics.sparse_trie_final_update_duration_histogram.record(start.elapsed());
|
||||
self.metrics.sparse_trie_total_duration_histogram.record(now.elapsed());
|
||||
let end = Instant::now();
|
||||
self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start));
|
||||
self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now));
|
||||
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
}
|
||||
@@ -156,7 +157,14 @@ where
|
||||
let started_at = Instant::now();
|
||||
|
||||
// Reveal new accounts and storage slots.
|
||||
trie.reveal_decoded_multiproof(multiproof)?;
|
||||
match multiproof {
|
||||
ProofResult::Legacy(decoded) => {
|
||||
trie.reveal_decoded_multiproof(decoded)?;
|
||||
}
|
||||
ProofResult::V2(decoded_v2) => {
|
||||
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
|
||||
}
|
||||
}
|
||||
let reveal_multiproof_elapsed = started_at.elapsed();
|
||||
trace!(
|
||||
target: "engine::root::sparse",
|
||||
|
||||
@@ -622,7 +622,8 @@ where
|
||||
.without_state_clear()
|
||||
.build();
|
||||
|
||||
let evm = self.evm_config.evm_with_env(&mut db, env.evm_env.clone());
|
||||
let spec_id = *env.evm_env.spec_id();
|
||||
let evm = self.evm_config.evm_with_env(&mut db, env.evm_env);
|
||||
let ctx =
|
||||
self.execution_ctx_for(input).map_err(|e| InsertBlockErrorKind::Other(Box::new(e)))?;
|
||||
let mut executor = self.evm_config.create_executor(evm, ctx);
|
||||
@@ -638,7 +639,7 @@ where
|
||||
CachedPrecompile::wrap(
|
||||
precompile,
|
||||
self.precompile_cache_map.cache_for_address(*address),
|
||||
*env.evm_env.spec_id(),
|
||||
spec_id,
|
||||
Some(metrics),
|
||||
)
|
||||
});
|
||||
|
||||
@@ -38,6 +38,7 @@ tempfile.workspace = true
|
||||
default = []
|
||||
|
||||
otlp = ["reth-tracing/otlp", "reth-node-core/otlp"]
|
||||
otlp-logs = ["reth-tracing/otlp-logs", "reth-node-core/otlp-logs"]
|
||||
|
||||
dev = ["reth-cli-commands/arbitrary"]
|
||||
|
||||
@@ -58,7 +59,8 @@ jemalloc-symbols = [
|
||||
"jemalloc-prof",
|
||||
"reth-node-metrics/jemalloc-symbols",
|
||||
]
|
||||
tracy-allocator = []
|
||||
tracy-allocator = ["tracy"]
|
||||
tracy = ["reth-tracing/tracy", "reth-node-core/tracy"]
|
||||
|
||||
# Because jemalloc is default and preferred over snmalloc when both features are
|
||||
# enabled, `--no-default-features` should be used when enabling snmalloc or
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_db::DatabaseEnv;
|
||||
use reth_node_api::NodePrimitives;
|
||||
use reth_node_builder::{NodeBuilder, WithLaunchContext};
|
||||
use reth_node_core::{
|
||||
args::{LogArgs, OtlpInitStatus, TraceArgs},
|
||||
args::{LogArgs, OtlpInitStatus, OtlpLogsStatus, TraceArgs},
|
||||
version::version_metadata,
|
||||
};
|
||||
use reth_node_metrics::recorder::install_prometheus_recorder;
|
||||
@@ -223,16 +223,19 @@ impl<
|
||||
/// If file logging is enabled, this function returns a guard that must be kept alive to ensure
|
||||
/// that all logs are flushed to disk.
|
||||
///
|
||||
/// If an OTLP endpoint is specified, it will export metrics to the configured collector.
|
||||
/// If an OTLP endpoint is specified, it will export traces and logs to the configured
|
||||
/// collector.
|
||||
pub fn init_tracing(
|
||||
&mut self,
|
||||
runner: &CliRunner,
|
||||
mut layers: Layers,
|
||||
) -> eyre::Result<Option<FileWorkerGuard>> {
|
||||
let otlp_status = runner.block_on(self.traces.init_otlp_tracing(&mut layers))?;
|
||||
let otlp_logs_status = runner.block_on(self.traces.init_otlp_logs(&mut layers))?;
|
||||
|
||||
let guard = self.logs.init_tracing_with_layers(layers)?;
|
||||
info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.logs.log_file_directory);
|
||||
|
||||
match otlp_status {
|
||||
OtlpInitStatus::Started(endpoint) => {
|
||||
info!(target: "reth::cli", "Started OTLP {:?} tracing export to {endpoint}", self.traces.protocol);
|
||||
@@ -243,6 +246,16 @@ impl<
|
||||
OtlpInitStatus::Disabled => {}
|
||||
}
|
||||
|
||||
match otlp_logs_status {
|
||||
OtlpLogsStatus::Started(endpoint) => {
|
||||
info!(target: "reth::cli", "Started OTLP {:?} logs export to {endpoint}", self.traces.protocol);
|
||||
}
|
||||
OtlpLogsStatus::NoFeature => {
|
||||
warn!(target: "reth::cli", "Provided OTLP logs arguments do not have effect, compile with the `otlp-logs` feature")
|
||||
}
|
||||
OtlpLogsStatus::Disabled => {}
|
||||
}
|
||||
|
||||
Ok(guard)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -303,6 +303,8 @@ where
|
||||
let eth_config =
|
||||
EthConfigHandler::new(ctx.node.provider().clone(), ctx.node.evm_config().clone());
|
||||
|
||||
let testing_skip_invalid_transactions = ctx.config.rpc.testing_skip_invalid_transactions;
|
||||
|
||||
self.inner
|
||||
.launch_add_ons_with(ctx, move |container| {
|
||||
container.modules.merge_if_module_configured(
|
||||
@@ -316,14 +318,16 @@ where
|
||||
|
||||
// testing_buildBlockV1: only wire when the hidden testing module is explicitly
|
||||
// requested on any transport. Default stays disabled to honor security guidance.
|
||||
let testing_api = TestingApi::new(
|
||||
let mut testing_api = TestingApi::new(
|
||||
container.registry.eth_api().clone(),
|
||||
container.registry.evm_config().clone(),
|
||||
)
|
||||
.into_rpc();
|
||||
);
|
||||
if testing_skip_invalid_transactions {
|
||||
testing_api = testing_api.with_skip_invalid_transactions();
|
||||
}
|
||||
container
|
||||
.modules
|
||||
.merge_if_module_configured(RethRpcModule::Testing, testing_api)?;
|
||||
.merge_if_module_configured(RethRpcModule::Testing, testing_api.into_rpc())?;
|
||||
|
||||
Ok(())
|
||||
})
|
||||
|
||||
@@ -1,6 +1,10 @@
|
||||
use crate::utils::eth_payload_attributes;
|
||||
use alloy_eips::eip7685::RequestsOrHash;
|
||||
use alloy_genesis::Genesis;
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_rpc_types_engine::{PayloadAttributes, PayloadStatusEnum};
|
||||
use jsonrpsee_core::client::ClientT;
|
||||
use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET};
|
||||
use reth_e2e_test_utils::{
|
||||
node::NodeTestContext, setup, transaction::TransactionTestContext, wallet::Wallet,
|
||||
};
|
||||
@@ -8,6 +12,7 @@ use reth_node_builder::{NodeBuilder, NodeHandle};
|
||||
use reth_node_core::{args::RpcServerArgs, node_config::NodeConfig};
|
||||
use reth_node_ethereum::EthereumNode;
|
||||
use reth_provider::BlockNumReader;
|
||||
use reth_rpc_api::TestingBuildBlockRequestV1;
|
||||
use reth_tasks::TaskManager;
|
||||
use std::sync::Arc;
|
||||
|
||||
@@ -180,3 +185,74 @@ async fn test_engine_graceful_shutdown() -> eyre::Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
let tasks = TaskManager::current();
|
||||
let exec = tasks.executor();
|
||||
|
||||
let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).osaka_activated().build(),
|
||||
);
|
||||
let genesis_hash = chain_spec.genesis_hash();
|
||||
|
||||
let node_config =
|
||||
NodeConfig::test().with_chain(chain_spec.clone()).with_unused_ports().with_rpc(
|
||||
RpcServerArgs::default()
|
||||
.with_unused_ports()
|
||||
.with_http()
|
||||
.with_http_api(reth_rpc_server_types::RpcModuleSelection::All),
|
||||
);
|
||||
|
||||
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config)
|
||||
.testing_node(exec)
|
||||
.node(EthereumNode::default())
|
||||
.launch()
|
||||
.await?;
|
||||
|
||||
let node = NodeTestContext::new(node, eth_payload_attributes).await?;
|
||||
|
||||
let wallet = Wallet::default();
|
||||
let raw_tx = TransactionTestContext::transfer_tx_bytes(1, wallet.inner).await;
|
||||
|
||||
let payload_attributes = PayloadAttributes {
|
||||
timestamp: chain_spec.genesis().timestamp + 1,
|
||||
prev_randao: B256::ZERO,
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
};
|
||||
|
||||
let request = TestingBuildBlockRequestV1 {
|
||||
parent_block_hash: genesis_hash,
|
||||
payload_attributes,
|
||||
transactions: vec![raw_tx],
|
||||
extra_data: None,
|
||||
};
|
||||
|
||||
let envelope = node.testing_build_block_v1(request).await?;
|
||||
|
||||
let engine_client = node.auth_server_handle().http_client();
|
||||
let payload = envelope.execution_payload.clone();
|
||||
let block_hash = payload.payload_inner.payload_inner.block_hash;
|
||||
|
||||
let versioned_hashes: Vec<B256> = Vec::new();
|
||||
let parent_beacon_block_root = B256::ZERO;
|
||||
let execution_requests = RequestsOrHash::Requests(envelope.execution_requests);
|
||||
|
||||
let status: alloy_rpc_types_engine::PayloadStatus = engine_client
|
||||
.request(
|
||||
"engine_newPayloadV4",
|
||||
(payload, versioned_hashes, parent_beacon_block_root, execution_requests),
|
||||
)
|
||||
.await?;
|
||||
assert_eq!(status.status, PayloadStatusEnum::Valid);
|
||||
|
||||
node.update_forkchoice(genesis_hash, block_hash).await?;
|
||||
|
||||
node.wait_block(1, block_hash, false).await?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -70,10 +70,11 @@ impl<T> Clone for UnboundedMeteredSender<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A wrapper type around [Receiver](mpsc::UnboundedReceiver) that updates metrics on receive.
|
||||
/// A wrapper type around [`UnboundedReceiver`](mpsc::UnboundedReceiver) that updates metrics on
|
||||
/// receive.
|
||||
#[derive(Debug)]
|
||||
pub struct UnboundedMeteredReceiver<T> {
|
||||
/// The [Receiver](mpsc::UnboundedReceiver) that this wraps around
|
||||
/// The [`UnboundedReceiver`](mpsc::UnboundedReceiver) that this wraps around
|
||||
receiver: mpsc::UnboundedReceiver<T>,
|
||||
/// Holds metrics for this type
|
||||
metrics: MeteredReceiverMetrics,
|
||||
|
||||
@@ -34,7 +34,6 @@ secp256k1 = { workspace = true, features = ["global-context", "std", "recovery",
|
||||
rand_08.workspace = true
|
||||
concat-kdf.workspace = true
|
||||
sha2.workspace = true
|
||||
sha3.workspace = true
|
||||
aes.workspace = true
|
||||
hmac.workspace = true
|
||||
block-padding.workspace = true
|
||||
|
||||
@@ -9,12 +9,12 @@ use crate::{
|
||||
use aes::{cipher::StreamCipher, Aes128, Aes256};
|
||||
use alloy_primitives::{
|
||||
bytes::{BufMut, Bytes, BytesMut},
|
||||
B128, B256, B512 as PeerId,
|
||||
Keccak256, B128, B256, B512 as PeerId,
|
||||
};
|
||||
use alloy_rlp::{Encodable, Rlp, RlpEncodable, RlpMaxEncodedLen};
|
||||
use byteorder::{BigEndian, ByteOrder, ReadBytesExt};
|
||||
use ctr::Ctr64BE;
|
||||
use digest::{crypto_common::KeyIvInit, Digest};
|
||||
use digest::crypto_common::KeyIvInit;
|
||||
use rand_08::{thread_rng as rng, Rng};
|
||||
use reth_network_peers::{id2pk, pk2id};
|
||||
use secp256k1::{
|
||||
@@ -22,7 +22,6 @@ use secp256k1::{
|
||||
PublicKey, SecretKey, SECP256K1,
|
||||
};
|
||||
use sha2::Sha256;
|
||||
use sha3::Keccak256;
|
||||
|
||||
const PROTOCOL_VERSION: usize = 4;
|
||||
|
||||
|
||||
@@ -10,11 +10,10 @@
|
||||
//! For more information, refer to the [Ethereum MAC specification](https://github.com/ethereum/devp2p/blob/master/rlpx.md#mac).
|
||||
|
||||
use aes::Aes256Enc;
|
||||
use alloy_primitives::{B128, B256};
|
||||
use alloy_primitives::{Keccak256, B128, B256};
|
||||
use block_padding::NoPadding;
|
||||
use cipher::BlockEncrypt;
|
||||
use digest::KeyInit;
|
||||
use sha3::{Digest, Keccak256};
|
||||
|
||||
/// [`Ethereum MAC`](https://github.com/ethereum/devp2p/blob/master/rlpx.md#mac) state.
|
||||
///
|
||||
|
||||
@@ -637,7 +637,7 @@ impl<Pool: TransactionPool, N: NetworkPrimitives> TransactionsManager<Pool, N> {
|
||||
//
|
||||
// known txns have already been successfully fetched or received over gossip.
|
||||
//
|
||||
// most hashes will be filtered out here since this the mempool protocol is a gossip
|
||||
// most hashes will be filtered out here since the mempool protocol is a gossip
|
||||
// protocol, healthy peers will send many of the same hashes.
|
||||
//
|
||||
let hashes_count_pre_pool_filter = partially_valid_msg.len();
|
||||
|
||||
@@ -79,9 +79,12 @@ use reth_stages::{
|
||||
};
|
||||
use reth_static_file::StaticFileProducer;
|
||||
use reth_tasks::TaskExecutor;
|
||||
use reth_tracing::tracing::{debug, error, info, warn};
|
||||
use reth_tracing::{
|
||||
throttle,
|
||||
tracing::{debug, error, info, warn},
|
||||
};
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
use std::{sync::Arc, thread::available_parallelism};
|
||||
use std::{sync::Arc, thread::available_parallelism, time::Duration};
|
||||
use tokio::sync::{
|
||||
mpsc::{unbounded_channel, UnboundedSender},
|
||||
oneshot, watch,
|
||||
@@ -167,7 +170,8 @@ impl LaunchContext {
|
||||
toml_config.peers.trusted_nodes_only = config.network.trusted_only;
|
||||
|
||||
// Merge static file CLI arguments with config file, giving priority to CLI
|
||||
toml_config.static_files = config.static_files.merge_with_config(toml_config.static_files);
|
||||
toml_config.static_files =
|
||||
config.static_files.merge_with_config(toml_config.static_files, config.pruning.minimal);
|
||||
|
||||
Ok(toml_config)
|
||||
}
|
||||
@@ -479,7 +483,7 @@ where
|
||||
let static_file_provider =
|
||||
StaticFileProviderBuilder::read_write(self.data_dir().static_files())
|
||||
.with_metrics()
|
||||
.with_blocks_per_file_for_segments(static_files_config.as_blocks_per_file_map())
|
||||
.with_blocks_per_file_for_segments(&static_files_config.as_blocks_per_file_map())
|
||||
.with_genesis_block_number(self.chain_spec().genesis().number.unwrap_or_default())
|
||||
.build()?;
|
||||
|
||||
@@ -650,23 +654,13 @@ where
|
||||
},
|
||||
ChainSpecInfo { name: self.chain_id().to_string() },
|
||||
self.task_executor().clone(),
|
||||
Hooks::builder()
|
||||
.with_hook({
|
||||
let db = self.database().clone();
|
||||
move || db.report_metrics()
|
||||
})
|
||||
.with_hook({
|
||||
let sfp = self.static_file_provider();
|
||||
move || {
|
||||
if let Err(error) = sfp.report_metrics() {
|
||||
error!(%error, "Failed to report metrics for the static file provider");
|
||||
}
|
||||
}
|
||||
})
|
||||
.build(),
|
||||
metrics_hooks(self.provider_factory()),
|
||||
self.data_dir().pprof_dumps(),
|
||||
)
|
||||
.with_push_gateway(self.node_config().metrics.push_gateway_url.clone(), self.node_config().metrics.push_gateway_interval);
|
||||
.with_push_gateway(
|
||||
self.node_config().metrics.push_gateway_url.clone(),
|
||||
self.node_config().metrics.push_gateway_interval,
|
||||
);
|
||||
|
||||
MetricServer::new(config).serve().await?;
|
||||
}
|
||||
@@ -952,7 +946,7 @@ where
|
||||
error!(
|
||||
"Op-mainnet has been launched without importing the pre-Bedrock state. The chain can't progress without this. See also https://reth.rs/run/sync-op-mainnet.html?minimal-bootstrap-recommended"
|
||||
);
|
||||
return Err(ProviderError::BestBlockNotFound)
|
||||
return Err(ProviderError::BestBlockNotFound);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1266,6 +1260,26 @@ where
|
||||
head: Head,
|
||||
}
|
||||
|
||||
/// Returns the metrics hooks for the node.
|
||||
pub fn metrics_hooks<N: NodeTypesWithDB>(provider_factory: &ProviderFactory<N>) -> Hooks {
|
||||
Hooks::builder()
|
||||
.with_hook({
|
||||
let db = provider_factory.db_ref().clone();
|
||||
move || throttle!(Duration::from_secs(5 * 60), || db.report_metrics())
|
||||
})
|
||||
.with_hook({
|
||||
let sfp = provider_factory.static_file_provider();
|
||||
move || {
|
||||
throttle!(Duration::from_secs(5 * 60), || {
|
||||
if let Err(error) = sfp.report_metrics() {
|
||||
error!(%error, "Failed to report metrics from static file provider");
|
||||
}
|
||||
})
|
||||
}
|
||||
})
|
||||
.build()
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::{LaunchContext, NodeConfig};
|
||||
@@ -1288,6 +1302,7 @@ mod tests {
|
||||
let node_config = NodeConfig {
|
||||
pruning: PruningArgs {
|
||||
full: true,
|
||||
minimal: false,
|
||||
block_interval: None,
|
||||
sender_recovery_full: false,
|
||||
sender_recovery_distance: None,
|
||||
|
||||
@@ -81,7 +81,9 @@ tokio.workspace = true
|
||||
jemalloc = ["reth-cli-util/jemalloc"]
|
||||
asm-keccak = ["alloy-primitives/asm-keccak"]
|
||||
keccak-cache-global = ["alloy-primitives/keccak-cache-global"]
|
||||
otlp = ["reth-tracing/otlp"]
|
||||
otlp = ["reth-tracing/otlp", "reth-tracing-otlp/otlp"]
|
||||
otlp-logs = ["reth-tracing/otlp-logs", "reth-tracing-otlp/otlp-logs"]
|
||||
tracy = ["reth-tracing/tracy"]
|
||||
|
||||
min-error-logs = ["tracing/release_max_level_error"]
|
||||
min-warn-logs = ["tracing/release_max_level_warn"]
|
||||
@@ -89,6 +91,9 @@ min-info-logs = ["tracing/release_max_level_info"]
|
||||
min-debug-logs = ["tracing/release_max_level_debug"]
|
||||
min-trace-logs = ["tracing/release_max_level_trace"]
|
||||
|
||||
# Marker feature for edge/unstable builds - captured by vergen in build.rs
|
||||
edge = []
|
||||
|
||||
[build-dependencies]
|
||||
vergen = { workspace = true, features = ["build", "cargo", "emit_and_set"] }
|
||||
vergen-git2.workspace = true
|
||||
|
||||
@@ -75,6 +75,20 @@ pub struct LogArgs {
|
||||
)]
|
||||
pub samply_filter: String,
|
||||
|
||||
/// Emit traces to tracy. Only useful when profiling.
|
||||
#[arg(long = "log.tracy", global = true, hide = true)]
|
||||
pub tracy: bool,
|
||||
|
||||
/// The filter to use for traces emitted to tracy.
|
||||
#[arg(
|
||||
long = "log.tracy.filter",
|
||||
value_name = "FILTER",
|
||||
global = true,
|
||||
default_value = "debug",
|
||||
hide = true
|
||||
)]
|
||||
pub tracy_filter: String,
|
||||
|
||||
/// Sets whether or not the formatter emits ANSI terminal escape codes for colors and other
|
||||
/// text formatting.
|
||||
#[arg(
|
||||
@@ -148,6 +162,12 @@ impl LogArgs {
|
||||
tracer = tracer.with_samply(config);
|
||||
}
|
||||
|
||||
#[cfg(feature = "tracy")]
|
||||
if self.tracy {
|
||||
let config = self.layer_info(LogFormat::Terminal, self.tracy_filter.clone(), false);
|
||||
tracer = tracer.with_tracy(config);
|
||||
}
|
||||
|
||||
let guard = tracer.init_with_layers(layers)?;
|
||||
Ok(guard)
|
||||
}
|
||||
|
||||
@@ -26,7 +26,7 @@ pub use log::{ColorMode, LogArgs, Verbosity};
|
||||
|
||||
/// `TraceArgs` for tracing and spans support
|
||||
mod trace;
|
||||
pub use trace::{OtlpInitStatus, TraceArgs};
|
||||
pub use trace::{OtlpInitStatus, OtlpLogsStatus, TraceArgs};
|
||||
|
||||
/// `MetricArgs` to configure metrics.
|
||||
mod metric;
|
||||
@@ -78,7 +78,7 @@ pub use era::{DefaultEraHost, EraArgs, EraSourceArgs};
|
||||
|
||||
/// `StaticFilesArgs` for configuring static files.
|
||||
mod static_files;
|
||||
pub use static_files::StaticFilesArgs;
|
||||
pub use static_files::{StaticFilesArgs, MINIMAL_BLOCKS_PER_FILE};
|
||||
|
||||
mod error;
|
||||
pub mod types;
|
||||
|
||||
@@ -16,9 +16,18 @@ use std::{collections::BTreeMap, ops::Not};
|
||||
#[command(next_help_heading = "Pruning")]
|
||||
pub struct PruningArgs {
|
||||
/// Run full node. Only the most recent [`MINIMUM_PRUNING_DISTANCE`] block states are stored.
|
||||
#[arg(long, default_value_t = false)]
|
||||
#[arg(long, default_value_t = false, conflicts_with = "minimal")]
|
||||
pub full: bool,
|
||||
|
||||
/// Run minimal storage mode with maximum pruning and smaller static files.
|
||||
///
|
||||
/// This mode configures the node to use minimal disk space by:
|
||||
/// - Fully pruning sender recovery, transaction lookup, receipts
|
||||
/// - Leaving 10,064 blocks for account, storage history and block bodies
|
||||
/// - Using 10,000 blocks per static file segment
|
||||
#[arg(long, default_value_t = false, conflicts_with = "full")]
|
||||
pub minimal: bool,
|
||||
|
||||
/// Minimum pruning interval measured in blocks.
|
||||
#[arg(long = "prune.block-interval", alias = "block-interval", value_parser = RangedU64ValueParser::<u64>::new().range(1..))]
|
||||
pub block_interval: Option<u64>,
|
||||
@@ -140,6 +149,23 @@ impl PruningArgs {
|
||||
}
|
||||
}
|
||||
|
||||
// If --minimal is set, use minimal storage mode with aggressive pruning.
|
||||
if self.minimal {
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
receipts: Some(PruneMode::Full),
|
||||
account_history: Some(PruneMode::Distance(10064)),
|
||||
storage_history: Some(PruneMode::Distance(10064)),
|
||||
bodies_history: Some(PruneMode::Distance(10064)),
|
||||
merkle_changesets: PruneMode::Distance(MERKLE_CHANGESETS_RETENTION_BLOCKS),
|
||||
receipts_log_filter: Default::default(),
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// Override with any explicitly set prune.* flags.
|
||||
if let Some(block_interval) = self.block_interval {
|
||||
config.block_interval = block_interval as usize;
|
||||
|
||||
@@ -605,7 +605,7 @@ pub struct RpcServerArgs {
|
||||
pub rpc_eth_proof_window: u64,
|
||||
|
||||
/// Maximum number of concurrent getproof requests.
|
||||
#[arg(long = "rpc.proof-permits", alias = "rpc-proof-permits", value_name = "COUNT", default_value_t = constants::DEFAULT_PROOF_PERMITS)]
|
||||
#[arg(long = "rpc.proof-permits", alias = "rpc-proof-permits", value_name = "COUNT", default_value_t = DefaultRpcServerArgs::get_global().rpc_proof_permits)]
|
||||
pub rpc_proof_permits: usize,
|
||||
|
||||
/// Configures the pending block behavior for RPC responses.
|
||||
@@ -640,6 +640,13 @@ pub struct RpcServerArgs {
|
||||
value_parser = parse_duration_from_secs_or_ms,
|
||||
)]
|
||||
pub rpc_send_raw_transaction_sync_timeout: Duration,
|
||||
|
||||
/// Skip invalid transactions in `testing_buildBlockV1` instead of failing.
|
||||
///
|
||||
/// When enabled, transactions that fail execution will be skipped, and all subsequent
|
||||
/// transactions from the same sender will also be skipped.
|
||||
#[arg(long = "testing.skip-invalid-transactions", default_value_t = false)]
|
||||
pub testing_skip_invalid_transactions: bool,
|
||||
}
|
||||
|
||||
impl RpcServerArgs {
|
||||
@@ -852,6 +859,7 @@ impl Default for RpcServerArgs {
|
||||
rpc_state_cache,
|
||||
gas_price_oracle,
|
||||
rpc_send_raw_transaction_sync_timeout,
|
||||
testing_skip_invalid_transactions: false,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1026,6 +1034,7 @@ mod tests {
|
||||
default_suggested_fee: None,
|
||||
},
|
||||
rpc_send_raw_transaction_sync_timeout: std::time::Duration::from_secs(30),
|
||||
testing_skip_invalid_transactions: true,
|
||||
};
|
||||
|
||||
let parsed_args = CommandParser::<RpcServerArgs>::parse_from([
|
||||
@@ -1114,6 +1123,7 @@ mod tests {
|
||||
"60",
|
||||
"--rpc.send-raw-transaction-sync-timeout",
|
||||
"30s",
|
||||
"--testing.skip-invalid-transactions",
|
||||
])
|
||||
.args;
|
||||
|
||||
|
||||
@@ -4,6 +4,11 @@ use clap::Args;
|
||||
use reth_config::config::{BlocksPerFileConfig, StaticFilesConfig};
|
||||
use reth_provider::StorageSettings;
|
||||
|
||||
/// Blocks per static file when running in `--minimal` node.
|
||||
///
|
||||
/// 10000 blocks per static file allows us to prune all history every 10k blocks.
|
||||
pub const MINIMAL_BLOCKS_PER_FILE: u64 = 10000;
|
||||
|
||||
/// Parameters for static files configuration
|
||||
#[derive(Debug, Args, PartialEq, Eq, Default, Clone, Copy)]
|
||||
#[command(next_help_heading = "Static Files")]
|
||||
@@ -61,14 +66,25 @@ pub struct StaticFilesArgs {
|
||||
impl StaticFilesArgs {
|
||||
/// Merges the CLI arguments with an existing [`StaticFilesConfig`], giving priority to CLI
|
||||
/// args.
|
||||
pub fn merge_with_config(&self, config: StaticFilesConfig) -> StaticFilesConfig {
|
||||
///
|
||||
/// If `minimal` is true, uses [`MINIMAL_BLOCKS_PER_FILE`] blocks per file as the default for
|
||||
/// headers, transactions, and receipts segments.
|
||||
pub fn merge_with_config(&self, config: StaticFilesConfig, minimal: bool) -> StaticFilesConfig {
|
||||
let minimal_blocks_per_file = minimal.then_some(MINIMAL_BLOCKS_PER_FILE);
|
||||
StaticFilesConfig {
|
||||
blocks_per_file: BlocksPerFileConfig {
|
||||
headers: self.blocks_per_file_headers.or(config.blocks_per_file.headers),
|
||||
headers: self
|
||||
.blocks_per_file_headers
|
||||
.or(minimal_blocks_per_file)
|
||||
.or(config.blocks_per_file.headers),
|
||||
transactions: self
|
||||
.blocks_per_file_transactions
|
||||
.or(minimal_blocks_per_file)
|
||||
.or(config.blocks_per_file.transactions),
|
||||
receipts: self.blocks_per_file_receipts.or(config.blocks_per_file.receipts),
|
||||
receipts: self
|
||||
.blocks_per_file_receipts
|
||||
.or(minimal_blocks_per_file)
|
||||
.or(config.blocks_per_file.receipts),
|
||||
transaction_senders: self
|
||||
.blocks_per_file_transaction_senders
|
||||
.or(config.blocks_per_file.transaction_senders),
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Opentelemetry tracing configuration through CLI args.
|
||||
//! Opentelemetry tracing and logging configuration through CLI args.
|
||||
|
||||
use clap::Parser;
|
||||
use eyre::WrapErr;
|
||||
@@ -6,7 +6,7 @@ use reth_tracing::{tracing_subscriber::EnvFilter, Layers};
|
||||
use reth_tracing_otlp::OtlpProtocol;
|
||||
use url::Url;
|
||||
|
||||
/// CLI arguments for configuring `Opentelemetry` trace and span export.
|
||||
/// CLI arguments for configuring `Opentelemetry` trace and logs export.
|
||||
#[derive(Debug, Clone, Parser)]
|
||||
pub struct TraceArgs {
|
||||
/// Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
@@ -30,9 +30,29 @@ pub struct TraceArgs {
|
||||
)]
|
||||
pub otlp: Option<Url>,
|
||||
|
||||
/// OTLP transport protocol to use for exporting traces.
|
||||
/// Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
///
|
||||
/// - `http`: expects endpoint path to end with `/v1/traces`
|
||||
/// If no value provided, defaults based on protocol:
|
||||
/// - HTTP: `http://localhost:4318/v1/logs`
|
||||
/// - gRPC: `http://localhost:4317`
|
||||
///
|
||||
/// Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
#[arg(
|
||||
long = "logs-otlp",
|
||||
env = "OTEL_EXPORTER_OTLP_LOGS_ENDPOINT",
|
||||
global = true,
|
||||
value_name = "URL",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "http://localhost:4318/v1/logs",
|
||||
require_equals = true,
|
||||
value_parser = parse_otlp_endpoint,
|
||||
help_heading = "Logging"
|
||||
)]
|
||||
pub logs_otlp: Option<Url>,
|
||||
|
||||
/// OTLP transport protocol to use for exporting traces and logs.
|
||||
///
|
||||
/// - `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs`
|
||||
/// - `grpc`: expects endpoint without a path
|
||||
///
|
||||
/// Defaults to HTTP if not specified.
|
||||
@@ -62,6 +82,22 @@ pub struct TraceArgs {
|
||||
)]
|
||||
pub otlp_filter: EnvFilter,
|
||||
|
||||
/// Set a filter directive for the OTLP logs exporter. This controls the verbosity
|
||||
/// of logs sent to the OTLP endpoint. It follows the same syntax as the
|
||||
/// `RUST_LOG` environment variable.
|
||||
///
|
||||
/// Example: --logs-otlp.filter=info,reth=debug
|
||||
///
|
||||
/// Defaults to INFO if not specified.
|
||||
#[arg(
|
||||
long = "logs-otlp.filter",
|
||||
global = true,
|
||||
value_name = "FILTER",
|
||||
default_value = "info",
|
||||
help_heading = "Logging"
|
||||
)]
|
||||
pub logs_otlp_filter: EnvFilter,
|
||||
|
||||
/// Service name to use for OTLP tracing export.
|
||||
///
|
||||
/// This name will be used to identify the service in distributed tracing systems
|
||||
@@ -101,8 +137,10 @@ impl Default for TraceArgs {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
otlp: None,
|
||||
logs_otlp: None,
|
||||
protocol: OtlpProtocol::Http,
|
||||
otlp_filter: EnvFilter::from_default_env(),
|
||||
logs_otlp_filter: EnvFilter::try_new("info").expect("valid filter"),
|
||||
sample_ratio: None,
|
||||
service_name: "reth".to_string(),
|
||||
}
|
||||
@@ -150,6 +188,37 @@ impl TraceArgs {
|
||||
Ok(OtlpInitStatus::Disabled)
|
||||
}
|
||||
}
|
||||
|
||||
/// Initialize OTLP logs export with the given layers.
|
||||
///
|
||||
/// This method handles OTLP logs initialization based on the configured options,
|
||||
/// including validation and protocol selection.
|
||||
///
|
||||
/// Returns the initialization status to allow callers to log appropriate messages.
|
||||
pub async fn init_otlp_logs(&mut self, _layers: &mut Layers) -> eyre::Result<OtlpLogsStatus> {
|
||||
if let Some(endpoint) = self.logs_otlp.as_mut() {
|
||||
self.protocol.validate_logs_endpoint(endpoint)?;
|
||||
|
||||
#[cfg(feature = "otlp-logs")]
|
||||
{
|
||||
let config = reth_tracing_otlp::OtlpLogsConfig::new(
|
||||
self.service_name.clone(),
|
||||
endpoint.clone(),
|
||||
self.protocol,
|
||||
)?;
|
||||
|
||||
_layers.with_log_layer(config.clone(), self.logs_otlp_filter.clone())?;
|
||||
|
||||
Ok(OtlpLogsStatus::Started(config.endpoint().clone()))
|
||||
}
|
||||
#[cfg(not(feature = "otlp-logs"))]
|
||||
{
|
||||
Ok(OtlpLogsStatus::NoFeature)
|
||||
}
|
||||
} else {
|
||||
Ok(OtlpLogsStatus::Disabled)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Status of OTLP tracing initialization.
|
||||
@@ -163,6 +232,17 @@ pub enum OtlpInitStatus {
|
||||
NoFeature,
|
||||
}
|
||||
|
||||
/// Status of OTLP logs initialization.
|
||||
#[derive(Debug)]
|
||||
pub enum OtlpLogsStatus {
|
||||
/// OTLP logs export was successfully started with the given endpoint.
|
||||
Started(Url),
|
||||
/// OTLP logs export is disabled (no endpoint configured).
|
||||
Disabled,
|
||||
/// OTLP logs arguments provided but feature is not compiled.
|
||||
NoFeature,
|
||||
}
|
||||
|
||||
// Parses an OTLP endpoint url.
|
||||
fn parse_otlp_endpoint(arg: &str) -> eyre::Result<Url> {
|
||||
Url::parse(arg).wrap_err("Invalid URL for OTLP trace output")
|
||||
|
||||
@@ -24,12 +24,12 @@ pub mod primitives {
|
||||
|
||||
/// Re-export of `reth_rpc_*` crates.
|
||||
pub mod rpc {
|
||||
/// Re-exported from `reth_rpc::rpc`.
|
||||
/// Re-exported from `reth_rpc_server_types::result`.
|
||||
pub mod result {
|
||||
pub use reth_rpc_server_types::result::*;
|
||||
}
|
||||
|
||||
/// Re-exported from `reth_rpc::eth`.
|
||||
/// Re-exported from `reth_rpc_convert`.
|
||||
pub mod compat {
|
||||
pub use reth_rpc_convert::*;
|
||||
}
|
||||
|
||||
@@ -64,6 +64,6 @@ pub enum EthStatsError {
|
||||
DataFetchError(String),
|
||||
|
||||
/// The request sent to the server was invalid or malformed
|
||||
#[error("Inivalid request")]
|
||||
#[error("Invalid request")]
|
||||
InvalidRequest,
|
||||
}
|
||||
|
||||
@@ -38,10 +38,12 @@ js-tracer = [
|
||||
jemalloc = ["reth-cli-util/jemalloc", "reth-optimism-cli/jemalloc"]
|
||||
jemalloc-prof = ["jemalloc", "reth-cli-util/jemalloc-prof", "reth-optimism-cli/jemalloc-prof"]
|
||||
jemalloc-symbols = ["jemalloc-prof", "reth-optimism-cli/jemalloc-symbols"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
|
||||
tracy = ["reth-optimism-cli/tracy"]
|
||||
|
||||
asm-keccak = ["reth-optimism-cli/asm-keccak", "reth-optimism-node/asm-keccak"]
|
||||
keccak-cache-global = [
|
||||
"reth-optimism-cli/keccak-cache-global",
|
||||
"reth-optimism-node/keccak-cache-global",
|
||||
]
|
||||
dev = [
|
||||
|
||||
@@ -76,8 +76,9 @@ reth-optimism-chainspec = { workspace = true, features = ["std", "superchain-con
|
||||
[features]
|
||||
default = []
|
||||
|
||||
# Opentelemtry feature to activate metrics export
|
||||
# Opentelemetry feature to activate tracing and logs export
|
||||
otlp = ["reth-tracing/otlp", "reth-node-core/otlp"]
|
||||
otlp-logs = ["reth-tracing/otlp-logs", "reth-node-core/otlp-logs"]
|
||||
|
||||
asm-keccak = [
|
||||
"alloy-primitives/asm-keccak",
|
||||
@@ -85,6 +86,12 @@ asm-keccak = [
|
||||
"reth-optimism-node/asm-keccak",
|
||||
]
|
||||
|
||||
keccak-cache-global = [
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
"reth-node-core/keccak-cache-global",
|
||||
"reth-optimism-node/keccak-cache-global",
|
||||
]
|
||||
|
||||
# Jemalloc feature for vergen to generate correct env vars
|
||||
jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
@@ -99,6 +106,8 @@ jemalloc-symbols = [
|
||||
"reth-node-metrics/jemalloc-symbols",
|
||||
]
|
||||
|
||||
tracy = ["reth-tracing/tracy", "reth-node-core/tracy"]
|
||||
|
||||
dev = [
|
||||
"dep:proptest",
|
||||
"reth-cli-commands/arbitrary",
|
||||
@@ -115,4 +124,4 @@ serde = [
|
||||
"reth-optimism-chainspec/serde",
|
||||
]
|
||||
|
||||
edge = ["reth-cli-commands/edge"]
|
||||
edge = ["reth-cli-commands/edge", "reth-node-core/edge"]
|
||||
|
||||
@@ -3,7 +3,7 @@ use eyre::{eyre, Result};
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_commands::launcher::Launcher;
|
||||
use reth_cli_runner::CliRunner;
|
||||
use reth_node_core::args::OtlpInitStatus;
|
||||
use reth_node_core::args::{OtlpInitStatus, OtlpLogsStatus};
|
||||
use reth_node_metrics::recorder::install_prometheus_recorder;
|
||||
use reth_optimism_chainspec::OpChainSpec;
|
||||
use reth_optimism_consensus::OpBeaconConsensus;
|
||||
@@ -124,9 +124,11 @@ where
|
||||
let mut layers = self.layers.take().unwrap_or_default();
|
||||
|
||||
let otlp_status = runner.block_on(self.cli.traces.init_otlp_tracing(&mut layers))?;
|
||||
let otlp_logs_status = runner.block_on(self.cli.traces.init_otlp_logs(&mut layers))?;
|
||||
|
||||
self.guard = self.cli.logs.init_tracing_with_layers(layers)?;
|
||||
info!(target: "reth::cli", "Initialized tracing, debug log directory: {}", self.cli.logs.log_file_directory);
|
||||
|
||||
match otlp_status {
|
||||
OtlpInitStatus::Started(endpoint) => {
|
||||
info!(target: "reth::cli", "Started OTLP {:?} tracing export to {endpoint}", self.cli.traces.protocol);
|
||||
@@ -136,6 +138,16 @@ where
|
||||
}
|
||||
OtlpInitStatus::Disabled => {}
|
||||
}
|
||||
|
||||
match otlp_logs_status {
|
||||
OtlpLogsStatus::Started(endpoint) => {
|
||||
info!(target: "reth::cli", "Started OTLP {:?} logs export to {endpoint}", self.cli.traces.protocol);
|
||||
}
|
||||
OtlpLogsStatus::NoFeature => {
|
||||
warn!(target: "reth::cli", "Provided OTLP logs arguments do not have effect, compile with the `otlp-logs` feature")
|
||||
}
|
||||
OtlpLogsStatus::Disabled => {}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -61,6 +61,7 @@ pub static DEV_HARDFORKS: LazyLock<ChainHardforks> = LazyLock::new(|| {
|
||||
(OpHardfork::Ecotone.boxed(), ForkCondition::Timestamp(0)),
|
||||
(OpHardfork::Fjord.boxed(), ForkCondition::Timestamp(0)),
|
||||
(OpHardfork::Granite.boxed(), ForkCondition::Timestamp(0)),
|
||||
(OpHardfork::Holocene.boxed(), ForkCondition::Timestamp(0)),
|
||||
(EthereumHardfork::Prague.boxed(), ForkCondition::Timestamp(0)),
|
||||
(OpHardfork::Isthmus.boxed(), ForkCondition::Timestamp(0)),
|
||||
(OpHardfork::Jovian.boxed(), JOVIAN_TIMESTAMP),
|
||||
|
||||
@@ -306,8 +306,7 @@ mod test {
|
||||
use alloy_op_hardforks::BASE_SEPOLIA_JOVIAN_TIMESTAMP;
|
||||
use alloy_primitives::{b64, Address, B256, B64};
|
||||
use alloy_rpc_types_engine::PayloadAttributes;
|
||||
use reth_chainspec::ChainSpec;
|
||||
use reth_optimism_chainspec::{OpChainSpec, BASE_SEPOLIA};
|
||||
use reth_optimism_chainspec::BASE_SEPOLIA;
|
||||
use reth_provider::noop::NoopProvider;
|
||||
use reth_trie_common::KeccakKeyHasher;
|
||||
|
||||
@@ -323,24 +322,6 @@ mod test {
|
||||
}};
|
||||
}
|
||||
|
||||
fn get_chainspec() -> Arc<OpChainSpec> {
|
||||
let base_sepolia_spec = BASE_SEPOLIA.inner.clone();
|
||||
|
||||
Arc::new(OpChainSpec {
|
||||
inner: ChainSpec {
|
||||
chain: base_sepolia_spec.chain,
|
||||
genesis: base_sepolia_spec.genesis,
|
||||
genesis_header: base_sepolia_spec.genesis_header,
|
||||
paris_block_and_final_difficulty: base_sepolia_spec
|
||||
.paris_block_and_final_difficulty,
|
||||
hardforks: base_sepolia_spec.hardforks,
|
||||
base_fee_params: base_sepolia_spec.base_fee_params,
|
||||
prune_delete_limit: 10000,
|
||||
..Default::default()
|
||||
},
|
||||
})
|
||||
}
|
||||
|
||||
const fn get_attributes(
|
||||
eip_1559_params: Option<B64>,
|
||||
min_base_fee: Option<u64>,
|
||||
@@ -364,8 +345,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_pre_holocene() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(None, None, 1732633199);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -378,8 +361,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_holocene_no_eip1559_params() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(None, None, 1732633200);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -392,8 +377,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_holocene_eip1559_params_zero_denominator() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(Some(b64!("0000000000000008")), None, 1732633200);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -406,8 +393,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_holocene_eip1559_params_zero_elasticity() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(Some(b64!("0000000800000000")), None, 1732633200);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -420,8 +409,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_holocene_valid() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(Some(b64!("0000000800000008")), None, 1732633200);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -434,8 +425,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_holocene_valid_all_zero() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(Some(b64!("0000000000000000")), None, 1732633200);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -448,8 +441,10 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn test_well_formed_attributes_jovian_valid() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes =
|
||||
get_attributes(Some(b64!("0000000000000000")), Some(1), BASE_SEPOLIA_JOVIAN_TIMESTAMP);
|
||||
|
||||
@@ -464,8 +459,10 @@ mod test {
|
||||
/// After Jovian (and holocene), eip1559 params must be Some
|
||||
#[test]
|
||||
fn test_malformed_attributes_jovian_with_eip_1559_params_none() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(None, Some(1), BASE_SEPOLIA_JOVIAN_TIMESTAMP);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -479,8 +476,10 @@ mod test {
|
||||
/// Before Jovian, min base fee must be None
|
||||
#[test]
|
||||
fn test_malformed_attributes_pre_jovian_with_min_base_fee() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes = get_attributes(Some(b64!("0000000000000000")), Some(1), 1732633200);
|
||||
|
||||
let result = <engine::OpEngineValidator<_, _, _> as EngineApiValidator<
|
||||
@@ -494,8 +493,10 @@ mod test {
|
||||
/// After Jovian, min base fee must be Some
|
||||
#[test]
|
||||
fn test_malformed_attributes_post_jovian_with_min_base_fee_none() {
|
||||
let validator =
|
||||
OpEngineValidator::new::<KeccakKeyHasher>(get_chainspec(), NoopProvider::default());
|
||||
let validator = OpEngineValidator::new::<KeccakKeyHasher>(
|
||||
BASE_SEPOLIA.clone(),
|
||||
NoopProvider::default(),
|
||||
);
|
||||
let attributes =
|
||||
get_attributes(Some(b64!("0000000000000000")), None, BASE_SEPOLIA_JOVIAN_TIMESTAMP);
|
||||
|
||||
|
||||
@@ -77,6 +77,7 @@ arbitrary = [
|
||||
keccak-cache-global = [
|
||||
"reth-optimism-node?/keccak-cache-global",
|
||||
"reth-node-core?/keccak-cache-global",
|
||||
"reth-optimism-cli?/keccak-cache-global",
|
||||
]
|
||||
test-utils = [
|
||||
"reth-chainspec/test-utils",
|
||||
|
||||
@@ -158,13 +158,13 @@ where
|
||||
.ok_or_else(|| PayloadBuilderError::MissingParentHeader(attributes.parent()))?
|
||||
};
|
||||
|
||||
let config = PayloadConfig::new(Arc::new(parent_header.clone()), attributes);
|
||||
let cached_reads = self.maybe_pre_cached(parent_header.hash());
|
||||
|
||||
let config = PayloadConfig::new(Arc::new(parent_header), attributes);
|
||||
|
||||
let until = self.job_deadline(config.attributes.timestamp());
|
||||
let deadline = Box::pin(tokio::time::sleep_until(until));
|
||||
|
||||
let cached_reads = self.maybe_pre_cached(parent_header.hash());
|
||||
|
||||
let mut job = BasicPayloadJob {
|
||||
config,
|
||||
executor: self.executor.clone(),
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
//! Sealed block types
|
||||
|
||||
use crate::{
|
||||
block::{error::BlockRecoveryError, RecoveredBlock},
|
||||
transaction::signed::RecoveryError,
|
||||
block::{error::BlockRecoveryError, header::BlockHeader, RecoveredBlock},
|
||||
transaction::signed::{RecoveryError, SignedTransaction},
|
||||
Block, BlockBody, GotExpected, InMemorySize, SealedHeader,
|
||||
};
|
||||
use alloc::vec::Vec;
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_consensus::BlockHeader as _;
|
||||
use alloy_eips::{eip1898::BlockWithParent, BlockNumHash};
|
||||
use alloy_primitives::{Address, BlockHash, Sealable, Sealed, B256};
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
@@ -327,6 +327,31 @@ impl<B: Block> From<SealedBlock<B>> for Sealed<B> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<B: Block> From<Sealed<B>> for SealedBlock<B> {
|
||||
fn from(value: Sealed<B>) -> Self {
|
||||
let (block, hash) = value.into_parts();
|
||||
Self::new_unchecked(block, hash)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T, H> SealedBlock<alloy_consensus::Block<T, H>>
|
||||
where
|
||||
T: Decodable + SignedTransaction,
|
||||
H: BlockHeader,
|
||||
{
|
||||
/// Decodes the block from RLP, computing the header hash directly from the RLP bytes.
|
||||
///
|
||||
/// This is more efficient than decoding and then sealing, as the header hash is computed
|
||||
/// from the raw RLP bytes without re-encoding.
|
||||
///
|
||||
/// This leverages [`alloy_consensus::Block::decode_sealed`].
|
||||
pub fn decode_sealed(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
|
||||
let sealed = alloy_consensus::Block::<T, H>::decode_sealed(buf)?;
|
||||
let (block, hash) = sealed.into_parts();
|
||||
Ok(Self::new_unchecked(block, hash))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "arbitrary"))]
|
||||
impl<'a, B> arbitrary::Arbitrary<'a> for SealedBlock<B>
|
||||
where
|
||||
@@ -555,4 +580,96 @@ mod tests {
|
||||
assert_eq!(sealed_block.header().state_root, decoded.header().state_root);
|
||||
assert_eq!(sealed_block.body().transactions.len(), decoded.body().transactions.len());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_decode_sealed_produces_correct_hash() {
|
||||
// Create a sample block using alloy_consensus::Block
|
||||
let header = alloy_consensus::Header {
|
||||
parent_hash: B256::ZERO,
|
||||
ommers_hash: B256::ZERO,
|
||||
beneficiary: Address::ZERO,
|
||||
state_root: B256::ZERO,
|
||||
transactions_root: B256::ZERO,
|
||||
receipts_root: B256::ZERO,
|
||||
logs_bloom: Default::default(),
|
||||
difficulty: Default::default(),
|
||||
number: 42,
|
||||
gas_limit: 30_000_000,
|
||||
gas_used: 21_000,
|
||||
timestamp: 1_000_000,
|
||||
extra_data: Default::default(),
|
||||
mix_hash: B256::ZERO,
|
||||
nonce: Default::default(),
|
||||
base_fee_per_gas: Some(1_000_000_000),
|
||||
withdrawals_root: None,
|
||||
blob_gas_used: None,
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
};
|
||||
|
||||
// Create a simple transaction
|
||||
let tx = alloy_consensus::TxLegacy {
|
||||
chain_id: Some(1),
|
||||
nonce: 0,
|
||||
gas_price: 21_000_000_000,
|
||||
gas_limit: 21_000,
|
||||
to: alloy_primitives::TxKind::Call(Address::ZERO),
|
||||
value: alloy_primitives::U256::from(100),
|
||||
input: alloy_primitives::Bytes::default(),
|
||||
};
|
||||
|
||||
let tx_signed =
|
||||
alloy_consensus::TxEnvelope::Legacy(alloy_consensus::Signed::new_unchecked(
|
||||
tx,
|
||||
alloy_primitives::Signature::test_signature(),
|
||||
B256::ZERO,
|
||||
));
|
||||
|
||||
// Create block body with the transaction
|
||||
let body = alloy_consensus::BlockBody {
|
||||
transactions: vec![tx_signed],
|
||||
ommers: vec![],
|
||||
withdrawals: Some(Default::default()),
|
||||
};
|
||||
|
||||
// Create the block
|
||||
let block = alloy_consensus::Block::new(header, body);
|
||||
let expected_hash = block.header.hash_slow();
|
||||
|
||||
// Encode the block
|
||||
let mut encoded = Vec::new();
|
||||
block.encode(&mut encoded);
|
||||
|
||||
// Decode using decode_sealed - this should compute hash from raw RLP
|
||||
let decoded =
|
||||
SealedBlock::<alloy_consensus::Block<alloy_consensus::TxEnvelope>>::decode_sealed(
|
||||
&mut encoded.as_slice(),
|
||||
)
|
||||
.expect("Failed to decode sealed block");
|
||||
|
||||
// Verify the hash matches
|
||||
assert_eq!(decoded.hash(), expected_hash);
|
||||
assert_eq!(decoded.header().number, 42);
|
||||
assert_eq!(decoded.body().transactions.len(), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_sealed_block_from_sealed() {
|
||||
let header = alloy_consensus::Header::default();
|
||||
let body = alloy_consensus::BlockBody::<alloy_consensus::TxEnvelope>::default();
|
||||
let block = alloy_consensus::Block::new(header, body);
|
||||
let hash = block.header.hash_slow();
|
||||
|
||||
// Create Sealed<Block>
|
||||
let sealed: Sealed<alloy_consensus::Block<alloy_consensus::TxEnvelope>> =
|
||||
Sealed::new_unchecked(block.clone(), hash);
|
||||
|
||||
// Convert to SealedBlock
|
||||
let sealed_block: SealedBlock<alloy_consensus::Block<alloy_consensus::TxEnvelope>> =
|
||||
SealedBlock::from(sealed);
|
||||
|
||||
assert_eq!(sealed_block.hash(), hash);
|
||||
assert_eq!(sealed_block.header().number, block.header.number);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ pub type BlockBody<T = TransactionSigned, H = Header> = alloy_consensus::BlockBo
|
||||
pub type SealedBlock<B = Block> = reth_primitives_traits::block::SealedBlock<B>;
|
||||
|
||||
/// Helper type for constructing the block
|
||||
#[deprecated(note = "Use `RecoveredBlock` instead")]
|
||||
#[deprecated(note = "Use `SealedBlock` instead")]
|
||||
pub type SealedBlockFor<B = Block> = reth_primitives_traits::block::SealedBlock<B>;
|
||||
|
||||
/// Ethereum recovered block
|
||||
|
||||
@@ -42,15 +42,15 @@ impl PruneMode {
|
||||
purpose: PrunePurpose,
|
||||
) -> Result<Option<(BlockNumber, Self)>, PruneSegmentError> {
|
||||
let result = match self {
|
||||
Self::Full if segment.min_blocks(purpose) == 0 => Some((tip, *self)),
|
||||
Self::Full if segment.min_blocks() == 0 => Some((tip, *self)),
|
||||
Self::Distance(distance) if *distance > tip => None, // Nothing to prune yet
|
||||
Self::Distance(distance) if *distance >= segment.min_blocks(purpose) => {
|
||||
Self::Distance(distance) if *distance >= segment.min_blocks() => {
|
||||
Some((tip - distance, *self))
|
||||
}
|
||||
Self::Before(n) if *n == tip + 1 && purpose.is_static_file() => Some((tip, *self)),
|
||||
Self::Before(n) if *n > tip => None, // Nothing to prune yet
|
||||
Self::Before(n) => {
|
||||
(tip - n >= segment.min_blocks(purpose)).then(|| ((*n).saturating_sub(1), *self))
|
||||
(tip - n >= segment.min_blocks()).then(|| ((*n).saturating_sub(1), *self))
|
||||
}
|
||||
_ => return Err(PruneSegmentError::Configuration(segment)),
|
||||
};
|
||||
@@ -93,7 +93,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_prune_target_block() {
|
||||
let tip = 20000;
|
||||
let segment = PruneSegment::Receipts;
|
||||
let segment = PruneSegment::AccountHistory;
|
||||
|
||||
let tests = vec![
|
||||
// MINIMUM_PRUNING_DISTANCE makes this impossible
|
||||
@@ -101,8 +101,8 @@ mod tests {
|
||||
// Nothing to prune
|
||||
(PruneMode::Distance(tip + 1), Ok(None)),
|
||||
(
|
||||
PruneMode::Distance(segment.min_blocks(PrunePurpose::User) + 1),
|
||||
Ok(Some(tip - (segment.min_blocks(PrunePurpose::User) + 1))),
|
||||
PruneMode::Distance(segment.min_blocks() + 1),
|
||||
Ok(Some(tip - (segment.min_blocks() + 1))),
|
||||
),
|
||||
// Nothing to prune
|
||||
(PruneMode::Before(tip + 1), Ok(None)),
|
||||
|
||||
@@ -61,15 +61,12 @@ impl PruneSegment {
|
||||
}
|
||||
|
||||
/// Returns minimum number of blocks to keep in the database for this segment.
|
||||
pub const fn min_blocks(&self, purpose: PrunePurpose) -> u64 {
|
||||
pub const fn min_blocks(&self) -> u64 {
|
||||
match self {
|
||||
Self::SenderRecovery | Self::TransactionLookup => 0,
|
||||
Self::Receipts if purpose.is_static_file() => 0,
|
||||
Self::ContractLogs |
|
||||
Self::AccountHistory |
|
||||
Self::StorageHistory |
|
||||
Self::Bodies |
|
||||
Self::Receipts => MINIMUM_PRUNING_DISTANCE,
|
||||
Self::SenderRecovery | Self::TransactionLookup | Self::Receipts | Self::Bodies => 0,
|
||||
Self::ContractLogs | Self::AccountHistory | Self::StorageHistory => {
|
||||
MINIMUM_PRUNING_DISTANCE
|
||||
}
|
||||
Self::MerkleChangeSets => MERKLE_CHANGESETS_RETENTION_BLOCKS,
|
||||
#[expect(deprecated)]
|
||||
#[expect(clippy::match_same_arms)]
|
||||
|
||||
@@ -58,13 +58,7 @@ pub struct PruneModes {
|
||||
pub transaction_lookup: Option<PruneMode>,
|
||||
/// Receipts pruning configuration. This setting overrides `receipts_log_filter`
|
||||
/// and offers improved performance.
|
||||
#[cfg_attr(
|
||||
any(test, feature = "serde"),
|
||||
serde(
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<MINIMUM_PRUNING_DISTANCE, _>"
|
||||
)
|
||||
)]
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
|
||||
pub receipts: Option<PruneMode>,
|
||||
/// Account History pruning configuration.
|
||||
#[cfg_attr(
|
||||
@@ -85,13 +79,7 @@ pub struct PruneModes {
|
||||
)]
|
||||
pub storage_history: Option<PruneMode>,
|
||||
/// Bodies History pruning configuration.
|
||||
#[cfg_attr(
|
||||
any(test, feature = "serde"),
|
||||
serde(
|
||||
skip_serializing_if = "Option::is_none",
|
||||
deserialize_with = "deserialize_opt_prune_mode_with_min_blocks::<MINIMUM_PRUNING_DISTANCE, _>"
|
||||
)
|
||||
)]
|
||||
#[cfg_attr(any(test, feature = "serde"), serde(skip_serializing_if = "Option::is_none",))]
|
||||
pub bodies_history: Option<PruneMode>,
|
||||
/// Merkle Changesets pruning configuration for `AccountsTrieChangeSets` and
|
||||
/// `StoragesTrieChangeSets`.
|
||||
@@ -188,7 +176,7 @@ impl PruneModes {
|
||||
if let Some(PruneMode::Distance(limit)) = prune_mode {
|
||||
// check if distance exceeds the configured limit
|
||||
if distance > *limit {
|
||||
// but only if have haven't pruned the target yet, if we dont have a checkpoint
|
||||
// but only if we haven't pruned the target yet, if we don't have a checkpoint
|
||||
// yet, it's fully unpruned yet
|
||||
let pruned_height = checkpoint
|
||||
.and_then(|checkpoint| checkpoint.1.block_number)
|
||||
|
||||
@@ -1525,7 +1525,7 @@ impl TransportRpcModuleConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the [`RpcModuleSelection`] for the http transport.
|
||||
/// Sets the [`RpcModuleSelection`] for the ipc transport.
|
||||
pub fn with_ipc(mut self, ipc: impl Into<RpcModuleSelection>) -> Self {
|
||||
self.ipc = Some(ipc.into());
|
||||
self
|
||||
@@ -1663,7 +1663,7 @@ impl TransportRpcModules {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the [`RpcModule`] for the http transport.
|
||||
/// Sets the [`RpcModule`] for the ipc transport.
|
||||
/// This will overwrite current module, if any.
|
||||
pub fn with_ipc(mut self, ipc: RpcModule<()>) -> Self {
|
||||
self.ipc = Some(ipc);
|
||||
|
||||
@@ -13,7 +13,9 @@ use reth_rpc_eth_types::{
|
||||
fee_history::calculate_reward_percentiles_for_block, utils::checked_blob_gas_used_ratio,
|
||||
EthApiError, FeeHistoryCache, FeeHistoryEntry, GasPriceOracle, RpcInvalidTransactionError,
|
||||
};
|
||||
use reth_storage_api::{BlockIdReader, BlockReaderIdExt, HeaderProvider, ProviderHeader};
|
||||
use reth_storage_api::{
|
||||
BlockIdReader, BlockNumReader, BlockReaderIdExt, HeaderProvider, ProviderHeader,
|
||||
};
|
||||
use tracing::debug;
|
||||
|
||||
/// Fee related functions for the [`EthApiServer`](crate::EthApiServer) trait in the
|
||||
@@ -92,6 +94,17 @@ pub trait EthFees:
|
||||
newest_block = BlockNumberOrTag::Latest;
|
||||
}
|
||||
|
||||
// For explicit block numbers, validate against chain head before resolution
|
||||
if let BlockNumberOrTag::Number(requested) = newest_block {
|
||||
let latest_block =
|
||||
self.provider().best_block_number().map_err(Self::Error::from_eth_err)?;
|
||||
if requested > latest_block {
|
||||
return Err(
|
||||
EthApiError::RequestBeyondHead { requested, head: latest_block }.into()
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
let end_block = self
|
||||
.provider()
|
||||
.block_number_for_id(newest_block.into())
|
||||
|
||||
@@ -56,6 +56,7 @@ metrics.workspace = true
|
||||
|
||||
# misc
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
url = { workspace = true, features = ["serde"] }
|
||||
thiserror.workspace = true
|
||||
derive_more.workspace = true
|
||||
schnellru.workspace = true
|
||||
|
||||
@@ -92,6 +92,14 @@ pub enum EthApiError {
|
||||
/// When an invalid block range is provided
|
||||
#[error("invalid block range")]
|
||||
InvalidBlockRange,
|
||||
/// Requested block number is beyond the head block
|
||||
#[error("request beyond head block: requested {requested}, head {head}")]
|
||||
RequestBeyondHead {
|
||||
/// The requested block number
|
||||
requested: u64,
|
||||
/// The current head block number
|
||||
head: u64,
|
||||
},
|
||||
/// Thrown when the target block for proof computation exceeds the maximum configured window.
|
||||
#[error("distance to target block exceeds maximum proof window")]
|
||||
ExceedsMaxProofWindow,
|
||||
@@ -268,6 +276,7 @@ impl From<EthApiError> for jsonrpsee_types::error::ErrorObject<'static> {
|
||||
EthApiError::InvalidTransactionSignature |
|
||||
EthApiError::EmptyRawTransactionData |
|
||||
EthApiError::InvalidBlockRange |
|
||||
EthApiError::RequestBeyondHead { .. } |
|
||||
EthApiError::ExceedsMaxProofWindow |
|
||||
EthApiError::ConflictingFeeFieldsInRequest |
|
||||
EthApiError::Signing(_) |
|
||||
|
||||
@@ -8,6 +8,9 @@
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
|
||||
// `url` is needed for serde support on `reqwest::Url`
|
||||
use url as _;
|
||||
|
||||
pub mod block;
|
||||
pub mod builder;
|
||||
pub mod cache;
|
||||
|
||||
@@ -134,7 +134,9 @@ where
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::eth::helpers::types::EthRpcConverter;
|
||||
use alloy_consensus::{Block, Header, SidecarBuilder, SimpleCoder, Transaction};
|
||||
use alloy_consensus::{
|
||||
BlobTransactionSidecar, Block, Header, SidecarBuilder, SimpleCoder, Transaction,
|
||||
};
|
||||
use alloy_primitives::{Address, U256};
|
||||
use alloy_rpc_types_eth::request::TransactionRequest;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder};
|
||||
@@ -332,7 +334,9 @@ mod tests {
|
||||
let tx_req = TransactionRequest {
|
||||
from: Some(address),
|
||||
to: Some(Address::random().into()),
|
||||
sidecar: Some(builder.build().unwrap().into()),
|
||||
sidecar: Some(BlobTransactionSidecarVariant::from(
|
||||
builder.build::<BlobTransactionSidecar>().unwrap(),
|
||||
)),
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -370,7 +374,9 @@ mod tests {
|
||||
from: Some(address),
|
||||
to: Some(Address::random().into()),
|
||||
transaction_type: Some(3), // EIP-4844
|
||||
sidecar: Some(builder.build().unwrap().into()),
|
||||
sidecar: Some(BlobTransactionSidecarVariant::from(
|
||||
builder.build::<BlobTransactionSidecar>().unwrap(),
|
||||
)),
|
||||
max_fee_per_blob_gas: Some(provided_blob_fee), // Already set
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -19,7 +19,7 @@ impl MinerApiServer for MinerApi {
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
fn set_gas_limit(&self, _gas_price: U128) -> RpcResult<bool> {
|
||||
fn set_gas_limit(&self, _gas_limit: U128) -> RpcResult<bool> {
|
||||
Ok(false)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
|
||||
use alloy_consensus::{Header, Transaction};
|
||||
use alloy_evm::Evm;
|
||||
use alloy_primitives::U256;
|
||||
use alloy_primitives::{map::HashSet, Address, U256};
|
||||
use alloy_rpc_types_engine::ExecutionPayloadEnvelopeV5;
|
||||
use async_trait::async_trait;
|
||||
use jsonrpsee::core::RpcResult;
|
||||
@@ -19,19 +19,31 @@ use reth_rpc_eth_api::{helpers::Call, FromEthApiError};
|
||||
use reth_rpc_eth_types::{utils::recover_raw_transaction, EthApiError};
|
||||
use reth_storage_api::{BlockReader, HeaderProvider};
|
||||
use revm::context::Block;
|
||||
use revm_primitives::map::DefaultHashBuilder;
|
||||
use std::sync::Arc;
|
||||
use tracing::debug;
|
||||
|
||||
/// Testing API handler.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct TestingApi<Eth, Evm> {
|
||||
eth_api: Eth,
|
||||
evm_config: Evm,
|
||||
/// If true, skip invalid transactions instead of failing.
|
||||
skip_invalid_transactions: bool,
|
||||
}
|
||||
|
||||
impl<Eth, Evm> TestingApi<Eth, Evm> {
|
||||
/// Create a new testing API handler.
|
||||
pub const fn new(eth_api: Eth, evm_config: Evm) -> Self {
|
||||
Self { eth_api, evm_config }
|
||||
Self { eth_api, evm_config, skip_invalid_transactions: false }
|
||||
}
|
||||
|
||||
/// Enable skipping invalid transactions instead of failing.
|
||||
/// When a transaction fails, all subsequent transactions from the same sender are also
|
||||
/// skipped.
|
||||
pub const fn with_skip_invalid_transactions(mut self) -> Self {
|
||||
self.skip_invalid_transactions = true;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -46,6 +58,7 @@ where
|
||||
request: TestingBuildBlockRequestV1,
|
||||
) -> Result<ExecutionPayloadEnvelopeV5, Eth::Error> {
|
||||
let evm_config = self.evm_config.clone();
|
||||
let skip_invalid_transactions = self.skip_invalid_transactions;
|
||||
self.eth_api
|
||||
.spawn_with_state_at_block(request.parent_block_hash, move |eth_api, state| {
|
||||
let state = state.database.0;
|
||||
@@ -79,11 +92,33 @@ where
|
||||
let mut total_fees = U256::ZERO;
|
||||
let base_fee = builder.evm_mut().block().basefee();
|
||||
|
||||
let mut invalid_senders: HashSet<Address, DefaultHashBuilder> = HashSet::default();
|
||||
|
||||
for tx in request.transactions {
|
||||
let tx: Recovered<TxTy<Evm::Primitives>> = recover_raw_transaction(&tx)?;
|
||||
let sender = tx.signer();
|
||||
|
||||
if skip_invalid_transactions && invalid_senders.contains(&sender) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let tip = tx.effective_tip_per_gas(base_fee).unwrap_or_default();
|
||||
let gas_used =
|
||||
builder.execute_transaction(tx).map_err(Eth::Error::from_eth_err)?;
|
||||
let gas_used = match builder.execute_transaction(tx) {
|
||||
Ok(gas_used) => gas_used,
|
||||
Err(err) => {
|
||||
if skip_invalid_transactions {
|
||||
debug!(
|
||||
target: "rpc::testing",
|
||||
?sender,
|
||||
error = ?err,
|
||||
"Skipping invalid transaction"
|
||||
);
|
||||
invalid_senders.insert(sender);
|
||||
continue;
|
||||
}
|
||||
return Err(Eth::Error::from_eth_err(err));
|
||||
}
|
||||
};
|
||||
|
||||
total_fees += U256::from(tip) * U256::from(gas_used);
|
||||
}
|
||||
|
||||
@@ -91,7 +91,7 @@ pub enum StageError {
|
||||
/// Database is ahead of static file data.
|
||||
#[error("missing static file data for block number: {number}", number = block.block.number)]
|
||||
MissingStaticFileData {
|
||||
/// Starting block with missing data.
|
||||
/// Starting block with missing data.
|
||||
block: Box<BlockWithParent>,
|
||||
/// Static File segment
|
||||
segment: StaticFileSegment,
|
||||
|
||||
@@ -7,7 +7,7 @@ use reth_db_api::{
|
||||
table::Value,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
DbTxUnwindExt, RawValue,
|
||||
RawValue,
|
||||
};
|
||||
use reth_primitives_traits::{GotExpected, NodePrimitives, SignedTransaction};
|
||||
use reth_provider::{
|
||||
@@ -140,6 +140,9 @@ where
|
||||
recover_range(range, block_numbers, provider, tx_batch_sender.clone(), &mut writer)?;
|
||||
}
|
||||
|
||||
// Advance the static file header to the end of this range to account for empty blocks.
|
||||
writer.ensure_at_block(end_block)?;
|
||||
|
||||
Ok(ExecOutput {
|
||||
checkpoint: StageCheckpoint::new(end_block)
|
||||
.with_entities_stage_checkpoint(stage_checkpoint(provider)?),
|
||||
@@ -155,12 +158,13 @@ where
|
||||
) -> Result<UnwindOutput, StageError> {
|
||||
let (_, unwind_to, _) = input.unwind_block_range_with_threshold(self.commit_threshold);
|
||||
|
||||
// Lookup latest tx id that we should unwind to
|
||||
let latest_tx_id = provider
|
||||
// Lookup the next tx id after unwind_to block (first tx to remove)
|
||||
let unwind_tx_from = provider
|
||||
.block_body_indices(unwind_to)?
|
||||
.ok_or(ProviderError::BlockBodyIndicesNotFound(unwind_to))?
|
||||
.last_tx_num();
|
||||
provider.tx_ref().unwind_table_by_num::<tables::TransactionSenders>(latest_tx_id)?;
|
||||
.next_tx_num();
|
||||
|
||||
EitherWriter::new_senders(provider, unwind_to)?.prune_senders(unwind_tx_from, unwind_to)?;
|
||||
|
||||
Ok(UnwindOutput {
|
||||
checkpoint: StageCheckpoint::new(unwind_to)
|
||||
@@ -415,7 +419,7 @@ mod tests {
|
||||
};
|
||||
use alloy_primitives::{BlockNumber, B256};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_db_api::cursor::DbCursorRO;
|
||||
use reth_db_api::{cursor::DbCursorRO, models::StorageSettings};
|
||||
use reth_ethereum_primitives::{Block, TransactionSigned};
|
||||
use reth_primitives_traits::{SealedBlock, SignerRecoverable};
|
||||
use reth_provider::{
|
||||
@@ -424,6 +428,7 @@ mod tests {
|
||||
};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneMode};
|
||||
use reth_stages_api::StageUnitCheckpoint;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_testing_utils::generators::{
|
||||
self, random_block, random_block_range, BlockParams, BlockRangeParams,
|
||||
};
|
||||
@@ -481,6 +486,50 @@ mod tests {
|
||||
assert!(runner.validate_execution(input, result.ok()).is_ok(), "execution validation");
|
||||
}
|
||||
|
||||
/// Ensure the static file header advances to trailing empty blocks.
|
||||
#[tokio::test]
|
||||
async fn execute_advances_static_file_for_trailing_empty_blocks() {
|
||||
let (stage_progress, target) = (0, 3);
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let runner = SenderRecoveryTestRunner::default();
|
||||
runner.db.factory.set_storage_settings_cache(
|
||||
StorageSettings::legacy().with_transaction_senders_in_static_files(true),
|
||||
);
|
||||
let input = ExecInput {
|
||||
target: Some(target),
|
||||
checkpoint: Some(StageCheckpoint::new(stage_progress)),
|
||||
};
|
||||
|
||||
let non_empty_block_number = stage_progress + 1;
|
||||
let blocks = (stage_progress..=input.target())
|
||||
.map(|number| {
|
||||
random_block(
|
||||
&mut rng,
|
||||
number,
|
||||
BlockParams {
|
||||
tx_count: Some((number == non_empty_block_number) as u8),
|
||||
..Default::default()
|
||||
},
|
||||
)
|
||||
})
|
||||
.collect::<Vec<_>>();
|
||||
runner
|
||||
.db
|
||||
.insert_blocks(blocks.iter(), StorageKind::Static)
|
||||
.expect("failed to insert blocks");
|
||||
|
||||
let result = runner.execute(input).await.unwrap();
|
||||
assert_matches!(result, Ok(ExecOutput { checkpoint, done: true }) if checkpoint.block_number == target);
|
||||
|
||||
let highest_block = runner
|
||||
.db
|
||||
.factory
|
||||
.static_file_provider()
|
||||
.get_highest_static_file_block(StaticFileSegment::TransactionSenders);
|
||||
assert_eq!(Some(target), highest_block);
|
||||
}
|
||||
|
||||
/// Execute the stage twice with input range that exceeds the commit threshold
|
||||
#[tokio::test]
|
||||
async fn execute_intermediate_commit() {
|
||||
|
||||
@@ -15,6 +15,7 @@ workspace = true
|
||||
alloy-primitives.workspace = true
|
||||
|
||||
clap = { workspace = true, features = ["derive"], optional = true }
|
||||
fixed-map.workspace = true
|
||||
derive_more.workspace = true
|
||||
serde = { workspace = true, features = ["alloc", "derive"] }
|
||||
strum = { workspace = true, features = ["derive"] }
|
||||
@@ -32,5 +33,6 @@ std = [
|
||||
"serde/std",
|
||||
"strum/std",
|
||||
"serde_json/std",
|
||||
"fixed-map/std",
|
||||
]
|
||||
clap = ["dep:clap"]
|
||||
|
||||
@@ -21,6 +21,9 @@ use core::ops::RangeInclusive;
|
||||
pub use event::StaticFileProducerEvent;
|
||||
pub use segment::{SegmentConfig, SegmentHeader, SegmentRangeInclusive, StaticFileSegment};
|
||||
|
||||
/// Map keyed by [`StaticFileSegment`].
|
||||
pub type StaticFileMap<T> = alloc::boxed::Box<fixed_map::Map<StaticFileSegment, T>>;
|
||||
|
||||
/// Default static file block count.
|
||||
pub const DEFAULT_BLOCKS_PER_STATIC_FILE: u64 = 500_000;
|
||||
|
||||
|
||||
@@ -22,6 +22,7 @@ use strum::{EnumIs, EnumString};
|
||||
EnumIs,
|
||||
Serialize,
|
||||
Deserialize,
|
||||
fixed_map::Key,
|
||||
)]
|
||||
#[strum(serialize_all = "kebab-case")]
|
||||
#[cfg_attr(feature = "clap", derive(clap::ValueEnum))]
|
||||
|
||||
@@ -92,10 +92,10 @@ impl DbTx for TxMock {
|
||||
|
||||
/// Commits the transaction.
|
||||
///
|
||||
/// **Mock behavior**: Always returns `Ok(true)`, indicating successful commit.
|
||||
/// **Mock behavior**: Always returns `Ok(())`, indicating successful commit.
|
||||
/// No actual data is persisted since this is a mock implementation.
|
||||
fn commit(self) -> Result<bool, DatabaseError> {
|
||||
Ok(true)
|
||||
fn commit(self) -> Result<(), DatabaseError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Aborts the transaction.
|
||||
|
||||
@@ -35,7 +35,7 @@ pub trait DbTx: Debug + Send {
|
||||
) -> Result<Option<T::Value>, DatabaseError>;
|
||||
/// Commit for read only transaction will consume and free transaction and allows
|
||||
/// freeing of memory pages
|
||||
fn commit(self) -> Result<bool, DatabaseError>;
|
||||
fn commit(self) -> Result<(), DatabaseError>;
|
||||
/// Aborts transaction
|
||||
fn abort(self);
|
||||
/// Iterate over read only values in table.
|
||||
|
||||
@@ -248,7 +248,7 @@ where
|
||||
println!(
|
||||
"{:?}\n",
|
||||
tx.inner
|
||||
.db_stat(&table_db)
|
||||
.db_stat(table_db.dbi())
|
||||
.map_err(|_| format!("Could not find table: {}", T::NAME))
|
||||
.map(|stats| {
|
||||
let num_pages =
|
||||
|
||||
@@ -278,7 +278,7 @@ impl DatabaseMetrics for DatabaseEnv {
|
||||
|
||||
let stats = tx
|
||||
.inner
|
||||
.db_stat(&table_db)
|
||||
.db_stat(table_db.dbi())
|
||||
.wrap_err(format!("Could not find table: {table}"))?;
|
||||
|
||||
let page_size = stats.page_size() as usize;
|
||||
|
||||
@@ -67,18 +67,25 @@ impl<K: TransactionKind> Tx<K> {
|
||||
self.metrics_handler.as_ref().map_or_else(|| self.inner.id(), |handler| Ok(handler.txn_id))
|
||||
}
|
||||
|
||||
/// Gets a table database handle if it exists, otherwise creates it.
|
||||
pub fn get_dbi<T: Table>(&self) -> Result<MDBX_dbi, DatabaseError> {
|
||||
if let Some(dbi) = self.dbis.get(T::NAME) {
|
||||
/// Gets a table database handle by name if it exists, otherwise, check the
|
||||
/// database, opening the DB if it exists.
|
||||
pub fn get_dbi_raw(&self, name: &str) -> Result<MDBX_dbi, DatabaseError> {
|
||||
if let Some(dbi) = self.dbis.get(name) {
|
||||
Ok(*dbi)
|
||||
} else {
|
||||
self.inner
|
||||
.open_db(Some(T::NAME))
|
||||
.open_db(Some(name))
|
||||
.map(|db| db.dbi())
|
||||
.map_err(|e| DatabaseError::Open(e.into()))
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a table database handle by name if it exists, otherwise, check the
|
||||
/// database, opening the DB if it exists.
|
||||
pub fn get_dbi<T: Table>(&self) -> Result<MDBX_dbi, DatabaseError> {
|
||||
self.get_dbi_raw(T::NAME)
|
||||
}
|
||||
|
||||
/// Create db Cursor
|
||||
pub fn new_cursor<T: Table>(&self) -> Result<Cursor<K, T>, DatabaseError> {
|
||||
let inner = self
|
||||
@@ -295,10 +302,10 @@ impl<K: TransactionKind> DbTx for Tx<K> {
|
||||
})
|
||||
}
|
||||
|
||||
fn commit(self) -> Result<bool, DatabaseError> {
|
||||
fn commit(self) -> Result<(), DatabaseError> {
|
||||
self.execute_with_close_transaction_metric(TransactionOutcome::Commit, |this| {
|
||||
match this.inner.commit().map_err(|e| DatabaseError::Commit(e.into())) {
|
||||
Ok((v, latency)) => (Ok(v), Some(latency)),
|
||||
Ok(latency) => (Ok(()), Some(latency)),
|
||||
Err(e) => (Err(e), None),
|
||||
}
|
||||
})
|
||||
|
||||
@@ -1,20 +1,22 @@
|
||||
//! reth's static file database table import and access
|
||||
|
||||
use std::{collections::HashMap, path::Path};
|
||||
use reth_nippy_jar::{NippyJar, NippyJarError};
|
||||
use reth_static_file_types::{
|
||||
SegmentHeader, SegmentRangeInclusive, StaticFileMap, StaticFileSegment,
|
||||
};
|
||||
use std::path::Path;
|
||||
|
||||
mod cursor;
|
||||
pub use cursor::StaticFileCursor;
|
||||
|
||||
mod mask;
|
||||
pub use mask::*;
|
||||
use reth_nippy_jar::{NippyJar, NippyJarError};
|
||||
|
||||
mod masks;
|
||||
pub use masks::*;
|
||||
use reth_static_file_types::{SegmentHeader, SegmentRangeInclusive, StaticFileSegment};
|
||||
|
||||
/// Alias type for a map of [`StaticFileSegment`] and sorted lists of existing static file ranges.
|
||||
type SortedStaticFiles = HashMap<StaticFileSegment, Vec<(SegmentRangeInclusive, SegmentHeader)>>;
|
||||
type SortedStaticFiles = StaticFileMap<Vec<(SegmentRangeInclusive, SegmentHeader)>>;
|
||||
|
||||
/// Given the `static_files` directory path, it returns a list over the existing `static_files`
|
||||
/// organized by [`StaticFileSegment`]. Each segment has a sorted list of block ranges and
|
||||
@@ -44,8 +46,8 @@ pub fn iter_static_files(path: &Path) -> Result<SortedStaticFiles, NippyJarError
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by block end range.
|
||||
for range_list in static_files.values_mut() {
|
||||
// Sort by block end range.
|
||||
range_list.sort_by_key(|(block_range, _)| block_range.end());
|
||||
}
|
||||
|
||||
|
||||
@@ -32,7 +32,7 @@ pub enum DatabaseError {
|
||||
/// Failed to commit transaction changes into the database.
|
||||
#[error("failed to commit transaction changes: {_0}")]
|
||||
Commit(DatabaseErrorInfo),
|
||||
/// Failed to initiate a transaction.
|
||||
/// Failed to initialize a transaction.
|
||||
#[error("failed to initialize a transaction: {_0}")]
|
||||
InitTx(DatabaseErrorInfo),
|
||||
/// Failed to initialize a cursor.
|
||||
|
||||
@@ -20,6 +20,9 @@ pub enum ProviderError {
|
||||
/// Pruning error.
|
||||
#[error(transparent)]
|
||||
Pruning(#[from] PruneSegmentError),
|
||||
/// Static file writer error.
|
||||
#[error(transparent)]
|
||||
StaticFileWriter(#[from] StaticFileWriterError),
|
||||
/// RLP error.
|
||||
#[error("{_0}")]
|
||||
Rlp(alloy_rlp::Error),
|
||||
@@ -183,7 +186,7 @@ impl ProviderError {
|
||||
other.downcast_ref()
|
||||
}
|
||||
|
||||
/// Returns true if the this type is a [`ProviderError::Other`] of that error
|
||||
/// Returns true if this type is a [`ProviderError::Other`] of that error
|
||||
/// type. Returns false otherwise.
|
||||
pub fn is_other<T: core::error::Error + 'static>(&self) -> bool {
|
||||
self.as_other().map(|err| err.is::<T>()).unwrap_or(false)
|
||||
@@ -216,18 +219,21 @@ pub struct RootMismatch {
|
||||
pub block_hash: BlockHash,
|
||||
}
|
||||
|
||||
/// A Static File Write Error.
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{message}")]
|
||||
pub struct StaticFileWriterError {
|
||||
/// The error message.
|
||||
pub message: String,
|
||||
/// A Static File Writer Error.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, thiserror::Error)]
|
||||
pub enum StaticFileWriterError {
|
||||
/// Cannot call `sync_all` or `finalize` when prune is queued.
|
||||
#[error("cannot call sync_all or finalize when prune is queued, use commit() instead")]
|
||||
FinalizeWithPruneQueued,
|
||||
/// Other error with message.
|
||||
#[error("{0}")]
|
||||
Other(String),
|
||||
}
|
||||
|
||||
impl StaticFileWriterError {
|
||||
/// Creates a new [`StaticFileWriterError`] with the given message.
|
||||
/// Creates a new [`StaticFileWriterError::Other`] with the given message.
|
||||
pub fn new(message: impl Into<String>) -> Self {
|
||||
Self { message: message.into() }
|
||||
Self::Other(message.into())
|
||||
}
|
||||
}
|
||||
/// Consistent database view error.
|
||||
|
||||
@@ -12,10 +12,10 @@ fn bench_get_seq_iter(c: &mut Criterion) {
|
||||
let (_dir, env) = setup_bench_db(n);
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
|
||||
let dbi = db.dbi();
|
||||
c.bench_function("bench_get_seq_iter", |b| {
|
||||
b.iter(|| {
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
let mut i = 0;
|
||||
let mut count = 0u32;
|
||||
|
||||
@@ -54,11 +54,11 @@ fn bench_get_seq_cursor(c: &mut Criterion) {
|
||||
let (_dir, env) = setup_bench_db(n);
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
|
||||
let dbi = db.dbi();
|
||||
c.bench_function("bench_get_seq_cursor", |b| {
|
||||
b.iter(|| {
|
||||
let (i, count) = txn
|
||||
.cursor(&db)
|
||||
.cursor(dbi)
|
||||
.unwrap()
|
||||
.iter::<ObjectLength, ObjectLength>()
|
||||
.map(Result::unwrap)
|
||||
|
||||
@@ -42,7 +42,9 @@ impl TableObject for Cow<'_, [u8]> {
|
||||
#[cfg(not(feature = "return-borrowed"))]
|
||||
{
|
||||
let is_dirty = (!K::IS_READ_ONLY) &&
|
||||
crate::error::mdbx_result(ffi::mdbx_is_dirty(_txn, data_val.iov_base))?;
|
||||
crate::error::mdbx_result(unsafe {
|
||||
ffi::mdbx_is_dirty(_txn, data_val.iov_base)
|
||||
})?;
|
||||
|
||||
Ok(if is_dirty { Cow::Owned(s.to_vec()) } else { Cow::Borrowed(s) })
|
||||
}
|
||||
|
||||
@@ -211,15 +211,14 @@ impl Environment {
|
||||
let mut freelist: usize = 0;
|
||||
let txn = self.begin_ro_txn()?;
|
||||
let db = Database::freelist_db();
|
||||
let cursor = txn.cursor(&db)?;
|
||||
let cursor = txn.cursor(db.dbi())?;
|
||||
|
||||
for result in cursor.iter_slices() {
|
||||
let (_key, value) = result?;
|
||||
if value.len() < size_of::<usize>() {
|
||||
if value.len() < size_of::<u32>() {
|
||||
return Err(Error::Corrupted)
|
||||
}
|
||||
|
||||
let s = &value[..size_of::<usize>()];
|
||||
let s = &value[..size_of::<u32>()];
|
||||
freelist += NativeEndian::read_u32(s) as usize;
|
||||
}
|
||||
|
||||
@@ -990,7 +989,10 @@ mod tests {
|
||||
result @ Err(_) => result.unwrap(),
|
||||
}
|
||||
}
|
||||
tx.commit().unwrap();
|
||||
// The transaction may be in an error state after hitting MapFull,
|
||||
// so commit could fail. We don't care about the result here since
|
||||
// the purpose of this test is to verify the HSR callback was called.
|
||||
let _ = tx.commit();
|
||||
}
|
||||
|
||||
// Expect the HSR to be called
|
||||
|
||||
@@ -123,6 +123,12 @@ pub enum Error {
|
||||
/// Read transaction has been timed out.
|
||||
#[error("read transaction has been timed out")]
|
||||
ReadTransactionTimeout,
|
||||
/// The transaction commit was aborted due to previous errors.
|
||||
///
|
||||
/// This can happen in exceptionally rare cases and it signals the problem coming from inside
|
||||
/// of mdbx.
|
||||
#[error("botched transaction")]
|
||||
BotchedTransaction,
|
||||
/// Permission defined
|
||||
#[error("permission denied to setup database")]
|
||||
Permission,
|
||||
@@ -204,6 +210,7 @@ impl Error {
|
||||
Self::WriteTransactionUnsupportedInReadOnlyMode |
|
||||
Self::NestedTransactionsUnsupportedWithWriteMap => ffi::MDBX_EACCESS,
|
||||
Self::ReadTransactionTimeout => -96000, // Custom non-MDBX error code
|
||||
Self::BotchedTransaction => -96001,
|
||||
Self::Permission => ffi::MDBX_EPERM,
|
||||
Self::Other(err_code) => *err_code,
|
||||
}
|
||||
@@ -216,6 +223,14 @@ impl From<Error> for i32 {
|
||||
}
|
||||
}
|
||||
|
||||
/// Parses an MDBX error code into a result type.
|
||||
///
|
||||
/// Note that this function returns `Ok(false)` on `MDBX_SUCCESS` and
|
||||
/// `Ok(true)` on `MDBX_RESULT_TRUE`. The return value requires extra
|
||||
/// care since its interpretation depends on the callee being called.
|
||||
///
|
||||
/// The most unintuitive case is `mdbx_txn_commit` which returns `Ok(true)`
|
||||
/// when the commit has been aborted.
|
||||
#[inline]
|
||||
pub(crate) const fn mdbx_result(err_code: c_int) -> Result<bool> {
|
||||
match err_code {
|
||||
|
||||
@@ -170,8 +170,8 @@ where
|
||||
/// Commits the transaction.
|
||||
///
|
||||
/// Any pending operations will be saved.
|
||||
pub fn commit(self) -> Result<(bool, CommitLatency)> {
|
||||
let result = self.txn_execute(|txn| {
|
||||
pub fn commit(self) -> Result<CommitLatency> {
|
||||
match self.txn_execute(|txn| {
|
||||
if K::IS_READ_ONLY {
|
||||
#[cfg(feature = "read-tx-timeouts")]
|
||||
self.env().txn_manager().remove_active_read_transaction(txn);
|
||||
@@ -186,10 +186,21 @@ where
|
||||
.send_message(TxnManagerMessage::Commit { tx: TxnPtr(txn), sender });
|
||||
rx.recv().unwrap()
|
||||
}
|
||||
})?;
|
||||
|
||||
self.inner.set_committed();
|
||||
result
|
||||
})? {
|
||||
//
|
||||
Ok((false, lat)) => {
|
||||
self.inner.set_committed();
|
||||
Ok(lat)
|
||||
}
|
||||
Ok((true, _)) => {
|
||||
// MDBX_RESULT_TRUE means the transaction was aborted due to prior errors.
|
||||
// The transaction is still finished/freed by MDBX, so we must mark it as
|
||||
// committed to prevent the Drop impl from trying to abort it again.
|
||||
self.inner.set_committed();
|
||||
Err(Error::BotchedTransaction)
|
||||
}
|
||||
Err(e) => Err(e),
|
||||
}
|
||||
}
|
||||
|
||||
/// Opens a handle to an MDBX database.
|
||||
@@ -208,11 +219,11 @@ where
|
||||
}
|
||||
|
||||
/// Gets the option flags for the given database in the transaction.
|
||||
pub fn db_flags(&self, db: &Database) -> Result<DatabaseFlags> {
|
||||
pub fn db_flags(&self, dbi: ffi::MDBX_dbi) -> Result<DatabaseFlags> {
|
||||
let mut flags: c_uint = 0;
|
||||
unsafe {
|
||||
self.txn_execute(|txn| {
|
||||
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, db.dbi(), &mut flags, ptr::null_mut()))
|
||||
mdbx_result(ffi::mdbx_dbi_flags_ex(txn, dbi, &mut flags, ptr::null_mut()))
|
||||
})??;
|
||||
}
|
||||
|
||||
@@ -222,8 +233,8 @@ where
|
||||
}
|
||||
|
||||
/// Retrieves database statistics.
|
||||
pub fn db_stat(&self, db: &Database) -> Result<Stat> {
|
||||
self.db_stat_with_dbi(db.dbi())
|
||||
pub fn db_stat(&self, dbi: ffi::MDBX_dbi) -> Result<Stat> {
|
||||
self.db_stat_with_dbi(dbi)
|
||||
}
|
||||
|
||||
/// Retrieves database statistics by the given dbi.
|
||||
@@ -238,8 +249,8 @@ where
|
||||
}
|
||||
|
||||
/// Open a new cursor on the given database.
|
||||
pub fn cursor(&self, db: &Database) -> Result<Cursor<K>> {
|
||||
Cursor::new(self.clone(), db.dbi())
|
||||
pub fn cursor(&self, dbi: ffi::MDBX_dbi) -> Result<Cursor<K>> {
|
||||
Cursor::new(self.clone(), dbi)
|
||||
}
|
||||
|
||||
/// Open a new cursor on the given dbi.
|
||||
@@ -400,7 +411,7 @@ impl Transaction<RW> {
|
||||
#[allow(clippy::mut_from_ref)]
|
||||
pub fn reserve(
|
||||
&self,
|
||||
db: &Database,
|
||||
dbi: ffi::MDBX_dbi,
|
||||
key: impl AsRef<[u8]>,
|
||||
len: usize,
|
||||
flags: WriteFlags,
|
||||
@@ -412,13 +423,7 @@ impl Transaction<RW> {
|
||||
ffi::MDBX_val { iov_len: len, iov_base: ptr::null_mut::<c_void>() };
|
||||
unsafe {
|
||||
mdbx_result(self.txn_execute(|txn| {
|
||||
ffi::mdbx_put(
|
||||
txn,
|
||||
db.dbi(),
|
||||
&key_val,
|
||||
&mut data_val,
|
||||
flags.bits() | ffi::MDBX_RESERVE,
|
||||
)
|
||||
ffi::mdbx_put(txn, dbi, &key_val, &mut data_val, flags.bits() | ffi::MDBX_RESERVE)
|
||||
})?)?;
|
||||
Ok(slice::from_raw_parts_mut(data_val.iov_base as *mut u8, data_val.iov_len))
|
||||
}
|
||||
@@ -473,10 +478,10 @@ impl Transaction<RW> {
|
||||
/// Drops the database from the environment.
|
||||
///
|
||||
/// # Safety
|
||||
/// Caller must close ALL other [Database] and [Cursor] instances pointing to the same dbi
|
||||
/// BEFORE calling this function.
|
||||
pub unsafe fn drop_db(&self, db: Database) -> Result<()> {
|
||||
mdbx_result(self.txn_execute(|txn| unsafe { ffi::mdbx_drop(txn, db.dbi(), true) })?)?;
|
||||
/// Caller must close ALL other [Database] and [Cursor] instances pointing
|
||||
/// to the same dbi BEFORE calling this function.
|
||||
pub unsafe fn drop_db(&self, dbi: ffi::MDBX_dbi) -> Result<()> {
|
||||
mdbx_result(self.txn_execute(|txn| unsafe { ffi::mdbx_drop(txn, dbi, true) })?)?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -488,8 +493,8 @@ impl Transaction<RO> {
|
||||
/// # Safety
|
||||
/// Caller must close ALL other [Database] and [Cursor] instances pointing to the same dbi
|
||||
/// BEFORE calling this function.
|
||||
pub unsafe fn close_db(&self, db: Database) -> Result<()> {
|
||||
mdbx_result(unsafe { ffi::mdbx_dbi_close(self.env().env_ptr(), db.dbi()) })?;
|
||||
pub unsafe fn close_db(&self, dbi: ffi::MDBX_dbi) -> Result<()> {
|
||||
mdbx_result(unsafe { ffi::mdbx_dbi_close(self.env().env_ptr(), dbi) })?;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -9,15 +9,15 @@ fn test_get() {
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
|
||||
assert_eq!(None, txn.cursor(&db).unwrap().first::<(), ()>().unwrap());
|
||||
assert_eq!(None, txn.cursor(dbi).unwrap().first::<(), ()>().unwrap());
|
||||
|
||||
txn.put(db.dbi(), b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key3", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key3", b"val3", WriteFlags::empty()).unwrap();
|
||||
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
assert_eq!(cursor.first().unwrap(), Some((*b"key1", *b"val1")));
|
||||
assert_eq!(cursor.get_current().unwrap(), Some((*b"key1", *b"val1")));
|
||||
assert_eq!(cursor.next().unwrap(), Some((*b"key2", *b"val2")));
|
||||
@@ -34,15 +34,15 @@ fn test_get_dup() {
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.create_db(None, DatabaseFlags::DUP_SORT).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val3", WriteFlags::empty()).unwrap();
|
||||
let dbi = txn.create_db(None, DatabaseFlags::DUP_SORT).unwrap().dbi();
|
||||
txn.put(dbi, b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val3", WriteFlags::empty()).unwrap();
|
||||
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
assert_eq!(cursor.first().unwrap(), Some((*b"key1", *b"val1")));
|
||||
assert_eq!(cursor.first_dup().unwrap(), Some(*b"val1"));
|
||||
assert_eq!(cursor.get_current().unwrap(), Some((*b"key1", *b"val1")));
|
||||
@@ -78,15 +78,16 @@ fn test_get_dupfixed() {
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.create_db(None, DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val4", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val5", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val6", WriteFlags::empty()).unwrap();
|
||||
let dbi =
|
||||
txn.create_db(None, DatabaseFlags::DUP_SORT | DatabaseFlags::DUP_FIXED).unwrap().dbi();
|
||||
txn.put(dbi, b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val4", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val5", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val6", WriteFlags::empty()).unwrap();
|
||||
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
assert_eq!(cursor.first().unwrap(), Some((*b"key1", *b"val1")));
|
||||
assert_eq!(cursor.get_multiple().unwrap(), Some(*b"val1val2val3"));
|
||||
assert_eq!(cursor.next_multiple::<(), ()>().unwrap(), None);
|
||||
@@ -110,12 +111,12 @@ fn test_iter() {
|
||||
for (key, data) in &items {
|
||||
txn.put(db.dbi(), key, data, WriteFlags::empty()).unwrap();
|
||||
}
|
||||
assert!(!txn.commit().unwrap().0);
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
|
||||
// Because Result implements FromIterator, we can collect the iterator
|
||||
// of items of type Result<_, E> into a Result<Vec<_, E>> by specifying
|
||||
@@ -155,8 +156,8 @@ fn test_iter_empty_database() {
|
||||
let dir = tempdir().unwrap();
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
|
||||
assert!(cursor.iter::<(), ()>().next().is_none());
|
||||
assert!(cursor.iter_start::<(), ()>().next().is_none());
|
||||
@@ -173,8 +174,8 @@ fn test_iter_empty_dup_database() {
|
||||
txn.commit().unwrap();
|
||||
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
|
||||
assert!(cursor.iter::<(), ()>().next().is_none());
|
||||
assert!(cursor.iter_start::<(), ()>().next().is_none());
|
||||
@@ -223,8 +224,8 @@ fn test_iter_dup() {
|
||||
}
|
||||
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
assert_eq!(items, cursor.iter_dup().flatten().collect::<Result<Vec<_>>>().unwrap());
|
||||
|
||||
cursor.set::<()>(b"b").unwrap();
|
||||
@@ -271,9 +272,9 @@ fn test_iter_del_get() {
|
||||
let items = vec![(*b"a", *b"1"), (*b"b", *b"2")];
|
||||
{
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.create_db(None, DatabaseFlags::DUP_SORT).unwrap();
|
||||
let dbi = txn.create_db(None, DatabaseFlags::DUP_SORT).unwrap().dbi();
|
||||
assert_eq!(
|
||||
txn.cursor(&db)
|
||||
txn.cursor(dbi)
|
||||
.unwrap()
|
||||
.iter_dup_of::<(), ()>(b"a")
|
||||
.collect::<Result<Vec<_>>>()
|
||||
@@ -294,8 +295,8 @@ fn test_iter_del_get() {
|
||||
}
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
assert_eq!(items, cursor.iter_dup().flatten().collect::<Result<Vec<_>>>().unwrap());
|
||||
|
||||
assert_eq!(
|
||||
@@ -316,8 +317,8 @@ fn test_put_del() {
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let mut cursor = txn.cursor(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let mut cursor = txn.cursor(dbi).unwrap();
|
||||
|
||||
cursor.put(b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
cursor.put(b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
|
||||
@@ -50,9 +50,9 @@ fn test_put_get_del_multi() {
|
||||
txn.commit().unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
{
|
||||
let mut cur = txn.cursor(&db).unwrap();
|
||||
let mut cur = txn.cursor(dbi).unwrap();
|
||||
let iter = cur.iter_dup_of::<(), [u8; 4]>(b"key1");
|
||||
let vals = iter.map(|x| x.unwrap()).map(|(_, x)| x).collect::<Vec<_>>();
|
||||
assert_eq!(vals, vec![*b"val1", *b"val2", *b"val3"]);
|
||||
@@ -66,9 +66,9 @@ fn test_put_get_del_multi() {
|
||||
txn.commit().unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
{
|
||||
let mut cur = txn.cursor(&db).unwrap();
|
||||
let mut cur = txn.cursor(dbi).unwrap();
|
||||
let iter = cur.iter_dup_of::<(), [u8; 4]>(b"key1");
|
||||
let vals = iter.map(|x| x.unwrap()).map(|(_, x)| x).collect::<Vec<_>>();
|
||||
assert_eq!(vals, vec![*b"val1", *b"val3"]);
|
||||
@@ -103,9 +103,9 @@ fn test_reserve() {
|
||||
let env = Environment::builder().open(dir.path()).unwrap();
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
{
|
||||
let mut writer = txn.reserve(&db, b"key1", 4, WriteFlags::empty()).unwrap();
|
||||
let mut writer = txn.reserve(dbi, b"key1", 4, WriteFlags::empty()).unwrap();
|
||||
writer.write_all(b"val1").unwrap();
|
||||
}
|
||||
txn.commit().unwrap();
|
||||
@@ -148,13 +148,13 @@ fn test_clear_db() {
|
||||
{
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
txn.put(txn.open_db(None).unwrap().dbi(), b"key", b"val", WriteFlags::empty()).unwrap();
|
||||
assert!(!txn.commit().unwrap().0);
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
|
||||
{
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
txn.clear_db(txn.open_db(None).unwrap().dbi()).unwrap();
|
||||
assert!(!txn.commit().unwrap().0);
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
@@ -178,16 +178,16 @@ fn test_drop_db() {
|
||||
.unwrap();
|
||||
// Workaround for MDBX dbi drop issue
|
||||
txn.create_db(Some("canary"), DatabaseFlags::empty()).unwrap();
|
||||
assert!(!txn.commit().unwrap().0);
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
{
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.open_db(Some("test")).unwrap();
|
||||
let dbi = txn.open_db(Some("test")).unwrap().dbi();
|
||||
unsafe {
|
||||
txn.drop_db(db).unwrap();
|
||||
txn.drop_db(dbi).unwrap();
|
||||
}
|
||||
assert!(matches!(txn.open_db(Some("test")).unwrap_err(), Error::NotFound));
|
||||
assert!(!txn.commit().unwrap().0);
|
||||
txn.commit().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -291,8 +291,8 @@ fn test_stat() {
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let stat = txn.db_stat(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let stat = txn.db_stat(dbi).unwrap();
|
||||
assert_eq!(stat.entries(), 3);
|
||||
}
|
||||
|
||||
@@ -304,8 +304,8 @@ fn test_stat() {
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let stat = txn.db_stat(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let stat = txn.db_stat(dbi).unwrap();
|
||||
assert_eq!(stat.entries(), 1);
|
||||
}
|
||||
|
||||
@@ -318,8 +318,8 @@ fn test_stat() {
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let db = txn.open_db(None).unwrap();
|
||||
let stat = txn.db_stat(&db).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let stat = txn.db_stat(dbi).unwrap();
|
||||
assert_eq!(stat.entries(), 4);
|
||||
}
|
||||
}
|
||||
@@ -331,20 +331,22 @@ fn test_stat_dupsort() {
|
||||
|
||||
let txn = env.begin_rw_txn().unwrap();
|
||||
let db = txn.create_db(None, DatabaseFlags::DUP_SORT).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key2", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key3", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key3", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(db.dbi(), b"key3", b"val3", WriteFlags::empty()).unwrap();
|
||||
let dbi = db.dbi();
|
||||
txn.put(dbi, b"key1", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key1", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key2", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key3", b"val1", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key3", b"val2", WriteFlags::empty()).unwrap();
|
||||
txn.put(dbi, b"key3", b"val3", WriteFlags::empty()).unwrap();
|
||||
txn.commit().unwrap();
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.db_stat(&txn.open_db(None).unwrap()).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let stat = txn.db_stat(dbi).unwrap();
|
||||
assert_eq!(stat.entries(), 9);
|
||||
}
|
||||
|
||||
@@ -356,7 +358,8 @@ fn test_stat_dupsort() {
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.db_stat(&txn.open_db(None).unwrap()).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let stat = txn.db_stat(dbi).unwrap();
|
||||
assert_eq!(stat.entries(), 5);
|
||||
}
|
||||
|
||||
@@ -369,7 +372,8 @@ fn test_stat_dupsort() {
|
||||
|
||||
{
|
||||
let txn = env.begin_ro_txn().unwrap();
|
||||
let stat = txn.db_stat(&txn.open_db(None).unwrap()).unwrap();
|
||||
let dbi = txn.open_db(None).unwrap().dbi();
|
||||
let stat = txn.db_stat(dbi).unwrap();
|
||||
assert_eq!(stat.entries(), 8);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -17,7 +17,7 @@ use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
error::Error as StdError,
|
||||
fs::File,
|
||||
io::{Read, Write},
|
||||
io::{self, Read, Write},
|
||||
ops::Range,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
@@ -201,11 +201,11 @@ impl<H: NippyJarHeader> NippyJar<H> {
|
||||
let config_path = path.with_extension(CONFIG_FILE_EXTENSION);
|
||||
let config_file = File::open(&config_path)
|
||||
.inspect_err(|e| {
|
||||
warn!( ?path, %e, "Failed to load static file jar");
|
||||
warn!(?path, %e, "Failed to load static file jar");
|
||||
})
|
||||
.map_err(|err| reth_fs_util::FsPathError::open(err, config_path))?;
|
||||
|
||||
let mut obj = Self::load_from_reader(config_file)?;
|
||||
let mut obj = Self::load_from_reader(io::BufReader::new(config_file))?;
|
||||
obj.path = path.to_path_buf();
|
||||
Ok(obj)
|
||||
}
|
||||
@@ -418,10 +418,15 @@ impl DataReader {
|
||||
&self.data_mmap[range]
|
||||
}
|
||||
|
||||
/// Returns total size of data
|
||||
/// Returns total size of data file.
|
||||
pub fn size(&self) -> usize {
|
||||
self.data_mmap.len()
|
||||
}
|
||||
|
||||
/// Returns total size of offsets file.
|
||||
pub fn offsets_size(&self) -> usize {
|
||||
self.offset_mmap.len()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -347,11 +347,27 @@ impl<H: NippyJarHeader> NippyJarWriter<H> {
|
||||
|
||||
/// Commits configuration and offsets to disk. It drains the internal offset list.
|
||||
pub fn commit(&mut self) -> Result<(), NippyJarError> {
|
||||
self.sync_all()?;
|
||||
self.finalize()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Syncs data and offsets to disk.
|
||||
///
|
||||
/// This does NOT commit the configuration. Call [`Self::finalize`] after to write the
|
||||
/// configuration and mark the writer as clean.
|
||||
pub fn sync_all(&mut self) -> Result<(), NippyJarError> {
|
||||
self.data_file.flush()?;
|
||||
self.data_file.get_ref().sync_all()?;
|
||||
|
||||
self.commit_offsets()?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Commits configuration to disk and marks the writer as clean.
|
||||
///
|
||||
/// Must be called after [`Self::sync_all`] to complete the commit.
|
||||
pub fn finalize(&mut self) -> Result<(), NippyJarError> {
|
||||
// Flushes `max_row_size` and total `rows` to disk.
|
||||
self.jar.freeze_config()?;
|
||||
self.dirty = false;
|
||||
|
||||
@@ -17,7 +17,6 @@ reth-chainspec.workspace = true
|
||||
reth-execution-types.workspace = true
|
||||
reth-ethereum-primitives = { workspace = true, features = ["reth-codec"] }
|
||||
reth-primitives-traits = { workspace = true, features = ["reth-codec", "secp256k1"] }
|
||||
reth-fs-util.workspace = true
|
||||
reth-errors.workspace = true
|
||||
reth-storage-errors.workspace = true
|
||||
reth-storage-api = { workspace = true, features = ["std", "db-api"] }
|
||||
|
||||
@@ -10,7 +10,7 @@ use std::{
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
use crate::providers::rocksdb::RocksDBBatch;
|
||||
use crate::{
|
||||
providers::{StaticFileProvider, StaticFileProviderRWRefMut},
|
||||
providers::{history_info, HistoryInfo, StaticFileProvider, StaticFileProviderRWRefMut},
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use alloy_primitives::{map::HashMap, Address, BlockNumber, TxHash, TxNumber};
|
||||
@@ -708,7 +708,7 @@ impl<CURSOR, N: NodePrimitives> EitherReader<'_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRO<tables::StoragesHistory>,
|
||||
{
|
||||
/// Gets a storage history entry.
|
||||
/// Gets a storage history shard entry for the given [`StorageShardedKey`], if present.
|
||||
pub fn get_storage_history(
|
||||
&mut self,
|
||||
key: StorageShardedKey,
|
||||
@@ -720,13 +720,43 @@ where
|
||||
Self::RocksDB(tx) => tx.get::<tables::StoragesHistory>(key),
|
||||
}
|
||||
}
|
||||
|
||||
/// Lookup storage history and return [`HistoryInfo`].
|
||||
pub fn storage_history_info(
|
||||
&mut self,
|
||||
address: Address,
|
||||
storage_key: alloy_primitives::B256,
|
||||
block_number: BlockNumber,
|
||||
lowest_available_block_number: Option<BlockNumber>,
|
||||
) -> ProviderResult<HistoryInfo> {
|
||||
match self {
|
||||
Self::Database(cursor, _) => {
|
||||
let key = StorageShardedKey::new(address, storage_key, block_number);
|
||||
history_info::<tables::StoragesHistory, _, _>(
|
||||
cursor,
|
||||
key,
|
||||
block_number,
|
||||
|k| k.address == address && k.sharded_key.key == storage_key,
|
||||
lowest_available_block_number,
|
||||
)
|
||||
}
|
||||
Self::StaticFile(_, _) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(tx) => tx.storage_history_info(
|
||||
address,
|
||||
storage_key,
|
||||
block_number,
|
||||
lowest_available_block_number,
|
||||
),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> EitherReader<'_, CURSOR, N>
|
||||
where
|
||||
CURSOR: DbCursorRO<tables::AccountsHistory>,
|
||||
{
|
||||
/// Gets an account history entry.
|
||||
/// Gets an account history shard entry for the given [`ShardedKey`], if present.
|
||||
pub fn get_account_history(
|
||||
&mut self,
|
||||
key: ShardedKey<Address>,
|
||||
@@ -738,6 +768,32 @@ where
|
||||
Self::RocksDB(tx) => tx.get::<tables::AccountsHistory>(key),
|
||||
}
|
||||
}
|
||||
|
||||
/// Lookup account history and return [`HistoryInfo`].
|
||||
pub fn account_history_info(
|
||||
&mut self,
|
||||
address: Address,
|
||||
block_number: BlockNumber,
|
||||
lowest_available_block_number: Option<BlockNumber>,
|
||||
) -> ProviderResult<HistoryInfo> {
|
||||
match self {
|
||||
Self::Database(cursor, _) => {
|
||||
let key = ShardedKey::new(address, block_number);
|
||||
history_info::<tables::AccountsHistory, _, _>(
|
||||
cursor,
|
||||
key,
|
||||
block_number,
|
||||
|k| k.key == address,
|
||||
lowest_available_block_number,
|
||||
)
|
||||
}
|
||||
Self::StaticFile(_, _) => Err(ProviderError::UnsupportedProvider),
|
||||
#[cfg(all(unix, feature = "rocksdb"))]
|
||||
Self::RocksDB(tx) => {
|
||||
tx.account_history_info(address, block_number, lowest_available_block_number)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<CURSOR, N: NodePrimitives> EitherReader<'_, CURSOR, N>
|
||||
@@ -894,8 +950,11 @@ mod rocksdb_tests {
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, IntegerList, ShardedKey},
|
||||
tables,
|
||||
transaction::DbTxMut,
|
||||
};
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_storage_api::{DatabaseProviderFactory, StorageSettings};
|
||||
use std::marker::PhantomData;
|
||||
use tempfile::TempDir;
|
||||
|
||||
fn create_rocksdb_provider() -> (TempDir, RocksDBProvider) {
|
||||
@@ -1125,10 +1184,391 @@ mod rocksdb_tests {
|
||||
assert_eq!(provider.get::<tables::AccountsHistory>(key).unwrap(), None);
|
||||
}
|
||||
|
||||
/// Test that `RocksDB` commits happen at `provider.commit()` level, not at writer level.
|
||||
// ==================== Parametrized Backend Equivalence Tests ====================
|
||||
//
|
||||
// These tests verify that MDBX and RocksDB produce identical results for history lookups.
|
||||
// Each scenario sets up the same data in both backends and asserts identical HistoryInfo.
|
||||
|
||||
/// Query parameters for a history lookup test case.
|
||||
struct HistoryQuery {
|
||||
block_number: BlockNumber,
|
||||
lowest_available: Option<BlockNumber>,
|
||||
expected: HistoryInfo,
|
||||
}
|
||||
|
||||
// Type aliases for cursor types (needed for EitherWriter/EitherReader type inference)
|
||||
type AccountsHistoryWriteCursor =
|
||||
reth_db::mdbx::cursor::Cursor<reth_db::mdbx::RW, tables::AccountsHistory>;
|
||||
type StoragesHistoryWriteCursor =
|
||||
reth_db::mdbx::cursor::Cursor<reth_db::mdbx::RW, tables::StoragesHistory>;
|
||||
type AccountsHistoryReadCursor =
|
||||
reth_db::mdbx::cursor::Cursor<reth_db::mdbx::RO, tables::AccountsHistory>;
|
||||
type StoragesHistoryReadCursor =
|
||||
reth_db::mdbx::cursor::Cursor<reth_db::mdbx::RO, tables::StoragesHistory>;
|
||||
|
||||
/// Runs the same account history queries against both MDBX and `RocksDB` backends,
|
||||
/// asserting they produce identical results.
|
||||
fn run_account_history_scenario(
|
||||
scenario_name: &str,
|
||||
address: Address,
|
||||
shards: &[(BlockNumber, Vec<BlockNumber>)], // (shard_highest_block, blocks_in_shard)
|
||||
queries: &[HistoryQuery],
|
||||
) {
|
||||
// Setup MDBX and RocksDB with identical data using EitherWriter
|
||||
let factory = create_test_provider_factory();
|
||||
let mdbx_provider = factory.database_provider_rw().unwrap();
|
||||
let (temp_dir, rocks_provider) = create_rocksdb_provider();
|
||||
|
||||
// Create writers for both backends
|
||||
let mut mdbx_writer: EitherWriter<'_, AccountsHistoryWriteCursor, EthPrimitives> =
|
||||
EitherWriter::Database(
|
||||
mdbx_provider.tx_ref().cursor_write::<tables::AccountsHistory>().unwrap(),
|
||||
);
|
||||
let mut rocks_writer: EitherWriter<'_, AccountsHistoryWriteCursor, EthPrimitives> =
|
||||
EitherWriter::RocksDB(rocks_provider.batch());
|
||||
|
||||
// Write identical data to both backends in a single loop
|
||||
for (highest_block, blocks) in shards {
|
||||
let key = ShardedKey::new(address, *highest_block);
|
||||
let value = IntegerList::new(blocks.clone()).unwrap();
|
||||
mdbx_writer.put_account_history(key.clone(), &value).unwrap();
|
||||
rocks_writer.put_account_history(key, &value).unwrap();
|
||||
}
|
||||
|
||||
// Commit both backends
|
||||
drop(mdbx_writer);
|
||||
mdbx_provider.commit().unwrap();
|
||||
if let EitherWriter::RocksDB(batch) = rocks_writer {
|
||||
batch.commit().unwrap();
|
||||
}
|
||||
|
||||
// Run queries against both backends using EitherReader
|
||||
let mdbx_ro = factory.database_provider_ro().unwrap();
|
||||
let rocks_tx = rocks_provider.tx();
|
||||
|
||||
for (i, query) in queries.iter().enumerate() {
|
||||
// MDBX query via EitherReader
|
||||
let mut mdbx_reader: EitherReader<'_, AccountsHistoryReadCursor, EthPrimitives> =
|
||||
EitherReader::Database(
|
||||
mdbx_ro.tx_ref().cursor_read::<tables::AccountsHistory>().unwrap(),
|
||||
PhantomData,
|
||||
);
|
||||
let mdbx_result = mdbx_reader
|
||||
.account_history_info(address, query.block_number, query.lowest_available)
|
||||
.unwrap();
|
||||
|
||||
// RocksDB query via EitherReader
|
||||
let mut rocks_reader: EitherReader<'_, AccountsHistoryReadCursor, EthPrimitives> =
|
||||
EitherReader::RocksDB(&rocks_tx);
|
||||
let rocks_result = rocks_reader
|
||||
.account_history_info(address, query.block_number, query.lowest_available)
|
||||
.unwrap();
|
||||
|
||||
// Assert both backends produce identical results
|
||||
assert_eq!(
|
||||
mdbx_result,
|
||||
rocks_result,
|
||||
"Backend mismatch in scenario '{}' query {}: block={}, lowest={:?}\n\
|
||||
MDBX: {:?}, RocksDB: {:?}",
|
||||
scenario_name,
|
||||
i,
|
||||
query.block_number,
|
||||
query.lowest_available,
|
||||
mdbx_result,
|
||||
rocks_result
|
||||
);
|
||||
|
||||
// Also verify against expected result
|
||||
assert_eq!(
|
||||
mdbx_result,
|
||||
query.expected,
|
||||
"Unexpected result in scenario '{}' query {}: block={}, lowest={:?}\n\
|
||||
Got: {:?}, Expected: {:?}",
|
||||
scenario_name,
|
||||
i,
|
||||
query.block_number,
|
||||
query.lowest_available,
|
||||
mdbx_result,
|
||||
query.expected
|
||||
);
|
||||
}
|
||||
|
||||
rocks_tx.rollback().unwrap();
|
||||
drop(temp_dir);
|
||||
}
|
||||
|
||||
/// Runs the same storage history queries against both MDBX and `RocksDB` backends,
|
||||
/// asserting they produce identical results.
|
||||
fn run_storage_history_scenario(
|
||||
scenario_name: &str,
|
||||
address: Address,
|
||||
storage_key: B256,
|
||||
shards: &[(BlockNumber, Vec<BlockNumber>)], // (shard_highest_block, blocks_in_shard)
|
||||
queries: &[HistoryQuery],
|
||||
) {
|
||||
// Setup MDBX and RocksDB with identical data using EitherWriter
|
||||
let factory = create_test_provider_factory();
|
||||
let mdbx_provider = factory.database_provider_rw().unwrap();
|
||||
let (temp_dir, rocks_provider) = create_rocksdb_provider();
|
||||
|
||||
// Create writers for both backends
|
||||
let mut mdbx_writer: EitherWriter<'_, StoragesHistoryWriteCursor, EthPrimitives> =
|
||||
EitherWriter::Database(
|
||||
mdbx_provider.tx_ref().cursor_write::<tables::StoragesHistory>().unwrap(),
|
||||
);
|
||||
let mut rocks_writer: EitherWriter<'_, StoragesHistoryWriteCursor, EthPrimitives> =
|
||||
EitherWriter::RocksDB(rocks_provider.batch());
|
||||
|
||||
// Write identical data to both backends in a single loop
|
||||
for (highest_block, blocks) in shards {
|
||||
let key = StorageShardedKey::new(address, storage_key, *highest_block);
|
||||
let value = IntegerList::new(blocks.clone()).unwrap();
|
||||
mdbx_writer.put_storage_history(key.clone(), &value).unwrap();
|
||||
rocks_writer.put_storage_history(key, &value).unwrap();
|
||||
}
|
||||
|
||||
// Commit both backends
|
||||
drop(mdbx_writer);
|
||||
mdbx_provider.commit().unwrap();
|
||||
if let EitherWriter::RocksDB(batch) = rocks_writer {
|
||||
batch.commit().unwrap();
|
||||
}
|
||||
|
||||
// Run queries against both backends using EitherReader
|
||||
let mdbx_ro = factory.database_provider_ro().unwrap();
|
||||
let rocks_tx = rocks_provider.tx();
|
||||
|
||||
for (i, query) in queries.iter().enumerate() {
|
||||
// MDBX query via EitherReader
|
||||
let mut mdbx_reader: EitherReader<'_, StoragesHistoryReadCursor, EthPrimitives> =
|
||||
EitherReader::Database(
|
||||
mdbx_ro.tx_ref().cursor_read::<tables::StoragesHistory>().unwrap(),
|
||||
PhantomData,
|
||||
);
|
||||
let mdbx_result = mdbx_reader
|
||||
.storage_history_info(
|
||||
address,
|
||||
storage_key,
|
||||
query.block_number,
|
||||
query.lowest_available,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// RocksDB query via EitherReader
|
||||
let mut rocks_reader: EitherReader<'_, StoragesHistoryReadCursor, EthPrimitives> =
|
||||
EitherReader::RocksDB(&rocks_tx);
|
||||
let rocks_result = rocks_reader
|
||||
.storage_history_info(
|
||||
address,
|
||||
storage_key,
|
||||
query.block_number,
|
||||
query.lowest_available,
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
// Assert both backends produce identical results
|
||||
assert_eq!(
|
||||
mdbx_result,
|
||||
rocks_result,
|
||||
"Backend mismatch in scenario '{}' query {}: block={}, lowest={:?}\n\
|
||||
MDBX: {:?}, RocksDB: {:?}",
|
||||
scenario_name,
|
||||
i,
|
||||
query.block_number,
|
||||
query.lowest_available,
|
||||
mdbx_result,
|
||||
rocks_result
|
||||
);
|
||||
|
||||
// Also verify against expected result
|
||||
assert_eq!(
|
||||
mdbx_result,
|
||||
query.expected,
|
||||
"Unexpected result in scenario '{}' query {}: block={}, lowest={:?}\n\
|
||||
Got: {:?}, Expected: {:?}",
|
||||
scenario_name,
|
||||
i,
|
||||
query.block_number,
|
||||
query.lowest_available,
|
||||
mdbx_result,
|
||||
query.expected
|
||||
);
|
||||
}
|
||||
|
||||
rocks_tx.rollback().unwrap();
|
||||
drop(temp_dir);
|
||||
}
|
||||
|
||||
/// Tests account history lookups across both MDBX and `RocksDB` backends.
|
||||
///
|
||||
/// This ensures all storage commits (MDBX, static files, `RocksDB`) happen atomically
|
||||
/// in a single place, making it easier to reason about commit ordering and consistency.
|
||||
/// Covers the following scenarios from PR2's `RocksDB`-only tests:
|
||||
/// 1. Single shard - basic lookups within one shard
|
||||
/// 2. Multiple shards - `prev()` shard detection and transitions
|
||||
/// 3. No history - query address with no entries
|
||||
/// 4. Pruning boundary - `lowest_available` boundary behavior (block at/after boundary)
|
||||
#[test]
|
||||
fn test_account_history_info_both_backends() {
|
||||
let address = Address::from([0x42; 20]);
|
||||
|
||||
// Scenario 1: Single shard with blocks [100, 200, 300]
|
||||
run_account_history_scenario(
|
||||
"single_shard",
|
||||
address,
|
||||
&[(u64::MAX, vec![100, 200, 300])],
|
||||
&[
|
||||
// Before first entry -> NotYetWritten
|
||||
HistoryQuery {
|
||||
block_number: 50,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::NotYetWritten,
|
||||
},
|
||||
// Between entries -> InChangeset(next_write)
|
||||
HistoryQuery {
|
||||
block_number: 150,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InChangeset(200),
|
||||
},
|
||||
// Exact match on entry -> InChangeset(same_block)
|
||||
HistoryQuery {
|
||||
block_number: 300,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InChangeset(300),
|
||||
},
|
||||
// After last entry in last shard -> InPlainState
|
||||
HistoryQuery {
|
||||
block_number: 500,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InPlainState,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// Scenario 2: Multiple shards - tests prev() shard detection
|
||||
run_account_history_scenario(
|
||||
"multiple_shards",
|
||||
address,
|
||||
&[
|
||||
(500, vec![100, 200, 300, 400, 500]), // First shard ends at 500
|
||||
(u64::MAX, vec![600, 700, 800]), // Last shard
|
||||
],
|
||||
&[
|
||||
// Before first shard, no prev -> NotYetWritten
|
||||
HistoryQuery {
|
||||
block_number: 50,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::NotYetWritten,
|
||||
},
|
||||
// Within first shard
|
||||
HistoryQuery {
|
||||
block_number: 150,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InChangeset(200),
|
||||
},
|
||||
// Between shards - prev() should find first shard
|
||||
HistoryQuery {
|
||||
block_number: 550,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InChangeset(600),
|
||||
},
|
||||
// After all entries
|
||||
HistoryQuery {
|
||||
block_number: 900,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InPlainState,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// Scenario 3: No history for address
|
||||
let address_without_history = Address::from([0x43; 20]);
|
||||
run_account_history_scenario(
|
||||
"no_history",
|
||||
address_without_history,
|
||||
&[], // No shards for this address
|
||||
&[HistoryQuery {
|
||||
block_number: 150,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::NotYetWritten,
|
||||
}],
|
||||
);
|
||||
|
||||
// Scenario 4: Query at pruning boundary
|
||||
// Note: We test block >= lowest_available because HistoricalStateProviderRef
|
||||
// errors on blocks below the pruning boundary before doing the lookup.
|
||||
// The RocksDB implementation doesn't have this check at the same level.
|
||||
// This tests that when pruning IS available, both backends agree.
|
||||
run_account_history_scenario(
|
||||
"with_pruning_boundary",
|
||||
address,
|
||||
&[(u64::MAX, vec![100, 200, 300])],
|
||||
&[
|
||||
// At pruning boundary -> InChangeset(first entry after block)
|
||||
HistoryQuery {
|
||||
block_number: 100,
|
||||
lowest_available: Some(100),
|
||||
expected: HistoryInfo::InChangeset(100),
|
||||
},
|
||||
// After pruning boundary, between entries
|
||||
HistoryQuery {
|
||||
block_number: 150,
|
||||
lowest_available: Some(100),
|
||||
expected: HistoryInfo::InChangeset(200),
|
||||
},
|
||||
],
|
||||
);
|
||||
}
|
||||
|
||||
/// Tests storage history lookups across both MDBX and `RocksDB` backends.
|
||||
#[test]
|
||||
fn test_storage_history_info_both_backends() {
|
||||
let address = Address::from([0x42; 20]);
|
||||
let storage_key = B256::from([0x01; 32]);
|
||||
let other_storage_key = B256::from([0x02; 32]);
|
||||
|
||||
// Single shard with blocks [100, 200, 300]
|
||||
run_storage_history_scenario(
|
||||
"storage_single_shard",
|
||||
address,
|
||||
storage_key,
|
||||
&[(u64::MAX, vec![100, 200, 300])],
|
||||
&[
|
||||
// Before first entry -> NotYetWritten
|
||||
HistoryQuery {
|
||||
block_number: 50,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::NotYetWritten,
|
||||
},
|
||||
// Between entries -> InChangeset(next_write)
|
||||
HistoryQuery {
|
||||
block_number: 150,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InChangeset(200),
|
||||
},
|
||||
// After last entry -> InPlainState
|
||||
HistoryQuery {
|
||||
block_number: 500,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::InPlainState,
|
||||
},
|
||||
],
|
||||
);
|
||||
|
||||
// No history for different storage key
|
||||
run_storage_history_scenario(
|
||||
"storage_no_history",
|
||||
address,
|
||||
other_storage_key,
|
||||
&[], // No shards for this storage key
|
||||
&[HistoryQuery {
|
||||
block_number: 150,
|
||||
lowest_available: None,
|
||||
expected: HistoryInfo::NotYetWritten,
|
||||
}],
|
||||
);
|
||||
}
|
||||
|
||||
/// Test that `RocksDB` batches created via `EitherWriter` are only made visible when
|
||||
/// `provider.commit()` is called, not when the writer is dropped.
|
||||
#[test]
|
||||
fn test_rocksdb_commits_at_provider_level() {
|
||||
let factory = create_test_provider_factory();
|
||||
|
||||
@@ -125,7 +125,7 @@ impl<DB: Database, N: NodeTypes> AsRef<DatabaseProvider<<DB as Database>::TXMut,
|
||||
|
||||
impl<DB: Database, N: NodeTypes + 'static> DatabaseProviderRW<DB, N> {
|
||||
/// Commit database transaction and static file if it exists.
|
||||
pub fn commit(self) -> ProviderResult<bool> {
|
||||
pub fn commit(self) -> ProviderResult<()> {
|
||||
self.0.commit()
|
||||
}
|
||||
|
||||
@@ -3422,7 +3422,7 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
}
|
||||
|
||||
/// Commit database transaction, static files, and pending `RocksDB` batches.
|
||||
fn commit(self) -> ProviderResult<bool> {
|
||||
fn commit(self) -> ProviderResult<()> {
|
||||
// For unwinding it makes more sense to commit the database first, since if
|
||||
// it is interrupted before the static files commit, we can just
|
||||
// truncate the static files according to the
|
||||
@@ -3453,7 +3453,7 @@ impl<TX: DbTx + 'static, N: NodeTypes + 'static> DBProvider for DatabaseProvider
|
||||
self.tx.commit()?;
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -16,8 +16,8 @@ pub use static_file::{
|
||||
mod state;
|
||||
pub use state::{
|
||||
historical::{
|
||||
needs_prev_shard_check, HistoricalStateProvider, HistoricalStateProviderRef, HistoryInfo,
|
||||
LowestAvailableBlocks,
|
||||
history_info, needs_prev_shard_check, HistoricalStateProvider, HistoricalStateProviderRef,
|
||||
HistoryInfo, LowestAvailableBlocks,
|
||||
},
|
||||
latest::{LatestStateProvider, LatestStateProviderRef},
|
||||
overlay::{OverlayStateProvider, OverlayStateProviderFactory},
|
||||
|
||||
@@ -12,8 +12,8 @@ use reth_storage_errors::{
|
||||
};
|
||||
use rocksdb::{
|
||||
BlockBasedOptions, Cache, ColumnFamilyDescriptor, CompactionPri, DBCompressionType,
|
||||
DBRawIteratorWithThreadMode, IteratorMode, Options, Transaction, TransactionDB,
|
||||
TransactionDBOptions, TransactionOptions, WriteBatchWithTransaction, WriteOptions,
|
||||
DBRawIteratorWithThreadMode, IteratorMode, OptimisticTransactionDB,
|
||||
OptimisticTransactionOptions, Options, Transaction, WriteBatchWithTransaction, WriteOptions,
|
||||
};
|
||||
use std::{
|
||||
fmt,
|
||||
@@ -200,20 +200,17 @@ impl RocksDBBuilder {
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Use TransactionDB for MDBX-like transaction semantics (read-your-writes, rollback)
|
||||
let txn_db_options = TransactionDBOptions::default();
|
||||
let db = TransactionDB::open_cf_descriptors(
|
||||
&options,
|
||||
&txn_db_options,
|
||||
&self.path,
|
||||
cf_descriptors,
|
||||
)
|
||||
.map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Open(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))
|
||||
})?;
|
||||
// Use OptimisticTransactionDB for MDBX-like transaction semantics (read-your-writes,
|
||||
// rollback) OptimisticTransactionDB uses optimistic concurrency control (conflict
|
||||
// detection at commit) and is backed by DBCommon, giving us access to
|
||||
// cancel_all_background_work for clean shutdown.
|
||||
let db = OptimisticTransactionDB::open_cf_descriptors(&options, &self.path, cf_descriptors)
|
||||
.map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Open(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))
|
||||
})?;
|
||||
|
||||
let metrics = self.enable_metrics.then(RocksDBMetrics::default);
|
||||
|
||||
@@ -241,8 +238,8 @@ pub struct RocksDBProvider(Arc<RocksDBProviderInner>);
|
||||
|
||||
/// Inner state for `RocksDB` provider.
|
||||
struct RocksDBProviderInner {
|
||||
/// `RocksDB` database instance with transaction support.
|
||||
db: TransactionDB,
|
||||
/// `RocksDB` database instance with optimistic transaction support.
|
||||
db: OptimisticTransactionDB,
|
||||
/// Metrics latency & operations.
|
||||
metrics: Option<RocksDBMetrics>,
|
||||
}
|
||||
@@ -250,12 +247,20 @@ struct RocksDBProviderInner {
|
||||
impl fmt::Debug for RocksDBProviderInner {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
f.debug_struct("RocksDBProviderInner")
|
||||
.field("db", &"<TransactionDB>")
|
||||
.field("db", &"<OptimisticTransactionDB>")
|
||||
.field("metrics", &self.metrics)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for RocksDBProviderInner {
|
||||
fn drop(&mut self) {
|
||||
// Cancel all background work (compaction, flush) before dropping.
|
||||
// This prevents pthread lock errors during shutdown.
|
||||
self.db.cancel_all_background_work(true);
|
||||
}
|
||||
}
|
||||
|
||||
impl Clone for RocksDBProvider {
|
||||
fn clone(&self) -> Self {
|
||||
Self(self.0.clone())
|
||||
@@ -274,9 +279,12 @@ impl RocksDBProvider {
|
||||
}
|
||||
|
||||
/// Creates a new transaction with MDBX-like semantics (read-your-writes, rollback).
|
||||
///
|
||||
/// Note: With `OptimisticTransactionDB`, commits may fail if there are conflicts.
|
||||
/// Conflict detection happens at commit time, not at write time.
|
||||
pub fn tx(&self) -> RocksTx<'_> {
|
||||
let write_options = WriteOptions::default();
|
||||
let txn_options = TransactionOptions::default();
|
||||
let txn_options = OptimisticTransactionOptions::default();
|
||||
let inner = self.0.db.transaction_opt(&write_options, &txn_options);
|
||||
RocksTx { inner, provider: self }
|
||||
}
|
||||
@@ -564,7 +572,7 @@ impl<'a> RocksDBBatch<'a> {
|
||||
/// Note: `Transaction` is `Send` but NOT `Sync`. This wrapper does not implement
|
||||
/// `DbTx`/`DbTxMut` traits directly; use RocksDB-specific methods instead.
|
||||
pub struct RocksTx<'db> {
|
||||
inner: Transaction<'db, TransactionDB>,
|
||||
inner: Transaction<'db, OptimisticTransactionDB>,
|
||||
provider: &'db RocksDBProvider,
|
||||
}
|
||||
|
||||
@@ -747,7 +755,7 @@ impl<'db> RocksTx<'db> {
|
||||
})?;
|
||||
|
||||
// Create a raw iterator to access key bytes directly.
|
||||
let mut iter: DBRawIteratorWithThreadMode<'_, Transaction<'_, TransactionDB>> =
|
||||
let mut iter: DBRawIteratorWithThreadMode<'_, Transaction<'_, OptimisticTransactionDB>> =
|
||||
self.inner.raw_iterator_cf(&cf);
|
||||
|
||||
// Seek to the smallest key >= encoded_key.
|
||||
@@ -825,7 +833,7 @@ impl<'db> RocksTx<'db> {
|
||||
|
||||
/// Returns an error if the raw iterator is in an invalid state due to an I/O error.
|
||||
fn raw_iter_status_ok(
|
||||
iter: &DBRawIteratorWithThreadMode<'_, Transaction<'_, TransactionDB>>,
|
||||
iter: &DBRawIteratorWithThreadMode<'_, Transaction<'_, OptimisticTransactionDB>>,
|
||||
) -> ProviderResult<()> {
|
||||
iter.status().map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
@@ -840,7 +848,7 @@ impl<'db> RocksTx<'db> {
|
||||
///
|
||||
/// Yields decoded `(Key, Value)` pairs in key order.
|
||||
pub struct RocksDBIter<'db, T: Table> {
|
||||
inner: rocksdb::DBIteratorWithThreadMode<'db, TransactionDB>,
|
||||
inner: rocksdb::DBIteratorWithThreadMode<'db, OptimisticTransactionDB>,
|
||||
_marker: std::marker::PhantomData<T>,
|
||||
}
|
||||
|
||||
@@ -884,7 +892,7 @@ impl<T: Table> Iterator for RocksDBIter<'_, T> {
|
||||
///
|
||||
/// Yields decoded `(Key, Value)` pairs. Sees uncommitted writes.
|
||||
pub struct RocksTxIter<'tx, T: Table> {
|
||||
inner: rocksdb::DBIteratorWithThreadMode<'tx, Transaction<'tx, TransactionDB>>,
|
||||
inner: rocksdb::DBIteratorWithThreadMode<'tx, Transaction<'tx, OptimisticTransactionDB>>,
|
||||
_marker: std::marker::PhantomData<T>,
|
||||
}
|
||||
|
||||
@@ -1092,7 +1100,7 @@ mod tests {
|
||||
.build()
|
||||
.unwrap();
|
||||
|
||||
// Do operations - data should be immediately readable with TransactionDB
|
||||
// Do operations - data should be immediately readable with OptimisticTransactionDB
|
||||
for i in 0..10 {
|
||||
let value = vec![i as u8];
|
||||
provider.put::<TestTable>(i, &value).unwrap();
|
||||
@@ -1107,7 +1115,7 @@ mod tests {
|
||||
let provider =
|
||||
RocksDBBuilder::new(temp_dir.path()).with_table::<TestTable>().build().unwrap();
|
||||
|
||||
// Insert data - TransactionDB writes are immediately visible
|
||||
// Insert data - OptimisticTransactionDB writes are immediately visible
|
||||
let value = vec![42u8; 1000];
|
||||
for i in 0..100 {
|
||||
provider.put::<TestTable>(i, &value).unwrap();
|
||||
@@ -1264,101 +1272,9 @@ mod tests {
|
||||
assert_eq!(last, Some((20, b"value_20".to_vec())));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_history_info_single_shard() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
let address = Address::from([0x42; 20]);
|
||||
|
||||
// Create a single shard with blocks [100, 200, 300] and highest_block = u64::MAX
|
||||
// This is the "last shard" invariant
|
||||
let chunk = IntegerList::new([100, 200, 300]).unwrap();
|
||||
let shard_key = ShardedKey::new(address, u64::MAX);
|
||||
provider.put::<tables::AccountsHistory>(shard_key, &chunk).unwrap();
|
||||
|
||||
let tx = provider.tx();
|
||||
|
||||
// Query for block 150: should find block 200 in changeset
|
||||
let result = tx.account_history_info(address, 150, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InChangeset(200));
|
||||
|
||||
// Query for block 50: should return NotYetWritten (before first entry, no prev shard)
|
||||
let result = tx.account_history_info(address, 50, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::NotYetWritten);
|
||||
|
||||
// Query for block 300: should return InChangeset(300) - exact match means look at
|
||||
// changeset at that block for the previous value
|
||||
let result = tx.account_history_info(address, 300, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InChangeset(300));
|
||||
|
||||
// Query for block 500: should return InPlainState (after last entry in last shard)
|
||||
let result = tx.account_history_info(address, 500, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InPlainState);
|
||||
|
||||
tx.rollback().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_history_info_multiple_shards() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
let address = Address::from([0x42; 20]);
|
||||
|
||||
// Create two shards: first shard ends at block 500, second is the last shard
|
||||
let chunk1 = IntegerList::new([100, 200, 300, 400, 500]).unwrap();
|
||||
let shard_key1 = ShardedKey::new(address, 500);
|
||||
provider.put::<tables::AccountsHistory>(shard_key1, &chunk1).unwrap();
|
||||
|
||||
let chunk2 = IntegerList::new([600, 700, 800]).unwrap();
|
||||
let shard_key2 = ShardedKey::new(address, u64::MAX);
|
||||
provider.put::<tables::AccountsHistory>(shard_key2, &chunk2).unwrap();
|
||||
|
||||
let tx = provider.tx();
|
||||
|
||||
// Query for block 50: should return NotYetWritten (before first shard, no prev)
|
||||
let result = tx.account_history_info(address, 50, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::NotYetWritten);
|
||||
|
||||
// Query for block 150: should find block 200 in first shard's changeset
|
||||
let result = tx.account_history_info(address, 150, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InChangeset(200));
|
||||
|
||||
// Query for block 550: should find block 600 in second shard's changeset
|
||||
// prev() should detect first shard exists
|
||||
let result = tx.account_history_info(address, 550, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InChangeset(600));
|
||||
|
||||
// Query for block 900: should return InPlainState (after last entry in last shard)
|
||||
let result = tx.account_history_info(address, 900, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InPlainState);
|
||||
|
||||
tx.rollback().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_account_history_info_no_history() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
let address1 = Address::from([0x42; 20]);
|
||||
let address2 = Address::from([0x43; 20]);
|
||||
|
||||
// Only add history for address1
|
||||
let chunk = IntegerList::new([100, 200, 300]).unwrap();
|
||||
let shard_key = ShardedKey::new(address1, u64::MAX);
|
||||
provider.put::<tables::AccountsHistory>(shard_key, &chunk).unwrap();
|
||||
|
||||
let tx = provider.tx();
|
||||
|
||||
// Query for address2 (no history exists): should return NotYetWritten
|
||||
let result = tx.account_history_info(address2, 150, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::NotYetWritten);
|
||||
|
||||
tx.rollback().unwrap();
|
||||
}
|
||||
|
||||
/// Tests the edge case where block < `lowest_available_block_number`.
|
||||
/// This case cannot be tested via `HistoricalStateProviderRef` (which errors before lookup),
|
||||
/// so we keep this RocksDB-specific test to verify the low-level behavior.
|
||||
#[test]
|
||||
fn test_account_history_info_pruned_before_first_entry() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
@@ -1382,39 +1298,4 @@ mod tests {
|
||||
|
||||
tx.rollback().unwrap();
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_storage_history_info() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let provider = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
let address = Address::from([0x42; 20]);
|
||||
let storage_key = B256::from([0x01; 32]);
|
||||
|
||||
// Create a single shard for this storage slot
|
||||
let chunk = IntegerList::new([100, 200, 300]).unwrap();
|
||||
let shard_key = StorageShardedKey::new(address, storage_key, u64::MAX);
|
||||
provider.put::<tables::StoragesHistory>(shard_key, &chunk).unwrap();
|
||||
|
||||
let tx = provider.tx();
|
||||
|
||||
// Query for block 150: should find block 200 in changeset
|
||||
let result = tx.storage_history_info(address, storage_key, 150, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InChangeset(200));
|
||||
|
||||
// Query for block 50: should return NotYetWritten
|
||||
let result = tx.storage_history_info(address, storage_key, 50, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::NotYetWritten);
|
||||
|
||||
// Query for block 500: should return InPlainState
|
||||
let result = tx.storage_history_info(address, storage_key, 500, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::InPlainState);
|
||||
|
||||
// Query for different storage key (no history): should return NotYetWritten
|
||||
let other_key = B256::from([0x02; 32]);
|
||||
let result = tx.storage_history_info(address, other_key, 150, None).unwrap();
|
||||
assert_eq!(result, HistoryInfo::NotYetWritten);
|
||||
|
||||
tx.rollback().unwrap();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -135,7 +135,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + BlockNumReader>
|
||||
|
||||
// history key to search IntegerList of block number changesets.
|
||||
let history_key = ShardedKey::new(address, self.block_number);
|
||||
self.history_info::<tables::AccountsHistory, _>(
|
||||
self.history_info_lookup::<tables::AccountsHistory, _>(
|
||||
history_key,
|
||||
|key| key.key == address,
|
||||
self.lowest_available_blocks.account_history_block_number,
|
||||
@@ -154,7 +154,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + BlockNumReader>
|
||||
|
||||
// history key to search IntegerList of block number changesets.
|
||||
let history_key = StorageShardedKey::new(address, storage_key, self.block_number);
|
||||
self.history_info::<tables::StoragesHistory, _>(
|
||||
self.history_info_lookup::<tables::StoragesHistory, _>(
|
||||
history_key,
|
||||
|key| key.address == address && key.sharded_key.key == storage_key,
|
||||
self.lowest_available_blocks.storage_history_block_number,
|
||||
@@ -204,7 +204,7 @@ impl<'b, Provider: DBProvider + ChangeSetReader + BlockNumReader>
|
||||
Ok(HashedStorage::from_reverts(self.tx(), address, self.block_number)?)
|
||||
}
|
||||
|
||||
fn history_info<T, K>(
|
||||
fn history_info_lookup<T, K>(
|
||||
&self,
|
||||
key: K,
|
||||
key_filter: impl Fn(&K) -> bool,
|
||||
@@ -214,45 +214,13 @@ impl<'b, Provider: DBProvider + ChangeSetReader + BlockNumReader>
|
||||
T: Table<Key = K, Value = BlockNumberList>,
|
||||
{
|
||||
let mut cursor = self.tx().cursor_read::<T>()?;
|
||||
|
||||
// Lookup the history chunk in the history index. If the key does not appear in the
|
||||
// index, the first chunk for the next key will be returned so we filter out chunks that
|
||||
// have a different key.
|
||||
if let Some(chunk) = cursor.seek(key)?.filter(|(key, _)| key_filter(key)).map(|x| x.1) {
|
||||
// Get the rank of the first entry before or equal to our block.
|
||||
let mut rank = chunk.rank(self.block_number);
|
||||
|
||||
// Adjust the rank, so that we have the rank of the first entry strictly before our
|
||||
// block (not equal to it).
|
||||
if rank.checked_sub(1).and_then(|r| chunk.select(r)) == Some(self.block_number) {
|
||||
rank -= 1;
|
||||
}
|
||||
|
||||
let found_block = chunk.select(rank);
|
||||
|
||||
// If our block is before the first entry in the index chunk and this first entry
|
||||
// doesn't equal to our block, it might be before the first write ever. To check, we
|
||||
// look at the previous entry and check if the key is the same.
|
||||
// This check is worth it, the `cursor.prev()` check is rarely triggered (the if will
|
||||
// short-circuit) and when it passes we save a full seek into the changeset/plain state
|
||||
// table.
|
||||
let is_before_first_write =
|
||||
needs_prev_shard_check(rank, found_block, self.block_number) &&
|
||||
!cursor.prev()?.is_some_and(|(key, _)| key_filter(&key));
|
||||
|
||||
Ok(HistoryInfo::from_lookup(
|
||||
found_block,
|
||||
is_before_first_write,
|
||||
lowest_available_block_number,
|
||||
))
|
||||
} else if lowest_available_block_number.is_some() {
|
||||
// The key may have been written, but due to pruning we may not have changesets and
|
||||
// history, so we need to make a plain state lookup.
|
||||
Ok(HistoryInfo::MaybeInPlainState)
|
||||
} else {
|
||||
// The key has not been written to at all.
|
||||
Ok(HistoryInfo::NotYetWritten)
|
||||
}
|
||||
history_info::<T, K, _>(
|
||||
&mut cursor,
|
||||
key,
|
||||
self.block_number,
|
||||
key_filter,
|
||||
lowest_available_block_number,
|
||||
)
|
||||
}
|
||||
|
||||
/// Set the lowest block number at which the account history is available.
|
||||
@@ -570,6 +538,60 @@ pub fn needs_prev_shard_check(
|
||||
rank == 0 && found_block != Some(block_number)
|
||||
}
|
||||
|
||||
/// Generic history lookup for sharded history tables.
|
||||
///
|
||||
/// Seeks to the shard containing `block_number`, verifies the key via `key_filter`,
|
||||
/// and checks previous shard to detect if we're before the first write.
|
||||
pub fn history_info<T, K, C>(
|
||||
cursor: &mut C,
|
||||
key: K,
|
||||
block_number: BlockNumber,
|
||||
key_filter: impl Fn(&K) -> bool,
|
||||
lowest_available_block_number: Option<BlockNumber>,
|
||||
) -> ProviderResult<HistoryInfo>
|
||||
where
|
||||
T: Table<Key = K, Value = BlockNumberList>,
|
||||
C: DbCursorRO<T>,
|
||||
{
|
||||
// Lookup the history chunk in the history index. If the key does not appear in the
|
||||
// index, the first chunk for the next key will be returned so we filter out chunks that
|
||||
// have a different key.
|
||||
if let Some(chunk) = cursor.seek(key)?.filter(|(k, _)| key_filter(k)).map(|x| x.1) {
|
||||
// Get the rank of the first entry before or equal to our block.
|
||||
let mut rank = chunk.rank(block_number);
|
||||
|
||||
// Adjust the rank, so that we have the rank of the first entry strictly before our
|
||||
// block (not equal to it).
|
||||
if rank.checked_sub(1).and_then(|r| chunk.select(r)) == Some(block_number) {
|
||||
rank -= 1;
|
||||
}
|
||||
|
||||
let found_block = chunk.select(rank);
|
||||
|
||||
// If our block is before the first entry in the index chunk and this first entry
|
||||
// doesn't equal to our block, it might be before the first write ever. To check, we
|
||||
// look at the previous entry and check if the key is the same.
|
||||
// This check is worth it, the `cursor.prev()` check is rarely triggered (the if will
|
||||
// short-circuit) and when it passes we save a full seek into the changeset/plain state
|
||||
// table.
|
||||
let is_before_first_write = needs_prev_shard_check(rank, found_block, block_number) &&
|
||||
!cursor.prev()?.is_some_and(|(k, _)| key_filter(&k));
|
||||
|
||||
Ok(HistoryInfo::from_lookup(
|
||||
found_block,
|
||||
is_before_first_write,
|
||||
lowest_available_block_number,
|
||||
))
|
||||
} else if lowest_available_block_number.is_some() {
|
||||
// The key may have been written, but due to pruning we may not have changesets and
|
||||
// history, so we need to make a plain state lookup.
|
||||
Ok(HistoryInfo::MaybeInPlainState)
|
||||
} else {
|
||||
// The key has not been written to at all.
|
||||
Ok(HistoryInfo::NotYetWritten)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::needs_prev_shard_check;
|
||||
|
||||
@@ -85,6 +85,11 @@ impl<'a, N: NodePrimitives> StaticFileJarProvider<'a, N> {
|
||||
self.metrics = Some(metrics);
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the total size of the data and offsets files (from the in-memory mmap).
|
||||
pub fn size(&self) -> usize {
|
||||
self.jar.value().size()
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives<BlockHeader: Value>> HeaderProvider for StaticFileJarProvider<'_, N> {
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user