mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
70 Commits
push
...
fix-filter
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
54c2cd753e | ||
|
|
6ff4f947c8 | ||
|
|
719bbc2543 | ||
|
|
a9a6044bc5 | ||
|
|
6f9a3242ef | ||
|
|
e89bf483bc | ||
|
|
61038449c8 | ||
|
|
48b2cd970f | ||
|
|
fb90051010 | ||
|
|
a0a622a155 | ||
|
|
8db352dfd2 | ||
|
|
117b212e2e | ||
|
|
df9e3669aa | ||
|
|
0464cddfb0 | ||
|
|
e21a174737 | ||
|
|
e972d9d8c7 | ||
|
|
7f00ebfafe | ||
|
|
883e9ae8cc | ||
|
|
a1e4132c2d | ||
|
|
4ecb0d5680 | ||
|
|
5b8808e5fd | ||
|
|
2eec519bf9 | ||
|
|
02513ecf3b | ||
|
|
10c6bdb5ff | ||
|
|
20ae9ac405 | ||
|
|
881500e592 | ||
|
|
8db125daff | ||
|
|
bf2071f773 | ||
|
|
ee5ec069cd | ||
|
|
8722277d6e | ||
|
|
57148eac9f | ||
|
|
74abad29ad | ||
|
|
997af404a5 | ||
|
|
314a92e93c | ||
|
|
f0c4be108b | ||
|
|
9265e8e46c | ||
|
|
7594e1513a | ||
|
|
7f5acc2723 | ||
|
|
60d0430c2b | ||
|
|
d49f828998 | ||
|
|
2f78bcd7b5 | ||
|
|
f60febfa62 | ||
|
|
317f858bd4 | ||
|
|
11acd97982 | ||
|
|
f5cf90227b | ||
|
|
0dd47af250 | ||
|
|
0142769191 | ||
|
|
e1dc93e24f | ||
|
|
33ac869a85 | ||
|
|
ec982f8686 | ||
|
|
47cef33a0d | ||
|
|
9529de4cf2 | ||
|
|
5a9dd02301 | ||
|
|
d71a0c0c7b | ||
|
|
2be3788481 | ||
|
|
adbec3218d | ||
|
|
2e5560b444 | ||
|
|
1f3fd5da2e | ||
|
|
3ab7cb98aa | ||
|
|
d3088e171c | ||
|
|
2c443a3dcb | ||
|
|
4b444069a5 | ||
|
|
25d371817a | ||
|
|
4b0fa8a330 | ||
|
|
df22d38224 | ||
|
|
e4ec836a46 | ||
|
|
d3c42fc718 | ||
|
|
8171cee927 | ||
|
|
61cfcd8195 | ||
|
|
b646f4559c |
10
.changelog/brave-dogs-fly.md
Normal file
10
.changelog/brave-dogs-fly.md
Normal file
@@ -0,0 +1,10 @@
|
||||
---
|
||||
reth-engine-primitives: patch
|
||||
reth-engine-tree: patch
|
||||
reth-node-core: patch
|
||||
reth-trie-parallel: minor
|
||||
---
|
||||
|
||||
Removed legacy proof calculation system and V2-specific configuration flags.
|
||||
|
||||
Removed the legacy (non-V2) proof calculation code paths, simplified multiproof task architecture by removing the dual-mode system, and cleaned up V2-specific CLI flags (`--engine.disable-proof-v2`, `--engine.disable-trie-cache`) that are no longer needed. The codebase now exclusively uses V2 proofs with the sparse trie cache.
|
||||
5
.changelog/calm-clams-buzz.md
Normal file
5
.changelog/calm-clams-buzz.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Refactored sparse trie node state tracking to use RLP nodes instead of hashes. Replaced `Option<B256>` hash fields with `SparseNodeState` enum that tracks either dirty nodes or cached RLP nodes with optional database storage flags. Added debug assertions to validate leaf path lengths and improved pruning logic to use node paths directly instead of path-hash tuples.
|
||||
5
.changelog/fast-seals-play.md
Normal file
5
.changelog/fast-seals-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-transaction-pool: minor
|
||||
---
|
||||
|
||||
Added `consensus_ref` method to `PoolTransaction` trait for borrowing consensus transactions without cloning.
|
||||
5
.changelog/fix-simulate-revert-code.md
Normal file
5
.changelog/fix-simulate-revert-code.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-rpc-eth-types: patch
|
||||
---
|
||||
|
||||
Updated `eth_simulateV1` revert error code from `-32000` to `3` to be consistent with `eth_call`, per [execution-apis#748](https://github.com/ethereum/execution-apis/pull/748).
|
||||
6
.changelog/merry-koalas-nod.md
Normal file
6
.changelog/merry-koalas-nod.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth-rpc-eth-api: minor
|
||||
reth-rpc-server-types: minor
|
||||
---
|
||||
|
||||
Added `eth_getStorageValues` RPC method for batch storage slot retrieval across multiple addresses.
|
||||
5
.changelog/odd-snails-play.md
Normal file
5
.changelog/odd-snails-play.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-node-core: minor
|
||||
---
|
||||
|
||||
Added `with_dev_block_time` helper method to `NodeConfig` for configuring dev miner block production interval.
|
||||
5
.changelog/proud-wolves-spin.md
Normal file
5
.changelog/proud-wolves-spin.md
Normal file
@@ -0,0 +1,5 @@
|
||||
---
|
||||
reth-storage-api: patch
|
||||
---
|
||||
|
||||
Added `Arc` to `auto_impl` derive for storage-api traits to support automatic `Arc` wrapper implementations.
|
||||
8
.changelog/quick-mice-jump.md
Normal file
8
.changelog/quick-mice-jump.md
Normal file
@@ -0,0 +1,8 @@
|
||||
---
|
||||
reth: patch
|
||||
reth-engine-tree: patch
|
||||
reth-node-builder: patch
|
||||
reth-trie-sparse: minor
|
||||
---
|
||||
|
||||
Added `trie-debug` feature for recording sparse trie mutations to aid in debugging state root mismatches.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Fixed a bug where trie nodes could appear in both `updated_nodes` and `removed_nodes` simultaneously by removing entries from `removed_nodes` when a node is inserted as updated.
|
||||
6
.changelog/warm-donkeys-dry.md
Normal file
6
.changelog/warm-donkeys-dry.md
Normal file
@@ -0,0 +1,6 @@
|
||||
---
|
||||
reth-engine-local: minor
|
||||
reth-node-builder: minor
|
||||
---
|
||||
|
||||
Added trigger-based `MiningMode` variant that allows blocks to be built on-demand via custom streams, and exposed `with_mining_mode` method on `DebugNodeLauncherFuture` to override default mining configuration.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-trie: patch
|
||||
---
|
||||
|
||||
Fixed a potential panic in `ProofCalculator` by clearing internal computation state (`branch_stack`, `child_stack`, `branch_path`, etc.) after errors, preventing stale state from causing `usize` underflow panics when the calculator is reused. Added a test verifying correct behavior after simulated mid-computation errors.
|
||||
@@ -1,6 +1,6 @@
|
||||
[profile.default]
|
||||
retries = { backoff = "exponential", count = 2, delay = "2s", jitter = true }
|
||||
slow-timeout = { period = "30s", terminate-after = 4 }
|
||||
slow-timeout = { period = "30s", terminate-after = 2 }
|
||||
|
||||
[[profile.default.overrides]]
|
||||
filter = "test(general_state_tests)"
|
||||
|
||||
66
.github/scripts/bench-reth-build.sh
vendored
Executable file
66
.github/scripts/bench-reth-build.sh
vendored
Executable file
@@ -0,0 +1,66 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds (or fetches from cache) reth binaries for benchmarking.
|
||||
#
|
||||
# Usage: bench-reth-build.sh <main|branch> <commit> [branch-sha]
|
||||
#
|
||||
# main — build/fetch the baseline binary at <commit> (merge-base)
|
||||
# branch — build/fetch the candidate binary + reth-bench at <commit>
|
||||
# optional branch-sha is the PR head commit for cache key
|
||||
#
|
||||
# Outputs:
|
||||
# main: target/profiling-baseline/reth
|
||||
# branch: target/profiling/reth, reth-bench installed to cargo bin
|
||||
#
|
||||
# Required: mc (MinIO client) configured at /home/ubuntu/.mc
|
||||
set -euo pipefail
|
||||
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MODE="$1"
|
||||
COMMIT="$2"
|
||||
|
||||
case "$MODE" in
|
||||
main)
|
||||
BUCKET="minio/reth-binaries/${COMMIT}"
|
||||
mkdir -p target/profiling-baseline
|
||||
|
||||
if $MC stat "${BUCKET}/reth" &>/dev/null; then
|
||||
echo "Cache hit for main (${COMMIT}), downloading binary..."
|
||||
$MC cp "${BUCKET}/reth" target/profiling-baseline/reth
|
||||
chmod +x target/profiling-baseline/reth
|
||||
else
|
||||
echo "Cache miss for main (${COMMIT}), building from source..."
|
||||
CURRENT_REF=$(git rev-parse HEAD)
|
||||
git checkout "${COMMIT}"
|
||||
cargo build --profile profiling --bin reth
|
||||
cp target/profiling/reth target/profiling-baseline/reth
|
||||
$MC cp target/profiling-baseline/reth "${BUCKET}/reth"
|
||||
git checkout "${CURRENT_REF}"
|
||||
fi
|
||||
;;
|
||||
|
||||
branch)
|
||||
BRANCH_SHA="${3:-$COMMIT}"
|
||||
BUCKET="minio/reth-binaries/${BRANCH_SHA}"
|
||||
|
||||
if $MC stat "${BUCKET}/reth" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
|
||||
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
|
||||
mkdir -p target/profiling
|
||||
$MC cp "${BUCKET}/reth" target/profiling/reth
|
||||
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench
|
||||
chmod +x target/profiling/reth /home/ubuntu/.cargo/bin/reth-bench
|
||||
else
|
||||
echo "Cache miss for ${BRANCH_SHA}, building from source..."
|
||||
rustup show active-toolchain || rustup default stable
|
||||
make profiling
|
||||
make install-reth-bench
|
||||
$MC cp target/profiling/reth "${BUCKET}/reth"
|
||||
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 <main|branch> <commit> [branch-sha]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
232
.github/scripts/bench-reth-charts.py
vendored
Normal file
232
.github/scripts/bench-reth-charts.py
vendored
Normal file
@@ -0,0 +1,232 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Generate benchmark charts from reth-bench CSV output.
|
||||
|
||||
Usage:
|
||||
bench-engine-charts.py <combined_csv> --output-dir <dir> [--baseline <baseline_csv>]
|
||||
|
||||
Generates three PNG charts:
|
||||
1. newPayload latency + Ggas/s per block (+ latency diff when baseline present)
|
||||
2. Wait breakdown (persistence, execution cache, sparse trie) per block
|
||||
3. Scatter plot of gas used vs latency
|
||||
|
||||
When --baseline is provided, charts overlay both datasets for comparison.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use("Agg")
|
||||
import matplotlib.pyplot as plt
|
||||
except ImportError:
|
||||
print("matplotlib is required: pip install matplotlib", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
GIGAGAS = 1_000_000_000
|
||||
|
||||
|
||||
def parse_combined_csv(path: str) -> list[dict]:
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(
|
||||
{
|
||||
"block_number": int(row["block_number"]),
|
||||
"gas_used": int(row["gas_used"]),
|
||||
"new_payload_latency_us": int(row["new_payload_latency"]),
|
||||
"persistence_wait_us": int(row["persistence_wait"])
|
||||
if row.get("persistence_wait")
|
||||
else None,
|
||||
"execution_cache_wait_us": int(row.get("execution_cache_wait", 0)),
|
||||
"sparse_trie_wait_us": int(row.get("sparse_trie_wait", 0)),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def plot_latency_and_throughput(
|
||||
feature: list[dict], baseline: list[dict] | None, out: Path,
|
||||
baseline_name: str = "main", branch_name: str = "branch",
|
||||
):
|
||||
num_plots = 3 if baseline else 2
|
||||
fig, axes = plt.subplots(num_plots, 1, figsize=(12, 4 * num_plots), sharex=True)
|
||||
ax1, ax2 = axes[0], axes[1]
|
||||
|
||||
feat_x = [r["block_number"] for r in feature]
|
||||
feat_lat = [r["new_payload_latency_us"] / 1_000 for r in feature]
|
||||
feat_ggas = []
|
||||
for r in feature:
|
||||
lat_s = r["new_payload_latency_us"] / 1_000_000
|
||||
feat_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
|
||||
|
||||
if baseline:
|
||||
base_x = [r["block_number"] for r in baseline]
|
||||
base_lat = [r["new_payload_latency_us"] / 1_000 for r in baseline]
|
||||
base_ggas = []
|
||||
for r in baseline:
|
||||
lat_s = r["new_payload_latency_us"] / 1_000_000
|
||||
base_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
|
||||
ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
|
||||
ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
|
||||
|
||||
ax1.plot(feat_x, feat_lat, linewidth=0.8, label=branch_name)
|
||||
ax1.set_ylabel("Latency (ms)")
|
||||
ax1.set_title("newPayload Latency per Block")
|
||||
ax1.grid(True, alpha=0.3)
|
||||
if baseline:
|
||||
ax1.legend()
|
||||
|
||||
ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=branch_name)
|
||||
ax2.set_ylabel("Ggas/s")
|
||||
ax2.set_title("Execution Throughput per Block")
|
||||
ax2.grid(True, alpha=0.3)
|
||||
if baseline:
|
||||
ax2.legend()
|
||||
|
||||
if baseline:
|
||||
ax3 = axes[2]
|
||||
base_by_block = {r["block_number"]: r["new_payload_latency_us"] for r in baseline}
|
||||
blocks, diffs = [], []
|
||||
for r in feature:
|
||||
bn = r["block_number"]
|
||||
if bn in base_by_block and base_by_block[bn] > 0:
|
||||
pct = (r["new_payload_latency_us"] - base_by_block[bn]) / base_by_block[bn] * 100
|
||||
blocks.append(bn)
|
||||
diffs.append(pct)
|
||||
if blocks:
|
||||
colors = ["green" if d <= 0 else "red" for d in diffs]
|
||||
ax3.bar(blocks, diffs, width=1.0, color=colors, alpha=0.7, edgecolor="none")
|
||||
ax3.axhline(0, color="black", linewidth=0.5)
|
||||
ax3.set_ylabel("Δ Latency (%)")
|
||||
ax3.set_title("Per-Block newPayload Latency Change (branch vs main)")
|
||||
ax3.grid(True, alpha=0.3, axis="y")
|
||||
|
||||
axes[-1].set_xlabel("Block Number")
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def plot_wait_breakdown(
|
||||
feature: list[dict], baseline: list[dict] | None, out: Path,
|
||||
baseline_name: str = "main", branch_name: str = "branch",
|
||||
):
|
||||
series = [
|
||||
("Persistence Wait", "persistence_wait_us"),
|
||||
("State Cache Wait", "execution_cache_wait_us"),
|
||||
("Trie Cache Wait", "sparse_trie_wait_us"),
|
||||
]
|
||||
|
||||
fig, axes = plt.subplots(len(series), 1, figsize=(12, 3 * len(series)), sharex=True)
|
||||
for ax, (label, key) in zip(axes, series):
|
||||
if baseline:
|
||||
bx = [r["block_number"] for r in baseline if r[key] is not None]
|
||||
by = [r[key] / 1_000 for r in baseline if r[key] is not None]
|
||||
if bx:
|
||||
ax.plot(bx, by, linewidth=0.8, label=baseline_name, alpha=0.7)
|
||||
|
||||
fx = [r["block_number"] for r in feature if r[key] is not None]
|
||||
fy = [r[key] / 1_000 for r in feature if r[key] is not None]
|
||||
if fx:
|
||||
ax.plot(fx, fy, linewidth=0.8, label=branch_name)
|
||||
|
||||
ax.set_ylabel("ms")
|
||||
ax.set_title(label)
|
||||
ax.grid(True, alpha=0.3)
|
||||
if baseline:
|
||||
ax.legend()
|
||||
|
||||
axes[-1].set_xlabel("Block Number")
|
||||
fig.suptitle("Wait Time Breakdown per Block", fontsize=14, y=1.01)
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150, bbox_inches="tight")
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def _add_regression(ax, x, y, color, label):
|
||||
"""Add a linear regression line to the axes."""
|
||||
if len(x) < 2:
|
||||
return
|
||||
xa, ya = np.array(x), np.array(y)
|
||||
m, b = np.polyfit(xa, ya, 1)
|
||||
x_range = np.linspace(xa.min(), xa.max(), 100)
|
||||
ax.plot(x_range, m * x_range + b, color=color, linewidth=1.5, alpha=0.8,
|
||||
label=label)
|
||||
|
||||
|
||||
def plot_gas_vs_latency(
|
||||
feature: list[dict], baseline: list[dict] | None, out: Path,
|
||||
baseline_name: str = "main", branch_name: str = "branch",
|
||||
):
|
||||
fig, ax = plt.subplots(figsize=(8, 6))
|
||||
|
||||
if baseline:
|
||||
bgas = [r["gas_used"] / 1_000_000 for r in baseline]
|
||||
blat = [r["new_payload_latency_us"] / 1_000 for r in baseline]
|
||||
ax.scatter(bgas, blat, s=8, alpha=0.5)
|
||||
_add_regression(ax, bgas, blat, "tab:blue", baseline_name)
|
||||
|
||||
fgas = [r["gas_used"] / 1_000_000 for r in feature]
|
||||
flat = [r["new_payload_latency_us"] / 1_000 for r in feature]
|
||||
ax.scatter(fgas, flat, s=8, alpha=0.6)
|
||||
_add_regression(ax, fgas, flat, "tab:orange", branch_name)
|
||||
|
||||
ax.set_xlabel("Gas Used (Mgas)")
|
||||
ax.set_ylabel("newPayload Latency (ms)")
|
||||
ax.set_title("Gas Used vs Latency")
|
||||
ax.grid(True, alpha=0.3)
|
||||
ax.legend()
|
||||
fig.tight_layout()
|
||||
fig.savefig(out, dpi=150)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Generate benchmark charts")
|
||||
parser.add_argument("combined_csv", help="Path to combined_latency.csv (feature)")
|
||||
parser.add_argument(
|
||||
"--output-dir", required=True, help="Output directory for PNG charts"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--baseline", help="Path to baseline (main) combined_latency.csv"
|
||||
)
|
||||
parser.add_argument("--baseline-name", default="main", help="Label for baseline")
|
||||
parser.add_argument("--branch-name", default="branch", help="Label for branch")
|
||||
args = parser.parse_args()
|
||||
|
||||
feature = parse_combined_csv(args.combined_csv)
|
||||
if not feature:
|
||||
print("No results found in combined CSV", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
baseline = None
|
||||
if args.baseline:
|
||||
baseline = parse_combined_csv(args.baseline)
|
||||
if not baseline:
|
||||
print(
|
||||
"Warning: no results in baseline CSV, skipping comparison",
|
||||
file=sys.stderr,
|
||||
)
|
||||
baseline = None
|
||||
|
||||
out_dir = Path(args.output_dir)
|
||||
out_dir.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
bname = args.baseline_name
|
||||
fname = args.branch_name
|
||||
plot_latency_and_throughput(feature, baseline, out_dir / "latency_throughput.png", bname, fname)
|
||||
plot_wait_breakdown(feature, baseline, out_dir / "wait_breakdown.png", bname, fname)
|
||||
plot_gas_vs_latency(feature, baseline, out_dir / "gas_vs_latency.png", bname, fname)
|
||||
|
||||
print(f"Charts written to {out_dir}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
94
.github/scripts/bench-reth-run.sh
vendored
Executable file
94
.github/scripts/bench-reth-run.sh
vendored
Executable file
@@ -0,0 +1,94 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Runs a single reth-bench cycle: mount snapshot → start node → warmup →
|
||||
# benchmark → stop node → recover snapshot.
|
||||
#
|
||||
# Usage: bench-reth-run.sh <label> <binary> <output-dir>
|
||||
#
|
||||
# Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
|
||||
set -euo pipefail
|
||||
|
||||
LABEL="$1"
|
||||
BINARY="$2"
|
||||
OUTPUT_DIR="$3"
|
||||
DATADIR="$SCHELK_MOUNT/datadir"
|
||||
LOG="/tmp/reth-bench-node-${LABEL}.log"
|
||||
|
||||
cleanup() {
|
||||
kill "$TAIL_PID" 2>/dev/null || true
|
||||
if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
|
||||
sudo kill "$RETH_PID"
|
||||
for i in $(seq 1 30); do
|
||||
sudo kill -0 "$RETH_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
sudo kill -9 "$RETH_PID" 2>/dev/null || true
|
||||
fi
|
||||
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
|
||||
}
|
||||
TAIL_PID=
|
||||
trap cleanup EXIT
|
||||
|
||||
# Mount
|
||||
sudo schelk mount -y
|
||||
sync
|
||||
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
|
||||
echo "=== Cache state after drop ==="
|
||||
free -h
|
||||
grep Cached /proc/meminfo
|
||||
|
||||
# Start reth
|
||||
# CPU layout: core 0 = OS/IRQs/reth-bench/aux, cores 1+ = reth node
|
||||
RETH_BENCH="$(which reth-bench)"
|
||||
ONLINE=$(nproc --all)
|
||||
RETH_CPUS="1-$(( ONLINE - 1 ))"
|
||||
sudo taskset -c "$RETH_CPUS" nice -n -20 "$BINARY" node \
|
||||
--datadir "$DATADIR" \
|
||||
--engine.accept-execution-requests-hash \
|
||||
--http \
|
||||
--http.port 8545 \
|
||||
--ws \
|
||||
--ws.api all \
|
||||
--authrpc.port 8551 \
|
||||
--disable-discovery \
|
||||
--no-persist-peers \
|
||||
> "$LOG" 2>&1 &
|
||||
|
||||
RETH_PID=$!
|
||||
stdbuf -oL tail -f "$LOG" | sed -u "s/^/[reth] /" &
|
||||
TAIL_PID=$!
|
||||
|
||||
for i in $(seq 1 60); do
|
||||
if curl -sf http://127.0.0.1:8545 -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
> /dev/null 2>&1; then
|
||||
echo "reth (${LABEL}) is ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
echo "::error::reth (${LABEL}) failed to start within 60s"
|
||||
cat "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Warmup
|
||||
sudo nice -n -20 "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
|
||||
--reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
|
||||
|
||||
# Benchmark
|
||||
sudo nice -n -20 "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "$BENCH_BLOCKS" \
|
||||
--reth-new-payload \
|
||||
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
|
||||
|
||||
# cleanup runs via trap
|
||||
415
.github/scripts/bench-reth-summary.py
vendored
Executable file
415
.github/scripts/bench-reth-summary.py
vendored
Executable file
@@ -0,0 +1,415 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Parse reth-bench CSV output and generate a summary JSON + markdown comparison.
|
||||
|
||||
Usage:
|
||||
bench-reth-summary.py <combined_csv> <gas_csv> \
|
||||
--output-summary <summary.json> \
|
||||
--output-markdown <comment.md> \
|
||||
--baseline-csv <baseline_combined.csv> \
|
||||
[--repo <owner/repo>] \
|
||||
[--baseline-ref <sha>] \
|
||||
[--branch-name <name>] \
|
||||
[--branch-sha <sha>]
|
||||
|
||||
Generates a paired statistical comparison between baseline (main) and branch.
|
||||
Matches blocks by number and computes per-block diffs to cancel out gas
|
||||
variance. Fails if baseline or branch CSV is missing or empty.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import csv
|
||||
import json
|
||||
import math
|
||||
import random
|
||||
import sys
|
||||
|
||||
GIGAGAS = 1_000_000_000
|
||||
T_CRITICAL = 1.96 # two-tailed 95% confidence
|
||||
BOOTSTRAP_ITERATIONS = 10_000
|
||||
|
||||
|
||||
def parse_combined_csv(path: str) -> list[dict]:
|
||||
"""Parse combined_latency.csv into a list of per-block dicts."""
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(
|
||||
{
|
||||
"block_number": int(row["block_number"]),
|
||||
"gas_used": int(row["gas_used"]),
|
||||
"gas_limit": int(row["gas_limit"]),
|
||||
"transaction_count": int(row["transaction_count"]),
|
||||
"new_payload_latency_us": int(row["new_payload_latency"]),
|
||||
"fcu_latency_us": int(row["fcu_latency"]),
|
||||
"total_latency_us": int(row["total_latency"]),
|
||||
"persistence_wait_us": int(row["persistence_wait"])
|
||||
if row.get("persistence_wait")
|
||||
else None,
|
||||
"execution_cache_wait_us": int(row.get("execution_cache_wait", 0)),
|
||||
"sparse_trie_wait_us": int(row.get("sparse_trie_wait", 0)),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def parse_gas_csv(path: str) -> list[dict]:
|
||||
"""Parse total_gas.csv into a list of per-block dicts."""
|
||||
rows = []
|
||||
with open(path) as f:
|
||||
reader = csv.DictReader(f)
|
||||
for row in reader:
|
||||
rows.append(
|
||||
{
|
||||
"block_number": int(row["block_number"]),
|
||||
"gas_used": int(row["gas_used"]),
|
||||
"time_us": int(row["time"]),
|
||||
}
|
||||
)
|
||||
return rows
|
||||
|
||||
|
||||
def stddev(values: list[float], mean: float) -> float:
|
||||
if len(values) < 2:
|
||||
return 0.0
|
||||
return math.sqrt(sum((v - mean) ** 2 for v in values) / (len(values) - 1))
|
||||
|
||||
|
||||
def percentile(sorted_vals: list[float], pct: int) -> float:
|
||||
if not sorted_vals:
|
||||
return 0.0
|
||||
idx = int(len(sorted_vals) * pct / 100)
|
||||
idx = min(idx, len(sorted_vals) - 1)
|
||||
return sorted_vals[idx]
|
||||
|
||||
|
||||
def compute_stats(combined: list[dict]) -> dict:
|
||||
"""Compute per-run statistics from parsed CSV data."""
|
||||
n = len(combined)
|
||||
if n == 0:
|
||||
return {}
|
||||
|
||||
latencies_ms = [r["new_payload_latency_us"] / 1_000 for r in combined]
|
||||
sorted_lat = sorted(latencies_ms)
|
||||
mean_lat = sum(latencies_ms) / n
|
||||
std_lat = stddev(latencies_ms, mean_lat)
|
||||
|
||||
mgas_s_values = []
|
||||
for r in combined:
|
||||
lat_s = r["new_payload_latency_us"] / 1_000_000
|
||||
if lat_s > 0:
|
||||
mgas_s_values.append(r["gas_used"] / lat_s / 1_000_000)
|
||||
mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_ms": mean_lat,
|
||||
"stddev_ms": std_lat,
|
||||
"p50_ms": percentile(sorted_lat, 50),
|
||||
"p90_ms": percentile(sorted_lat, 90),
|
||||
"p99_ms": percentile(sorted_lat, 99),
|
||||
"mean_mgas_s": mean_mgas_s,
|
||||
}
|
||||
|
||||
|
||||
def _paired_data(
|
||||
baseline: list[dict], branch: list[dict]
|
||||
) -> tuple[list[tuple[float, float]], list[float], list[float]]:
|
||||
"""Match blocks and return paired latencies and per-block diffs.
|
||||
|
||||
Returns:
|
||||
pairs: list of (baseline_ms, branch_ms) tuples
|
||||
lat_diffs_ms: list of branch − baseline latency diffs in ms
|
||||
mgas_diffs: list of branch − baseline Mgas/s diffs
|
||||
"""
|
||||
baseline_by_block = {r["block_number"]: r for r in baseline}
|
||||
branch_by_block = {r["block_number"]: r for r in branch}
|
||||
common_blocks = sorted(set(baseline_by_block) & set(branch_by_block))
|
||||
|
||||
pairs = []
|
||||
lat_diffs_ms = []
|
||||
mgas_diffs = []
|
||||
for bn in common_blocks:
|
||||
b = baseline_by_block[bn]
|
||||
f = branch_by_block[bn]
|
||||
b_ms = b["new_payload_latency_us"] / 1_000
|
||||
f_ms = f["new_payload_latency_us"] / 1_000
|
||||
pairs.append((b_ms, f_ms))
|
||||
lat_diffs_ms.append(f_ms - b_ms)
|
||||
b_lat_s = b["new_payload_latency_us"] / 1_000_000
|
||||
f_lat_s = f["new_payload_latency_us"] / 1_000_000
|
||||
if b_lat_s > 0 and f_lat_s > 0:
|
||||
mgas_diffs.append(
|
||||
f["gas_used"] / f_lat_s / 1_000_000
|
||||
- b["gas_used"] / b_lat_s / 1_000_000
|
||||
)
|
||||
return pairs, lat_diffs_ms, mgas_diffs
|
||||
|
||||
|
||||
def compute_paired_stats(
|
||||
baseline_runs: list[list[dict]],
|
||||
branch_runs: list[list[dict]],
|
||||
) -> dict:
|
||||
"""Compute paired statistics between baseline and branch runs.
|
||||
|
||||
Each pair (baseline_runs[i], branch_runs[i]) produces per-block diffs.
|
||||
All diffs are pooled for the final CI.
|
||||
"""
|
||||
all_pairs = []
|
||||
all_lat_diffs = []
|
||||
all_mgas_diffs = []
|
||||
for baseline, branch in zip(baseline_runs, branch_runs):
|
||||
pairs, lat_diffs, mgas_diffs = _paired_data(baseline, branch)
|
||||
all_pairs.extend(pairs)
|
||||
all_lat_diffs.extend(lat_diffs)
|
||||
all_mgas_diffs.extend(mgas_diffs)
|
||||
|
||||
if not all_lat_diffs:
|
||||
return {}
|
||||
|
||||
n = len(all_lat_diffs)
|
||||
mean_diff = sum(all_lat_diffs) / n
|
||||
std_diff = stddev(all_lat_diffs, mean_diff)
|
||||
se = std_diff / math.sqrt(n) if n > 0 else 0.0
|
||||
ci = T_CRITICAL * se
|
||||
|
||||
# Bootstrap CI on difference-of-percentiles (resample paired blocks)
|
||||
base_lats = sorted([p[0] for p in all_pairs])
|
||||
branch_lats = sorted([p[1] for p in all_pairs])
|
||||
p50_diff = percentile(branch_lats, 50) - percentile(base_lats, 50)
|
||||
p90_diff = percentile(branch_lats, 90) - percentile(base_lats, 90)
|
||||
p99_diff = percentile(branch_lats, 99) - percentile(base_lats, 99)
|
||||
|
||||
rng = random.Random(42)
|
||||
p50_boot, p90_boot, p99_boot = [], [], []
|
||||
for _ in range(BOOTSTRAP_ITERATIONS):
|
||||
sample = rng.choices(all_pairs, k=n)
|
||||
b_sorted = sorted(p[0] for p in sample)
|
||||
f_sorted = sorted(p[1] for p in sample)
|
||||
p50_boot.append(percentile(f_sorted, 50) - percentile(b_sorted, 50))
|
||||
p90_boot.append(percentile(f_sorted, 90) - percentile(b_sorted, 90))
|
||||
p99_boot.append(percentile(f_sorted, 99) - percentile(b_sorted, 99))
|
||||
p50_boot.sort()
|
||||
p90_boot.sort()
|
||||
p99_boot.sort()
|
||||
lo = int(BOOTSTRAP_ITERATIONS * 0.025)
|
||||
hi = int(BOOTSTRAP_ITERATIONS * 0.975)
|
||||
|
||||
mean_mgas_diff = sum(all_mgas_diffs) / len(all_mgas_diffs) if all_mgas_diffs else 0.0
|
||||
std_mgas_diff = stddev(all_mgas_diffs, mean_mgas_diff) if len(all_mgas_diffs) > 1 else 0.0
|
||||
mgas_se = std_mgas_diff / math.sqrt(len(all_mgas_diffs)) if all_mgas_diffs else 0.0
|
||||
mgas_ci = T_CRITICAL * mgas_se
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_diff_ms": mean_diff,
|
||||
"ci_ms": ci,
|
||||
"p50_diff_ms": p50_diff,
|
||||
"p50_ci_ms": (p50_boot[hi] - p50_boot[lo]) / 2,
|
||||
"p90_diff_ms": p90_diff,
|
||||
"p90_ci_ms": (p90_boot[hi] - p90_boot[lo]) / 2,
|
||||
"p99_diff_ms": p99_diff,
|
||||
"p99_ci_ms": (p99_boot[hi] - p99_boot[lo]) / 2,
|
||||
"mean_mgas_diff": mean_mgas_diff,
|
||||
"mgas_ci": mgas_ci,
|
||||
}
|
||||
|
||||
|
||||
def compute_summary(combined: list[dict], gas: list[dict]) -> dict:
|
||||
"""Compute aggregate metrics from parsed CSV data."""
|
||||
blocks = len(combined)
|
||||
return {
|
||||
"blocks": blocks,
|
||||
}
|
||||
|
||||
|
||||
def format_duration(seconds: float) -> str:
|
||||
if seconds >= 60:
|
||||
return f"{seconds / 60:.1f}min"
|
||||
return f"{seconds}s"
|
||||
|
||||
|
||||
def format_gas(gas: int) -> str:
|
||||
if gas >= GIGAGAS:
|
||||
return f"{gas / GIGAGAS:.1f}G"
|
||||
if gas >= 1_000_000:
|
||||
return f"{gas / 1_000_000:.1f}M"
|
||||
return f"{gas:,}"
|
||||
|
||||
|
||||
|
||||
def fmt_ms(v: float) -> str:
|
||||
return f"{v:.2f}ms"
|
||||
|
||||
|
||||
def fmt_mgas(v: float) -> str:
|
||||
return f"{v:.2f}"
|
||||
|
||||
|
||||
def change_str(pct: float, ci_pct: float, lower_is_better: bool) -> str:
|
||||
"""Format change% with paired CI significance.
|
||||
|
||||
Significant if the CI doesn't cross zero (i.e. |pct| > ci_pct).
|
||||
"""
|
||||
significant = abs(pct) > ci_pct
|
||||
if not significant:
|
||||
emoji = "⚪"
|
||||
elif (pct < 0) == lower_is_better:
|
||||
emoji = "✅"
|
||||
else:
|
||||
emoji = "❌"
|
||||
|
||||
return f"{pct:+.2f}% {emoji} (±{ci_pct:.2f}%)"
|
||||
|
||||
|
||||
def generate_comparison_table(
|
||||
run1: dict,
|
||||
run2: dict,
|
||||
paired: dict,
|
||||
repo: str,
|
||||
baseline_ref: str,
|
||||
branch_name: str,
|
||||
branch_sha: str,
|
||||
) -> str:
|
||||
"""Generate a markdown comparison table between baseline (main) and branch."""
|
||||
n = paired["n"]
|
||||
|
||||
def pct(base: float, feat: float) -> float:
|
||||
return (feat - base) / base * 100.0 if base > 0 else 0.0
|
||||
|
||||
mean_pct = pct(run1["mean_ms"], run2["mean_ms"])
|
||||
gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"])
|
||||
|
||||
p50_pct = pct(run1["p50_ms"], run2["p50_ms"])
|
||||
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
|
||||
p99_pct = pct(run1["p99_ms"], run2["p99_ms"])
|
||||
|
||||
# Bootstrap CIs as % of baseline percentile
|
||||
p50_ci_pct = paired["p50_ci_ms"] / run1["p50_ms"] * 100.0 if run1["p50_ms"] > 0 else 0.0
|
||||
p90_ci_pct = paired["p90_ci_ms"] / run1["p90_ms"] * 100.0 if run1["p90_ms"] > 0 else 0.0
|
||||
p99_ci_pct = paired["p99_ci_ms"] / run1["p99_ms"] * 100.0 if run1["p99_ms"] > 0 else 0.0
|
||||
|
||||
# CI as a percentage of baseline mean
|
||||
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
|
||||
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
|
||||
|
||||
base_url = f"https://github.com/{repo}/commit"
|
||||
baseline_label = f"[`main`]({base_url}/{baseline_ref})"
|
||||
branch_label = f"[`{branch_name}`]({base_url}/{branch_sha})"
|
||||
|
||||
lines = [
|
||||
f"| Metric | {baseline_label} | {branch_label} | Change |",
|
||||
"|--------|------|--------|--------|",
|
||||
f"| Mean | {fmt_ms(run1['mean_ms'])} | {fmt_ms(run2['mean_ms'])} | {change_str(mean_pct, lat_ci_pct, lower_is_better=True)} |",
|
||||
f"| StdDev | {fmt_ms(run1['stddev_ms'])} | {fmt_ms(run2['stddev_ms'])} | |",
|
||||
f"| P50 | {fmt_ms(run1['p50_ms'])} | {fmt_ms(run2['p50_ms'])} | {change_str(p50_pct, p50_ci_pct, lower_is_better=True)} |",
|
||||
f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, lower_is_better=True)} |",
|
||||
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
|
||||
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
|
||||
"",
|
||||
f"*{n} blocks*",
|
||||
]
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def generate_markdown(
|
||||
summary: dict, comparison_table: str,
|
||||
behind_main: int = 0, repo: str = "", baseline_ref: str = "",
|
||||
) -> str:
|
||||
"""Generate a markdown comment body."""
|
||||
lines = ["## Benchmark Results", "", comparison_table]
|
||||
if behind_main > 0:
|
||||
s = "s" if behind_main > 1 else ""
|
||||
diff_link = f"https://github.com/{repo}/compare/{baseline_ref[:12]}...main"
|
||||
lines.append("")
|
||||
lines.append(f"> ⚠️ Branch is [**{behind_main} commit{s} behind `main`**]({diff_link}). Consider rebasing for accurate results.")
|
||||
return "\n".join(lines)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser(description="Parse reth-bench ABBA results")
|
||||
parser.add_argument(
|
||||
"--baseline-csv", nargs="+", required=True,
|
||||
help="Baseline combined_latency.csv files (A1, A2)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--branch-csv", nargs="+", required=True,
|
||||
help="Branch combined_latency.csv files (B1, B2)",
|
||||
)
|
||||
parser.add_argument("--gas-csv", required=True, help="Path to total_gas.csv")
|
||||
parser.add_argument(
|
||||
"--output-summary", required=True, help="Output JSON summary path"
|
||||
)
|
||||
parser.add_argument("--output-markdown", required=True, help="Output markdown path")
|
||||
parser.add_argument(
|
||||
"--repo", default="paradigmxyz/reth", help="GitHub repo (owner/name)"
|
||||
)
|
||||
parser.add_argument("--baseline-ref", default=None, help="Baseline commit SHA")
|
||||
parser.add_argument("--branch-name", default=None, help="Branch name")
|
||||
parser.add_argument("--branch-sha", default=None, help="Branch commit SHA")
|
||||
parser.add_argument("--behind-main", type=int, default=0, help="Commits behind main")
|
||||
args = parser.parse_args()
|
||||
|
||||
if len(args.baseline_csv) != len(args.branch_csv):
|
||||
print("Must provide equal number of baseline and branch CSVs", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
baseline_runs = []
|
||||
branch_runs = []
|
||||
for path in args.baseline_csv:
|
||||
data = parse_combined_csv(path)
|
||||
if not data:
|
||||
print(f"No results in {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
baseline_runs.append(data)
|
||||
for path in args.branch_csv:
|
||||
data = parse_combined_csv(path)
|
||||
if not data:
|
||||
print(f"No results in {path}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
branch_runs.append(data)
|
||||
|
||||
gas = parse_gas_csv(args.gas_csv)
|
||||
|
||||
all_baseline = [r for run in baseline_runs for r in run]
|
||||
all_branch = [r for run in branch_runs for r in run]
|
||||
|
||||
summary = compute_summary(all_branch, gas)
|
||||
with open(args.output_summary, "w") as f:
|
||||
json.dump(summary, f, indent=2)
|
||||
print(f"Summary written to {args.output_summary}")
|
||||
|
||||
baseline_stats = compute_stats(all_baseline)
|
||||
branch_stats = compute_stats(all_branch)
|
||||
paired_stats = compute_paired_stats(baseline_runs, branch_runs)
|
||||
|
||||
if not paired_stats:
|
||||
print("No common blocks between baseline and branch runs", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
comparison_table = generate_comparison_table(
|
||||
baseline_stats,
|
||||
branch_stats,
|
||||
paired_stats,
|
||||
repo=args.repo,
|
||||
baseline_ref=args.baseline_ref or "main",
|
||||
branch_name=args.branch_name or "branch",
|
||||
branch_sha=args.branch_sha or "unknown",
|
||||
)
|
||||
print(f"Generated comparison ({paired_stats['n']} paired blocks, "
|
||||
f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")
|
||||
|
||||
markdown = generate_markdown(
|
||||
summary, comparison_table,
|
||||
behind_main=args.behind_main,
|
||||
repo=args.repo,
|
||||
baseline_ref=args.baseline_ref or "",
|
||||
)
|
||||
|
||||
with open(args.output_markdown, "w") as f:
|
||||
f.write(markdown)
|
||||
print(f"Markdown written to {args.output_markdown}")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
369
.github/workflows/bench.yml
vendored
369
.github/workflows/bench.yml
vendored
@@ -1,11 +1,25 @@
|
||||
# Runs benchmarks.
|
||||
#
|
||||
# The reth-bench job replays real blocks via the Engine API against a reth node
|
||||
# backed by a local snapshot managed with schelk.
|
||||
#
|
||||
# It runs the main (baseline) binary and the branch (candidate) binary on the
|
||||
# same block range (snapshot recovered between runs) to compare performance.
|
||||
|
||||
on:
|
||||
pull_request:
|
||||
# TODO: Disabled temporarily for https://github.com/CodSpeedHQ/runner/issues/55
|
||||
# merge_group:
|
||||
push:
|
||||
branches: [main]
|
||||
issue_comment:
|
||||
types: [created, edited]
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
blocks:
|
||||
description: "Number of blocks to benchmark"
|
||||
required: false
|
||||
default: "50"
|
||||
type: string
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
@@ -14,8 +28,18 @@ env:
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
name: bench
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
|
||||
concurrency:
|
||||
group: bench-${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
codspeed:
|
||||
if: github.event_name != 'issue_comment'
|
||||
runs-on: depot-ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -31,6 +55,7 @@ jobs:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/merge', github.event.issue.number) || '' }}
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
@@ -49,3 +74,345 @@ jobs:
|
||||
run: cargo codspeed run ${{ matrix.crates }}
|
||||
mode: instrumentation
|
||||
token: ${{ secrets.CODSPEED_TOKEN }}
|
||||
|
||||
reth-bench:
|
||||
if: github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, 'derek bench')
|
||||
name: reth-bench
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
SCHELK_MOUNT: /reth-bench
|
||||
steps:
|
||||
- name: Check org membership
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
const user = context.payload.comment.user.login;
|
||||
try {
|
||||
const { status } = await github.rest.orgs.checkMembershipForUser({
|
||||
org: 'paradigmxyz',
|
||||
username: user,
|
||||
});
|
||||
if (status !== 204 && status !== 302) {
|
||||
core.setFailed(`@${user} is not a member of paradigmxyz`);
|
||||
}
|
||||
} catch (e) {
|
||||
core.setFailed(`@${user} is not a member of paradigmxyz`);
|
||||
}
|
||||
|
||||
- name: Parse arguments
|
||||
id: args
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const body = context.payload.comment.body.trim();
|
||||
const known = new Set(['blocks', 'warmup']);
|
||||
const defaults = { blocks: '500', warmup: '100' };
|
||||
const unknown = [];
|
||||
const invalid = [];
|
||||
const args = body.replace(/^derek bench\s*/, '');
|
||||
for (const part of args.split(/\s+/).filter(Boolean)) {
|
||||
const eq = part.indexOf('=');
|
||||
if (eq === -1) {
|
||||
unknown.push(part);
|
||||
continue;
|
||||
}
|
||||
const key = part.slice(0, eq);
|
||||
const value = part.slice(eq + 1);
|
||||
if (!known.has(key)) {
|
||||
unknown.push(key);
|
||||
} else if (!/^\d+$/.test(value)) {
|
||||
invalid.push(`\`${key}=${value}\` (must be a positive integer)`);
|
||||
} else {
|
||||
defaults[key] = value;
|
||||
}
|
||||
}
|
||||
const errors = [];
|
||||
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
|
||||
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
|
||||
if (errors.length) {
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`derek bench [blocks=N] [warmup=N]\``;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body: msg,
|
||||
});
|
||||
core.setFailed(msg);
|
||||
return;
|
||||
}
|
||||
core.setOutput('blocks', defaults.blocks);
|
||||
core.setOutput('warmup', defaults.warmup);
|
||||
core.exportVariable('BENCH_BLOCKS', defaults.blocks);
|
||||
core.exportVariable('BENCH_WARMUP_BLOCKS', defaults.warmup);
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
await github.rest.reactions.createForIssueComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: context.payload.comment.id,
|
||||
content: 'eyes',
|
||||
});
|
||||
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const blocks = '${{ steps.args.outputs.blocks }}';
|
||||
const warmup = '${{ steps.args.outputs.warmup }}';
|
||||
const { data: comment } = await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body: `🚀 Benchmark started! [View run](${runUrl})\n\n⏳ **Status:** Building binaries...\n\n**Config:** ${blocks} blocks, ${warmup} warmup blocks`,
|
||||
});
|
||||
core.setOutput('comment-id', comment.id);
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ format('refs/pull/{0}/merge', github.event.issue.number) }}
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
continue-on-error: true
|
||||
|
||||
# Verify all required tools are available
|
||||
- name: Check dependencies
|
||||
run: |
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
echo "::error::Missing required tools: ${missing[*]}"
|
||||
exit 1
|
||||
fi
|
||||
echo "All dependencies found"
|
||||
|
||||
# Build binaries
|
||||
- name: Fetch or build main binaries
|
||||
run: |
|
||||
MERGE_BASE=$(git merge-base HEAD origin/main 2>/dev/null || echo "${{ github.sha }}")
|
||||
.github/scripts/bench-reth-build.sh main "$MERGE_BASE"
|
||||
- name: Fetch or build branch binaries
|
||||
run: |
|
||||
BRANCH_SHA="${{ github.sha }}"
|
||||
.github/scripts/bench-reth-build.sh branch "$BRANCH_SHA"
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
sudo cpupower frequency-set -g performance || true
|
||||
# Disable turbo boost (Intel and AMD paths)
|
||||
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
|
||||
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
|
||||
sudo swapoff -a || true
|
||||
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
|
||||
# Disable SMT (hyperthreading)
|
||||
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
|
||||
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
|
||||
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
|
||||
if [ "$current" != "$first" ]; then
|
||||
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
|
||||
fi
|
||||
done
|
||||
echo "Online CPUs: $(nproc)"
|
||||
# Disable transparent huge pages (compaction causes latency spikes)
|
||||
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
||||
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
||||
done || true
|
||||
# Prevent deep C-states (avoids wake-up latency jitter)
|
||||
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
||||
# Move all IRQs to core 0 (housekeeping core)
|
||||
for irq in /proc/irq/*/smp_affinity_list; do
|
||||
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
||||
done
|
||||
# Stop noisy background services
|
||||
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
|
||||
# Log environment for reproducibility
|
||||
echo "=== Benchmark environment ==="
|
||||
uname -r
|
||||
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
|
||||
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
|
||||
free -h
|
||||
|
||||
# Clean up any leftover state
|
||||
- name: Pre-flight cleanup
|
||||
run: |
|
||||
pkill -9 reth || true
|
||||
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
|
||||
|
||||
- name: Update status (running benchmarks)
|
||||
if: steps.ack.outputs.comment-id
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: ${{ steps.ack.outputs.comment-id || 0 }},
|
||||
body: `🚀 Benchmark started! [View run](${runUrl})\n\n⏳ **Status:** Running benchmarks (2 runs)...`,
|
||||
});
|
||||
|
||||
- name: "Run benchmark: baseline"
|
||||
run: taskset -c 0 .github/scripts/bench-reth-run.sh baseline target/profiling-baseline/reth /tmp/bench-results-baseline
|
||||
|
||||
- name: "Run benchmark: branch"
|
||||
run: taskset -c 0 .github/scripts/bench-reth-run.sh branch target/profiling/reth /tmp/bench-results-branch
|
||||
|
||||
# Results & charts
|
||||
- name: Parse results
|
||||
id: results
|
||||
if: success()
|
||||
env:
|
||||
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
|
||||
BRANCH_SHA: ${{ github.sha }}
|
||||
run: |
|
||||
git fetch origin main --quiet
|
||||
# Use the actual PR head commit, not HEAD (which is the merge commit
|
||||
# refs/pull/N/merge and always has origin/main as a parent).
|
||||
MERGE_BASE=$(git merge-base "${BRANCH_SHA}" origin/main 2>/dev/null || echo "${{ github.sha }}")
|
||||
MAIN_HEAD=$(git rev-parse origin/main 2>/dev/null || echo "")
|
||||
BEHIND_MAIN=0
|
||||
if [ -n "$MAIN_HEAD" ] && [ "$MERGE_BASE" != "$MAIN_HEAD" ]; then
|
||||
BEHIND_MAIN=$(git rev-list --count "${MERGE_BASE}..${MAIN_HEAD}" 2>/dev/null || echo "0")
|
||||
fi
|
||||
|
||||
SUMMARY_ARGS="--output-summary /tmp/bench-summary.json"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown /tmp/bench-comment.md"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${MERGE_BASE}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --branch-name ${BRANCH_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --branch-sha ${BRANCH_SHA}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv /tmp/bench-results-baseline/combined_latency.csv"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --branch-csv /tmp/bench-results-branch/combined_latency.csv"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv /tmp/bench-results-branch/total_gas.csv"
|
||||
if [ "$BEHIND_MAIN" -gt 0 ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --behind-main $BEHIND_MAIN"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
|
||||
|
||||
- name: Generate charts
|
||||
if: success()
|
||||
env:
|
||||
BRANCH_NAME: ${{ github.head_ref || github.ref_name }}
|
||||
run: |
|
||||
CHART_ARGS="/tmp/bench-results-branch/combined_latency.csv --output-dir /tmp/bench-charts"
|
||||
CHART_ARGS="$CHART_ARGS --baseline /tmp/bench-results-baseline/combined_latency.csv"
|
||||
CHART_ARGS="$CHART_ARGS --branch-name ${BRANCH_NAME}"
|
||||
# shellcheck disable=SC2086
|
||||
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
|
||||
|
||||
- name: Upload results
|
||||
if: success()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: bench-reth-results
|
||||
path: |
|
||||
/tmp/bench-results-baseline/
|
||||
/tmp/bench-results-branch/
|
||||
/tmp/bench-summary.json
|
||||
/tmp/bench-charts/
|
||||
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success()
|
||||
run: |
|
||||
PR_NUMBER=${{ github.event.issue.number }}
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}"
|
||||
|
||||
if git fetch origin bench-charts 2>/dev/null; then
|
||||
git checkout bench-charts
|
||||
else
|
||||
git checkout --orphan bench-charts
|
||||
git rm -rf . 2>/dev/null || true
|
||||
fi
|
||||
|
||||
mkdir -p "${CHART_DIR}"
|
||||
cp /tmp/bench-charts/*.png "${CHART_DIR}/"
|
||||
git add "${CHART_DIR}"
|
||||
git -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
||||
commit -m "bench charts for PR #${PR_NUMBER} run ${RUN_ID}"
|
||||
git push origin bench-charts
|
||||
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Compare & comment
|
||||
if: success()
|
||||
uses: actions/github-script@v7
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
|
||||
let comment = '';
|
||||
try {
|
||||
comment = fs.readFileSync('/tmp/bench-comment.md', 'utf8');
|
||||
} catch (e) {
|
||||
comment = '⚠️ Engine benchmark completed but failed to generate comparison.';
|
||||
}
|
||||
|
||||
const sha = '${{ steps.push-charts.outputs.sha }}';
|
||||
const prNumber = context.issue.number;
|
||||
const runId = '${{ github.run_id }}';
|
||||
const baseUrl = `https://raw.githubusercontent.com/${context.repo.owner}/${context.repo.repo}/${sha}/pr/${prNumber}/${runId}`;
|
||||
|
||||
const charts = [
|
||||
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
|
||||
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
|
||||
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
|
||||
];
|
||||
|
||||
let chartMarkdown = '\n\n### Charts\n\n';
|
||||
for (const chart of charts) {
|
||||
chartMarkdown += `<details><summary>${chart.label}</summary>\n\n`;
|
||||
chartMarkdown += `\n\n`;
|
||||
chartMarkdown += `</details>\n\n`;
|
||||
}
|
||||
|
||||
comment += chartMarkdown;
|
||||
|
||||
const requestedBy = '${{ github.event.comment.user.login }}';
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
const body = `cc @${requestedBy}\n\n✅ Benchmark complete! [View run](${runUrl})\n\n${comment}`;
|
||||
const ackCommentId = '${{ steps.ack.outputs.comment-id }}';
|
||||
|
||||
if (ackCommentId) {
|
||||
await github.rest.issues.updateComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
comment_id: parseInt(ackCommentId),
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
- name: Upload node log
|
||||
if: failure()
|
||||
uses: actions/upload-artifact@v4
|
||||
with:
|
||||
name: reth-node-log
|
||||
path: |
|
||||
/tmp/reth-bench-node-baseline.log
|
||||
/tmp/reth-bench-node-branch.log
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|
||||
|
||||
2
.github/workflows/book.yml
vendored
2
.github/workflows/book.yml
vendored
@@ -15,7 +15,7 @@ env:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: depot-ubuntu-latest-8
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Checkout
|
||||
|
||||
2
.github/workflows/check-alloy.yml
vendored
2
.github/workflows/check-alloy.yml
vendored
@@ -25,7 +25,7 @@ env:
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: depot-ubuntu-latest-16
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
2
.github/workflows/compact.yml
vendored
2
.github/workflows/compact.yml
vendored
@@ -18,7 +18,7 @@ env:
|
||||
name: compact-codec
|
||||
jobs:
|
||||
compact-codec:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
matrix:
|
||||
bin:
|
||||
|
||||
33
.github/workflows/docker-test.yml
vendored
33
.github/workflows/docker-test.yml
vendored
@@ -15,7 +15,6 @@ on:
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
@@ -31,10 +30,22 @@ jobs:
|
||||
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
|
||||
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Detect fork
|
||||
id: fork
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
# Depot build (upstream only)
|
||||
- name: Set up Depot CLI
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/setup-action@v1
|
||||
|
||||
- name: Build reth image
|
||||
- name: Build reth image (Depot)
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/bake-action@v1
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
@@ -46,6 +57,24 @@ jobs:
|
||||
targets: ${{ inputs.hive_target }}
|
||||
push: false
|
||||
|
||||
# Docker build (forks)
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build reth image (Docker)
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/bake-action@v6
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
with:
|
||||
files: docker-bake.hcl
|
||||
targets: ${{ inputs.hive_target }}
|
||||
push: false
|
||||
set: |
|
||||
*.dockerfile=Dockerfile
|
||||
|
||||
- name: Upload reth image
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
|
||||
4
.github/workflows/e2e.yml
vendored
4
.github/workflows/e2e.yml
vendored
@@ -20,7 +20,7 @@ concurrency:
|
||||
jobs:
|
||||
test:
|
||||
name: e2e-testsuite
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 90
|
||||
@@ -47,7 +47,7 @@ jobs:
|
||||
|
||||
rocksdb:
|
||||
name: e2e-rocksdb
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
|
||||
4
.github/workflows/hive.yml
vendored
4
.github/workflows/hive.yml
vendored
@@ -32,7 +32,7 @@ jobs:
|
||||
prepare-hive:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Checkout hive tests
|
||||
@@ -188,7 +188,7 @@ jobs:
|
||||
- build-reth-edge
|
||||
- prepare-hive
|
||||
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
issues: write
|
||||
steps:
|
||||
|
||||
2
.github/workflows/integration.yml
vendored
2
.github/workflows/integration.yml
vendored
@@ -24,7 +24,7 @@ jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
strategy:
|
||||
|
||||
2
.github/workflows/kurtosis.yml
vendored
2
.github/workflows/kurtosis.yml
vendored
@@ -31,7 +31,7 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
name: run kurtosis
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
needs:
|
||||
- build-reth
|
||||
steps:
|
||||
|
||||
26
.github/workflows/lint.yml
vendored
26
.github/workflows/lint.yml
vendored
@@ -13,7 +13,7 @@ env:
|
||||
jobs:
|
||||
clippy-binaries:
|
||||
name: clippy binaries / ${{ matrix.type }}
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -42,7 +42,7 @@ jobs:
|
||||
|
||||
clippy:
|
||||
name: clippy
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -59,7 +59,7 @@ jobs:
|
||||
RUSTFLAGS: -D warnings
|
||||
|
||||
wasm:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -79,7 +79,7 @@ jobs:
|
||||
.github/scripts/check_wasm.sh
|
||||
|
||||
riscv:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -98,7 +98,7 @@ jobs:
|
||||
|
||||
crate-checks:
|
||||
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2, 3]
|
||||
@@ -117,14 +117,14 @@ jobs:
|
||||
|
||||
msrv:
|
||||
name: MSRV
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: "1.88" # MSRV
|
||||
toolchain: "1.93" # MSRV
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
@@ -135,7 +135,7 @@ jobs:
|
||||
|
||||
docs:
|
||||
name: docs
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -153,7 +153,7 @@ jobs:
|
||||
|
||||
fmt:
|
||||
name: fmt
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -167,7 +167,7 @@ jobs:
|
||||
|
||||
udeps:
|
||||
name: udeps
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -182,7 +182,7 @@ jobs:
|
||||
|
||||
book:
|
||||
name: book
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -240,7 +240,7 @@ jobs:
|
||||
# Checks that selected crates can compile with power set of features
|
||||
features:
|
||||
name: features
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -264,7 +264,7 @@ jobs:
|
||||
|
||||
# Check crates correctly propagate features
|
||||
feature-propagation:
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@@ -102,7 +102,7 @@ jobs:
|
||||
- name: Install cross main
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --git https://github.com/cross-rs/cross
|
||||
cargo install cross --locked --git https://github.com/cross-rs/cross
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
4
.github/workflows/stage.yml
vendored
4
.github/workflows/stage.yml
vendored
@@ -23,7 +23,7 @@ jobs:
|
||||
name: stage-run-test
|
||||
# Only run stage commands test in merge groups
|
||||
if: github.event_name == 'merge_group'
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -38,7 +38,7 @@ jobs:
|
||||
cache-on-failure: true
|
||||
- name: Build reth
|
||||
run: |
|
||||
cargo install --path bin/reth
|
||||
cargo install --locked --path bin/reth
|
||||
- name: Run headers stage
|
||||
run: |
|
||||
reth stage run headers --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
|
||||
|
||||
2
.github/workflows/sync-era.yml
vendored
2
.github/workflows/sync-era.yml
vendored
@@ -19,7 +19,7 @@ jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
2
.github/workflows/sync.yml
vendored
2
.github/workflows/sync.yml
vendored
@@ -19,7 +19,7 @@ jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
|
||||
6
.github/workflows/unit.yml
vendored
6
.github/workflows/unit.yml
vendored
@@ -20,7 +20,7 @@ concurrency:
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.type }} / ${{ matrix.storage }}
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
|
||||
@@ -57,7 +57,7 @@ jobs:
|
||||
|
||||
state:
|
||||
name: Ethereum state tests
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -92,7 +92,7 @@ jobs:
|
||||
|
||||
doc:
|
||||
name: doc tests
|
||||
runs-on: depot-ubuntu-latest
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 30
|
||||
|
||||
68
CLAUDE.md
68
CLAUDE.md
@@ -313,6 +313,74 @@ GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst);
|
||||
Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful?
|
||||
|
||||
|
||||
#### Rust Style Guides
|
||||
|
||||
##### Type Ordering in Files
|
||||
|
||||
When defining structs, traits, and functions in a file, follow this ordering convention. The file's primary type (matching the file name) comes first, followed by supporting public types, then private types and helpers.
|
||||
|
||||
```rust
|
||||
use ...;
|
||||
|
||||
/// The primary type of this file (matches filename).
|
||||
pub struct PayloadProcessor { ... }
|
||||
|
||||
impl PayloadProcessor { ... }
|
||||
|
||||
// Followed by public auxiliary types that support the primary type
|
||||
|
||||
/// Configuration for the processor.
|
||||
pub struct PayloadProcessorConfig { ... }
|
||||
|
||||
/// Result type returned by processor operations.
|
||||
pub struct ProcessorResult { ... }
|
||||
|
||||
// Followed by public traits related to the primary type
|
||||
|
||||
pub trait ProcessorExt { ... }
|
||||
|
||||
// Followed by private helper types
|
||||
|
||||
struct InternalState { ... }
|
||||
|
||||
// Followed by private helper functions
|
||||
|
||||
fn validate_input() { ... }
|
||||
```
|
||||
|
||||
❌ **Bad**: Adding new traits and auxiliary types **above** the file's primary type (see [#22133](https://github.com/paradigmxyz/reth/pull/22133)):
|
||||
|
||||
```rust
|
||||
use ...;
|
||||
|
||||
// ❌ BAD - new auxiliary struct added before the file's main type
|
||||
pub struct CacheWaitDurations { ... }
|
||||
|
||||
// ❌ BAD - new trait added before the file's main type
|
||||
pub trait WaitForCaches { ... }
|
||||
|
||||
// The file's primary type is buried below unrelated additions
|
||||
pub struct PayloadProcessor { ... }
|
||||
```
|
||||
|
||||
✅ **Good**: New types go **after** the primary type:
|
||||
|
||||
```rust
|
||||
use ...;
|
||||
|
||||
// ✅ The file's primary type stays at the top
|
||||
pub struct PayloadProcessor { ... }
|
||||
|
||||
impl PayloadProcessor { ... }
|
||||
|
||||
// ✅ Auxiliary types follow the primary type
|
||||
pub struct CacheWaitDurations { ... }
|
||||
|
||||
pub trait WaitForCaches { ... }
|
||||
|
||||
impl WaitForCaches for PayloadProcessor { ... }
|
||||
```
|
||||
|
||||
### Example Contribution Workflow
|
||||
|
||||
Let's say you want to fix a bug where external IP resolution fails on startup:
|
||||
|
||||
626
Cargo.lock
generated
626
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
57
Cargo.toml
57
Cargo.toml
@@ -1,7 +1,7 @@
|
||||
[workspace.package]
|
||||
version = "1.11.3"
|
||||
version = "1.11.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.88"
|
||||
rust-version = "1.93"
|
||||
license = "MIT OR Apache-2.0"
|
||||
homepage = "https://paradigmxyz.github.io/reth"
|
||||
repository = "https://github.com/paradigmxyz/reth"
|
||||
@@ -27,7 +27,6 @@ members = [
|
||||
"crates/engine/invalid-block-hooks/",
|
||||
"crates/engine/local",
|
||||
"crates/engine/primitives/",
|
||||
"crates/engine/service",
|
||||
"crates/engine/tree/",
|
||||
"crates/engine/util/",
|
||||
"crates/era",
|
||||
@@ -349,7 +348,6 @@ reth-ecies = { path = "crates/net/ecies" }
|
||||
reth-engine-local = { path = "crates/engine/local" }
|
||||
reth-engine-primitives = { path = "crates/engine/primitives", default-features = false }
|
||||
reth-engine-tree = { path = "crates/engine/tree" }
|
||||
reth-engine-service = { path = "crates/engine/service" }
|
||||
reth-engine-util = { path = "crates/engine/util" }
|
||||
reth-era = { path = "crates/era" }
|
||||
reth-era-downloader = { path = "crates/era-downloader" }
|
||||
@@ -449,18 +447,14 @@ revm-inspectors = "0.34.2"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.6"
|
||||
alloy-primitives = { version = "1.5.6", default-features = false, features = [
|
||||
"map-foldhash",
|
||||
] }
|
||||
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.27.2", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = [
|
||||
"core-net",
|
||||
] }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
@@ -472,15 +466,10 @@ alloy-genesis = { version = "1.6.3", default-features = false }
|
||||
alloy-json-rpc = { version = "1.6.3", default-features = false }
|
||||
alloy-network = { version = "1.6.3", default-features = false }
|
||||
alloy-network-primitives = { version = "1.6.3", default-features = false }
|
||||
alloy-provider = { version = "1.6.3", features = [
|
||||
"reqwest",
|
||||
"debug-api",
|
||||
], default-features = false }
|
||||
alloy-provider = { version = "1.6.3", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-client = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types = { version = "1.6.3", features = [
|
||||
"eth",
|
||||
], default-features = false }
|
||||
alloy-rpc-types = { version = "1.6.3", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.6.3", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.6.3", default-features = false }
|
||||
@@ -494,9 +483,7 @@ alloy-serde = { version = "1.6.3", default-features = false }
|
||||
alloy-signer = { version = "1.6.3", default-features = false }
|
||||
alloy-signer-local = { version = "1.6.3", default-features = false }
|
||||
alloy-transport = { version = "1.6.3" }
|
||||
alloy-transport-http = { version = "1.6.3", features = [
|
||||
"reqwest-rustls-tls",
|
||||
], default-features = false }
|
||||
alloy-transport-http = { version = "1.6.3", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.6.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.6.3", default-features = false }
|
||||
|
||||
@@ -515,10 +502,7 @@ either = { version = "1.15.0", default-features = false }
|
||||
arrayvec = { version = "0.7.6", default-features = false }
|
||||
aquamarine = "0.6"
|
||||
auto_impl = "1"
|
||||
backon = { version = "1.2", default-features = false, features = [
|
||||
"std-blocking-sleep",
|
||||
"tokio-sleep",
|
||||
] }
|
||||
backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] }
|
||||
bincode = "1.3"
|
||||
bitflags = "2.4"
|
||||
boyer-moore-magiclen = "0.2.16"
|
||||
@@ -540,14 +524,11 @@ itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
lz4 = "1.28.1"
|
||||
modular-bitfield = "0.13.1"
|
||||
notify = { version = "8.0.0", default-features = false, features = [
|
||||
"macos_fsevent",
|
||||
] }
|
||||
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
|
||||
nybbles = { version = "0.4.8", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = [
|
||||
"critical-section",
|
||||
] }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
parking_lot = "0.12"
|
||||
quanta = "0.12"
|
||||
paste = "1.0"
|
||||
rand = "0.9"
|
||||
rayon = "1.7"
|
||||
@@ -565,9 +546,7 @@ strum_macros = "0.27"
|
||||
syn = "2.0"
|
||||
thiserror = { version = "2.0.0", default-features = false }
|
||||
tar = "0.4.44"
|
||||
tracing = { version = "0.1.0", default-features = false, features = [
|
||||
"attributes",
|
||||
] }
|
||||
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
|
||||
tracing-appender = "0.2"
|
||||
url = { version = "2.3", default-features = false }
|
||||
zstd = "0.13"
|
||||
@@ -605,11 +584,7 @@ futures-util = { version = "0.3", default-features = false }
|
||||
hyper = "1.3"
|
||||
hyper-util = "0.1.5"
|
||||
pin-project = "1.0.12"
|
||||
reqwest = { version = "0.12", default-features = false, features = [
|
||||
"rustls-tls",
|
||||
"rustls-tls-native-roots",
|
||||
"stream",
|
||||
] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
|
||||
tracing-futures = "0.2"
|
||||
tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
@@ -634,10 +609,7 @@ proptest-arbitrary-interop = "0.1.0"
|
||||
# crypto
|
||||
enr = { version = "0.13", default-features = false }
|
||||
k256 = { version = "0.13", default-features = false, features = ["ecdsa"] }
|
||||
secp256k1 = { version = "0.30", default-features = false, features = [
|
||||
"global-context",
|
||||
"recovery",
|
||||
] }
|
||||
secp256k1 = { version = "0.30", default-features = false, features = ["global-context", "recovery"] }
|
||||
# rand 8 for secp256k1
|
||||
rand_08 = { package = "rand", version = "0.8" }
|
||||
|
||||
@@ -692,6 +664,7 @@ cipher = "0.4.3"
|
||||
comfy-table = "7.0"
|
||||
concat-kdf = "0.1.0"
|
||||
crossbeam-channel = "0.5.13"
|
||||
crossbeam-utils = "0.8"
|
||||
crossterm = "0.29.0"
|
||||
csv = "1.3.0"
|
||||
ctrlc = "3.4"
|
||||
|
||||
@@ -19,10 +19,11 @@ pre-build = [
|
||||
image = "ubuntu:24.04"
|
||||
pre-build = [
|
||||
"apt update",
|
||||
"apt install --yes gcc gcc-riscv64-linux-gnu libclang-dev make",
|
||||
"apt install --yes gcc gcc-riscv64-linux-gnu g++-riscv64-linux-gnu libclang-dev make",
|
||||
]
|
||||
env.passthrough = [
|
||||
"CARGO_TARGET_RISCV64GC_UNKNOWN_LINUX_GNU_LINKER=riscv64-linux-gnu-gcc",
|
||||
"CXX_riscv64gc_unknown_linux_gnu=riscv64-linux-gnu-g++",
|
||||
]
|
||||
|
||||
[build.env]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# syntax=docker.io/docker/dockerfile:1.7-labs
|
||||
|
||||
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
|
||||
FROM lukemathwalker/cargo-chef:latest-rust-1.93 AS chef
|
||||
WORKDIR /app
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
# Usage:
|
||||
# reth: --build-arg BINARY=reth
|
||||
|
||||
FROM rust:1 AS builder
|
||||
FROM rust:1.93 AS builder
|
||||
WORKDIR /app
|
||||
|
||||
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
|
||||
|
||||
4
Makefile
4
Makefile
@@ -80,7 +80,7 @@ build-native-%:
|
||||
#
|
||||
# These commands require that:
|
||||
#
|
||||
# - `cross` is installed (`cargo install cross`).
|
||||
# - `cross` is installed (`cargo install --locked cross`).
|
||||
# - Docker is running.
|
||||
# - The current user is in the `docker` group.
|
||||
#
|
||||
@@ -261,7 +261,7 @@ lint-typos: ensure-typos
|
||||
|
||||
ensure-typos:
|
||||
@if ! command -v typos &> /dev/null; then \
|
||||
echo "typos not found. Please install it by running the command 'cargo install typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \
|
||||
echo "typos not found. Please install it by running the command 'cargo install --locked typos-cli' or refer to the following link for more information: https://github.com/crate-ci/typos"; \
|
||||
exit 1; \
|
||||
fi
|
||||
|
||||
|
||||
@@ -93,7 +93,7 @@ When updating this, also update:
|
||||
- .github/workflows/lint.yml
|
||||
-->
|
||||
|
||||
The Minimum Supported Rust Version (MSRV) of this project is [1.88.0](https://blog.rust-lang.org/2025/06/26/Rust-1.88.0/).
|
||||
The Minimum Supported Rust Version (MSRV) of this project is [1.93.0](https://blog.rust-lang.org/2026/01/22/Rust-1.93.0/).
|
||||
|
||||
See the docs for detailed instructions on how to [build from source](https://reth.rs/installation/source/).
|
||||
|
||||
|
||||
@@ -29,6 +29,8 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) next_block: u64,
|
||||
/// Whether the chain is an OP rollup.
|
||||
pub(crate) is_optimism: bool,
|
||||
/// Whether to use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
pub(crate) use_reth_namespace: bool,
|
||||
}
|
||||
|
||||
impl BenchContext {
|
||||
@@ -140,6 +142,14 @@ impl BenchContext {
|
||||
};
|
||||
|
||||
let next_block = first_block.header.number + 1;
|
||||
Ok(Self { auth_provider, block_provider, benchmark_mode, next_block, is_optimism })
|
||||
let use_reth_namespace = bench_args.reth_new_payload;
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
benchmark_mode,
|
||||
next_block,
|
||||
is_optimism,
|
||||
use_reth_namespace,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use crate::{
|
||||
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
|
||||
output::GasRampPayloadFile,
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload_with_reth, payload_to_new_payload},
|
||||
};
|
||||
use alloy_eips::BlockNumberOrTag;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
@@ -47,6 +47,14 @@ pub struct Command {
|
||||
/// Output directory for benchmark results and generated payloads.
|
||||
#[arg(long, value_name = "OUTPUT")]
|
||||
output: PathBuf,
|
||||
|
||||
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
///
|
||||
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
|
||||
/// directly, waits for persistence and cache updates to complete before processing,
|
||||
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
@@ -138,6 +146,9 @@ impl Command {
|
||||
);
|
||||
}
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload endpoint");
|
||||
}
|
||||
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
@@ -163,7 +174,7 @@ impl Command {
|
||||
// Regenerate the payload from the modified block, but keep the original sidecar
|
||||
// which contains the actual execution requests data (not just the hash)
|
||||
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
|
||||
let (version, params) = payload_to_new_payload(
|
||||
let (version, params, execution_data) = payload_to_new_payload(
|
||||
payload,
|
||||
sidecar,
|
||||
false,
|
||||
@@ -174,13 +185,18 @@ impl Command {
|
||||
// Save payload to file with version info for replay
|
||||
let payload_path =
|
||||
self.output.join(format!("payload_block_{}.json", block.header.number));
|
||||
let file =
|
||||
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
|
||||
let file = GasRampPayloadFile {
|
||||
version: version as u8,
|
||||
block_hash,
|
||||
params: params.clone(),
|
||||
execution_data: Some(execution_data.clone()),
|
||||
};
|
||||
let payload_json = serde_json::to_string_pretty(&file)?;
|
||||
std::fs::write(&payload_path, &payload_json)?;
|
||||
info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
|
||||
|
||||
call_new_payload(&provider, version, params).await?;
|
||||
let reth_data = self.reth_new_payload.then_some(execution_data);
|
||||
let _ = call_new_payload_with_reth(&provider, version, params, reth_data).await?;
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
|
||||
@@ -20,7 +20,7 @@ use crate::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
},
|
||||
},
|
||||
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
|
||||
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload_with_reth},
|
||||
};
|
||||
use alloy_provider::Provider;
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
@@ -150,10 +150,15 @@ impl Command {
|
||||
auth_provider,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
..
|
||||
use_reth_namespace,
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
|
||||
if use_reth_namespace {
|
||||
info!("Using reth_newPayload endpoint");
|
||||
}
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -230,16 +235,40 @@ impl Command {
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(block, is_optimism)?;
|
||||
let (version, params, execution_data) = block_to_new_payload(block, is_optimism)?;
|
||||
let start = Instant::now();
|
||||
call_new_payload(&auth_provider, version, params).await?;
|
||||
let reth_data = use_reth_namespace.then_some(execution_data);
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params, reth_data).await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
gas_used,
|
||||
latency: np_latency,
|
||||
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
|
||||
execution_cache_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.execution_cache_wait)
|
||||
.unwrap_or_default(),
|
||||
sparse_trie_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.sparse_trie_wait)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency = start.elapsed();
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
let total_latency = if server_timings.is_some() {
|
||||
// When using server-side latency for newPayload, derive total from the
|
||||
// independently measured components to avoid mixing server-side and
|
||||
// client-side (network-inclusive) timings.
|
||||
np_latency + fcu_latency
|
||||
} else {
|
||||
start.elapsed()
|
||||
};
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
|
||||
@@ -8,7 +8,7 @@ use crate::{
|
||||
NEW_PAYLOAD_OUTPUT_SUFFIX,
|
||||
},
|
||||
},
|
||||
valid_payload::{block_to_new_payload, call_new_payload},
|
||||
valid_payload::{block_to_new_payload, call_new_payload_with_reth},
|
||||
};
|
||||
use alloy_provider::Provider;
|
||||
use clap::Parser;
|
||||
@@ -49,10 +49,15 @@ impl Command {
|
||||
auth_provider,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
..
|
||||
use_reth_namespace,
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
|
||||
if use_reth_namespace {
|
||||
info!("Using reth_newPayload endpoint");
|
||||
}
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
@@ -100,12 +105,28 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let (version, params) = block_to_new_payload(block, is_optimism)?;
|
||||
let (version, params, execution_data) = block_to_new_payload(block, is_optimism)?;
|
||||
|
||||
let start = Instant::now();
|
||||
call_new_payload(&auth_provider, version, params).await?;
|
||||
let reth_data = use_reth_namespace.then_some(execution_data);
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params, reth_data).await?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
let latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
gas_used,
|
||||
latency,
|
||||
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
|
||||
execution_cache_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.execution_cache_wait)
|
||||
.unwrap_or_default(),
|
||||
sparse_trie_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.sparse_trie_wait)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
blocks_processed += 1;
|
||||
let progress = match total_blocks {
|
||||
Some(total) => format!("{blocks_processed}/{total}"),
|
||||
|
||||
@@ -27,6 +27,9 @@ pub(crate) struct GasRampPayloadFile {
|
||||
pub(crate) block_hash: B256,
|
||||
/// The params to pass to newPayload.
|
||||
pub(crate) params: serde_json::Value,
|
||||
/// The execution data for `reth_newPayload`.
|
||||
#[serde(skip_serializing_if = "Option::is_none", default)]
|
||||
pub(crate) execution_data: Option<alloy_rpc_types_engine::ExecutionData>,
|
||||
}
|
||||
|
||||
/// This represents the results of a single `newPayload` call in the benchmark, containing the gas
|
||||
@@ -37,6 +40,12 @@ pub(crate) struct NewPayloadResult {
|
||||
pub(crate) gas_used: u64,
|
||||
/// The latency of the `newPayload` call.
|
||||
pub(crate) latency: Duration,
|
||||
/// Time spent waiting for persistence. `None` when no persistence was in-flight.
|
||||
pub(crate) persistence_wait: Option<Duration>,
|
||||
/// Time spent waiting for execution cache lock.
|
||||
pub(crate) execution_cache_wait: Duration,
|
||||
/// Time spent waiting for sparse trie lock.
|
||||
pub(crate) sparse_trie_wait: Duration,
|
||||
}
|
||||
|
||||
impl NewPayloadResult {
|
||||
@@ -67,9 +76,12 @@ impl Serialize for NewPayloadResult {
|
||||
{
|
||||
// convert the time to microseconds
|
||||
let time = self.latency.as_micros();
|
||||
let mut state = serializer.serialize_struct("NewPayloadResult", 2)?;
|
||||
let mut state = serializer.serialize_struct("NewPayloadResult", 5)?;
|
||||
state.serialize_field("gas_used", &self.gas_used)?;
|
||||
state.serialize_field("latency", &time)?;
|
||||
state.serialize_field("persistence_wait", &self.persistence_wait.map(|d| d.as_micros()))?;
|
||||
state.serialize_field("execution_cache_wait", &self.execution_cache_wait.as_micros())?;
|
||||
state.serialize_field("sparse_trie_wait", &self.sparse_trie_wait.as_micros())?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
@@ -126,7 +138,7 @@ impl Serialize for CombinedResult {
|
||||
let fcu_latency = self.fcu_latency.as_micros();
|
||||
let new_payload_latency = self.new_payload_result.latency.as_micros();
|
||||
let total_latency = self.total_latency.as_micros();
|
||||
let mut state = serializer.serialize_struct("CombinedResult", 7)?;
|
||||
let mut state = serializer.serialize_struct("CombinedResult", 10)?;
|
||||
|
||||
// flatten the new payload result because this is meant for CSV writing
|
||||
state.serialize_field("block_number", &self.block_number)?;
|
||||
@@ -136,6 +148,18 @@ impl Serialize for CombinedResult {
|
||||
state.serialize_field("new_payload_latency", &new_payload_latency)?;
|
||||
state.serialize_field("fcu_latency", &fcu_latency)?;
|
||||
state.serialize_field("total_latency", &total_latency)?;
|
||||
state.serialize_field(
|
||||
"persistence_wait",
|
||||
&self.new_payload_result.persistence_wait.map(|d| d.as_micros()),
|
||||
)?;
|
||||
state.serialize_field(
|
||||
"execution_cache_wait",
|
||||
&self.new_payload_result.execution_cache_wait.as_micros(),
|
||||
)?;
|
||||
state.serialize_field(
|
||||
"sparse_trie_wait",
|
||||
&self.new_payload_result.sparse_trie_wait.as_micros(),
|
||||
)?;
|
||||
state.end()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,12 +23,15 @@ use crate::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
},
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload},
|
||||
valid_payload::{call_forkchoice_updated, call_new_payload_with_reth},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadEnvelopeV4, ForkchoiceState, JwtSecret};
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV4, ExecutionPayloadSidecar,
|
||||
ForkchoiceState, JwtSecret, PraguePayloadFields,
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use reth_cli_runner::CliContext;
|
||||
@@ -124,6 +127,14 @@ pub struct Command {
|
||||
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
|
||||
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
|
||||
ws_rpc_url: Option<String>,
|
||||
|
||||
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
///
|
||||
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
|
||||
/// directly, waits for persistence and cache updates to complete before processing,
|
||||
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
}
|
||||
|
||||
/// A loaded payload ready for execution.
|
||||
@@ -163,6 +174,9 @@ impl Command {
|
||||
self.persistence_threshold
|
||||
);
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload endpoint");
|
||||
}
|
||||
|
||||
// Set up waiter based on configured options
|
||||
// When both are set: wait at least wait_time, and also wait for persistence if needed
|
||||
@@ -248,7 +262,15 @@ impl Command {
|
||||
"Executing gas ramp payload (newPayload + FCU)"
|
||||
);
|
||||
|
||||
call_new_payload(&auth_provider, payload.version, payload.file.params.clone()).await?;
|
||||
let reth_data =
|
||||
if self.reth_new_payload { payload.file.execution_data.clone() } else { None };
|
||||
let _ = call_new_payload_with_reth(
|
||||
&auth_provider,
|
||||
payload.version,
|
||||
payload.file.params.clone(),
|
||||
reth_data,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: payload.file.block_hash,
|
||||
@@ -303,20 +325,47 @@ impl Command {
|
||||
"Sending newPayload"
|
||||
);
|
||||
|
||||
let status = auth_provider
|
||||
.new_payload_v4(
|
||||
execution_payload.clone(),
|
||||
vec![],
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
)
|
||||
.await?;
|
||||
let params = serde_json::to_value((
|
||||
execution_payload.clone(),
|
||||
Vec::<B256>::new(),
|
||||
B256::ZERO,
|
||||
envelope.execution_requests.to_vec(),
|
||||
))?;
|
||||
|
||||
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
|
||||
let reth_data = self.reth_new_payload.then(|| ExecutionData {
|
||||
payload: execution_payload.clone().into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
parent_beacon_block_root: B256::ZERO,
|
||||
},
|
||||
PraguePayloadFields { requests: envelope.execution_requests.clone().into() },
|
||||
),
|
||||
});
|
||||
|
||||
if !status.is_valid() {
|
||||
return Err(eyre::eyre!("Payload rejected: {:?}", status));
|
||||
}
|
||||
let server_timings = call_new_payload_with_reth(
|
||||
&auth_provider,
|
||||
EngineApiMessageVersion::V4,
|
||||
params,
|
||||
reth_data,
|
||||
)
|
||||
.await?;
|
||||
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
gas_used,
|
||||
latency: np_latency,
|
||||
persistence_wait: server_timings.as_ref().and_then(|t| t.persistence_wait),
|
||||
execution_cache_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.execution_cache_wait)
|
||||
.unwrap_or_default(),
|
||||
sparse_trie_wait: server_timings
|
||||
.as_ref()
|
||||
.map(|t| t.sparse_trie_wait)
|
||||
.unwrap_or_default(),
|
||||
};
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
@@ -326,10 +375,12 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency = start.elapsed();
|
||||
let fcu_latency = total_latency - new_payload_result.latency;
|
||||
let total_latency =
|
||||
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
|
||||
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
@@ -352,7 +403,7 @@ impl Command {
|
||||
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
|
||||
results.push((gas_row, combined_result));
|
||||
|
||||
debug!(target: "reth-bench", ?status, ?fcu_result, "Payload executed successfully");
|
||||
debug!(target: "reth-bench", ?fcu_result, "Payload executed successfully");
|
||||
parent_hash = block_hash;
|
||||
}
|
||||
|
||||
|
||||
@@ -6,12 +6,14 @@ use alloy_eips::eip7685::Requests;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar, ForkchoiceState,
|
||||
ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
|
||||
ExecutionData, ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar,
|
||||
ForkchoiceState, ForkchoiceUpdated, PayloadAttributes, PayloadStatus,
|
||||
};
|
||||
use alloy_transport::TransportResult;
|
||||
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use serde::Deserialize;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, error};
|
||||
|
||||
/// An extension trait for providers that implement the engine API, to wait for a VALID response.
|
||||
@@ -161,10 +163,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
|
||||
///
|
||||
/// Returns `(version, versioned_params, execution_data)`.
|
||||
pub(crate) fn block_to_new_payload(
|
||||
block: AnyRpcBlock,
|
||||
is_optimism: bool,
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value, ExecutionData)> {
|
||||
let block = block
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
@@ -179,13 +184,19 @@ pub(crate) fn block_to_new_payload(
|
||||
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
|
||||
}
|
||||
|
||||
/// Converts an execution payload and sidecar into versioned engine API params and an
|
||||
/// [`ExecutionData`].
|
||||
///
|
||||
/// Returns `(version, versioned_params, execution_data)`.
|
||||
pub(crate) fn payload_to_new_payload(
|
||||
payload: ExecutionPayload,
|
||||
sidecar: ExecutionPayloadSidecar,
|
||||
is_optimism: bool,
|
||||
withdrawals_root: Option<B256>,
|
||||
target_version: Option<EngineApiMessageVersion>,
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value)> {
|
||||
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value, ExecutionData)> {
|
||||
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
|
||||
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar.cancun().unwrap();
|
||||
@@ -244,7 +255,7 @@ pub(crate) fn payload_to_new_payload(
|
||||
}
|
||||
};
|
||||
|
||||
Ok((version, params))
|
||||
Ok((version, params, execution_data))
|
||||
}
|
||||
|
||||
/// Calls the correct `engine_newPayload` method depending on the given [`ExecutionPayload`] and its
|
||||
@@ -252,32 +263,109 @@ pub(crate) fn payload_to_new_payload(
|
||||
///
|
||||
/// # Panics
|
||||
/// If the given payload is a V3 payload, but a parent beacon block root is provided as `None`.
|
||||
#[allow(dead_code)]
|
||||
pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
|
||||
provider: P,
|
||||
version: EngineApiMessageVersion,
|
||||
params: serde_json::Value,
|
||||
) -> TransportResult<()> {
|
||||
let method = version.method_name();
|
||||
) -> TransportResult<Option<NewPayloadTimingBreakdown>> {
|
||||
call_new_payload_with_reth(provider, version, params, None).await
|
||||
}
|
||||
|
||||
debug!(target: "reth-bench", method, "Sending newPayload");
|
||||
/// Response from `reth_newPayload` endpoint, which includes server-measured latency.
|
||||
#[derive(Debug, Deserialize)]
|
||||
struct RethPayloadStatus {
|
||||
#[serde(flatten)]
|
||||
status: PayloadStatus,
|
||||
latency_us: u64,
|
||||
#[serde(default)]
|
||||
persistence_wait_us: Option<u64>,
|
||||
#[serde(default)]
|
||||
execution_cache_wait_us: u64,
|
||||
#[serde(default)]
|
||||
sparse_trie_wait_us: u64,
|
||||
}
|
||||
|
||||
let mut status: PayloadStatus = provider.client().request(method, ¶ms).await?;
|
||||
/// Server-side timing breakdown from `reth_newPayload` endpoint.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub(crate) struct NewPayloadTimingBreakdown {
|
||||
/// Server-side execution latency.
|
||||
pub(crate) latency: Duration,
|
||||
/// Time spent waiting for persistence. `None` when no persistence was in-flight.
|
||||
pub(crate) persistence_wait: Option<Duration>,
|
||||
/// Time spent waiting for execution cache lock.
|
||||
pub(crate) execution_cache_wait: Duration,
|
||||
/// Time spent waiting for sparse trie lock.
|
||||
pub(crate) sparse_trie_wait: Duration,
|
||||
}
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(target: "reth-bench", ?status, ?params, "Invalid {method}",);
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(std::io::Error::other(
|
||||
format!("Invalid {method}: {status:?}"),
|
||||
))))
|
||||
/// Calls either `engine_newPayload*` or `reth_newPayload` depending on whether
|
||||
/// `reth_execution_data` is provided.
|
||||
///
|
||||
/// When `reth_execution_data` is `Some`, uses the `reth_newPayload` endpoint which takes
|
||||
/// `ExecutionData` directly and waits for persistence and cache updates to complete.
|
||||
///
|
||||
/// Returns the server-reported timing breakdown when using the reth namespace, or `None` for
|
||||
/// the standard engine namespace.
|
||||
pub(crate) async fn call_new_payload_with_reth<N: Network, P: Provider<N>>(
|
||||
provider: P,
|
||||
version: EngineApiMessageVersion,
|
||||
params: serde_json::Value,
|
||||
reth_execution_data: Option<ExecutionData>,
|
||||
) -> TransportResult<Option<NewPayloadTimingBreakdown>> {
|
||||
if let Some(execution_data) = reth_execution_data {
|
||||
let method = "reth_newPayload";
|
||||
let reth_params = serde_json::to_value((execution_data.clone(),))
|
||||
.expect("ExecutionData serialization cannot fail");
|
||||
|
||||
debug!(target: "reth-bench", method, "Sending newPayload");
|
||||
|
||||
let mut resp: RethPayloadStatus = provider.client().request(method, &reth_params).await?;
|
||||
|
||||
while !resp.status.is_valid() {
|
||||
if resp.status.is_invalid() {
|
||||
error!(target: "reth-bench", status=?resp.status, "Invalid {method}");
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
|
||||
std::io::Error::other(format!("Invalid {method}: {:?}", resp.status)),
|
||||
)))
|
||||
}
|
||||
if resp.status.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
"invalid range: no canonical state found for parent of requested block",
|
||||
))
|
||||
}
|
||||
resp = provider.client().request(method, &reth_params).await?;
|
||||
}
|
||||
if status.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
"invalid range: no canonical state found for parent of requested block",
|
||||
))
|
||||
|
||||
Ok(Some(NewPayloadTimingBreakdown {
|
||||
latency: Duration::from_micros(resp.latency_us),
|
||||
persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
|
||||
execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
|
||||
sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
|
||||
}))
|
||||
} else {
|
||||
let method = version.method_name();
|
||||
|
||||
debug!(target: "reth-bench", method, "Sending newPayload");
|
||||
|
||||
let mut status: PayloadStatus = provider.client().request(method, ¶ms).await?;
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(target: "reth-bench", ?status, ?params, "Invalid {method}",);
|
||||
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
|
||||
std::io::Error::other(format!("Invalid {method}: {status:?}")),
|
||||
)))
|
||||
}
|
||||
if status.is_syncing() {
|
||||
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
|
||||
"invalid range: no canonical state found for parent of requested block",
|
||||
))
|
||||
}
|
||||
status = provider.client().request(method, ¶ms).await?;
|
||||
}
|
||||
status = provider.client().request(method, ¶ms).await?;
|
||||
Ok(None)
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
|
||||
|
||||
@@ -190,6 +190,7 @@ min-trace-logs = [
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"]
|
||||
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
|
||||
edge = ["rocksdb"]
|
||||
|
||||
|
||||
@@ -312,11 +312,6 @@ impl DeferredTrieData {
|
||||
/// Given that invariant, circular wait dependencies are impossible.
|
||||
#[instrument(level = "debug", target = "engine::tree::deferred_trie", skip_all)]
|
||||
pub fn wait_cloned(&self) -> ComputedTrieData {
|
||||
#[cfg(feature = "rayon")]
|
||||
debug_assert!(
|
||||
rayon::current_thread_index().is_none(),
|
||||
"wait_cloned must not be called from a rayon worker thread"
|
||||
);
|
||||
let mut state = self.state.lock();
|
||||
match &mut *state {
|
||||
// If the deferred trie data is ready, return the cached result.
|
||||
|
||||
@@ -285,7 +285,6 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
|
||||
// (We can't just use `upsert` method with a dup cursor, it's not properly
|
||||
// supported)
|
||||
let nibbles = StoredNibblesSubKey(path);
|
||||
let entry = StorageTrieEntry { nibbles: nibbles.clone(), node };
|
||||
if storage_trie_cursor
|
||||
.seek_by_key_subkey(account, nibbles.clone())?
|
||||
.filter(|v| v.nibbles == nibbles)
|
||||
@@ -293,6 +292,7 @@ fn verify_and_repair<N: ProviderNodeTypes>(tool: &DbTool<N>) -> eyre::Result<()>
|
||||
{
|
||||
storage_trie_cursor.delete_current()?;
|
||||
}
|
||||
let entry = StorageTrieEntry { nibbles, node };
|
||||
storage_trie_cursor.upsert(account, &entry)?;
|
||||
}
|
||||
Output::Progress(path) => {
|
||||
|
||||
@@ -384,15 +384,19 @@ fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
let mut total_size: Option<u64> = None;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
let finalize_download = |size: u64| -> Result<(PathBuf, u64)> {
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
Ok((final_path.clone(), size))
|
||||
};
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
let existing_size = fs::metadata(&part_path).map(|m| m.len()).unwrap_or(0);
|
||||
|
||||
if let Some(total) = total_size &&
|
||||
existing_size >= total
|
||||
{
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, total));
|
||||
return finalize_download(total);
|
||||
}
|
||||
|
||||
if attempt > 1 {
|
||||
@@ -476,9 +480,7 @@ fn resumable_download(url: &str, target_dir: &Path) -> Result<(PathBuf, u64)> {
|
||||
continue;
|
||||
}
|
||||
|
||||
fs::rename(&part_path, &final_path)?;
|
||||
info!(target: "reth::cli", "Download complete: {}", final_path.display());
|
||||
return Ok((final_path, current_total));
|
||||
return finalize_download(current_total);
|
||||
}
|
||||
|
||||
Err(last_error
|
||||
|
||||
@@ -139,7 +139,7 @@ where
|
||||
total_decoded_blocks += file_client.headers_len();
|
||||
total_decoded_txns += file_client.total_transactions();
|
||||
|
||||
let (mut pipeline, events) = build_import_pipeline_impl(
|
||||
let (mut pipeline, events, _runtime) = build_import_pipeline_impl(
|
||||
config,
|
||||
provider_factory.clone(),
|
||||
&consensus,
|
||||
@@ -265,7 +265,11 @@ pub fn build_import_pipeline_impl<N, C, E>(
|
||||
static_file_producer: StaticFileProducer<ProviderFactory<N>>,
|
||||
disable_exec: bool,
|
||||
evm_config: E,
|
||||
) -> eyre::Result<(Pipeline<N>, impl futures::Stream<Item = NodeEvent<N::Primitives>> + use<N, C, E>)>
|
||||
) -> eyre::Result<(
|
||||
Pipeline<N>,
|
||||
impl futures::Stream<Item = NodeEvent<N::Primitives>> + use<N, C, E>,
|
||||
reth_tasks::Runtime,
|
||||
)>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
C: FullConsensus<N::Primitives> + 'static,
|
||||
@@ -281,9 +285,12 @@ where
|
||||
.sealed_header(last_block_number)?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(last_block_number.into()))?;
|
||||
|
||||
let runtime = reth_tasks::Runtime::with_existing_handle(tokio::runtime::Handle::current())
|
||||
.expect("failed to create runtime");
|
||||
|
||||
let mut header_downloader = ReverseHeadersDownloaderBuilder::new(config.stages.headers)
|
||||
.build(file_client.clone(), consensus.clone())
|
||||
.into_task();
|
||||
.into_task_with(&runtime);
|
||||
// TODO: The pipeline should correctly configure the downloader on its own.
|
||||
// Find the possibility to remove unnecessary pre-configuration.
|
||||
header_downloader.update_local_head(local_head);
|
||||
@@ -291,7 +298,7 @@ where
|
||||
|
||||
let mut body_downloader = BodiesDownloaderBuilder::new(config.stages.bodies)
|
||||
.build(file_client.clone(), consensus.clone(), provider_factory.clone())
|
||||
.into_task();
|
||||
.into_task_with(&runtime);
|
||||
// TODO: The pipeline should correctly configure the downloader on its own.
|
||||
// Find the possibility to remove unnecessary pre-configuration.
|
||||
body_downloader
|
||||
@@ -326,5 +333,5 @@ where
|
||||
|
||||
let events = pipeline.events().map(Into::into);
|
||||
|
||||
Ok((pipeline, events))
|
||||
Ok((pipeline, events, runtime))
|
||||
}
|
||||
|
||||
@@ -54,12 +54,20 @@ impl<T: PayloadTypes> PayloadTestContext<T> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Wait until the best built payload is ready
|
||||
/// Wait until the best built payload is ready.
|
||||
///
|
||||
/// Panics if the payload builder does not produce a non-empty payload within 30 seconds.
|
||||
pub async fn wait_for_built_payload(&self, payload_id: PayloadId) {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let payload =
|
||||
self.payload_builder.best_payload(payload_id).await.transpose().ok().flatten();
|
||||
if payload.is_none_or(|p| p.block().body().transactions().is_empty()) {
|
||||
assert!(
|
||||
start.elapsed() < std::time::Duration::from_secs(30),
|
||||
"timed out waiting for a non-empty payload for {payload_id} — \
|
||||
check that the chain spec supports all generated tx types"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
continue
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use alloy_primitives::{TxHash, B256};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use eyre::OptionExt;
|
||||
use futures_util::{stream::Fuse, StreamExt};
|
||||
use futures_util::{stream::Fuse, Stream, StreamExt};
|
||||
use reth_engine_primitives::ConsensusEngineHandle;
|
||||
use reth_payload_builder::PayloadBuilderHandle;
|
||||
use reth_payload_primitives::{
|
||||
@@ -14,6 +14,7 @@ use reth_storage_api::BlockReader;
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
fmt,
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
@@ -24,7 +25,6 @@ use tokio_stream::wrappers::ReceiverStream;
|
||||
use tracing::error;
|
||||
|
||||
/// A mining mode for the local dev engine.
|
||||
#[derive(Debug)]
|
||||
pub enum MiningMode<Pool: TransactionPool + Unpin> {
|
||||
/// In this mode a block is built as soon as
|
||||
/// a valid transaction reaches the pool.
|
||||
@@ -43,6 +43,25 @@ pub enum MiningMode<Pool: TransactionPool + Unpin> {
|
||||
},
|
||||
/// In this mode a block is built at a fixed interval.
|
||||
Interval(Interval),
|
||||
/// In this mode a block is built when the trigger stream yields a value.
|
||||
///
|
||||
/// This is a general-purpose trigger that can be fired on demand, for example via a channel
|
||||
/// or any other [`Stream`] implementation.
|
||||
Trigger(Pin<Box<dyn Stream<Item = ()> + Send + Sync>>),
|
||||
}
|
||||
|
||||
impl<Pool: TransactionPool + Unpin> fmt::Debug for MiningMode<Pool> {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Instant { max_transactions, accumulated, .. } => f
|
||||
.debug_struct("Instant")
|
||||
.field("max_transactions", max_transactions)
|
||||
.field("accumulated", accumulated)
|
||||
.finish(),
|
||||
Self::Interval(interval) => f.debug_tuple("Interval").field(interval).finish(),
|
||||
Self::Trigger(_) => f.debug_tuple("Trigger").finish(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<Pool: TransactionPool + Unpin> MiningMode<Pool> {
|
||||
@@ -57,6 +76,14 @@ impl<Pool: TransactionPool + Unpin> MiningMode<Pool> {
|
||||
let start = tokio::time::Instant::now() + duration;
|
||||
Self::Interval(tokio::time::interval_at(start, duration))
|
||||
}
|
||||
|
||||
/// Constructor for a [`MiningMode::Trigger`]
|
||||
///
|
||||
/// Accepts any stream that yields `()` values, each of which triggers a new block to be
|
||||
/// mined. This can be backed by a channel, a custom stream, or any other async source.
|
||||
pub fn trigger(trigger: impl Stream<Item = ()> + Send + Sync + 'static) -> Self {
|
||||
Self::Trigger(Box::pin(trigger))
|
||||
}
|
||||
}
|
||||
|
||||
impl<Pool: TransactionPool + Unpin> Future for MiningMode<Pool> {
|
||||
@@ -91,6 +118,12 @@ impl<Pool: TransactionPool + Unpin> Future for MiningMode<Pool> {
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
Self::Trigger(trigger) => {
|
||||
if trigger.poll_next_unpin(cx).is_ready() {
|
||||
return Poll::Ready(())
|
||||
}
|
||||
Poll::Pending
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -32,28 +32,18 @@ fn default_account_worker_count() -> usize {
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 60;
|
||||
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation when V2 proofs are
|
||||
/// enabled. This is 4x the default chunk size to take advantage of more efficient V2 proof
|
||||
/// computation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2: usize = DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE * 4;
|
||||
/// The size of proof targets chunk optimized for small blocks (≤20M gas used).
|
||||
/// Benchmarks: <https://gist.github.com/yongkangc/fda9c24846f0ba891376bcf81b002008>
|
||||
pub const SMALL_BLOCK_MULTIPROOF_CHUNK_SIZE: usize = 30;
|
||||
|
||||
/// Gas threshold below which the small block chunk size is used.
|
||||
pub const SMALL_BLOCK_GAS_THRESHOLD: u64 = 20_000_000;
|
||||
|
||||
/// Default number of reserved CPU cores for non-reth processes.
|
||||
///
|
||||
/// This will be deducted from the thread count of main reth global threadpool.
|
||||
pub const DEFAULT_RESERVED_CPU_CORES: usize = 1;
|
||||
|
||||
/// Returns the default maximum concurrency for prewarm task based on available parallelism.
|
||||
fn default_prewarm_max_concurrency() -> usize {
|
||||
#[cfg(feature = "std")]
|
||||
{
|
||||
std::thread::available_parallelism().map_or(16, |n| n.get())
|
||||
}
|
||||
#[cfg(not(feature = "std"))]
|
||||
{
|
||||
16
|
||||
}
|
||||
}
|
||||
|
||||
/// Default depth for sparse trie pruning.
|
||||
///
|
||||
/// Nodes at this depth and below are converted to hash stubs to reduce memory.
|
||||
@@ -161,20 +151,14 @@ pub struct TreeConfig {
|
||||
/// where immediate payload regeneration is desired despite the head not changing or moving to
|
||||
/// an ancestor.
|
||||
always_process_payload_attributes_on_canonical_head: bool,
|
||||
/// Maximum concurrency for the prewarm task.
|
||||
prewarm_max_concurrency: usize,
|
||||
/// Whether to unwind canonical header to ancestor during forkchoice updates.
|
||||
allow_unwind_canonical_header: bool,
|
||||
/// Number of storage proof worker threads.
|
||||
storage_worker_count: usize,
|
||||
/// Number of account proof worker threads.
|
||||
account_worker_count: usize,
|
||||
/// Whether to disable V2 storage proofs.
|
||||
disable_proof_v2: bool,
|
||||
/// Whether to disable cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
/// Whether to disable sparse trie cache.
|
||||
disable_trie_cache: bool,
|
||||
/// Depth for sparse trie pruning after state root computation.
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum number of storage tries to retain after pruning.
|
||||
@@ -209,13 +193,10 @@ impl Default for TreeConfig {
|
||||
precompile_cache_disabled: false,
|
||||
state_root_fallback: false,
|
||||
always_process_payload_attributes_on_canonical_head: false,
|
||||
prewarm_max_concurrency: default_prewarm_max_concurrency(),
|
||||
allow_unwind_canonical_header: false,
|
||||
storage_worker_count: default_storage_worker_count(),
|
||||
account_worker_count: default_account_worker_count(),
|
||||
disable_proof_v2: false,
|
||||
disable_cache_metrics: false,
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth: DEFAULT_SPARSE_TRIE_PRUNE_DEPTH,
|
||||
sparse_trie_max_storage_tries: DEFAULT_SPARSE_TRIE_MAX_STORAGE_TRIES,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
@@ -246,11 +227,9 @@ impl TreeConfig {
|
||||
precompile_cache_disabled: bool,
|
||||
state_root_fallback: bool,
|
||||
always_process_payload_attributes_on_canonical_head: bool,
|
||||
prewarm_max_concurrency: usize,
|
||||
allow_unwind_canonical_header: bool,
|
||||
storage_worker_count: usize,
|
||||
account_worker_count: usize,
|
||||
disable_proof_v2: bool,
|
||||
disable_cache_metrics: bool,
|
||||
sparse_trie_prune_depth: usize,
|
||||
sparse_trie_max_storage_tries: usize,
|
||||
@@ -275,13 +254,10 @@ impl TreeConfig {
|
||||
precompile_cache_disabled,
|
||||
state_root_fallback,
|
||||
always_process_payload_attributes_on_canonical_head,
|
||||
prewarm_max_concurrency,
|
||||
allow_unwind_canonical_header,
|
||||
storage_worker_count,
|
||||
account_worker_count,
|
||||
disable_proof_v2,
|
||||
disable_cache_metrics,
|
||||
disable_trie_cache: false,
|
||||
sparse_trie_prune_depth,
|
||||
sparse_trie_max_storage_tries,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
@@ -324,16 +300,9 @@ impl TreeConfig {
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
|
||||
/// Return the multiproof task chunk size, using the V2 default if V2 proofs are enabled
|
||||
/// and the chunk size is at the default value.
|
||||
/// Return the effective multiproof task chunk size.
|
||||
pub const fn effective_multiproof_chunk_size(&self) -> usize {
|
||||
if !self.disable_proof_v2 &&
|
||||
self.multiproof_chunk_size == DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE
|
||||
{
|
||||
DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE_V2
|
||||
} else {
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
self.multiproof_chunk_size
|
||||
}
|
||||
|
||||
/// Return the number of reserved CPU cores for non-reth processes
|
||||
@@ -533,17 +502,6 @@ impl TreeConfig {
|
||||
self.has_enough_parallelism && !self.legacy_state_root
|
||||
}
|
||||
|
||||
/// Setter for prewarm max concurrency.
|
||||
pub const fn with_prewarm_max_concurrency(mut self, prewarm_max_concurrency: usize) -> Self {
|
||||
self.prewarm_max_concurrency = prewarm_max_concurrency;
|
||||
self
|
||||
}
|
||||
|
||||
/// Return the prewarm max concurrency.
|
||||
pub const fn prewarm_max_concurrency(&self) -> usize {
|
||||
self.prewarm_max_concurrency
|
||||
}
|
||||
|
||||
/// Return the number of storage proof worker threads.
|
||||
pub const fn storage_worker_count(&self) -> usize {
|
||||
self.storage_worker_count
|
||||
@@ -580,17 +538,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Return whether V2 storage proofs are disabled.
|
||||
pub const fn disable_proof_v2(&self) -> bool {
|
||||
self.disable_proof_v2
|
||||
}
|
||||
|
||||
/// Setter for whether to disable V2 storage proofs.
|
||||
pub const fn with_disable_proof_v2(mut self, disable_proof_v2: bool) -> Self {
|
||||
self.disable_proof_v2 = disable_proof_v2;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether cache metrics recording is disabled.
|
||||
pub const fn disable_cache_metrics(&self) -> bool {
|
||||
self.disable_cache_metrics
|
||||
@@ -602,17 +549,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether sparse trie cache is disabled.
|
||||
pub const fn disable_trie_cache(&self) -> bool {
|
||||
self.disable_trie_cache
|
||||
}
|
||||
|
||||
/// Setter for whether to disable sparse trie cache.
|
||||
pub const fn with_disable_trie_cache(mut self, value: bool) -> Self {
|
||||
self.disable_trie_cache = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the sparse trie prune depth.
|
||||
pub const fn sparse_trie_prune_depth(&self) -> usize {
|
||||
self.sparse_trie_prune_depth
|
||||
|
||||
@@ -15,6 +15,7 @@ use futures::{future::Either, FutureExt, TryFutureExt};
|
||||
use reth_errors::RethResult;
|
||||
use reth_payload_builder_primitives::PayloadBuilderError;
|
||||
use reth_payload_primitives::{EngineApiMessageVersion, PayloadTypes};
|
||||
use std::time::Duration;
|
||||
use tokio::sync::{mpsc::UnboundedSender, oneshot};
|
||||
|
||||
/// Type alias for backwards compat
|
||||
@@ -142,6 +143,20 @@ impl Future for PendingPayloadId {
|
||||
}
|
||||
}
|
||||
|
||||
/// Timing breakdown for `reth_newPayload` responses.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub struct NewPayloadTimings {
|
||||
/// Server-side execution latency.
|
||||
pub latency: Duration,
|
||||
/// Time spent waiting for persistence to complete.
|
||||
/// `None` when no persistence was in-flight.
|
||||
pub persistence_wait: Option<Duration>,
|
||||
/// Time spent waiting for the execution cache lock.
|
||||
pub execution_cache_wait: Duration,
|
||||
/// Time spent waiting for the sparse trie lock.
|
||||
pub sparse_trie_wait: Duration,
|
||||
}
|
||||
|
||||
/// A message for the beacon engine from other components of the node (engine RPC API invoked by the
|
||||
/// consensus layer).
|
||||
#[derive(Debug)]
|
||||
@@ -153,6 +168,16 @@ pub enum BeaconEngineMessage<Payload: PayloadTypes> {
|
||||
/// The sender for returning payload status result.
|
||||
tx: oneshot::Sender<Result<PayloadStatus, BeaconOnNewPayloadError>>,
|
||||
},
|
||||
/// Message with new payload used by `reth_newPayload` endpoint.
|
||||
///
|
||||
/// Waits for persistence, execution cache, and sparse trie locks before processing,
|
||||
/// and returns detailed timing breakdown alongside the payload status.
|
||||
RethNewPayload {
|
||||
/// The execution payload received by Engine API.
|
||||
payload: Payload::ExecutionData,
|
||||
/// The sender for returning payload status result and timing breakdown.
|
||||
tx: oneshot::Sender<Result<(PayloadStatus, NewPayloadTimings), BeaconOnNewPayloadError>>,
|
||||
},
|
||||
/// Message with updated forkchoice state.
|
||||
ForkchoiceUpdated {
|
||||
/// The updated forkchoice state.
|
||||
@@ -178,6 +203,15 @@ impl<Payload: PayloadTypes> Display for BeaconEngineMessage<Payload> {
|
||||
payload.block_hash()
|
||||
)
|
||||
}
|
||||
Self::RethNewPayload { payload, .. } => {
|
||||
write!(
|
||||
f,
|
||||
"RethNewPayload(parent: {}, number: {}, hash: {})",
|
||||
payload.parent_hash(),
|
||||
payload.block_number(),
|
||||
payload.block_hash()
|
||||
)
|
||||
}
|
||||
Self::ForkchoiceUpdated { state, payload_attrs, .. } => {
|
||||
// we don't want to print the entire payload attributes, because for OP this
|
||||
// includes all txs
|
||||
@@ -223,6 +257,19 @@ where
|
||||
rx.await.map_err(|_| BeaconOnNewPayloadError::EngineUnavailable)?
|
||||
}
|
||||
|
||||
/// Sends a new payload message used by `reth_newPayload` endpoint.
|
||||
///
|
||||
/// Waits for persistence, execution cache, and sparse trie locks before processing,
|
||||
/// and returns detailed timing breakdown alongside the payload status.
|
||||
pub async fn reth_new_payload(
|
||||
&self,
|
||||
payload: Payload::ExecutionData,
|
||||
) -> Result<(PayloadStatus, NewPayloadTimings), BeaconOnNewPayloadError> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _ = self.to_engine.send(BeaconEngineMessage::RethNewPayload { payload, tx });
|
||||
rx.await.map_err(|_| BeaconOnNewPayloadError::EngineUnavailable)?
|
||||
}
|
||||
|
||||
/// Sends a forkchoice update message to the beacon consensus engine and waits for a response.
|
||||
///
|
||||
/// See also <https://github.com/ethereum/execution-apis/blob/3d627c95a4d3510a8187dd02e0250ecb4331d27e/src/engine/shanghai.md#engine_forkchoiceupdatedv2>
|
||||
|
||||
@@ -1,47 +0,0 @@
|
||||
[package]
|
||||
name = "reth-engine-service"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-consensus.workspace = true
|
||||
reth-engine-tree.workspace = true
|
||||
reth-evm.workspace = true
|
||||
reth-network-p2p.workspace = true
|
||||
reth-payload-builder.workspace = true
|
||||
reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-stages-api.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-node-types.workspace = true
|
||||
reth-chainspec.workspace = true
|
||||
reth-engine-primitives.workspace = true
|
||||
reth-trie-db.workspace = true
|
||||
|
||||
# async
|
||||
futures.workspace = true
|
||||
pin-project.workspace = true
|
||||
|
||||
# misc
|
||||
|
||||
[dev-dependencies]
|
||||
reth-engine-tree = { workspace = true, features = ["test-utils"] }
|
||||
reth-ethereum-consensus.workspace = true
|
||||
reth-ethereum-engine-primitives.workspace = true
|
||||
reth-evm-ethereum.workspace = true
|
||||
reth-exex-types.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-node-ethereum.workspace = true
|
||||
reth-trie-db.workspace = true
|
||||
|
||||
alloy-eips.workspace = true
|
||||
tokio = { workspace = true, features = ["sync"] }
|
||||
tokio-stream.workspace = true
|
||||
@@ -1,12 +0,0 @@
|
||||
//! Engine service implementation.
|
||||
|
||||
#![doc(
|
||||
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
|
||||
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
|
||||
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
|
||||
)]
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
|
||||
/// Engine Service
|
||||
pub mod service;
|
||||
@@ -1,229 +0,0 @@
|
||||
use futures::{Stream, StreamExt};
|
||||
use pin_project::pin_project;
|
||||
use reth_chainspec::EthChainSpec;
|
||||
use reth_consensus::FullConsensus;
|
||||
use reth_engine_primitives::{BeaconEngineMessage, ConsensusEngineEvent};
|
||||
use reth_engine_tree::{
|
||||
backfill::PipelineSync,
|
||||
download::BasicBlockDownloader,
|
||||
engine::{EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineHandler},
|
||||
persistence::PersistenceHandle,
|
||||
tree::{EngineApiTreeHandler, EngineValidator, TreeConfig},
|
||||
};
|
||||
pub use reth_engine_tree::{
|
||||
chain::{ChainEvent, ChainOrchestrator},
|
||||
engine::EngineApiEvent,
|
||||
};
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_network_p2p::BlockClient;
|
||||
use reth_node_types::{BlockTy, NodeTypes};
|
||||
use reth_payload_builder::PayloadBuilderHandle;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, ProviderNodeTypes},
|
||||
ProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune::PrunerWithFactory;
|
||||
use reth_stages_api::{MetricEventsSender, Pipeline};
|
||||
use reth_tasks::TaskSpawner;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use std::{
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
|
||||
/// Alias for consensus engine stream.
|
||||
pub type EngineMessageStream<T> = Pin<Box<dyn Stream<Item = BeaconEngineMessage<T>> + Send + Sync>>;
|
||||
|
||||
/// Alias for chain orchestrator.
|
||||
type EngineServiceType<N, Client> = ChainOrchestrator<
|
||||
EngineHandler<
|
||||
EngineApiRequestHandler<
|
||||
EngineApiRequest<<N as NodeTypes>::Payload, <N as NodeTypes>::Primitives>,
|
||||
<N as NodeTypes>::Primitives,
|
||||
>,
|
||||
EngineMessageStream<<N as NodeTypes>::Payload>,
|
||||
BasicBlockDownloader<Client, BlockTy<N>>,
|
||||
>,
|
||||
PipelineSync<N>,
|
||||
>;
|
||||
|
||||
/// The type that drives the chain forward and communicates progress.
|
||||
#[pin_project]
|
||||
#[expect(missing_debug_implementations)]
|
||||
// TODO(mattsse): remove hidden once fixed : <https://github.com/rust-lang/rust/issues/135363>
|
||||
// otherwise rustdoc fails to resolve the alias
|
||||
#[doc(hidden)]
|
||||
pub struct EngineService<N, Client>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
Client: BlockClient<Block = BlockTy<N>> + 'static,
|
||||
{
|
||||
orchestrator: EngineServiceType<N, Client>,
|
||||
}
|
||||
|
||||
impl<N, Client> EngineService<N, Client>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
Client: BlockClient<Block = BlockTy<N>> + 'static,
|
||||
{
|
||||
/// Constructor for `EngineService`.
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
pub fn new<V, C>(
|
||||
consensus: Arc<dyn FullConsensus<N::Primitives>>,
|
||||
chain_spec: Arc<N::ChainSpec>,
|
||||
client: Client,
|
||||
incoming_requests: EngineMessageStream<N::Payload>,
|
||||
pipeline: Pipeline<N>,
|
||||
pipeline_task_spawner: Box<dyn TaskSpawner>,
|
||||
provider: ProviderFactory<N>,
|
||||
blockchain_db: BlockchainProvider<N>,
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
payload_builder: PayloadBuilderHandle<N::Payload>,
|
||||
payload_validator: V,
|
||||
tree_config: TreeConfig,
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
evm_config: C,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> Self
|
||||
where
|
||||
V: EngineValidator<N::Payload>,
|
||||
C: ConfigureEvm<Primitives = N::Primitives> + 'static,
|
||||
{
|
||||
let engine_kind =
|
||||
if chain_spec.is_optimism() { EngineApiKind::OpStack } else { EngineApiKind::Ethereum };
|
||||
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
let use_hashed_state = provider.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
|
||||
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
|
||||
|
||||
let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new(
|
||||
blockchain_db,
|
||||
consensus,
|
||||
payload_validator,
|
||||
persistence_handle,
|
||||
payload_builder,
|
||||
canonical_in_memory_state,
|
||||
tree_config,
|
||||
engine_kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
);
|
||||
|
||||
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
|
||||
let handler = EngineHandler::new(engine_handler, downloader, incoming_requests);
|
||||
|
||||
let backfill_sync = PipelineSync::new(pipeline, pipeline_task_spawner);
|
||||
|
||||
Self { orchestrator: ChainOrchestrator::new(handler, backfill_sync) }
|
||||
}
|
||||
|
||||
/// Returns a mutable reference to the orchestrator.
|
||||
pub fn orchestrator_mut(&mut self) -> &mut EngineServiceType<N, Client> {
|
||||
&mut self.orchestrator
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, Client> Stream for EngineService<N, Client>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
Client: BlockClient<Block = BlockTy<N>> + 'static,
|
||||
{
|
||||
type Item = ChainEvent<ConsensusEngineEvent<N::Primitives>>;
|
||||
|
||||
fn poll_next(self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
let mut orchestrator = self.project().orchestrator;
|
||||
StreamExt::poll_next_unpin(&mut orchestrator, cx)
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_chainspec::{ChainSpecBuilder, MAINNET};
|
||||
use reth_engine_primitives::{BeaconEngineMessage, NoopInvalidBlockHook};
|
||||
use reth_engine_tree::{test_utils::TestPipelineBuilder, tree::BasicEngineValidator};
|
||||
use reth_ethereum_consensus::EthBeaconConsensus;
|
||||
use reth_ethereum_engine_primitives::EthEngineTypes;
|
||||
use reth_evm_ethereum::EthEvmConfig;
|
||||
use reth_exex_types::FinishedExExHeight;
|
||||
use reth_network_p2p::test_utils::TestFullBlockClient;
|
||||
use reth_node_ethereum::EthereumEngineValidator;
|
||||
use reth_primitives_traits::SealedHeader;
|
||||
use reth_provider::{
|
||||
providers::BlockchainProvider, test_utils::create_test_provider_factory_with_chain_spec,
|
||||
};
|
||||
use reth_prune::Pruner;
|
||||
use reth_tasks::TokioTaskExecutor;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use std::sync::Arc;
|
||||
use tokio::sync::{mpsc::unbounded_channel, watch};
|
||||
use tokio_stream::wrappers::UnboundedReceiverStream;
|
||||
|
||||
#[test]
|
||||
fn eth_chain_orchestrator_build() {
|
||||
let chain_spec = Arc::new(
|
||||
ChainSpecBuilder::default()
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(MAINNET.genesis.clone())
|
||||
.paris_activated()
|
||||
.build(),
|
||||
);
|
||||
let consensus = Arc::new(EthBeaconConsensus::new(chain_spec.clone()));
|
||||
|
||||
let client = TestFullBlockClient::default();
|
||||
|
||||
let (_tx, rx) = unbounded_channel::<BeaconEngineMessage<EthEngineTypes>>();
|
||||
let incoming_requests = UnboundedReceiverStream::new(rx);
|
||||
|
||||
let pipeline = TestPipelineBuilder::new().build(chain_spec.clone());
|
||||
let pipeline_task_spawner = Box::<TokioTaskExecutor>::default();
|
||||
let provider_factory = create_test_provider_factory_with_chain_spec(chain_spec.clone());
|
||||
|
||||
let blockchain_db =
|
||||
BlockchainProvider::with_latest(provider_factory.clone(), SealedHeader::default())
|
||||
.unwrap();
|
||||
let engine_payload_validator = EthereumEngineValidator::new(chain_spec.clone());
|
||||
let (_tx, rx) = watch::channel(FinishedExExHeight::NoExExs);
|
||||
let pruner = Pruner::new_with_factory(provider_factory.clone(), vec![], 0, 0, None, rx);
|
||||
let evm_config = EthEvmConfig::new(chain_spec.clone());
|
||||
|
||||
let changeset_cache = ChangesetCache::new();
|
||||
|
||||
let engine_validator = BasicEngineValidator::new(
|
||||
blockchain_db.clone(),
|
||||
consensus.clone(),
|
||||
evm_config.clone(),
|
||||
engine_payload_validator,
|
||||
TreeConfig::default(),
|
||||
Box::new(NoopInvalidBlockHook::default()),
|
||||
changeset_cache.clone(),
|
||||
reth_tasks::Runtime::test(),
|
||||
);
|
||||
|
||||
let (sync_metrics_tx, _sync_metrics_rx) = unbounded_channel();
|
||||
let (tx, _rx) = unbounded_channel();
|
||||
let _eth_service = EngineService::new(
|
||||
consensus,
|
||||
chain_spec,
|
||||
client,
|
||||
Box::pin(incoming_requests),
|
||||
pipeline,
|
||||
pipeline_task_spawner,
|
||||
provider_factory,
|
||||
blockchain_db,
|
||||
pruner,
|
||||
PayloadBuilderHandle::new(tx),
|
||||
engine_validator,
|
||||
TreeConfig::default(),
|
||||
sync_metrics_tx,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -29,7 +29,7 @@ reth-provider.workspace = true
|
||||
reth-prune.workspace = true
|
||||
reth-revm = { workspace = true, features = ["optional-balance-check"] }
|
||||
reth-stages-api.workspace = true
|
||||
reth-tasks.workspace = true
|
||||
reth-tasks = { workspace = true, features = ["rayon"] }
|
||||
reth-trie-parallel.workspace = true
|
||||
reth-trie-sparse = { workspace = true, features = ["std", "metrics"] }
|
||||
reth-trie.workspace = true
|
||||
@@ -54,7 +54,6 @@ thiserror.workspace = true
|
||||
tokio = { workspace = true, features = ["rt", "rt-multi-thread", "sync", "macros"] }
|
||||
fixed-cache.workspace = true
|
||||
moka = { workspace = true, features = ["sync"] }
|
||||
smallvec.workspace = true
|
||||
|
||||
# metrics
|
||||
metrics.workspace = true
|
||||
@@ -73,6 +72,7 @@ reth-prune-types = { workspace = true, optional = true }
|
||||
reth-stages = { workspace = true, optional = true }
|
||||
reth-static-file = { workspace = true, optional = true }
|
||||
reth-tracing = { workspace = true, optional = true }
|
||||
serde_json = { workspace = true, optional = true }
|
||||
|
||||
[dev-dependencies]
|
||||
# reth
|
||||
@@ -143,6 +143,7 @@ test-utils = [
|
||||
"reth-evm-ethereum/test-utils",
|
||||
"reth-tasks/test-utils",
|
||||
]
|
||||
trie-debug = ["reth-trie-sparse/trie-debug", "dep:serde_json"]
|
||||
rocksdb = [
|
||||
"reth-provider/rocksdb",
|
||||
"reth-prune/rocksdb",
|
||||
|
||||
@@ -10,7 +10,7 @@
|
||||
use futures::FutureExt;
|
||||
use reth_provider::providers::ProviderNodeTypes;
|
||||
use reth_stages_api::{ControlFlow, Pipeline, PipelineError, PipelineTarget, PipelineWithResult};
|
||||
use reth_tasks::TaskSpawner;
|
||||
use reth_tasks::Runtime;
|
||||
use std::task::{ready, Context, Poll};
|
||||
use tokio::sync::oneshot;
|
||||
use tracing::trace;
|
||||
@@ -80,7 +80,7 @@ pub enum BackfillEvent {
|
||||
#[derive(Debug)]
|
||||
pub struct PipelineSync<N: ProviderNodeTypes> {
|
||||
/// The type that can spawn the pipeline task.
|
||||
pipeline_task_spawner: Box<dyn TaskSpawner>,
|
||||
pipeline_task_spawner: Runtime,
|
||||
/// The current state of the pipeline.
|
||||
/// The pipeline is used for large ranges.
|
||||
pipeline_state: PipelineState<N>,
|
||||
@@ -90,7 +90,7 @@ pub struct PipelineSync<N: ProviderNodeTypes> {
|
||||
|
||||
impl<N: ProviderNodeTypes> PipelineSync<N> {
|
||||
/// Create a new instance.
|
||||
pub fn new(pipeline: Pipeline<N>, pipeline_task_spawner: Box<dyn TaskSpawner>) -> Self {
|
||||
pub fn new(pipeline: Pipeline<N>, pipeline_task_spawner: Runtime) -> Self {
|
||||
Self {
|
||||
pipeline_task_spawner,
|
||||
pipeline_state: PipelineState::Idle(Some(Box::new(pipeline))),
|
||||
@@ -140,10 +140,10 @@ impl<N: ProviderNodeTypes> PipelineSync<N> {
|
||||
let pipeline = pipeline.take().expect("exists");
|
||||
self.pipeline_task_spawner.spawn_critical_blocking_task(
|
||||
"pipeline task",
|
||||
Box::pin(async move {
|
||||
async move {
|
||||
let result = pipeline.run_as_fut(Some(target)).await;
|
||||
let _ = tx.send(result);
|
||||
}),
|
||||
},
|
||||
);
|
||||
self.pipeline_state = PipelineState::Running(rx);
|
||||
|
||||
@@ -241,7 +241,7 @@ mod tests {
|
||||
use reth_provider::test_utils::MockNodeTypesWithDB;
|
||||
use reth_stages::ExecOutput;
|
||||
use reth_stages_api::StageCheckpoint;
|
||||
use reth_tasks::TokioTaskExecutor;
|
||||
use reth_tasks::Runtime;
|
||||
use std::{collections::VecDeque, future::poll_fn, sync::Arc};
|
||||
|
||||
struct TestHarness {
|
||||
@@ -267,7 +267,7 @@ mod tests {
|
||||
})]))
|
||||
.build(chain_spec);
|
||||
|
||||
let pipeline_sync = PipelineSync::new(pipeline, Box::<TokioTaskExecutor>::default());
|
||||
let pipeline_sync = PipelineSync::new(pipeline, Runtime::test());
|
||||
let client = TestFullBlockClient::default();
|
||||
let header = Header {
|
||||
base_fee_per_gas: Some(7),
|
||||
|
||||
110
crates/engine/tree/src/launch.rs
Normal file
110
crates/engine/tree/src/launch.rs
Normal file
@@ -0,0 +1,110 @@
|
||||
//! Engine orchestrator launch helper.
|
||||
//!
|
||||
//! Provides [`build_engine_orchestrator`](crate::launch::build_engine_orchestrator) which wires
|
||||
//! together all engine components and returns a
|
||||
//! [`ChainOrchestrator`](crate::chain::ChainOrchestrator) ready to be polled as a `Stream`.
|
||||
|
||||
use crate::{
|
||||
backfill::PipelineSync,
|
||||
chain::ChainOrchestrator,
|
||||
download::BasicBlockDownloader,
|
||||
engine::{EngineApiKind, EngineApiRequest, EngineApiRequestHandler, EngineHandler},
|
||||
persistence::PersistenceHandle,
|
||||
tree::{EngineApiTreeHandler, EngineValidator, TreeConfig, WaitForCaches},
|
||||
};
|
||||
use futures::Stream;
|
||||
use reth_consensus::FullConsensus;
|
||||
use reth_engine_primitives::BeaconEngineMessage;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_network_p2p::BlockClient;
|
||||
use reth_payload_builder::PayloadBuilderHandle;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, ProviderNodeTypes},
|
||||
ProviderFactory, StorageSettingsCache,
|
||||
};
|
||||
use reth_prune::PrunerWithFactory;
|
||||
use reth_stages_api::{MetricEventsSender, Pipeline};
|
||||
use reth_tasks::Runtime;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use std::sync::Arc;
|
||||
|
||||
/// Builds the engine [`ChainOrchestrator`] that drives the chain forward.
|
||||
///
|
||||
/// This spawns and wires together the following components:
|
||||
///
|
||||
/// - **[`BasicBlockDownloader`]** — downloads blocks on demand from the network during live sync.
|
||||
/// - **[`PersistenceHandle`]** — spawns the persistence service on a background thread for writing
|
||||
/// blocks and performing pruning outside the critical consensus path.
|
||||
/// - **[`EngineApiTreeHandler`]** — spawns the tree handler that processes engine API requests
|
||||
/// (`newPayload`, `forkchoiceUpdated`) and maintains the in-memory chain state.
|
||||
/// - **[`EngineApiRequestHandler`]** + **[`EngineHandler`]** — glue that routes incoming CL
|
||||
/// messages to the tree handler and manages download requests.
|
||||
/// - **[`PipelineSync`]** — wraps the staged sync [`Pipeline`] for backfill sync when the node
|
||||
/// needs to catch up over large block ranges.
|
||||
///
|
||||
/// The returned orchestrator implements [`Stream`] and yields
|
||||
/// [`ChainEvent`]s.
|
||||
///
|
||||
/// [`ChainEvent`]: crate::chain::ChainEvent
|
||||
#[expect(clippy::too_many_arguments, clippy::type_complexity)]
|
||||
pub fn build_engine_orchestrator<N, Client, S, V, C>(
|
||||
engine_kind: EngineApiKind,
|
||||
consensus: Arc<dyn FullConsensus<N::Primitives>>,
|
||||
client: Client,
|
||||
incoming_requests: S,
|
||||
pipeline: Pipeline<N>,
|
||||
pipeline_task_spawner: Runtime,
|
||||
provider: ProviderFactory<N>,
|
||||
blockchain_db: BlockchainProvider<N>,
|
||||
pruner: PrunerWithFactory<ProviderFactory<N>>,
|
||||
payload_builder: PayloadBuilderHandle<N::Payload>,
|
||||
payload_validator: V,
|
||||
tree_config: TreeConfig,
|
||||
sync_metrics_tx: MetricEventsSender,
|
||||
evm_config: C,
|
||||
changeset_cache: ChangesetCache,
|
||||
) -> ChainOrchestrator<
|
||||
EngineHandler<
|
||||
EngineApiRequestHandler<EngineApiRequest<N::Payload, N::Primitives>, N::Primitives>,
|
||||
S,
|
||||
BasicBlockDownloader<Client, <N::Primitives as NodePrimitives>::Block>,
|
||||
>,
|
||||
PipelineSync<N>,
|
||||
>
|
||||
where
|
||||
N: ProviderNodeTypes,
|
||||
Client: BlockClient<Block = <N::Primitives as NodePrimitives>::Block> + 'static,
|
||||
S: Stream<Item = BeaconEngineMessage<N::Payload>> + Send + Sync + Unpin + 'static,
|
||||
V: EngineValidator<N::Payload> + WaitForCaches,
|
||||
C: ConfigureEvm<Primitives = N::Primitives> + 'static,
|
||||
{
|
||||
let downloader = BasicBlockDownloader::new(client, consensus.clone());
|
||||
let use_hashed_state = provider.cached_storage_settings().use_hashed_state();
|
||||
|
||||
let persistence_handle =
|
||||
PersistenceHandle::<N::Primitives>::spawn_service(provider, pruner, sync_metrics_tx);
|
||||
|
||||
let canonical_in_memory_state = blockchain_db.canonical_in_memory_state();
|
||||
|
||||
let (to_tree_tx, from_tree) = EngineApiTreeHandler::spawn_new(
|
||||
blockchain_db,
|
||||
consensus,
|
||||
payload_validator,
|
||||
persistence_handle,
|
||||
payload_builder,
|
||||
canonical_in_memory_state,
|
||||
tree_config,
|
||||
engine_kind,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
use_hashed_state,
|
||||
);
|
||||
|
||||
let engine_handler = EngineApiRequestHandler::new(to_tree_tx, from_tree);
|
||||
let handler = EngineHandler::new(engine_handler, downloader, incoming_requests);
|
||||
|
||||
let backfill_sync = PipelineSync::new(pipeline, pipeline_task_spawner);
|
||||
|
||||
ChainOrchestrator::new(handler, backfill_sync)
|
||||
}
|
||||
@@ -100,6 +100,8 @@ pub mod chain;
|
||||
pub mod download;
|
||||
/// Engine Api chain handler support.
|
||||
pub mod engine;
|
||||
/// Engine orchestrator launch helper.
|
||||
pub mod launch;
|
||||
/// Metrics support.
|
||||
pub mod metrics;
|
||||
/// The background writer service, coordinating write operations on static files and the database.
|
||||
|
||||
@@ -4,7 +4,7 @@ use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use reth_chain_state::ExecutedBlock;
|
||||
use reth_errors::ProviderError;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_primitives_traits::{FastInstant as Instant, NodePrimitives};
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, BlockExecutionWriter, BlockHashReader, ChainStateBlockWriter,
|
||||
DBProvider, DatabaseProviderFactory, ProviderFactory, SaveBlocksMode,
|
||||
@@ -18,7 +18,6 @@ use std::{
|
||||
Arc,
|
||||
},
|
||||
thread::JoinHandle,
|
||||
time::Instant,
|
||||
};
|
||||
use thiserror::Error;
|
||||
use tracing::{debug, error, instrument};
|
||||
@@ -119,7 +118,7 @@ where
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(new_tip_num))]
|
||||
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(%new_tip_num))]
|
||||
fn on_remove_blocks_above(
|
||||
&self,
|
||||
new_tip_num: u64,
|
||||
|
||||
@@ -845,10 +845,8 @@ impl SavedCache {
|
||||
self.caches.update_metrics(&self.metrics);
|
||||
}
|
||||
|
||||
/// Clears all caches, resetting them to empty state,
|
||||
/// and updates the hash of the block this cache belongs to.
|
||||
pub(crate) fn clear_with_hash(&mut self, hash: B256) {
|
||||
self.hash = hash;
|
||||
/// Clears all caches, resetting them to empty state.
|
||||
pub(crate) fn clear(&self) {
|
||||
self.caches.clear();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@ use alloy_primitives::{Address, StorageKey, StorageValue, B256};
|
||||
use metrics::{Gauge, Histogram};
|
||||
use reth_errors::ProviderResult;
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::{Account, Bytecode};
|
||||
use reth_primitives_traits::{Account, Bytecode, FastInstant as Instant};
|
||||
use reth_provider::{
|
||||
AccountReader, BlockHashReader, BytecodeReader, HashedPostStateProvider, StateProofProvider,
|
||||
StateProvider, StateRootProvider, StorageRootProvider,
|
||||
@@ -14,7 +14,7 @@ use reth_trie::{
|
||||
};
|
||||
use std::{
|
||||
sync::atomic::{AtomicU64, Ordering},
|
||||
time::{Duration, Instant},
|
||||
time::Duration,
|
||||
};
|
||||
|
||||
/// Nanoseconds per second
|
||||
|
||||
@@ -8,9 +8,9 @@ use reth_metrics::{
|
||||
metrics::{Counter, Gauge, Histogram},
|
||||
Metrics,
|
||||
};
|
||||
use reth_primitives_traits::constants::gas_units::MEGAGAS;
|
||||
use reth_primitives_traits::{constants::gas_units::MEGAGAS, FastInstant as Instant};
|
||||
use reth_trie::updates::TrieUpdates;
|
||||
use std::time::{Duration, Instant};
|
||||
use std::time::Duration;
|
||||
|
||||
/// Upper bounds for each gas bucket. The last bucket is a catch-all for
|
||||
/// everything above the final threshold: <5M, 5-10M, 10-20M, 20-30M, 30-40M, >40M.
|
||||
@@ -34,6 +34,10 @@ pub struct EngineApiMetrics {
|
||||
/// Metrics for EIP-7928 Block-Level Access Lists (BAL).
|
||||
#[allow(dead_code)]
|
||||
pub(crate) bal: BalMetrics,
|
||||
/// Gas-bucketed execution sub-phase metrics.
|
||||
pub(crate) execution_gas_buckets: ExecutionGasBucketMetrics,
|
||||
/// Gas-bucketed block validation sub-phase metrics.
|
||||
pub(crate) block_validation_gas_buckets: BlockValidationGasBucketMetrics,
|
||||
}
|
||||
|
||||
impl EngineApiMetrics {
|
||||
@@ -82,6 +86,22 @@ impl EngineApiMetrics {
|
||||
self.executor.post_execution_histogram.record(elapsed);
|
||||
}
|
||||
|
||||
/// Records execution duration into the gas-bucketed execution histogram.
|
||||
pub fn record_block_execution_gas_bucket(&self, gas_used: u64, elapsed: Duration) {
|
||||
let idx = GasBucketMetrics::bucket_index(gas_used);
|
||||
self.execution_gas_buckets.buckets[idx]
|
||||
.execution_gas_bucket_histogram
|
||||
.record(elapsed.as_secs_f64());
|
||||
}
|
||||
|
||||
/// Records state root duration into the gas-bucketed block validation histogram.
|
||||
pub fn record_state_root_gas_bucket(&self, gas_used: u64, elapsed_secs: f64) {
|
||||
let idx = GasBucketMetrics::bucket_index(gas_used);
|
||||
self.block_validation_gas_buckets.buckets[idx]
|
||||
.state_root_gas_bucket_histogram
|
||||
.record(elapsed_secs);
|
||||
}
|
||||
|
||||
/// Records the time spent waiting for the next transaction from the iterator.
|
||||
pub fn record_transaction_wait(&self, elapsed: Duration) {
|
||||
self.executor.transaction_wait_histogram.record(elapsed);
|
||||
@@ -280,7 +300,8 @@ impl GasBucketMetrics {
|
||||
.record(gas_used as f64 / elapsed.as_secs_f64());
|
||||
}
|
||||
|
||||
fn bucket_index(gas_used: u64) -> usize {
|
||||
/// Returns the bucket index for a given gas value.
|
||||
pub(crate) fn bucket_index(gas_used: u64) -> usize {
|
||||
GAS_BUCKET_THRESHOLDS
|
||||
.iter()
|
||||
.position(|&threshold| gas_used < threshold)
|
||||
@@ -288,7 +309,7 @@ impl GasBucketMetrics {
|
||||
}
|
||||
|
||||
/// Returns a human-readable label like `<5M`, `5-10M`, … `>40M`.
|
||||
fn bucket_label(index: usize) -> String {
|
||||
pub(crate) fn bucket_label(index: usize) -> String {
|
||||
if index == 0 {
|
||||
let hi = GAS_BUCKET_THRESHOLDS[0] / MEGAGAS;
|
||||
format!("<{hi}M")
|
||||
@@ -303,6 +324,56 @@ impl GasBucketMetrics {
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-gas-bucket execution duration metric.
|
||||
#[derive(Clone, Metrics)]
|
||||
#[metrics(scope = "sync.execution")]
|
||||
pub(crate) struct ExecutionGasBucketSeries {
|
||||
/// Gas-bucketed EVM execution duration.
|
||||
pub(crate) execution_gas_bucket_histogram: Histogram,
|
||||
}
|
||||
|
||||
/// Holds pre-initialized [`ExecutionGasBucketSeries`] instances, one per gas bucket.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct ExecutionGasBucketMetrics {
|
||||
buckets: [ExecutionGasBucketSeries; NUM_GAS_BUCKETS],
|
||||
}
|
||||
|
||||
impl Default for ExecutionGasBucketMetrics {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
buckets: std::array::from_fn(|i| {
|
||||
let label = GasBucketMetrics::bucket_label(i);
|
||||
ExecutionGasBucketSeries::new_with_labels(&[("gas_bucket", label)])
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Per-gas-bucket block validation metrics (state root).
|
||||
#[derive(Clone, Metrics)]
|
||||
#[metrics(scope = "sync.block_validation")]
|
||||
pub(crate) struct BlockValidationGasBucketSeries {
|
||||
/// Gas-bucketed state root computation duration.
|
||||
pub(crate) state_root_gas_bucket_histogram: Histogram,
|
||||
}
|
||||
|
||||
/// Holds pre-initialized [`BlockValidationGasBucketSeries`] instances, one per gas bucket.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct BlockValidationGasBucketMetrics {
|
||||
buckets: [BlockValidationGasBucketSeries; NUM_GAS_BUCKETS],
|
||||
}
|
||||
|
||||
impl Default for BlockValidationGasBucketMetrics {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
buckets: std::array::from_fn(|i| {
|
||||
let label = GasBucketMetrics::bucket_label(i);
|
||||
BlockValidationGasBucketSeries::new_with_labels(&[("gas_bucket", label)])
|
||||
}),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for engine newPayload responses.
|
||||
#[derive(Metrics)]
|
||||
#[metrics(scope = "consensus.engine.beacon")]
|
||||
|
||||
@@ -19,7 +19,7 @@ use reth_chain_state::{
|
||||
use reth_consensus::{Consensus, FullConsensus};
|
||||
use reth_engine_primitives::{
|
||||
BeaconEngineMessage, BeaconOnNewPayloadError, ConsensusEngineEvent, ExecutionPayload,
|
||||
ForkchoiceStateTracker, OnForkChoiceUpdated,
|
||||
ForkchoiceStateTracker, NewPayloadTimings, OnForkChoiceUpdated,
|
||||
};
|
||||
use reth_errors::{ConsensusError, ProviderResult};
|
||||
use reth_evm::ConfigureEvm;
|
||||
@@ -27,7 +27,9 @@ use reth_payload_builder::PayloadBuilderHandle;
|
||||
use reth_payload_primitives::{
|
||||
BuiltPayload, EngineApiMessageVersion, NewPayloadError, PayloadBuilderAttributes, PayloadTypes,
|
||||
};
|
||||
use reth_primitives_traits::{NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
use reth_primitives_traits::{
|
||||
FastInstant as Instant, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader,
|
||||
};
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockExecutionResult, BlockReader, ChangeSetReader,
|
||||
DatabaseProviderFactory, HashedPostStateProvider, ProviderError, StageCheckpointReader,
|
||||
@@ -40,7 +42,7 @@ use reth_tasks::spawn_os_thread;
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use revm::interpreter::debug_unreachable;
|
||||
use state::TreeState;
|
||||
use std::{fmt::Debug, ops, sync::Arc, time::Instant};
|
||||
use std::{fmt::Debug, ops, sync::Arc, time::Duration};
|
||||
|
||||
use crossbeam_channel::{Receiver, Sender};
|
||||
use tokio::sync::{
|
||||
@@ -321,7 +323,7 @@ where
|
||||
+ StorageSettingsCache,
|
||||
C: ConfigureEvm<Primitives = N> + 'static,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
V: EngineValidator<T>,
|
||||
V: EngineValidator<T> + WaitForCaches,
|
||||
{
|
||||
/// Creates a new [`EngineApiTreeHandler`].
|
||||
#[expect(clippy::too_many_arguments)]
|
||||
@@ -1411,7 +1413,7 @@ where
|
||||
// Spawn a background task to trigger computation so it's ready when the next payload
|
||||
// arrives.
|
||||
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
|
||||
tokio::task::spawn_blocking(move || {
|
||||
rayon::spawn(move || {
|
||||
let _ = overlay.get();
|
||||
});
|
||||
}
|
||||
@@ -1553,6 +1555,94 @@ where
|
||||
// handle the event if any
|
||||
self.on_maybe_tree_event(maybe_event)?;
|
||||
}
|
||||
BeaconEngineMessage::RethNewPayload { payload, tx } => {
|
||||
// Before processing the new payload, we wait for persistence and
|
||||
// cache updates to complete. We do it in parallel, spawning
|
||||
// persistence and cache update wait tasks with Tokio, so that we
|
||||
// can get an unbiased breakdown on how long did every step take.
|
||||
//
|
||||
// If we first wait for persistence, and only then for cache
|
||||
// updates, we will offset the cache update waits by the duration of
|
||||
// persistence, which is incorrect.
|
||||
debug!(target: "engine::tree", "Waiting for persistence and caches in parallel before processing reth_newPayload");
|
||||
|
||||
let pending_persistence = self.persistence_state.rx.take();
|
||||
let persistence_rx = if let Some((rx, start_time, _action)) =
|
||||
pending_persistence
|
||||
{
|
||||
let (persistence_tx, persistence_rx) =
|
||||
std::sync::mpsc::channel();
|
||||
tokio::task::spawn_blocking(move || {
|
||||
let start = Instant::now();
|
||||
let result =
|
||||
rx.recv().expect("persistence state channel closed");
|
||||
let _ = persistence_tx.send((
|
||||
result,
|
||||
start_time,
|
||||
start.elapsed(),
|
||||
));
|
||||
});
|
||||
Some(persistence_rx)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let cache_wait = self.payload_validator.wait_for_caches();
|
||||
|
||||
let persistence_wait = if let Some(persistence_rx) = persistence_rx
|
||||
{
|
||||
let (result, start_time, wait_duration) = persistence_rx
|
||||
.recv()
|
||||
.expect("persistence result channel closed");
|
||||
let _ = self.on_persistence_complete(result, start_time);
|
||||
Some(wait_duration)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
?persistence_wait,
|
||||
execution_cache_wait = ?cache_wait.execution_cache,
|
||||
sparse_trie_wait = ?cache_wait.sparse_trie,
|
||||
"Persistence finished and caches updated for reth_newPayload"
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
let gas_used = payload.gas_used();
|
||||
let num_hash = payload.num_hash();
|
||||
let mut output = self.on_new_payload(payload);
|
||||
let latency = start.elapsed();
|
||||
self.metrics.engine.new_payload.update_response_metrics(
|
||||
start,
|
||||
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
|
||||
&output,
|
||||
gas_used,
|
||||
);
|
||||
|
||||
let maybe_event =
|
||||
output.as_mut().ok().and_then(|out| out.event.take());
|
||||
|
||||
let timings = NewPayloadTimings {
|
||||
latency,
|
||||
persistence_wait,
|
||||
execution_cache_wait: cache_wait.execution_cache,
|
||||
sparse_trie_wait: cache_wait.sparse_trie,
|
||||
};
|
||||
if let Err(err) =
|
||||
tx.send(output.map(|o| (o.outcome, timings)).map_err(|e| {
|
||||
BeaconOnNewPayloadError::Internal(Box::new(e))
|
||||
}))
|
||||
{
|
||||
error!(target: "engine::tree", payload=?num_hash, elapsed=?start.elapsed(), "Failed to send event: {err:?}");
|
||||
self.metrics
|
||||
.engine
|
||||
.failed_new_payload_response_deliveries
|
||||
.increment(1);
|
||||
}
|
||||
|
||||
self.on_maybe_tree_event(maybe_event)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2602,7 +2692,7 @@ where
|
||||
/// Returns `InsertPayloadOk::Inserted(BlockStatus::Valid)` on successful execution,
|
||||
/// `InsertPayloadOk::AlreadySeen` if the block already exists, or
|
||||
/// `InsertPayloadOk::Inserted(BlockStatus::Disconnected)` if parent state is missing.
|
||||
#[instrument(level = "debug", target = "engine::tree", skip_all, fields(block_id))]
|
||||
#[instrument(level = "debug", target = "engine::tree", skip_all, fields(?block_id))]
|
||||
fn insert_block_or_payload<Input, Err>(
|
||||
&mut self,
|
||||
block_id: BlockWithParent,
|
||||
@@ -3046,3 +3136,23 @@ enum PersistTarget {
|
||||
/// Persist all blocks up to and including the canonical head.
|
||||
Head,
|
||||
}
|
||||
|
||||
/// Result of waiting for caches to become available.
|
||||
#[derive(Debug, Clone, Copy, Default)]
|
||||
pub struct CacheWaitDurations {
|
||||
/// Time spent waiting for the execution cache lock.
|
||||
pub execution_cache: Duration,
|
||||
/// Time spent waiting for the sparse trie lock.
|
||||
pub sparse_trie: Duration,
|
||||
}
|
||||
|
||||
/// Trait for types that can wait for caches to become available.
|
||||
///
|
||||
/// This is used by `reth_newPayload` endpoint to ensure that payload processing
|
||||
/// waits for any ongoing operations to complete before starting.
|
||||
pub trait WaitForCaches {
|
||||
/// Waits for cache updates to complete.
|
||||
///
|
||||
/// Returns the time spent waiting for each cache separately.
|
||||
fn wait_for_caches(&self) -> CacheWaitDurations;
|
||||
}
|
||||
|
||||
@@ -2,13 +2,13 @@
|
||||
|
||||
use super::precompile_cache::PrecompileCacheMap;
|
||||
use crate::tree::{
|
||||
cached_state::{CachedStateMetrics, CachedStateProvider, ExecutionCache, SavedCache},
|
||||
cached_state::{CachedStateMetrics, ExecutionCache, SavedCache},
|
||||
payload_processor::{
|
||||
prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
},
|
||||
sparse_trie::{SparseTrieCacheTask, SparseTrieTask, SpawnedSparseTrieTask},
|
||||
StateProviderBuilder, TreeConfig,
|
||||
sparse_trie::SparseTrieCacheTask,
|
||||
CacheWaitDurations, StateProviderBuilder, TreeConfig, WaitForCaches,
|
||||
};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal};
|
||||
@@ -16,10 +16,11 @@ use alloy_evm::block::StateChangeSource;
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use metrics::{Counter, Histogram};
|
||||
use multiproof::{SparseTrieUpdate, *};
|
||||
use multiproof::*;
|
||||
use parking_lot::RwLock;
|
||||
use prewarm::PrewarmMetrics;
|
||||
use rayon::prelude::*;
|
||||
use reth_engine_primitives::{SMALL_BLOCK_GAS_THRESHOLD, SMALL_BLOCK_MULTIPROOF_CHUNK_SIZE};
|
||||
use reth_evm::{
|
||||
block::ExecutableTxParts,
|
||||
execute::{ExecutableTxFor, WithTxEnv},
|
||||
@@ -27,13 +28,12 @@ use reth_evm::{
|
||||
SpecFor, TxEnvFor,
|
||||
};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_primitives_traits::{FastInstant as Instant, NodePrimitives};
|
||||
use reth_provider::{
|
||||
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProvider,
|
||||
StateProviderFactory, StateReader,
|
||||
BlockExecutionOutput, BlockReader, DatabaseProviderROFactory, StateProviderFactory, StateReader,
|
||||
};
|
||||
use reth_revm::{db::BundleState, state::EvmState};
|
||||
use reth_tasks::Runtime;
|
||||
use reth_tasks::{ForEachOrdered, Runtime};
|
||||
use reth_trie::{hashed_cursor::HashedCursorFactory, trie_cursor::TrieCursorFactory};
|
||||
use reth_trie_parallel::{
|
||||
proof_task::{ProofTaskCtx, ProofWorkerHandle},
|
||||
@@ -43,14 +43,13 @@ use reth_trie_sparse::{
|
||||
ParallelSparseTrie, ParallelismThresholds, RevealableSparseTrie, SparseStateTrie,
|
||||
};
|
||||
use std::{
|
||||
collections::BTreeMap,
|
||||
ops::Not,
|
||||
sync::{
|
||||
atomic::AtomicBool,
|
||||
mpsc::{self, channel},
|
||||
Arc,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
time::Duration,
|
||||
};
|
||||
use tracing::{debug, debug_span, instrument, warn, Span};
|
||||
|
||||
@@ -97,6 +96,7 @@ pub const SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY: usize = 1_000_000;
|
||||
/// Blocks with fewer transactions than this skip prewarming, since the fixed overhead of spawning
|
||||
/// prewarm workers exceeds the execution time saved.
|
||||
pub const SMALL_BLOCK_TX_THRESHOLD: usize = 5;
|
||||
|
||||
/// Type alias for [`PayloadHandle`] returned by payload processor spawn methods.
|
||||
type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
WithTxEnv<TxEnvFor<Evm>, <I as ExecutableTxIterator<Evm>>::Recovered>,
|
||||
@@ -132,8 +132,6 @@ where
|
||||
/// re-use allocated memory. Stored with the block hash it was computed for to enable trie
|
||||
/// preservation across sequential payload validations.
|
||||
sparse_state_trie: SharedPreservedSparseTrie,
|
||||
/// Maximum concurrency for prewarm task.
|
||||
prewarm_max_concurrency: usize,
|
||||
/// Sparse trie prune depth.
|
||||
sparse_trie_prune_depth: usize,
|
||||
/// Maximum storage tries to retain after pruning.
|
||||
@@ -172,7 +170,6 @@ where
|
||||
precompile_cache_disabled: config.precompile_cache_disabled(),
|
||||
precompile_cache_map,
|
||||
sparse_state_trie: SharedPreservedSparseTrie::default(),
|
||||
prewarm_max_concurrency: config.prewarm_max_concurrency(),
|
||||
sparse_trie_prune_depth: config.sparse_trie_prune_depth(),
|
||||
sparse_trie_max_storage_tries: config.sparse_trie_max_storage_tries(),
|
||||
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
|
||||
@@ -181,6 +178,46 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Evm> WaitForCaches for PayloadProcessor<Evm>
|
||||
where
|
||||
Evm: ConfigureEvm,
|
||||
{
|
||||
fn wait_for_caches(&self) -> CacheWaitDurations {
|
||||
debug!(target: "engine::tree::payload_processor", "Waiting for execution cache and sparse trie locks");
|
||||
|
||||
// Wait for both caches in parallel using std threads
|
||||
let execution_cache = self.execution_cache.clone();
|
||||
let sparse_trie = self.sparse_state_trie.clone();
|
||||
|
||||
// Use channels and spawn_blocking instead of std::thread::spawn
|
||||
let (execution_tx, execution_rx) = std::sync::mpsc::channel();
|
||||
let (sparse_trie_tx, sparse_trie_rx) = std::sync::mpsc::channel();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _ = execution_tx.send(execution_cache.wait_for_availability());
|
||||
});
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _ = sparse_trie_tx.send(sparse_trie.wait_for_availability());
|
||||
});
|
||||
|
||||
let execution_cache_duration =
|
||||
execution_rx.recv().expect("execution cache wait task failed to send result");
|
||||
let sparse_trie_duration =
|
||||
sparse_trie_rx.recv().expect("sparse trie wait task failed to send result");
|
||||
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
?execution_cache_duration,
|
||||
?sparse_trie_duration,
|
||||
"Execution cache and sparse trie locks acquired"
|
||||
);
|
||||
CacheWaitDurations {
|
||||
execution_cache: execution_cache_duration,
|
||||
sparse_trie: sparse_trie_duration,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
where
|
||||
N: NodePrimitives,
|
||||
@@ -211,7 +248,7 @@ where
|
||||
///
|
||||
/// ## Sparse trie task
|
||||
///
|
||||
/// Responsible for calculating the state root based on the received [`SparseTrieUpdate`].
|
||||
/// Responsible for calculating the state root.
|
||||
///
|
||||
/// This task runs until there are no further updates to process.
|
||||
///
|
||||
@@ -238,6 +275,7 @@ where
|
||||
F: DatabaseProviderROFactory<Provider: TrieCursorFactory + HashedCursorFactory>
|
||||
+ Clone
|
||||
+ Send
|
||||
+ Sync
|
||||
+ 'static,
|
||||
{
|
||||
// start preparing transactions immediately
|
||||
@@ -245,85 +283,34 @@ where
|
||||
self.spawn_tx_iterator(transactions, env.transaction_count);
|
||||
|
||||
let span = Span::current();
|
||||
let (to_sparse_trie, sparse_trie_rx) = channel();
|
||||
let (to_multi_proof, from_multi_proof) = crossbeam_channel::unbounded();
|
||||
|
||||
// Extract V2 proofs flag early so we can pass it to prewarm
|
||||
let v2_proofs_enabled = !config.disable_proof_v2();
|
||||
|
||||
// Capture parent_state_root before env is moved into spawn_caching_with
|
||||
let parent_state_root = env.parent_state_root;
|
||||
let transaction_count = env.transaction_count;
|
||||
let chunk_size = Self::adaptive_chunk_size(config, env.gas_used);
|
||||
let prewarm_handle = self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
provider_builder,
|
||||
Some(to_multi_proof.clone()),
|
||||
bal,
|
||||
);
|
||||
|
||||
// Handle BAL-based optimization if available
|
||||
let prewarm_handle = if let Some(bal) = bal {
|
||||
// When BAL is present, use BAL prewarming and send BAL to multiproof
|
||||
debug!(target: "engine::tree::payload_processor", "BAL present, using BAL prewarming");
|
||||
|
||||
// The prewarm task converts the BAL to HashedPostState and sends it on
|
||||
// to_multi_proof after slot prefetching completes.
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
Some(bal),
|
||||
v2_proofs_enabled,
|
||||
)
|
||||
} else {
|
||||
// Normal path: spawn with transaction prewarming
|
||||
self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
provider_builder.clone(),
|
||||
Some(to_multi_proof.clone()),
|
||||
None,
|
||||
v2_proofs_enabled,
|
||||
)
|
||||
};
|
||||
|
||||
// Create and spawn the storage proof task
|
||||
// Create and spawn the storage proof task.
|
||||
let task_ctx = ProofTaskCtx::new(multiproof_provider_factory);
|
||||
let proof_handle = ProofWorkerHandle::new(&self.executor, task_ctx, v2_proofs_enabled);
|
||||
|
||||
if config.disable_trie_cache() {
|
||||
let multi_proof_task = MultiProofTask::new(
|
||||
proof_handle.clone(),
|
||||
to_sparse_trie,
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size()),
|
||||
to_multi_proof.clone(),
|
||||
from_multi_proof.clone(),
|
||||
)
|
||||
.with_v2_proofs_enabled(v2_proofs_enabled);
|
||||
|
||||
// spawn multi-proof task
|
||||
let parent_span = span.clone();
|
||||
let saved_cache = prewarm_handle.saved_cache.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = parent_span.entered();
|
||||
// Build a state provider for the multiproof task
|
||||
let provider = provider_builder.build().expect("failed to build provider");
|
||||
let provider = if let Some(saved_cache) = saved_cache {
|
||||
let (cache, metrics, _disable_metrics) = saved_cache.split();
|
||||
Box::new(CachedStateProvider::new(provider, cache, metrics))
|
||||
as Box<dyn StateProvider>
|
||||
} else {
|
||||
Box::new(provider)
|
||||
};
|
||||
multi_proof_task.run(provider);
|
||||
});
|
||||
}
|
||||
let halve_workers = transaction_count <= Self::SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD;
|
||||
let proof_handle = ProofWorkerHandle::new(&self.executor, task_ctx, halve_workers);
|
||||
|
||||
// wire the sparse trie to the state root response receiver
|
||||
let (state_root_tx, state_root_rx) = channel();
|
||||
|
||||
// Spawn the sparse trie task using any stored trie and parallel trie configuration.
|
||||
self.spawn_sparse_trie_task(
|
||||
sparse_trie_rx,
|
||||
proof_handle,
|
||||
state_root_tx,
|
||||
from_multi_proof,
|
||||
config,
|
||||
parent_state_root,
|
||||
chunk_size,
|
||||
);
|
||||
|
||||
PayloadHandle {
|
||||
@@ -351,9 +338,7 @@ where
|
||||
{
|
||||
let (prewarm_rx, execution_rx) =
|
||||
self.spawn_tx_iterator(transactions, env.transaction_count);
|
||||
// This path doesn't use multiproof, so V2 proofs flag doesn't matter
|
||||
let prewarm_handle =
|
||||
self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal, false);
|
||||
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal);
|
||||
PayloadHandle {
|
||||
to_multi_proof: None,
|
||||
prewarm_handle,
|
||||
@@ -363,6 +348,10 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Transaction count threshold below which proof workers are halved, since fewer transactions
|
||||
/// produce fewer state changes and most workers would be idle overhead.
|
||||
const SMALL_BLOCK_PROOF_WORKER_TX_THRESHOLD: usize = 30;
|
||||
|
||||
/// Transaction count threshold below which sequential signature recovery is used.
|
||||
///
|
||||
/// For blocks with fewer than this many transactions, the rayon parallel iterator overhead
|
||||
@@ -371,22 +360,42 @@ where
|
||||
/// for small blocks.
|
||||
const SMALL_BLOCK_TX_THRESHOLD: usize = 30;
|
||||
|
||||
/// Returns the multiproof chunk size adapted to the block's gas usage.
|
||||
///
|
||||
/// For blocks with ≤20M gas used, a smaller chunk size (30) yields better throughput.
|
||||
/// For larger blocks, the configured default chunk size is used.
|
||||
const fn adaptive_chunk_size(config: &TreeConfig, gas_used: u64) -> Option<usize> {
|
||||
if !config.multiproof_chunking_enabled() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let size = if gas_used > 0 && gas_used <= SMALL_BLOCK_GAS_THRESHOLD {
|
||||
SMALL_BLOCK_MULTIPROOF_CHUNK_SIZE
|
||||
} else {
|
||||
config.multiproof_chunk_size()
|
||||
};
|
||||
|
||||
Some(size)
|
||||
}
|
||||
|
||||
/// Spawns a task advancing transaction env iterator and streaming updates through a channel.
|
||||
///
|
||||
/// For blocks with fewer than [`Self::SMALL_BLOCK_TX_THRESHOLD`] transactions, uses
|
||||
/// sequential iteration to avoid rayon overhead.
|
||||
/// sequential iteration to avoid rayon overhead. For larger blocks, uses rayon parallel
|
||||
/// iteration with [`ForEachOrdered`] to recover signatures in parallel while streaming
|
||||
/// results to execution in the original transaction order.
|
||||
#[expect(clippy::type_complexity)]
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip_all)]
|
||||
fn spawn_tx_iterator<I: ExecutableTxIterator<Evm>>(
|
||||
&self,
|
||||
transactions: I,
|
||||
transaction_count: usize,
|
||||
) -> (
|
||||
mpsc::Receiver<WithTxEnv<TxEnvFor<Evm>, I::Recovered>>,
|
||||
mpsc::Receiver<(usize, WithTxEnv<TxEnvFor<Evm>, I::Recovered>)>,
|
||||
mpsc::Receiver<Result<WithTxEnv<TxEnvFor<Evm>, I::Recovered>, I::Error>>,
|
||||
) {
|
||||
let (ooo_tx, ooo_rx) = mpsc::channel();
|
||||
let (prewarm_tx, prewarm_rx) = mpsc::channel();
|
||||
let (execute_tx, execute_rx) = mpsc::channel();
|
||||
let (prewarm_tx, prewarm_rx) = mpsc::sync_channel(transaction_count);
|
||||
let (execute_tx, execute_rx) = mpsc::sync_channel(transaction_count);
|
||||
|
||||
if transaction_count == 0 {
|
||||
// Empty block — nothing to do.
|
||||
@@ -399,6 +408,8 @@ where
|
||||
"using sequential sig recovery for small block"
|
||||
);
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor", "tx iterator").entered();
|
||||
let (transactions, convert) = transactions.into_parts();
|
||||
for (idx, tx) in transactions.into_iter().enumerate() {
|
||||
let tx = convert.convert(tx);
|
||||
@@ -407,67 +418,54 @@ where
|
||||
WithTxEnv { tx_env, tx: Arc::new(tx) }
|
||||
});
|
||||
if let Ok(tx) = &tx {
|
||||
let _ = prewarm_tx.send(tx.clone());
|
||||
let _ = prewarm_tx.send((idx, tx.clone()));
|
||||
}
|
||||
let _ = ooo_tx.send((idx, tx));
|
||||
let _ = execute_tx.send(tx);
|
||||
}
|
||||
});
|
||||
} else {
|
||||
// Parallel path — spawn on rayon for parallel signature recovery.
|
||||
rayon::spawn(move || {
|
||||
// Parallel path — recover signatures in parallel on rayon, stream results
|
||||
// to execution in order via `for_each_ordered`.
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor", "tx iterator").entered();
|
||||
let (transactions, convert) = transactions.into_parts();
|
||||
transactions.into_par_iter().enumerate().for_each_with(
|
||||
ooo_tx,
|
||||
|ooo_tx, (idx, tx)| {
|
||||
transactions
|
||||
.into_par_iter()
|
||||
.enumerate()
|
||||
.map(|(idx, tx)| {
|
||||
let tx = convert.convert(tx);
|
||||
let tx = tx.map(|tx| {
|
||||
tx.map(|tx| {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
WithTxEnv { tx_env, tx: Arc::new(tx) }
|
||||
});
|
||||
// Only send Ok(_) variants to prewarming task.
|
||||
if let Ok(tx) = &tx {
|
||||
let _ = prewarm_tx.send(tx.clone());
|
||||
}
|
||||
let _ = ooo_tx.send((idx, tx));
|
||||
},
|
||||
);
|
||||
let tx = WithTxEnv { tx_env, tx: Arc::new(tx) };
|
||||
// Send to prewarming out of order with the original index.
|
||||
let _ = prewarm_tx.send((idx, tx.clone()));
|
||||
tx
|
||||
})
|
||||
})
|
||||
.for_each_ordered(|tx| {
|
||||
let _ = execute_tx.send(tx);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// Spawn a task that processes out-of-order transactions from the task above and sends them
|
||||
// to the execution task in order.
|
||||
self.executor.spawn_blocking(move || {
|
||||
let mut next_for_execution = 0;
|
||||
let mut queue = BTreeMap::new();
|
||||
while let Ok((idx, tx)) = ooo_rx.recv() {
|
||||
if next_for_execution == idx {
|
||||
let _ = execute_tx.send(tx);
|
||||
next_for_execution += 1;
|
||||
|
||||
while let Some(entry) = queue.first_entry()
|
||||
&& *entry.key() == next_for_execution
|
||||
{
|
||||
let _ = execute_tx.send(entry.remove());
|
||||
next_for_execution += 1;
|
||||
}
|
||||
} else {
|
||||
queue.insert(idx, tx);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
(prewarm_rx, execute_rx)
|
||||
}
|
||||
|
||||
/// Spawn prewarming optionally wired to the multiproof task for target updates.
|
||||
#[instrument(
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor",
|
||||
skip_all,
|
||||
fields(bal=%bal.is_some())
|
||||
)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
env: ExecutionEnv<Evm>,
|
||||
transactions: mpsc::Receiver<impl ExecutableTxFor<Evm> + Clone + Send + 'static>,
|
||||
transactions: mpsc::Receiver<(usize, impl ExecutableTxFor<Evm> + Clone + Send + 'static)>,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
v2_proofs_enabled: bool,
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -487,7 +485,6 @@ where
|
||||
terminate_execution: Arc::new(AtomicBool::new(false)),
|
||||
precompile_cache_disabled: self.precompile_cache_disabled,
|
||||
precompile_cache_map: self.precompile_cache_map.clone(),
|
||||
v2_proofs_enabled,
|
||||
};
|
||||
|
||||
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
|
||||
@@ -495,10 +492,8 @@ where
|
||||
self.execution_cache.clone(),
|
||||
prewarm_ctx,
|
||||
to_multi_proof,
|
||||
self.prewarm_max_concurrency,
|
||||
);
|
||||
|
||||
// spawn pre-warm task
|
||||
{
|
||||
let to_prewarm_task = to_prewarm_task.clone();
|
||||
self.executor.spawn_blocking(move || {
|
||||
@@ -536,26 +531,22 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns the [`SparseTrieTask`] for this payload processor.
|
||||
/// Spawns the [`SparseTrieCacheTask`] for this payload processor.
|
||||
///
|
||||
/// The trie is preserved when the new payload is a child of the previous one.
|
||||
fn spawn_sparse_trie_task(
|
||||
&self,
|
||||
sparse_trie_rx: mpsc::Receiver<SparseTrieUpdate>,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
state_root_tx: mpsc::Sender<Result<StateRootComputeOutcome, ParallelStateRootError>>,
|
||||
from_multi_proof: CrossbeamReceiver<MultiProofMessage>,
|
||||
config: &TreeConfig,
|
||||
parent_state_root: B256,
|
||||
chunk_size: Option<usize>,
|
||||
) {
|
||||
let preserved_sparse_trie = self.sparse_state_trie.clone();
|
||||
let trie_metrics = self.trie_metrics.clone();
|
||||
let disable_trie_cache = config.disable_trie_cache();
|
||||
let prune_depth = self.sparse_trie_prune_depth;
|
||||
let max_storage_tries = self.sparse_trie_max_storage_tries;
|
||||
let disable_cache_pruning = self.disable_sparse_trie_cache_pruning;
|
||||
let chunk_size =
|
||||
config.multiproof_chunking_enabled().then_some(config.multiproof_chunk_size());
|
||||
let executor = self.executor.clone();
|
||||
|
||||
let parent_span = Span::current();
|
||||
@@ -591,23 +582,14 @@ where
|
||||
.with_updates(true)
|
||||
});
|
||||
|
||||
let mut task = if disable_trie_cache {
|
||||
SpawnedSparseTrieTask::Cleared(SparseTrieTask::new(
|
||||
sparse_trie_rx,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
))
|
||||
} else {
|
||||
SpawnedSparseTrieTask::Cached(SparseTrieCacheTask::new_with_trie(
|
||||
&executor,
|
||||
from_multi_proof,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie.with_skip_proof_node_filtering(true),
|
||||
chunk_size,
|
||||
))
|
||||
};
|
||||
let mut task = SparseTrieCacheTask::new_with_trie(
|
||||
&executor,
|
||||
from_multi_proof,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie.with_skip_proof_node_filtering(true),
|
||||
chunk_size,
|
||||
);
|
||||
|
||||
let result = task.run();
|
||||
// Capture the computed state_root before sending the result
|
||||
@@ -644,7 +626,7 @@ where
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor", "preserve").entered();
|
||||
let deferred = if let Some(state_root) = computed_state_root {
|
||||
let start = std::time::Instant::now();
|
||||
let start = Instant::now();
|
||||
let (trie, deferred) = task.into_trie_for_reuse(
|
||||
prune_depth,
|
||||
max_storage_tries,
|
||||
@@ -926,7 +908,7 @@ impl PayloadExecutionCache {
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip(self))]
|
||||
pub(crate) fn get_cache_for(&self, parent_hash: B256) -> Option<SavedCache> {
|
||||
let start = Instant::now();
|
||||
let mut cache = self.inner.write();
|
||||
let cache = self.inner.read();
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
self.metrics.execution_cache_wait_duration.record(elapsed.as_secs_f64());
|
||||
@@ -934,7 +916,7 @@ impl PayloadExecutionCache {
|
||||
warn!(blocked_for=?elapsed, "Blocked waiting for execution cache mutex");
|
||||
}
|
||||
|
||||
if let Some(c) = cache.as_mut() {
|
||||
if let Some(c) = cache.as_ref() {
|
||||
let cached_hash = c.executed_block_hash();
|
||||
// Check that the cache hash matches the parent hash of the current block. It won't
|
||||
// match in case it's a fork block.
|
||||
@@ -955,13 +937,13 @@ impl PayloadExecutionCache {
|
||||
);
|
||||
|
||||
if available {
|
||||
// If the has is available (no other threads are using it), but has a mismatching
|
||||
// parent hash, we can just clear it and keep using without re-creating from
|
||||
// scratch.
|
||||
if !hash_matches {
|
||||
// Fork block: clear and update the hash on the ORIGINAL before cloning.
|
||||
// This prevents the canonical chain from matching on the stale hash
|
||||
// and picking up polluted data if the fork block fails.
|
||||
c.clear_with_hash(parent_hash);
|
||||
c.clear();
|
||||
}
|
||||
return Some(c.clone());
|
||||
return Some(c.clone())
|
||||
} else if hash_matches {
|
||||
self.metrics.execution_cache_in_use.increment(1);
|
||||
}
|
||||
@@ -972,6 +954,12 @@ impl PayloadExecutionCache {
|
||||
None
|
||||
}
|
||||
|
||||
/// Clears the tracked cache
|
||||
#[expect(unused)]
|
||||
pub(crate) fn clear(&self) {
|
||||
self.inner.write().take();
|
||||
}
|
||||
|
||||
/// Waits until the execution cache becomes available for use.
|
||||
///
|
||||
/// This acquires a write lock to ensure exclusive access, then immediately releases it.
|
||||
@@ -1043,6 +1031,9 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
/// Used to determine parallel worker count for prewarming.
|
||||
/// A value of 0 indicates the count is unknown.
|
||||
pub transaction_count: usize,
|
||||
/// Total gas used by all transactions in the block.
|
||||
/// Used to adaptively select multiproof chunk size for optimal throughput.
|
||||
pub gas_used: u64,
|
||||
/// Withdrawals included in the block.
|
||||
/// Used to generate prefetch targets for withdrawal addresses.
|
||||
pub withdrawals: Option<Vec<Withdrawal>>,
|
||||
@@ -1059,6 +1050,7 @@ where
|
||||
parent_hash: Default::default(),
|
||||
parent_state_root: Default::default(),
|
||||
transaction_count: 0,
|
||||
gas_used: 0,
|
||||
withdrawals: None,
|
||||
}
|
||||
}
|
||||
@@ -1143,18 +1135,10 @@ mod tests {
|
||||
|
||||
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(hash)));
|
||||
|
||||
// When the parent hash doesn't match (fork block), the cache is cleared,
|
||||
// hash updated on the original, and clone returned for reuse
|
||||
// When the parent hash doesn't match, the cache is cleared and returned for reuse
|
||||
let different_hash = B256::from([4u8; 32]);
|
||||
let cache = execution_cache.get_cache_for(different_hash);
|
||||
assert!(cache.is_some(), "cache should be returned for reuse after clearing");
|
||||
|
||||
drop(cache);
|
||||
|
||||
// The stored cache now has the fork block's parent hash.
|
||||
// Canonical chain looking for original hash sees a mismatch → clears and reuses.
|
||||
let original = execution_cache.get_cache_for(hash);
|
||||
assert!(original.is_some(), "canonical chain gets cache back via mismatch+clear");
|
||||
assert!(cache.is_some(), "cache should be returned for reuse after clearing")
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1378,61 +1362,4 @@ mod tests {
|
||||
"State root mismatch: task={root_from_task}, base={root_from_regular}"
|
||||
);
|
||||
}
|
||||
|
||||
/// Tests the full prewarm lifecycle for a fork block:
|
||||
///
|
||||
/// 1. Cache is at canonical block 4.
|
||||
/// 2. Fork block (parent = block 2) checks out the cache via `get_cache_for`, simulating what
|
||||
/// `PrewarmCacheTask` does when it receives a `SavedCache`.
|
||||
/// 3. Prewarm populates the shared cache with fork-specific state.
|
||||
/// 4. While the prewarm clone is alive, the cache is unavailable (`usage_guard` > 1).
|
||||
/// 5. Prewarm drops without calling `save_cache` (fork block was invalid).
|
||||
/// 6. Canonical block 5 (parent = block 4) must get a cache with correct hash and no stale fork
|
||||
/// data.
|
||||
#[test]
|
||||
fn fork_prewarm_dropped_without_save_does_not_corrupt_cache() {
|
||||
let execution_cache = PayloadExecutionCache::default();
|
||||
|
||||
// Canonical chain at block 4.
|
||||
let block4_hash = B256::from([4u8; 32]);
|
||||
execution_cache.update_with_guard(|slot| *slot = Some(make_saved_cache(block4_hash)));
|
||||
|
||||
// Fork block arrives with parent = block 2. Prewarm task checks out the cache.
|
||||
// This simulates PrewarmCacheTask receiving a SavedCache clone from get_cache_for.
|
||||
let fork_parent = B256::from([2u8; 32]);
|
||||
let prewarm_cache = execution_cache.get_cache_for(fork_parent);
|
||||
assert!(prewarm_cache.is_some(), "prewarm should obtain cache for fork block");
|
||||
let prewarm_cache = prewarm_cache.unwrap();
|
||||
assert_eq!(prewarm_cache.executed_block_hash(), fork_parent);
|
||||
|
||||
// Prewarm populates cache with fork-specific state (ancestor data for block 2).
|
||||
// Since ExecutionCache uses Arc<Inner>, this data is shared with the stored original.
|
||||
let fork_addr = Address::from([0xBB; 20]);
|
||||
let fork_key = B256::from([0xCC; 32]);
|
||||
prewarm_cache.cache().insert_storage(fork_addr, fork_key, Some(U256::from(999)));
|
||||
|
||||
// While prewarm holds the clone, the usage_guard count > 1 → cache is in use.
|
||||
let during_prewarm = execution_cache.get_cache_for(block4_hash);
|
||||
assert!(
|
||||
during_prewarm.is_none(),
|
||||
"cache must be unavailable while prewarm holds a reference"
|
||||
);
|
||||
|
||||
// Fork block fails — prewarm task drops without calling save_cache/update_with_guard.
|
||||
drop(prewarm_cache);
|
||||
|
||||
// Canonical block 5 arrives (parent = block 4).
|
||||
// Stored hash = fork_parent (our fix), so get_cache_for sees a mismatch,
|
||||
// clears the stale fork data, and returns a cache with hash = block4_hash.
|
||||
let block5_cache = execution_cache.get_cache_for(block4_hash);
|
||||
assert!(
|
||||
block5_cache.is_some(),
|
||||
"canonical chain must get cache after fork prewarm is dropped"
|
||||
);
|
||||
assert_eq!(
|
||||
block5_cache.as_ref().unwrap().executed_block_hash(),
|
||||
block4_hash,
|
||||
"cache must carry the canonical parent hash, not the fork parent"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -3,7 +3,7 @@
|
||||
use alloy_primitives::B256;
|
||||
use parking_lot::Mutex;
|
||||
use reth_trie_sparse::SparseStateTrie;
|
||||
use std::sync::Arc;
|
||||
use std::{sync::Arc, time::Instant};
|
||||
use tracing::debug;
|
||||
|
||||
/// Type alias for the sparse trie type used in preservation.
|
||||
@@ -12,7 +12,7 @@ pub(super) type SparseTrie = SparseStateTrie;
|
||||
/// Shared handle to a preserved sparse trie that can be reused across payload validations.
|
||||
///
|
||||
/// This is stored in [`PayloadProcessor`](super::PayloadProcessor) and cloned to pass to
|
||||
/// [`SparseTrieTask`](super::sparse_trie::SparseTrieTask) for trie reuse.
|
||||
/// [`SparseTrieCacheTask`](super::sparse_trie::SparseTrieCacheTask) for trie reuse.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(super) struct SharedPreservedSparseTrie(Arc<Mutex<Option<PreservedSparseTrie>>>);
|
||||
|
||||
@@ -28,6 +28,27 @@ impl SharedPreservedSparseTrie {
|
||||
pub(super) fn lock(&self) -> PreservedTrieGuard<'_> {
|
||||
PreservedTrieGuard(self.0.lock())
|
||||
}
|
||||
|
||||
/// Waits until the sparse trie lock becomes available.
|
||||
///
|
||||
/// This acquires and immediately releases the lock, ensuring that any
|
||||
/// ongoing operations complete before returning. Useful for synchronization
|
||||
/// before starting payload processing.
|
||||
///
|
||||
/// Returns the time spent waiting for the lock.
|
||||
pub(super) fn wait_for_availability(&self) -> std::time::Duration {
|
||||
let start = Instant::now();
|
||||
let _guard = self.0.lock();
|
||||
let elapsed = start.elapsed();
|
||||
if elapsed.as_millis() > 5 {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
blocked_for=?elapsed,
|
||||
"Waited for preserved sparse trie to become available"
|
||||
);
|
||||
}
|
||||
elapsed
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that holds the lock on the preserved trie.
|
||||
|
||||
@@ -13,11 +13,7 @@
|
||||
|
||||
use crate::tree::{
|
||||
cached_state::{CachedStateProvider, SavedCache},
|
||||
payload_processor::{
|
||||
bal::{self, total_slots, BALSlotIter},
|
||||
multiproof::{MultiProofMessage, VersionedMultiProofTargets},
|
||||
PayloadExecutionCache,
|
||||
},
|
||||
payload_processor::{bal, multiproof::MultiProofMessage, PayloadExecutionCache},
|
||||
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
|
||||
ExecutionEnv, StateProviderBuilder,
|
||||
};
|
||||
@@ -25,35 +21,32 @@ use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_evm::Database;
|
||||
use alloy_primitives::{keccak256, map::B256Set, B256};
|
||||
use alloy_primitives::{keccak256, StorageKey, B256};
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use rayon::prelude::*;
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::NodePrimitives;
|
||||
use reth_primitives_traits::{FastInstant as Instant, NodePrimitives};
|
||||
use reth_provider::{
|
||||
AccountReader, BlockExecutionOutput, BlockReader, StateProvider, StateProviderFactory,
|
||||
StateReader,
|
||||
};
|
||||
use reth_revm::{database::StateProviderDatabase, state::EvmState};
|
||||
use reth_tasks::Runtime;
|
||||
use reth_trie::MultiProofTargets;
|
||||
use std::{
|
||||
ops::Range,
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc::{self, channel, Receiver, Sender},
|
||||
Arc,
|
||||
},
|
||||
time::Instant,
|
||||
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
|
||||
use std::sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
mpsc::{self, channel, Receiver, Sender, SyncSender},
|
||||
Arc,
|
||||
};
|
||||
use tracing::{debug, debug_span, instrument, trace, warn, Span};
|
||||
|
||||
/// Determines the prewarming mode: transaction-based, BAL-based, or skipped.
|
||||
#[derive(Debug)]
|
||||
pub enum PrewarmMode<Tx> {
|
||||
/// Prewarm by executing transactions from a stream.
|
||||
Transactions(Receiver<Tx>),
|
||||
/// Prewarm by executing transactions from a stream, each paired with its block index.
|
||||
Transactions(Receiver<(usize, Tx)>),
|
||||
/// Prewarm by prefetching slots from a Block Access List.
|
||||
BlockAccessList(Arc<BlockAccessList>),
|
||||
/// Transaction prewarming is skipped (e.g. small blocks where the overhead exceeds the
|
||||
@@ -86,8 +79,6 @@ where
|
||||
execution_cache: PayloadExecutionCache,
|
||||
/// Context provided to execution tasks
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
/// How many transactions should be executed in parallel
|
||||
max_concurrency: usize,
|
||||
/// Sender to emit evm state outcome messages, if any.
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
/// Receiver for events produced by tx execution
|
||||
@@ -108,13 +99,12 @@ where
|
||||
execution_cache: PayloadExecutionCache,
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
max_concurrency: usize,
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
max_concurrency,
|
||||
prewarming_threads = executor.prewarming_pool().current_num_threads(),
|
||||
transaction_count = ctx.env.transaction_count,
|
||||
"Initialized prewarm task"
|
||||
);
|
||||
@@ -124,7 +114,6 @@ where
|
||||
executor,
|
||||
execution_cache,
|
||||
ctx,
|
||||
max_concurrency,
|
||||
to_multi_proof,
|
||||
actions_rx,
|
||||
parent_span: Span::current(),
|
||||
@@ -140,7 +129,7 @@ where
|
||||
/// subsequent transactions in the block.
|
||||
fn spawn_all<Tx>(
|
||||
&self,
|
||||
pending: mpsc::Receiver<Tx>,
|
||||
pending: mpsc::Receiver<(usize, Tx)>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
) where
|
||||
@@ -148,30 +137,28 @@ where
|
||||
{
|
||||
let executor = self.executor.clone();
|
||||
let ctx = self.ctx.clone();
|
||||
let max_concurrency = self.max_concurrency;
|
||||
let span = Span::current();
|
||||
|
||||
self.executor.spawn_blocking(move || {
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor::prewarm", parent: span, "spawn_all").entered();
|
||||
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
|
||||
// When transaction_count is 0, it means the count is unknown. In this case, spawn
|
||||
// max workers to handle potentially many transactions in parallel rather
|
||||
// than bottlenecking on a single worker.
|
||||
let transaction_count = ctx.env.transaction_count;
|
||||
let workers_needed = if transaction_count == 0 {
|
||||
max_concurrency
|
||||
let pool_threads = executor.prewarming_pool().current_num_threads();
|
||||
// Don't spawn more workers than transactions. When transaction_count is 0
|
||||
// (unknown), use all pool threads.
|
||||
let workers_needed = if ctx.env.transaction_count > 0 {
|
||||
ctx.env.transaction_count.min(pool_threads)
|
||||
} else {
|
||||
transaction_count.min(max_concurrency)
|
||||
pool_threads
|
||||
};
|
||||
|
||||
let (done_tx, done_rx) = mpsc::sync_channel(workers_needed);
|
||||
|
||||
// Spawn workers
|
||||
let tx_sender = ctx.clone().spawn_workers(workers_needed, &executor, to_multi_proof.clone(), done_tx.clone());
|
||||
|
||||
// Distribute transactions to workers
|
||||
let mut tx_index = 0usize;
|
||||
while let Ok(tx) = pending.recv() {
|
||||
let mut tx_count = 0usize;
|
||||
while let Ok((tx_index, tx)) = pending.recv() {
|
||||
// Stop distributing if termination was requested
|
||||
if ctx.terminate_execution.load(Ordering::Relaxed) {
|
||||
trace!(
|
||||
@@ -188,7 +175,7 @@ where
|
||||
// exit early when signaled.
|
||||
let _ = tx_sender.send(indexed_tx);
|
||||
|
||||
tx_index += 1;
|
||||
tx_count += 1;
|
||||
}
|
||||
|
||||
// Send withdrawal prefetch targets after all transactions have been distributed
|
||||
@@ -196,9 +183,9 @@ where
|
||||
&& let Some(withdrawals) = &ctx.env.withdrawals
|
||||
&& !withdrawals.is_empty()
|
||||
{
|
||||
let targets =
|
||||
multiproof_targets_from_withdrawals(withdrawals, ctx.v2_proofs_enabled);
|
||||
let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(targets));
|
||||
let targets = multiproof_targets_from_withdrawals(withdrawals);
|
||||
let _ = to_multi_proof
|
||||
.send(MultiProofMessage::PrefetchProofs(targets));
|
||||
}
|
||||
|
||||
// drop sender and wait for all tasks to finish
|
||||
@@ -207,7 +194,7 @@ where
|
||||
while done_rx.recv().is_ok() {}
|
||||
|
||||
let _ = actions_tx
|
||||
.send(PrewarmTaskEvent::FinishedTxExecution { executed_transactions: tx_index });
|
||||
.send(PrewarmTaskEvent::FinishedTxExecution { executed_transactions: tx_count });
|
||||
});
|
||||
}
|
||||
|
||||
@@ -274,10 +261,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Runs BAL-based prewarming by spawning workers to prefetch storage slots.
|
||||
///
|
||||
/// Divides the total slots across `max_concurrency` workers, each responsible for
|
||||
/// prefetching a range of slots from the BAL.
|
||||
/// Runs BAL-based prewarming by using the prewarming pool's parallel iterator to prefetch
|
||||
/// accounts and storage slots.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn run_bal_prewarm(
|
||||
&self,
|
||||
@@ -296,59 +281,35 @@ where
|
||||
return;
|
||||
}
|
||||
|
||||
let total_slots = total_slots(&bal);
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
total_slots,
|
||||
max_concurrency = self.max_concurrency,
|
||||
"Starting BAL prewarm"
|
||||
);
|
||||
|
||||
if total_slots == 0 {
|
||||
if bal.is_empty() {
|
||||
self.send_bal_hashed_state(&bal);
|
||||
let _ =
|
||||
actions_tx.send(PrewarmTaskEvent::FinishedTxExecution { executed_transactions: 0 });
|
||||
return;
|
||||
}
|
||||
|
||||
let (done_tx, done_rx) = mpsc::channel();
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
accounts = bal.len(),
|
||||
"Starting BAL prewarm"
|
||||
);
|
||||
|
||||
// Calculate number of workers needed (at most max_concurrency)
|
||||
let workers_needed = total_slots.min(self.max_concurrency);
|
||||
|
||||
// Calculate slots per worker
|
||||
let slots_per_worker = total_slots / workers_needed;
|
||||
let remainder = total_slots % workers_needed;
|
||||
|
||||
// Spawn workers with their assigned ranges
|
||||
for i in 0..workers_needed {
|
||||
let start = i * slots_per_worker + i.min(remainder);
|
||||
let extra = if i < remainder { 1 } else { 0 };
|
||||
let end = start + slots_per_worker + extra;
|
||||
|
||||
self.ctx.spawn_bal_worker(
|
||||
i,
|
||||
&self.executor,
|
||||
Arc::clone(&bal),
|
||||
start..end,
|
||||
done_tx.clone(),
|
||||
let ctx = self.ctx.clone();
|
||||
self.executor.prewarming_pool().install_fn(|| {
|
||||
bal.par_iter().for_each_init(
|
||||
|| (ctx.clone(), None::<CachedStateProvider<reth_provider::StateProviderBox>>),
|
||||
|(ctx, provider), account| {
|
||||
if ctx.terminate_execution.load(Ordering::Relaxed) {
|
||||
return;
|
||||
}
|
||||
ctx.prefetch_bal_account(provider, account);
|
||||
},
|
||||
);
|
||||
}
|
||||
|
||||
// Drop our handle to done_tx so we can detect completion
|
||||
drop(done_tx);
|
||||
|
||||
// Wait for all workers to complete
|
||||
let mut completed_workers = 0;
|
||||
while done_rx.recv().is_ok() {
|
||||
completed_workers += 1;
|
||||
}
|
||||
});
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
completed_workers,
|
||||
"All BAL prewarm workers completed"
|
||||
"All BAL prewarm accounts completed"
|
||||
);
|
||||
|
||||
// Convert BAL to HashedPostState and send to multiproof task
|
||||
@@ -491,8 +452,6 @@ where
|
||||
pub precompile_cache_disabled: bool,
|
||||
/// The precompile cache map.
|
||||
pub precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether V2 proof calculation is enabled.
|
||||
pub v2_proofs_enabled: bool,
|
||||
}
|
||||
|
||||
impl<N, P, Evm> PrewarmContext<N, P, Evm>
|
||||
@@ -501,12 +460,9 @@ where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
Evm: ConfigureEvm<Primitives = N> + 'static,
|
||||
{
|
||||
/// Splits this context into an evm, an evm config, metrics, the atomic bool for terminating
|
||||
/// execution, and whether V2 proofs are enabled.
|
||||
/// Splits this context into an evm, metrics, and the atomic bool for terminating execution.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn evm_for_ctx(
|
||||
self,
|
||||
) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>, bool)> {
|
||||
fn evm_for_ctx(self) -> Option<(EvmFor<Evm, impl Database>, PrewarmMetrics, Arc<AtomicBool>)> {
|
||||
let Self {
|
||||
env,
|
||||
evm_config,
|
||||
@@ -516,7 +472,6 @@ where
|
||||
terminate_execution,
|
||||
precompile_cache_disabled,
|
||||
precompile_cache_map,
|
||||
v2_proofs_enabled,
|
||||
} = self;
|
||||
|
||||
let mut state_provider = match provider.build() {
|
||||
@@ -567,7 +522,7 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
Some((evm, metrics, terminate_execution, v2_proofs_enabled))
|
||||
Some((evm, metrics, terminate_execution))
|
||||
}
|
||||
|
||||
/// Accepts a [`CrossbeamReceiver`] of transactions and a handle to prewarm task. Executes
|
||||
@@ -580,25 +535,21 @@ where
|
||||
///
|
||||
/// Note: There are no ordering guarantees; this does not reflect the state produced by
|
||||
/// sequential execution.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn transact_batch<Tx>(
|
||||
self,
|
||||
txs: CrossbeamReceiver<IndexedTransaction<Tx>>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
done_tx: Sender<()>,
|
||||
done_tx: SyncSender<()>,
|
||||
) where
|
||||
Tx: ExecutableTxFor<Evm>,
|
||||
{
|
||||
let Some((mut evm, metrics, terminate_execution, v2_proofs_enabled)) = self.evm_for_ctx()
|
||||
else {
|
||||
return
|
||||
};
|
||||
let Some((mut evm, metrics, terminate_execution)) = self.evm_for_ctx() else { return };
|
||||
|
||||
while let Ok(IndexedTransaction { index, tx }) = txs.recv() {
|
||||
let _enter = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
"prewarm tx",
|
||||
index,
|
||||
i=index,
|
||||
)
|
||||
.entered();
|
||||
|
||||
@@ -637,8 +588,7 @@ where
|
||||
// Only send outcome for transactions after the first txn
|
||||
// as the main execution will be just as fast
|
||||
if index > 0 {
|
||||
let (targets, storage_targets) =
|
||||
multiproof_targets_from_state(res.state, v2_proofs_enabled);
|
||||
let (targets, storage_targets) = multiproof_targets_from_state(res.state);
|
||||
metrics.prefetch_storage_targets.record(storage_targets as f64);
|
||||
if let Some(to_multi_proof) = &to_multi_proof {
|
||||
let _ = to_multi_proof.send(MultiProofMessage::PrefetchProofs(targets));
|
||||
@@ -660,7 +610,7 @@ where
|
||||
workers_needed: usize,
|
||||
task_executor: &Runtime,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
done_tx: Sender<()>,
|
||||
done_tx: SyncSender<()>,
|
||||
) -> CrossbeamSender<IndexedTransaction<Tx>>
|
||||
where
|
||||
Tx: ExecutableTxFor<Evm> + Send + 'static,
|
||||
@@ -668,171 +618,72 @@ where
|
||||
let (tx_sender, tx_receiver) = crossbeam_channel::unbounded();
|
||||
|
||||
// Spawn workers that all pull from the shared receiver
|
||||
let executor = task_executor.clone();
|
||||
let span = Span::current();
|
||||
task_executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
for idx in 0..workers_needed {
|
||||
let ctx = self.clone();
|
||||
let to_multi_proof = to_multi_proof.clone();
|
||||
let done_tx = done_tx.clone();
|
||||
let rx = tx_receiver.clone();
|
||||
let span = debug_span!(target: "engine::tree::payload_processor::prewarm", "prewarm worker", idx);
|
||||
executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
ctx.transact_batch(rx, to_multi_proof, done_tx);
|
||||
});
|
||||
}
|
||||
});
|
||||
for idx in 0..workers_needed {
|
||||
let ctx = self.clone();
|
||||
let to_multi_proof = to_multi_proof.clone();
|
||||
let done_tx = done_tx.clone();
|
||||
let rx = tx_receiver.clone();
|
||||
let span = debug_span!(target: "engine::tree::payload_processor::prewarm", parent: &span, "prewarm worker", idx);
|
||||
task_executor.prewarming_pool().spawn(move || {
|
||||
let _enter = span.entered();
|
||||
ctx.transact_batch(rx, to_multi_proof, done_tx);
|
||||
});
|
||||
}
|
||||
|
||||
tx_sender
|
||||
}
|
||||
|
||||
/// Spawns a worker task for BAL slot prefetching.
|
||||
/// Prefetches a single account and all its storage slots from the BAL into the cache.
|
||||
///
|
||||
/// The worker iterates over the specified range of slots in the BAL and ensures
|
||||
/// each slot is loaded into the cache by accessing it through the state provider.
|
||||
fn spawn_bal_worker(
|
||||
/// The `provider` is lazily initialized on first call and reused across accounts on the same
|
||||
/// thread.
|
||||
fn prefetch_bal_account(
|
||||
&self,
|
||||
idx: usize,
|
||||
executor: &Runtime,
|
||||
bal: Arc<BlockAccessList>,
|
||||
range: Range<usize>,
|
||||
done_tx: Sender<()>,
|
||||
provider: &mut Option<CachedStateProvider<reth_provider::StateProviderBox>>,
|
||||
account: &alloy_eip7928::AccountChanges,
|
||||
) {
|
||||
let ctx = self.clone();
|
||||
let span = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
"bal prewarm worker",
|
||||
idx,
|
||||
range_start = range.start,
|
||||
range_end = range.end
|
||||
);
|
||||
|
||||
executor.spawn_blocking(move || {
|
||||
let _enter = span.entered();
|
||||
ctx.prefetch_bal_slots(bal, range, done_tx);
|
||||
});
|
||||
}
|
||||
|
||||
/// Prefetches storage slots from a BAL range into the cache.
|
||||
///
|
||||
/// This iterates through the specified range of slots and accesses them via the state
|
||||
/// provider to populate the cache.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn prefetch_bal_slots(
|
||||
self,
|
||||
bal: Arc<BlockAccessList>,
|
||||
range: Range<usize>,
|
||||
done_tx: Sender<()>,
|
||||
) {
|
||||
let Self { saved_cache, provider, metrics, .. } = self;
|
||||
|
||||
// Build state provider
|
||||
let state_provider = match provider.build() {
|
||||
Ok(provider) => provider,
|
||||
Err(err) => {
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
%err,
|
||||
"Failed to build state provider in BAL prewarm thread"
|
||||
);
|
||||
let _ = done_tx.send(());
|
||||
return;
|
||||
let state_provider = match provider {
|
||||
Some(p) => p,
|
||||
slot @ None => {
|
||||
let built = match self.provider.build() {
|
||||
Ok(p) => p,
|
||||
Err(err) => {
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
%err,
|
||||
"Failed to build state provider in BAL prewarm thread"
|
||||
);
|
||||
return;
|
||||
}
|
||||
};
|
||||
let saved_cache =
|
||||
self.saved_cache.as_ref().expect("BAL prewarm should only run with cache");
|
||||
let caches = saved_cache.cache().clone();
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
slot.insert(CachedStateProvider::new(built, caches, cache_metrics))
|
||||
}
|
||||
};
|
||||
|
||||
// Wrap with cache (guaranteed to be Some since run_bal_prewarm checks)
|
||||
let saved_cache = saved_cache.expect("BAL prewarm should only run with cache");
|
||||
let caches = saved_cache.cache().clone();
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
let state_provider = CachedStateProvider::new(state_provider, caches, cache_metrics);
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
// Track last seen address to avoid fetching the same account multiple times.
|
||||
let mut last_address = None;
|
||||
let _ = state_provider.basic_account(&account.address);
|
||||
|
||||
// Iterate through the assigned range of slots
|
||||
for (address, slot) in BALSlotIter::new(&bal, range.clone()) {
|
||||
// Fetch the account if this is a different address than the last one
|
||||
if last_address != Some(address) {
|
||||
let _ = state_provider.basic_account(&address);
|
||||
last_address = Some(address);
|
||||
}
|
||||
|
||||
// Access the slot to populate the cache
|
||||
let _ = state_provider.storage(address, slot);
|
||||
for slot in &account.storage_changes {
|
||||
let _ = state_provider.storage(account.address, StorageKey::from(slot.slot));
|
||||
}
|
||||
for &slot in &account.storage_reads {
|
||||
let _ = state_provider.storage(account.address, StorageKey::from(slot));
|
||||
}
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
|
||||
trace!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
?range,
|
||||
elapsed_ms = elapsed.as_millis(),
|
||||
"BAL prewarm worker completed"
|
||||
);
|
||||
|
||||
// Signal completion
|
||||
let _ = done_tx.send(());
|
||||
metrics.bal_slot_iteration_duration.record(elapsed.as_secs_f64());
|
||||
self.metrics.bal_slot_iteration_duration.record(start.elapsed().as_secs_f64());
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a set of [`VersionedMultiProofTargets`] and the total amount of storage targets, based
|
||||
/// on the given state.
|
||||
fn multiproof_targets_from_state(
|
||||
state: EvmState,
|
||||
v2_enabled: bool,
|
||||
) -> (VersionedMultiProofTargets, usize) {
|
||||
if v2_enabled {
|
||||
multiproof_targets_v2_from_state(state)
|
||||
} else {
|
||||
multiproof_targets_legacy_from_state(state)
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns legacy [`MultiProofTargets`] and the total amount of storage targets, based on the
|
||||
/// Returns a set of [`MultiProofTargetsV2`] and the total amount of storage targets, based on the
|
||||
/// given state.
|
||||
fn multiproof_targets_legacy_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
|
||||
let mut targets = MultiProofTargets::with_capacity(state.len());
|
||||
let mut storage_targets = 0;
|
||||
for (addr, account) in state {
|
||||
// if the account was not touched, or if the account was selfdestructed, do not
|
||||
// fetch proofs for it
|
||||
//
|
||||
// Since selfdestruct can only happen in the same transaction, we can skip
|
||||
// prefetching proofs for selfdestructed accounts
|
||||
//
|
||||
// See: https://eips.ethereum.org/EIPS/eip-6780
|
||||
if !account.is_touched() || account.is_selfdestructed() {
|
||||
continue
|
||||
}
|
||||
|
||||
let mut storage_set =
|
||||
B256Set::with_capacity_and_hasher(account.storage.len(), Default::default());
|
||||
for (key, slot) in account.storage {
|
||||
// do nothing if unchanged
|
||||
if !slot.is_changed() {
|
||||
continue
|
||||
}
|
||||
|
||||
storage_set.insert(keccak256(B256::new(key.to_be_bytes())));
|
||||
}
|
||||
|
||||
storage_targets += storage_set.len();
|
||||
targets.insert(keccak256(addr), storage_set);
|
||||
}
|
||||
|
||||
(VersionedMultiProofTargets::Legacy(targets), storage_targets)
|
||||
}
|
||||
|
||||
/// Returns V2 [`reth_trie_parallel::targets_v2::MultiProofTargetsV2`] and the total amount of
|
||||
/// storage targets, based on the given state.
|
||||
fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTargets, usize) {
|
||||
fn multiproof_targets_from_state(state: EvmState) -> (MultiProofTargetsV2, usize) {
|
||||
use reth_trie::proof_v2;
|
||||
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
|
||||
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
let mut storage_target_count = 0;
|
||||
@@ -868,27 +719,17 @@ fn multiproof_targets_v2_from_state(state: EvmState) -> (VersionedMultiProofTarg
|
||||
}
|
||||
}
|
||||
|
||||
(VersionedMultiProofTargets::V2(targets), storage_target_count)
|
||||
(targets, storage_target_count)
|
||||
}
|
||||
|
||||
/// Returns [`VersionedMultiProofTargets`] for withdrawal addresses.
|
||||
/// Returns [`MultiProofTargetsV2`] for withdrawal addresses.
|
||||
///
|
||||
/// Withdrawals only modify account balances (no storage), so the targets contain
|
||||
/// only account-level entries with empty storage sets.
|
||||
fn multiproof_targets_from_withdrawals(
|
||||
withdrawals: &[Withdrawal],
|
||||
v2_enabled: bool,
|
||||
) -> VersionedMultiProofTargets {
|
||||
use reth_trie_parallel::targets_v2::MultiProofTargetsV2;
|
||||
if v2_enabled {
|
||||
VersionedMultiProofTargets::V2(MultiProofTargetsV2 {
|
||||
account_targets: withdrawals.iter().map(|w| keccak256(w.address).into()).collect(),
|
||||
..Default::default()
|
||||
})
|
||||
} else {
|
||||
VersionedMultiProofTargets::Legacy(
|
||||
withdrawals.iter().map(|w| (keccak256(w.address), Default::default())).collect(),
|
||||
)
|
||||
fn multiproof_targets_from_withdrawals(withdrawals: &[Withdrawal]) -> MultiProofTargetsV2 {
|
||||
MultiProofTargetsV2 {
|
||||
account_targets: withdrawals.iter().map(|w| keccak256(w.address).into()).collect(),
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,212 +3,35 @@
|
||||
use crate::tree::{
|
||||
multiproof::{
|
||||
dispatch_with_chunking, evm_state_to_hashed_post_state, MultiProofMessage,
|
||||
VersionedMultiProofTargets, DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
},
|
||||
payload_processor::multiproof::{MultiProofTaskMetrics, SparseTrieUpdate},
|
||||
payload_processor::multiproof::MultiProofTaskMetrics,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
use rayon::iter::ParallelIterator;
|
||||
use reth_primitives_traits::{Account, ParallelBridgeBuffered};
|
||||
use reth_primitives_traits::{Account, FastInstant as Instant, ParallelBridgeBuffered};
|
||||
use reth_tasks::Runtime;
|
||||
use reth_trie::{
|
||||
proof_v2::Target, updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, Nibbles,
|
||||
TrieAccount, EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
proof_v2::Target, updates::TrieUpdates, DecodedMultiProofV2, HashedPostState, TrieAccount,
|
||||
EMPTY_ROOT_HASH, TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use reth_trie_parallel::{
|
||||
proof_task::{
|
||||
AccountMultiproofInput, ProofResult, ProofResultContext, ProofResultMessage,
|
||||
ProofWorkerHandle,
|
||||
AccountMultiproofInput, ProofResultContext, ProofResultMessage, ProofWorkerHandle,
|
||||
},
|
||||
root::ParallelStateRootError,
|
||||
targets_v2::MultiProofTargetsV2,
|
||||
};
|
||||
#[cfg(feature = "trie-debug")]
|
||||
use reth_trie_sparse::debug_recorder::TrieDebugRecorder;
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieResult, SparseTrieErrorKind, SparseTrieResult},
|
||||
provider::{TrieNodeProvider, TrieNodeProviderFactory},
|
||||
DeferredDrops, LeafUpdate, ParallelSparseTrie, SparseStateTrie, SparseTrie,
|
||||
errors::SparseTrieResult, DeferredDrops, LeafUpdate, ParallelSparseTrie, SparseStateTrie,
|
||||
SparseTrie,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use smallvec::SmallVec;
|
||||
use std::{
|
||||
sync::mpsc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{debug, debug_span, error, instrument, trace};
|
||||
|
||||
#[expect(clippy::large_enum_variant)]
|
||||
pub(super) enum SpawnedSparseTrieTask<BPF, A, S>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrie + Send + Sync + Default,
|
||||
S: SparseTrie + Send + Sync + Default + Clone,
|
||||
{
|
||||
Cleared(SparseTrieTask<BPF, A, S>),
|
||||
Cached(SparseTrieCacheTask<A, S>),
|
||||
}
|
||||
|
||||
impl<BPF, A, S> SpawnedSparseTrieTask<BPF, A, S>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync + Clone,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrie + Send + Sync + Default,
|
||||
S: SparseTrie + Send + Sync + Default + Clone,
|
||||
{
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
match self {
|
||||
Self::Cleared(task) => task.run(),
|
||||
Self::Cached(task) => task.run(),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_trie_for_reuse(
|
||||
self,
|
||||
prune_depth: usize,
|
||||
max_storage_tries: usize,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
disable_pruning: bool,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
match self {
|
||||
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
Self::Cached(task) => task.into_trie_for_reuse(
|
||||
prune_depth,
|
||||
max_storage_tries,
|
||||
max_nodes_capacity,
|
||||
max_values_capacity,
|
||||
disable_pruning,
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
pub(super) fn into_cleared_trie(
|
||||
self,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
match self {
|
||||
Self::Cleared(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
Self::Cached(task) => task.into_cleared_trie(max_nodes_capacity, max_values_capacity),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// A task responsible for populating the sparse trie.
|
||||
pub(super) struct SparseTrieTask<BPF, A = ParallelSparseTrie, S = ParallelSparseTrie>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
{
|
||||
/// Receives updates from the state root task.
|
||||
pub(super) updates: mpsc::Receiver<SparseTrieUpdate>,
|
||||
/// `SparseStateTrie` used for computing the state root.
|
||||
pub(super) trie: SparseStateTrie<A, S>,
|
||||
pub(super) metrics: MultiProofTaskMetrics,
|
||||
/// Trie node provider factory.
|
||||
blinded_provider_factory: BPF,
|
||||
}
|
||||
|
||||
impl<BPF, A, S> SparseTrieTask<BPF, A, S>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync + Clone,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrie + Send + Sync + Default,
|
||||
S: SparseTrie + Send + Sync + Default + Clone,
|
||||
{
|
||||
/// Creates a new sparse trie task with the given trie.
|
||||
pub(super) const fn new(
|
||||
updates: mpsc::Receiver<SparseTrieUpdate>,
|
||||
blinded_provider_factory: BPF,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
) -> Self {
|
||||
Self { updates, metrics, trie, blinded_provider_factory }
|
||||
}
|
||||
|
||||
/// Runs the sparse trie task to completion, computing the state root.
|
||||
///
|
||||
/// Receives [`SparseTrieUpdate`]s until the channel is closed, applying each update
|
||||
/// to the trie. Once all updates are processed, computes and returns the final state root.
|
||||
#[instrument(
|
||||
name = "SparseTrieTask::run",
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
pub(super) fn run(&mut self) -> Result<StateRootComputeOutcome, ParallelStateRootError> {
|
||||
let now = Instant::now();
|
||||
|
||||
let mut num_iterations = 0;
|
||||
|
||||
while let Ok(mut update) = self.updates.recv() {
|
||||
num_iterations += 1;
|
||||
let mut num_updates = 1;
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::sparse_trie", "drain updates")
|
||||
.entered();
|
||||
while let Ok(next) = self.updates.try_recv() {
|
||||
update.extend(next);
|
||||
num_updates += 1;
|
||||
}
|
||||
drop(_enter);
|
||||
|
||||
debug!(
|
||||
target: "engine::root",
|
||||
num_updates,
|
||||
account_proofs = update.multiproof.account_proofs_len(),
|
||||
storage_proofs = update.multiproof.storage_proofs_len(),
|
||||
"Updating sparse trie"
|
||||
);
|
||||
|
||||
let elapsed =
|
||||
update_sparse_trie(&mut self.trie, update, &self.blinded_provider_factory)
|
||||
.map_err(|e| {
|
||||
ParallelStateRootError::Other(format!(
|
||||
"could not calculate state root: {e:?}"
|
||||
))
|
||||
})?;
|
||||
self.metrics.sparse_trie_update_duration_histogram.record(elapsed);
|
||||
trace!(target: "engine::root", ?elapsed, num_iterations, "Root calculation completed");
|
||||
}
|
||||
|
||||
debug!(target: "engine::root", num_iterations, "All proofs processed, ending calculation");
|
||||
|
||||
let start = Instant::now();
|
||||
let (state_root, trie_updates) =
|
||||
self.trie.root_with_updates(&self.blinded_provider_factory).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
|
||||
})?;
|
||||
|
||||
let end = Instant::now();
|
||||
self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start));
|
||||
self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now));
|
||||
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
}
|
||||
|
||||
/// Clears and shrinks the trie, discarding all state.
|
||||
///
|
||||
/// Use this when the payload was invalid or cancelled - we don't want to preserve
|
||||
/// potentially invalid trie state, but we keep the allocations for reuse.
|
||||
pub(super) fn into_cleared_trie(
|
||||
self,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
let Self { mut trie, .. } = self;
|
||||
trie.clear();
|
||||
trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
let deferred = trie.take_deferred_drops();
|
||||
(trie, deferred)
|
||||
}
|
||||
}
|
||||
use tracing::{debug, debug_span, error, instrument};
|
||||
|
||||
/// Maximum number of pending/prewarm updates that we accumulate in memory before actually applying.
|
||||
const MAX_PENDING_UPDATES: usize = 100;
|
||||
@@ -335,7 +158,7 @@ where
|
||||
SparseTrieTaskMessage::PrefetchProofs(targets)
|
||||
}
|
||||
MultiProofMessage::StateUpdate(_, state) => {
|
||||
let _span = debug_span!(target: "engine::tree::payload_processor::sparse_trie", "hashing state update", update_len = state.len()).entered();
|
||||
let _span = debug_span!(target: "engine::tree::payload_processor::sparse_trie", "hashing state update", n = state.len()).entered();
|
||||
let hashed = evm_state_to_hashed_post_state(state);
|
||||
SparseTrieTaskMessage::HashedState(hashed)
|
||||
}
|
||||
@@ -429,14 +252,10 @@ where
|
||||
let Ok(result) = message else {
|
||||
unreachable!("we own the sender half")
|
||||
};
|
||||
let ProofResult::V2(mut result) = result.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
let mut result = result.result?;
|
||||
|
||||
while let Ok(next) = self.proof_result_rx.try_recv() {
|
||||
let ProofResult::V2(res) = next.result? else {
|
||||
unreachable!("sparse trie as cache must only be used with multiproof v2");
|
||||
};
|
||||
let res = next.result?;
|
||||
result.extend(res);
|
||||
}
|
||||
|
||||
@@ -478,11 +297,19 @@ where
|
||||
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
|
||||
})?;
|
||||
|
||||
#[cfg(feature = "trie-debug")]
|
||||
let debug_recorders = self.trie.take_debug_recorders();
|
||||
|
||||
let end = Instant::now();
|
||||
self.metrics.sparse_trie_final_update_duration_histogram.record(end.duration_since(start));
|
||||
self.metrics.sparse_trie_total_duration_histogram.record(end.duration_since(now));
|
||||
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates })
|
||||
Ok(StateRootComputeOutcome {
|
||||
state_root,
|
||||
trie_updates,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
debug_recorders,
|
||||
})
|
||||
}
|
||||
|
||||
/// Processes a [`SparseTrieTaskMessage`] from the hashing task.
|
||||
@@ -501,11 +328,7 @@ where
|
||||
target = "engine::tree::payload_processor::sparse_trie",
|
||||
skip_all
|
||||
)]
|
||||
fn on_prewarm_targets(&mut self, targets: VersionedMultiProofTargets) {
|
||||
let VersionedMultiProofTargets::V2(targets) = targets else {
|
||||
unreachable!("sparse trie as cache must only be used with V2 multiproof targets");
|
||||
};
|
||||
|
||||
fn on_prewarm_targets(&mut self, targets: MultiProofTargetsV2) {
|
||||
for target in targets.account_targets {
|
||||
// Only touch accounts that are not yet present in the updates set.
|
||||
self.new_account_updates.entry(target.key()).or_insert(LeafUpdate::Touched);
|
||||
@@ -593,18 +416,24 @@ where
|
||||
self.process_leaf_updates(true)?;
|
||||
|
||||
for (address, mut new) in self.new_storage_updates.drain() {
|
||||
let updates = self.storage_updates.entry(address).or_default();
|
||||
for (slot, new) in new.drain() {
|
||||
match updates.entry(slot) {
|
||||
Entry::Occupied(mut entry) => {
|
||||
// Only overwrite existing entries with new values
|
||||
if new.is_changed() {
|
||||
entry.insert(new);
|
||||
match self.storage_updates.entry(address) {
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(new); // insert the whole map at once, no per-slot loop
|
||||
}
|
||||
Entry::Occupied(mut entry) => {
|
||||
let updates = entry.get_mut();
|
||||
for (slot, new) in new.drain() {
|
||||
match updates.entry(slot) {
|
||||
Entry::Occupied(mut slot_entry) => {
|
||||
if new.is_changed() {
|
||||
slot_entry.insert(new);
|
||||
}
|
||||
}
|
||||
Entry::Vacant(slot_entry) => {
|
||||
slot_entry.insert(new);
|
||||
}
|
||||
}
|
||||
}
|
||||
Entry::Vacant(entry) => {
|
||||
entry.insert(new);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -649,7 +478,7 @@ where
|
||||
})
|
||||
.par_bridge_buffered()
|
||||
.map(|(address, updates, mut fetched, mut trie)| {
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie leaf updates", ?address).entered();
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie leaf updates", a=%address).entered();
|
||||
let mut targets = Vec::new();
|
||||
|
||||
trie.update_leaves(updates, |path, min_len| match fetched.entry(path) {
|
||||
@@ -852,7 +681,7 @@ where
|
||||
MultiProofTargetsV2::chunks,
|
||||
|proof_targets| {
|
||||
if let Err(e) = self.proof_worker_handle.dispatch_account_multiproof(
|
||||
AccountMultiproofInput::V2 {
|
||||
AccountMultiproofInput {
|
||||
targets: proof_targets,
|
||||
proof_result_sender: ProofResultContext::new(
|
||||
self.proof_result_tx.clone(),
|
||||
@@ -875,7 +704,7 @@ enum SparseTrieTaskMessage {
|
||||
/// A hashed state update ready to be processed.
|
||||
HashedState(HashedPostState),
|
||||
/// Prefetch proof targets (passed through directly).
|
||||
PrefetchProofs(VersionedMultiProofTargets),
|
||||
PrefetchProofs(MultiProofTargetsV2),
|
||||
/// Signals that all state updates have been received.
|
||||
FinishedStateUpdates,
|
||||
}
|
||||
@@ -888,160 +717,10 @@ pub struct StateRootComputeOutcome {
|
||||
pub state_root: B256,
|
||||
/// The trie updates.
|
||||
pub trie_updates: TrieUpdates,
|
||||
}
|
||||
|
||||
/// Updates the sparse trie with the given proofs and state, and returns the elapsed time.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::sparse_trie", skip_all)]
|
||||
pub(crate) fn update_sparse_trie<BPF, A, S>(
|
||||
trie: &mut SparseStateTrie<A, S>,
|
||||
SparseTrieUpdate { mut state, multiproof }: SparseTrieUpdate,
|
||||
blinded_provider_factory: &BPF,
|
||||
) -> SparseStateTrieResult<Duration>
|
||||
where
|
||||
BPF: TrieNodeProviderFactory + Send + Sync,
|
||||
BPF::AccountNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
BPF::StorageNodeProvider: TrieNodeProvider + Send + Sync,
|
||||
A: SparseTrie + Send + Sync + Default,
|
||||
S: SparseTrie + Send + Sync + Default + Clone,
|
||||
{
|
||||
trace!(target: "engine::root::sparse", "Updating sparse trie");
|
||||
let started_at = Instant::now();
|
||||
|
||||
// Reveal new accounts and storage slots.
|
||||
match multiproof {
|
||||
ProofResult::Legacy(decoded, _) => {
|
||||
trie.reveal_decoded_multiproof(decoded)?;
|
||||
}
|
||||
ProofResult::V2(decoded_v2) => {
|
||||
trie.reveal_decoded_multiproof_v2(decoded_v2)?;
|
||||
}
|
||||
}
|
||||
let reveal_multiproof_elapsed = started_at.elapsed();
|
||||
trace!(
|
||||
target: "engine::root::sparse",
|
||||
?reveal_multiproof_elapsed,
|
||||
"Done revealing multiproof"
|
||||
);
|
||||
|
||||
// Update storage slots with new values and calculate storage roots.
|
||||
let span = tracing::Span::current();
|
||||
let results: Vec<_> = state
|
||||
.storages
|
||||
.into_iter()
|
||||
.map(|(address, storage)| (address, storage, trie.take_storage_trie(&address)))
|
||||
.par_bridge_buffered()
|
||||
.map(|(address, storage, storage_trie)| {
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_processor::sparse_trie", parent: &span, "storage trie", ?address)
|
||||
.entered();
|
||||
|
||||
trace!(target: "engine::tree::payload_processor::sparse_trie", "Updating storage");
|
||||
let storage_provider = blinded_provider_factory.storage_node_provider(address);
|
||||
let mut storage_trie = storage_trie.ok_or(SparseTrieErrorKind::Blind)?;
|
||||
|
||||
if storage.wiped {
|
||||
trace!(target: "engine::tree::payload_processor::sparse_trie", "Wiping storage");
|
||||
storage_trie.wipe()?;
|
||||
}
|
||||
|
||||
// Defer leaf removals until after updates/additions, so that we don't delete an
|
||||
// intermediate branch node during a removal and then re-add that branch back during a
|
||||
// later leaf addition. This is an optimization, but also a requirement inherited from
|
||||
// multiproof generating, which can't know the order that leaf operations happen in.
|
||||
let mut removed_slots = SmallVec::<[Nibbles; 8]>::new();
|
||||
|
||||
for (slot, value) in storage.storage {
|
||||
let slot_nibbles = Nibbles::unpack(slot);
|
||||
|
||||
if value.is_zero() {
|
||||
removed_slots.push(slot_nibbles);
|
||||
continue;
|
||||
}
|
||||
|
||||
trace!(target: "engine::tree::payload_processor::sparse_trie", ?slot_nibbles, "Updating storage slot");
|
||||
storage_trie.update_leaf(
|
||||
slot_nibbles,
|
||||
alloy_rlp::encode_fixed_size(&value).to_vec(),
|
||||
&storage_provider,
|
||||
)?;
|
||||
}
|
||||
|
||||
for slot_nibbles in removed_slots {
|
||||
trace!(target: "engine::root::sparse", ?slot_nibbles, "Removing storage slot");
|
||||
storage_trie.remove_leaf(&slot_nibbles, &storage_provider)?;
|
||||
}
|
||||
|
||||
storage_trie.root();
|
||||
|
||||
SparseStateTrieResult::Ok((address, storage_trie))
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Defer leaf removals until after updates/additions, so that we don't delete an intermediate
|
||||
// branch node during a removal and then re-add that branch back during a later leaf addition.
|
||||
// This is an optimization, but also a requirement inherited from multiproof generating, which
|
||||
// can't know the order that leaf operations happen in.
|
||||
let mut removed_accounts = Vec::new();
|
||||
|
||||
// Update account storage roots
|
||||
let _enter =
|
||||
tracing::debug_span!(target: "engine::tree::payload_processor::sparse_trie", "account trie")
|
||||
.entered();
|
||||
for result in results {
|
||||
let (address, storage_trie) = result?;
|
||||
trie.insert_storage_trie(address, storage_trie);
|
||||
|
||||
if let Some(account) = state.accounts.remove(&address) {
|
||||
// If the account itself has an update, remove it from the state update and update in
|
||||
// one go instead of doing it down below.
|
||||
trace!(target: "engine::root::sparse", ?address, "Updating account and its storage root");
|
||||
if !trie.update_account(
|
||||
address,
|
||||
account.unwrap_or_default(),
|
||||
blinded_provider_factory,
|
||||
)? {
|
||||
removed_accounts.push(address);
|
||||
}
|
||||
} else if trie.is_account_revealed(address) {
|
||||
// Otherwise, if the account is revealed, only update its storage root.
|
||||
trace!(target: "engine::root::sparse", ?address, "Updating account storage root");
|
||||
if !trie.update_account_storage_root(address, blinded_provider_factory)? {
|
||||
removed_accounts.push(address);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Update accounts
|
||||
for (address, account) in state.accounts {
|
||||
trace!(target: "engine::root::sparse", ?address, "Updating account");
|
||||
if !trie.update_account(address, account.unwrap_or_default(), blinded_provider_factory)? {
|
||||
removed_accounts.push(address);
|
||||
}
|
||||
}
|
||||
|
||||
// Remove accounts
|
||||
for address in removed_accounts {
|
||||
trace!(target: "engine::root::sparse", ?address, "Removing account");
|
||||
let nibbles = Nibbles::unpack(address);
|
||||
trie.remove_account_leaf(&nibbles, blinded_provider_factory)?;
|
||||
}
|
||||
|
||||
let elapsed_before = started_at.elapsed();
|
||||
trace!(
|
||||
target: "engine::root::sparse",
|
||||
"Calculating subtries"
|
||||
);
|
||||
trie.calculate_subtries();
|
||||
|
||||
let elapsed = started_at.elapsed();
|
||||
let below_level_elapsed = elapsed - elapsed_before;
|
||||
trace!(
|
||||
target: "engine::root::sparse",
|
||||
?below_level_elapsed,
|
||||
"Intermediate nodes calculated"
|
||||
);
|
||||
|
||||
Ok(elapsed)
|
||||
/// Debug recorders taken from the sparse tries, keyed by `None` for account trie
|
||||
/// and `Some(address)` for storage tries.
|
||||
#[cfg(feature = "trie-debug")]
|
||||
pub debug_recorders: Vec<(Option<B256>, TrieDebugRecorder)>,
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
|
||||
@@ -7,14 +7,16 @@ use crate::tree::{
|
||||
payload_processor::PayloadProcessor,
|
||||
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle, StateProviderBuilder,
|
||||
StateProviderDatabase, TreeConfig,
|
||||
CacheWaitDurations, EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle,
|
||||
StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches,
|
||||
};
|
||||
use alloy_consensus::transaction::{Either, TxHashRef};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
use alloy_primitives::B256;
|
||||
#[cfg(feature = "trie-debug")]
|
||||
use reth_trie_sparse::debug_recorder::TrieDebugRecorder;
|
||||
|
||||
use crate::tree::payload_processor::receipt_root_task::{IndexedReceipt, ReceiptRootTaskHandle};
|
||||
use reth_chain_state::{CanonicalInMemoryState, DeferredTrieData, ExecutedBlock, LazyOverlay};
|
||||
@@ -31,8 +33,8 @@ use reth_payload_primitives::{
|
||||
BuiltPayload, InvalidPayloadAttributesError, NewPayloadError, PayloadTypes,
|
||||
};
|
||||
use reth_primitives_traits::{
|
||||
AlloyBlockHeader, BlockBody, BlockTy, GotExpected, NodePrimitives, RecoveredBlock, SealedBlock,
|
||||
SealedHeader, SignerRecoverable,
|
||||
AlloyBlockHeader, BlockBody, BlockTy, FastInstant as Instant, GotExpected, NodePrimitives,
|
||||
RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable,
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
@@ -49,7 +51,6 @@ use std::{
|
||||
collections::HashMap,
|
||||
panic::{self, AssertUnwindSafe},
|
||||
sync::{mpsc::RecvTimeoutError, Arc},
|
||||
time::Instant,
|
||||
};
|
||||
use tracing::{debug, debug_span, error, info, instrument, trace, warn};
|
||||
|
||||
@@ -327,9 +328,41 @@ where
|
||||
mut ctx: TreeCtx<'_, N>,
|
||||
) -> ValidationOutcome<N, InsertPayloadError<N::Block>>
|
||||
where
|
||||
V: PayloadValidator<T, Block = N::Block>,
|
||||
V: PayloadValidator<T, Block = N::Block> + Clone,
|
||||
Evm: ConfigureEngineEvm<T::ExecutionData, Primitives = N>,
|
||||
{
|
||||
// Spawn block conversion on a background thread so it runs concurrently with the
|
||||
// rest of the function (setup + execution). For payloads this overlaps the cost of
|
||||
// RLP decoding + header hashing; for already-converted blocks this is a no-op.
|
||||
let convert_to_block = match &input {
|
||||
BlockOrPayload::Payload(_) => {
|
||||
let payload_clone = input.clone();
|
||||
let validator = self.validator.clone();
|
||||
let (tx, rx) = tokio::sync::oneshot::channel();
|
||||
self.payload_processor.executor().spawn_blocking(move || {
|
||||
let BlockOrPayload::Payload(payload) = payload_clone else { unreachable!() };
|
||||
let _ = tx.send(validator.convert_payload_to_block(payload));
|
||||
});
|
||||
Either::Left(rx)
|
||||
}
|
||||
BlockOrPayload::Block(_) => Either::Right(()),
|
||||
};
|
||||
|
||||
// Returns the sealed block, either by awaiting the background conversion task (for
|
||||
// payloads) or by extracting the already-converted block directly.
|
||||
let convert_to_block =
|
||||
move |input: BlockOrPayload<T>| -> Result<SealedBlock<N::Block>, NewPayloadError> {
|
||||
match convert_to_block {
|
||||
Either::Left(rx) => rx.blocking_recv().map_err(|_| {
|
||||
NewPayloadError::Other("block conversion task panicked".into())
|
||||
})?,
|
||||
Either::Right(()) => {
|
||||
let BlockOrPayload::Block(block) = input else { unreachable!() };
|
||||
Ok(block)
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
/// A helper macro that returns the block in case there was an error
|
||||
/// This macro is used for early returns before block conversion
|
||||
macro_rules! ensure_ok {
|
||||
@@ -337,7 +370,7 @@ where
|
||||
match $expr {
|
||||
Ok(val) => val,
|
||||
Err(e) => {
|
||||
let block = self.convert_to_block(input)?;
|
||||
let block = convert_to_block(input)?;
|
||||
return Err(InsertBlockError::new(block, e.into()).into())
|
||||
}
|
||||
}
|
||||
@@ -368,7 +401,7 @@ where
|
||||
else {
|
||||
// this is pre-validated in the tree
|
||||
return Err(InsertBlockError::new(
|
||||
self.convert_to_block(input)?,
|
||||
convert_to_block(input)?,
|
||||
ProviderError::HeaderNotFound(parent_hash.into()).into(),
|
||||
)
|
||||
.into())
|
||||
@@ -381,7 +414,7 @@ where
|
||||
let Some(parent_block) = ensure_ok!(self.sealed_header_by_hash(parent_hash, ctx.state()))
|
||||
else {
|
||||
return Err(InsertBlockError::new(
|
||||
self.convert_to_block(input)?,
|
||||
convert_to_block(input)?,
|
||||
ProviderError::HeaderNotFound(parent_hash.into()).into(),
|
||||
)
|
||||
.into())
|
||||
@@ -397,6 +430,7 @@ where
|
||||
parent_hash: input.parent_hash(),
|
||||
parent_state_root: parent_block.state_root(),
|
||||
transaction_count: input.transaction_count(),
|
||||
gas_used: input.gas_used(),
|
||||
withdrawals: input.withdrawals().map(|w| w.to_vec()),
|
||||
};
|
||||
|
||||
@@ -473,7 +507,7 @@ where
|
||||
// needed. This frees up resources while state root computation continues.
|
||||
let valid_block_tx = handle.terminate_caching(Some(output.clone()));
|
||||
|
||||
let block = self.convert_to_block(input)?.with_senders(senders);
|
||||
let block = convert_to_block(input)?.with_senders(senders);
|
||||
|
||||
// Wait for the receipt root computation to complete.
|
||||
let receipt_root_bloom = receipt_root_rx
|
||||
@@ -500,6 +534,8 @@ where
|
||||
let root_time = Instant::now();
|
||||
let mut maybe_state_root = None;
|
||||
let mut state_root_task_failed = false;
|
||||
#[cfg(feature = "trie-debug")]
|
||||
let mut trie_debug_recorders = Vec::new();
|
||||
|
||||
match strategy {
|
||||
StateRootStrategy::StateRootTask => {
|
||||
@@ -515,17 +551,34 @@ where
|
||||
);
|
||||
|
||||
match task_result {
|
||||
Ok(StateRootComputeOutcome { state_root, trie_updates }) => {
|
||||
Ok(StateRootComputeOutcome {
|
||||
state_root,
|
||||
trie_updates,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
debug_recorders,
|
||||
}) => {
|
||||
let elapsed = root_time.elapsed();
|
||||
info!(target: "engine::tree::payload_validator", ?state_root, ?elapsed, "State root task finished");
|
||||
|
||||
#[cfg(feature = "trie-debug")]
|
||||
{
|
||||
trie_debug_recorders = debug_recorders;
|
||||
}
|
||||
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
self.compare_trie_updates_with_serial(
|
||||
let _has_diff = self.compare_trie_updates_with_serial(
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
trie_updates.clone(),
|
||||
);
|
||||
#[cfg(feature = "trie-debug")]
|
||||
if _has_diff {
|
||||
Self::write_trie_debug_recorders(
|
||||
block.header().number(),
|
||||
&trie_debug_recorders,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// we double check the state root here for good measure
|
||||
@@ -538,6 +591,11 @@ where
|
||||
block_state_root = ?block.header().state_root(),
|
||||
"State root task returned incorrect state root"
|
||||
);
|
||||
#[cfg(feature = "trie-debug")]
|
||||
Self::write_trie_debug_recorders(
|
||||
block.header().number(),
|
||||
&trie_debug_recorders,
|
||||
);
|
||||
state_root_task_failed = true;
|
||||
}
|
||||
}
|
||||
@@ -597,10 +655,15 @@ where
|
||||
};
|
||||
|
||||
self.metrics.block_validation.record_state_root(&trie_output, root_elapsed.as_secs_f64());
|
||||
self.metrics
|
||||
.record_state_root_gas_bucket(block.header().gas_used(), root_elapsed.as_secs_f64());
|
||||
debug!(target: "engine::tree::payload_validator", ?root_elapsed, "Calculated state root");
|
||||
|
||||
// ensure state root matches
|
||||
if state_root != block.header().state_root() {
|
||||
#[cfg(feature = "trie-debug")]
|
||||
Self::write_trie_debug_recorders(block.header().number(), &trie_debug_recorders);
|
||||
|
||||
// call post-block hook
|
||||
self.on_invalid_block(
|
||||
&parent_block,
|
||||
@@ -765,6 +828,7 @@ where
|
||||
|
||||
let execution_duration = execution_start.elapsed();
|
||||
self.metrics.record_block_execution(&output, execution_duration);
|
||||
self.metrics.record_block_execution_gas_bucket(output.result.gas_used, execution_duration);
|
||||
|
||||
debug!(target: "engine::tree::payload_validator", elapsed = ?execution_duration, "Executed block");
|
||||
Ok((output, senders, result_rx))
|
||||
@@ -972,7 +1036,12 @@ where
|
||||
))
|
||||
})?;
|
||||
let (state_root, trie_updates) = result?;
|
||||
return Ok(Ok(StateRootComputeOutcome { state_root, trie_updates }));
|
||||
return Ok(Ok(StateRootComputeOutcome {
|
||||
state_root,
|
||||
trie_updates,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
debug_recorders: Vec::new(),
|
||||
}));
|
||||
}
|
||||
Err(RecvTimeoutError::Timeout) => {}
|
||||
}
|
||||
@@ -984,7 +1053,12 @@ where
|
||||
"State root timeout race won"
|
||||
);
|
||||
let (state_root, trie_updates) = result?;
|
||||
return Ok(Ok(StateRootComputeOutcome { state_root, trie_updates }));
|
||||
return Ok(Ok(StateRootComputeOutcome {
|
||||
state_root,
|
||||
trie_updates,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
debug_recorders: Vec::new(),
|
||||
}));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1002,7 +1076,7 @@ where
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &HashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) {
|
||||
) -> bool {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
@@ -1026,6 +1100,7 @@ where
|
||||
%err,
|
||||
"Error comparing trie updates"
|
||||
);
|
||||
return true;
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
@@ -1045,6 +1120,45 @@ where
|
||||
);
|
||||
}
|
||||
}
|
||||
false
|
||||
}
|
||||
|
||||
/// Writes trie debug recorders to a JSON file for the given block number.
|
||||
///
|
||||
/// The file is written to the current working directory as
|
||||
/// `trie_debug_block_{block_number}.json`.
|
||||
#[cfg(feature = "trie-debug")]
|
||||
fn write_trie_debug_recorders(
|
||||
block_number: u64,
|
||||
recorders: &[(Option<B256>, TrieDebugRecorder)],
|
||||
) {
|
||||
let path = format!("trie_debug_block_{block_number}.json");
|
||||
match serde_json::to_string_pretty(recorders) {
|
||||
Ok(json) => match std::fs::write(&path, json) {
|
||||
Ok(()) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%path,
|
||||
"Wrote trie debug recorders to file"
|
||||
);
|
||||
}
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
%path,
|
||||
"Failed to write trie debug recorders"
|
||||
);
|
||||
}
|
||||
},
|
||||
Err(err) => {
|
||||
warn!(
|
||||
target: "engine::tree::payload_validator",
|
||||
%err,
|
||||
"Failed to serialize trie debug recorders"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Validates the block after execution.
|
||||
@@ -1142,7 +1256,7 @@ where
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_validator",
|
||||
skip_all,
|
||||
fields(strategy)
|
||||
fields(?strategy)
|
||||
)]
|
||||
fn spawn_payload_processor<T: ExecutableTxIterator<Evm>>(
|
||||
&mut self,
|
||||
@@ -1538,7 +1652,7 @@ where
|
||||
+ Clone
|
||||
+ 'static,
|
||||
N: NodePrimitives,
|
||||
V: PayloadValidator<Types, Block = N::Block>,
|
||||
V: PayloadValidator<Types, Block = N::Block> + Clone,
|
||||
Evm: ConfigureEngineEvm<Types::ExecutionData, Primitives = N> + 'static,
|
||||
Types: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
{
|
||||
@@ -1582,8 +1696,17 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<P, Evm, V> WaitForCaches for BasicEngineValidator<P, Evm, V>
|
||||
where
|
||||
Evm: ConfigureEvm,
|
||||
{
|
||||
fn wait_for_caches(&self) -> CacheWaitDurations {
|
||||
self.payload_processor.wait_for_caches()
|
||||
}
|
||||
}
|
||||
|
||||
/// Enum representing either block or payload being validated.
|
||||
#[derive(Debug)]
|
||||
#[derive(Debug, Clone)]
|
||||
pub enum BlockOrPayload<T: PayloadTypes> {
|
||||
/// Payload.
|
||||
Payload(T::ExecutionData),
|
||||
@@ -1659,4 +1782,15 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
Self::Block(block) => block.body().withdrawals().map(|w| w.as_slice()),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total gas used by the block.
|
||||
pub fn gas_used(&self) -> u64
|
||||
where
|
||||
T::ExecutionData: ExecutionPayload,
|
||||
{
|
||||
match self {
|
||||
Self::Payload(payload) => payload.gas_used(),
|
||||
Self::Block(block) => block.gas_used(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::Receiver as CrossbeamReceiver;
|
||||
use std::time::Instant;
|
||||
use reth_primitives_traits::FastInstant as Instant;
|
||||
use tracing::trace;
|
||||
|
||||
/// The state of the persistence task.
|
||||
|
||||
@@ -77,7 +77,8 @@ impl EngineMessageStore {
|
||||
})?,
|
||||
)?;
|
||||
}
|
||||
BeaconEngineMessage::NewPayload { payload, tx: _tx } => {
|
||||
BeaconEngineMessage::NewPayload { payload, .. } |
|
||||
BeaconEngineMessage::RethNewPayload { payload, .. } => {
|
||||
let filename = format!("{}-new_payload-{}.json", timestamp, payload.block_hash());
|
||||
fs::write(
|
||||
self.path.join(filename),
|
||||
|
||||
@@ -425,17 +425,9 @@ impl TotalDifficulty {
|
||||
|
||||
/// Convert to an [`Entry`]
|
||||
pub fn to_entry(&self) -> Entry {
|
||||
let mut data = [0u8; 32];
|
||||
|
||||
let be_bytes = self.value.to_be_bytes_vec();
|
||||
|
||||
if be_bytes.len() <= 32 {
|
||||
data[32 - be_bytes.len()..].copy_from_slice(&be_bytes);
|
||||
} else {
|
||||
data.copy_from_slice(&be_bytes[be_bytes.len() - 32..]);
|
||||
}
|
||||
|
||||
Entry::new(TOTAL_DIFFICULTY, data.to_vec())
|
||||
// era1 spec: `total-difficulty = { type: 0x0600, data: SSZ uint256 }` (little-endian)
|
||||
let data = self.value.to_le_bytes::<32>().to_vec();
|
||||
Entry::new(TOTAL_DIFFICULTY, data)
|
||||
}
|
||||
|
||||
/// Create from an [`Entry`]
|
||||
@@ -454,8 +446,8 @@ impl TotalDifficulty {
|
||||
)));
|
||||
}
|
||||
|
||||
// Convert 32-byte array to U256
|
||||
let value = U256::from_be_slice(&entry.data);
|
||||
// era1 spec: `total-difficulty = { type: 0x0600, data: SSZ uint256 }` (little-endian)
|
||||
let value = U256::from_le_slice(&entry.data);
|
||||
|
||||
Ok(Self { value })
|
||||
}
|
||||
@@ -608,6 +600,19 @@ mod tests {
|
||||
assert_eq!(recovered.value, value);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_total_difficulty_ssz_le_encoding() {
|
||||
// Verify that total-difficulty is encoded as SSZ uint256 (little-endian).
|
||||
// See https://github.com/eth-clients/e2store-format-specs/blob/main/formats/era1.md
|
||||
let value = U256::from(1u64);
|
||||
let td = TotalDifficulty::new(value);
|
||||
let entry = td.to_entry();
|
||||
|
||||
// Little-endian: least significant byte first [1, 0, 0, ..., 0]
|
||||
assert_eq!(entry.data[0], 1, "First byte must be 1 (little-endian)");
|
||||
assert_eq!(entry.data[31], 0, "Last byte must be 0 (little-endian)");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_compression_roundtrip() {
|
||||
let rlp_data = vec![1, 2, 3, 4, 5, 6, 7, 8, 9, 10];
|
||||
|
||||
@@ -158,6 +158,7 @@ where
|
||||
reserved_cpu_cores: command.engine.reserved_cpu_cores,
|
||||
proof_storage_worker_threads: command.engine.storage_worker_count,
|
||||
proof_account_worker_threads: command.engine.account_worker_count,
|
||||
prewarming_threads: command.engine.prewarming_threads,
|
||||
..Default::default()
|
||||
};
|
||||
let runner = CliRunner::try_with_runtime_config(
|
||||
|
||||
@@ -53,9 +53,7 @@ impl<
|
||||
<<Self::BuiltPayload as BuiltPayload>::Primitives as NodePrimitives>::Block,
|
||||
>,
|
||||
) -> Self::ExecutionData {
|
||||
let (payload, sidecar) =
|
||||
ExecutionPayload::from_block_unchecked(block.hash(), &block.into_block());
|
||||
ExecutionData { payload, sidecar }
|
||||
T::block_to_payload(block)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -285,7 +285,7 @@ where
|
||||
Arc::new(ctx.node.consensus().clone()),
|
||||
ctx.node.evm_config().clone(),
|
||||
ctx.config.rpc.flashbots_config(),
|
||||
Box::new(ctx.node.task_executor().clone()),
|
||||
ctx.node.task_executor().clone(),
|
||||
Arc::new(EthereumEngineValidator::new(ctx.config.chain.clone())),
|
||||
);
|
||||
|
||||
|
||||
@@ -214,7 +214,7 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> {
|
||||
TransactionTestContext::validate_sidecar(envelope);
|
||||
|
||||
// build last Prague payload
|
||||
node.payload.timestamp = current_timestamp + 11;
|
||||
node.payload.timestamp = current_timestamp + 1;
|
||||
let prague_payload = node.new_payload().await?;
|
||||
assert!(matches!(prague_payload.sidecars(), BlobSidecars::Eip4844(_)));
|
||||
|
||||
@@ -227,7 +227,7 @@ async fn blob_conversion_at_osaka() -> eyre::Result<()> {
|
||||
// validate sidecar
|
||||
TransactionTestContext::validate_sidecar(envelope);
|
||||
|
||||
tokio::time::sleep(Duration::from_secs(11)).await;
|
||||
tokio::time::sleep(Duration::from_secs(6)).await;
|
||||
|
||||
// fetch second blob tx from rpc again
|
||||
let envelope = node.rpc.envelope_by_hash(blob_tx_hash).await?;
|
||||
|
||||
@@ -282,6 +282,7 @@ async fn test_sparse_trie_reuse_across_blocks() -> eyre::Result<()> {
|
||||
.chain(MAINNET.chain)
|
||||
.genesis(serde_json::from_str(include_str!("../assets/genesis.json")).unwrap())
|
||||
.cancun_activated()
|
||||
.prague_activated()
|
||||
.build(),
|
||||
),
|
||||
false,
|
||||
|
||||
@@ -90,8 +90,8 @@ async fn test_fee_history() -> eyre::Result<()> {
|
||||
assert_eq!(block.header.gas_used, receipt.gas_used,);
|
||||
assert_eq!(block.header.base_fee_per_gas.unwrap(), expected_first_base_fee as u64);
|
||||
|
||||
for _ in 0..100 {
|
||||
let _ = GasWaster::deploy_builder(&provider, U256::from(rng.random_range(0..1000)))
|
||||
for _ in 0..20 {
|
||||
let _ = GasWaster::deploy_builder(&provider, U256::from(rng.random_range(0..100)))
|
||||
.send()
|
||||
.await?;
|
||||
|
||||
@@ -100,7 +100,7 @@ async fn test_fee_history() -> eyre::Result<()> {
|
||||
|
||||
let latest_block = provider.get_block_number().await?;
|
||||
|
||||
for _ in 0..100 {
|
||||
for _ in 0..20 {
|
||||
let latest_block = rng.random_range(0..=latest_block);
|
||||
let block_count = rng.random_range(1..=(latest_block + 1));
|
||||
|
||||
|
||||
@@ -2,8 +2,7 @@
|
||||
use alloy_consensus::BlockHeader;
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::{Block, RecoveredBlock};
|
||||
use std::time::Instant;
|
||||
use reth_primitives_traits::{Block, FastInstant as Instant, RecoveredBlock};
|
||||
|
||||
/// Executor metrics.
|
||||
#[derive(Metrics, Clone)]
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use reth_primitives_traits::{Account, Bytecode};
|
||||
use revm::database::BundleState;
|
||||
use revm::database::{states::BundleState, BundleAccount};
|
||||
|
||||
pub use alloy_evm::block::BlockExecutionResult;
|
||||
|
||||
@@ -37,6 +37,11 @@ impl<T> BlockExecutionOutput<T> {
|
||||
self.state.account(address).map(|a| a.info.as_ref().map(Into::into))
|
||||
}
|
||||
|
||||
/// Returns the state [`BundleAccount`] for the given address.
|
||||
pub fn account_state(&self, address: &Address) -> Option<&BundleAccount> {
|
||||
self.state.account(address)
|
||||
}
|
||||
|
||||
/// Get storage if value is known.
|
||||
///
|
||||
/// This means that depending on status we can potentially return `U256::ZERO`.
|
||||
|
||||
@@ -10,6 +10,7 @@ use reth_provider::{BlockReader, Chain, HeaderProvider, StateProviderFactory};
|
||||
use reth_stages_api::ExecutionStageThresholds;
|
||||
use reth_tracing::tracing::debug;
|
||||
use std::{
|
||||
collections::VecDeque,
|
||||
fmt::Debug,
|
||||
pin::Pin,
|
||||
sync::Arc,
|
||||
@@ -286,6 +287,9 @@ where
|
||||
backfill_job: Option<StreamBackfillJob<E, P, Chain<E::Primitives>>>,
|
||||
/// Custom thresholds for the backfill job, if set.
|
||||
backfill_thresholds: Option<ExecutionStageThresholds>,
|
||||
/// Notifications that arrived during backfill and need to be delivered after it completes.
|
||||
/// These are notifications for blocks beyond the backfill range that we must not drop.
|
||||
pending_notifications: VecDeque<ExExNotification<E::Primitives>>,
|
||||
}
|
||||
|
||||
impl<P, E> ExExNotificationsWithHead<P, E>
|
||||
@@ -312,6 +316,7 @@ where
|
||||
pending_check_backfill: true,
|
||||
backfill_job: None,
|
||||
backfill_thresholds: None,
|
||||
pending_notifications: VecDeque::new(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -448,6 +453,34 @@ where
|
||||
// 3. If backfill is in progress yield new notifications
|
||||
if let Some(backfill_job) = &mut this.backfill_job {
|
||||
debug!(target: "exex::notifications", "Polling backfill job");
|
||||
|
||||
// Drain the notification channel to prevent backpressure from stalling the
|
||||
// ExExManager. During backfill, the ExEx is not consuming from the channel,
|
||||
// so the capacity-1 channel fills up, which blocks the manager's PollSender,
|
||||
// which fills the manager's 1024-entry buffer, which blocks all upstream
|
||||
// senders. Notifications for blocks covered by the backfill range are
|
||||
// discarded (they'll be re-delivered by the backfill job), while
|
||||
// notifications beyond the backfill range are buffered for delivery after the
|
||||
// backfill completes.
|
||||
while let Poll::Ready(Some(notification)) = this.notifications.poll_recv(cx) {
|
||||
// Always buffer revert-containing notifications (ChainReverted,
|
||||
// ChainReorged) because the backfill job only re-delivers
|
||||
// ChainCommitted from the database. Discarding a reorg here would
|
||||
// leave the ExEx unaware of the fork switch.
|
||||
if notification.reverted_chain().is_some() {
|
||||
this.pending_notifications.push_back(notification);
|
||||
continue;
|
||||
}
|
||||
if let Some(committed) = notification.committed_chain() &&
|
||||
committed.tip().number() <= this.initial_local_head.number
|
||||
{
|
||||
// Covered by backfill range, safe to discard
|
||||
continue;
|
||||
}
|
||||
// Beyond the backfill range — buffer for delivery after backfill
|
||||
this.pending_notifications.push_back(notification);
|
||||
}
|
||||
|
||||
if let Some(chain) = ready!(backfill_job.poll_next_unpin(cx)).transpose()? {
|
||||
debug!(target: "exex::notifications", range = ?chain.range(), "Backfill job returned a chain");
|
||||
return Poll::Ready(Some(Ok(ExExNotification::ChainCommitted {
|
||||
@@ -459,13 +492,18 @@ where
|
||||
this.backfill_job = None;
|
||||
}
|
||||
|
||||
// 4. Otherwise advance the regular event stream
|
||||
// 4. Deliver any notifications that were buffered during backfill
|
||||
if let Some(notification) = this.pending_notifications.pop_front() {
|
||||
return Poll::Ready(Some(Ok(notification)))
|
||||
}
|
||||
|
||||
// 5. Otherwise advance the regular event stream
|
||||
loop {
|
||||
let Some(notification) = ready!(this.notifications.poll_recv(cx)) else {
|
||||
return Poll::Ready(None)
|
||||
};
|
||||
|
||||
// 5. In case the exex is ahead of the new tip, we must skip it
|
||||
// 6. In case the exex is ahead of the new tip, we must skip it
|
||||
if let Some(committed) = notification.committed_chain() {
|
||||
// inclusive check because we should start with `exex.head + 1`
|
||||
if this.initial_exex_head.block.number >= committed.tip().number() {
|
||||
@@ -789,4 +827,135 @@ mod tests {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Regression test for <https://github.com/paradigmxyz/reth/issues/19665>.
|
||||
///
|
||||
/// During backfill, `poll_next` must drain the notification channel so that
|
||||
/// the upstream `ExExManager` is never blocked by a full channel. Without
|
||||
/// the drain loop the capacity-1 channel stays full for the entire backfill
|
||||
/// duration, which stalls the manager's `PollSender` and eventually blocks
|
||||
/// all upstream senders once the 1024-entry buffer fills up.
|
||||
///
|
||||
/// The key assertion is the `try_send` after the first `poll_next`: it
|
||||
/// proves the channel was drained during the backfill poll. Without the
|
||||
/// fix this `try_send` fails because the notification is still sitting in
|
||||
/// the channel.
|
||||
#[tokio::test]
|
||||
async fn exex_notifications_backfill_drains_channel() -> eyre::Result<()> {
|
||||
let mut rng = generators::rng();
|
||||
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let wal = Wal::new(temp_dir.path()).unwrap();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let genesis_hash = init_genesis(&provider_factory)?;
|
||||
let genesis_block = provider_factory
|
||||
.block(genesis_hash.into())?
|
||||
.ok_or_else(|| eyre::eyre!("genesis block not found"))?;
|
||||
|
||||
let provider = BlockchainProvider::new(provider_factory.clone())?;
|
||||
|
||||
// Insert block 1 into the DB so there's something to backfill
|
||||
let node_head_block = random_block(
|
||||
&mut rng,
|
||||
genesis_block.number + 1,
|
||||
BlockParams { parent: Some(genesis_hash), tx_count: Some(0), ..Default::default() },
|
||||
)
|
||||
.try_recover()?;
|
||||
let node_head = node_head_block.num_hash();
|
||||
let provider_rw = provider_factory.provider_rw()?;
|
||||
provider_rw.insert_block(&node_head_block)?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
// ExEx head is at genesis — backfill will run for block 1
|
||||
let exex_head =
|
||||
ExExHead { block: BlockNumHash { number: genesis_block.number, hash: genesis_hash } };
|
||||
|
||||
// Notification for a block AFTER the backfill range (block 2).
|
||||
let post_backfill_notification = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![random_block(
|
||||
&mut rng,
|
||||
node_head.number + 1,
|
||||
BlockParams { parent: Some(node_head.hash), ..Default::default() },
|
||||
)
|
||||
.try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
// Another notification (block 3) used to probe channel capacity.
|
||||
let probe_notification = ExExNotification::ChainCommitted {
|
||||
new: Arc::new(Chain::new(
|
||||
vec![random_block(
|
||||
&mut rng,
|
||||
node_head.number + 2,
|
||||
BlockParams { parent: None, ..Default::default() },
|
||||
)
|
||||
.try_recover()?],
|
||||
Default::default(),
|
||||
BTreeMap::new(),
|
||||
)),
|
||||
};
|
||||
|
||||
let (notifications_tx, notifications_rx) = mpsc::channel(1);
|
||||
|
||||
// Fill the capacity-1 channel.
|
||||
notifications_tx.send(post_backfill_notification.clone()).await?;
|
||||
|
||||
// Confirm the channel is full — this is the precondition that causes the
|
||||
// stall in production: the ExExManager's PollSender would block here.
|
||||
assert!(
|
||||
notifications_tx.try_send(probe_notification.clone()).is_err(),
|
||||
"channel should be full before backfill poll"
|
||||
);
|
||||
|
||||
let mut notifications = ExExNotificationsWithoutHead::new(
|
||||
node_head,
|
||||
provider,
|
||||
EthEvmConfig::mainnet(),
|
||||
notifications_rx,
|
||||
wal.handle(),
|
||||
)
|
||||
.with_head(exex_head);
|
||||
|
||||
// Poll once — this returns the backfill result for block 1. Crucially,
|
||||
// the drain loop in poll_next runs in this same call, consuming the
|
||||
// notification from the channel and buffering it.
|
||||
let backfill_result = notifications.next().await.transpose()?;
|
||||
assert_eq!(
|
||||
backfill_result,
|
||||
Some(ExExNotification::ChainCommitted {
|
||||
new: Arc::new(
|
||||
BackfillJobFactory::new(
|
||||
notifications.evm_config.clone(),
|
||||
notifications.provider.clone()
|
||||
)
|
||||
.backfill(1..=1)
|
||||
.next()
|
||||
.ok_or_eyre("failed to backfill")??
|
||||
)
|
||||
})
|
||||
);
|
||||
|
||||
// KEY ASSERTION: the channel was drained during the backfill poll above.
|
||||
// Without the drain loop this try_send fails because the original
|
||||
// notification is still occupying the capacity-1 channel.
|
||||
assert!(
|
||||
notifications_tx.try_send(probe_notification.clone()).is_ok(),
|
||||
"channel should have been drained during backfill poll"
|
||||
);
|
||||
|
||||
// The first buffered notification (block 2) was drained from the channel
|
||||
// during backfill and is delivered now.
|
||||
let buffered = notifications.next().await.transpose()?;
|
||||
assert_eq!(buffered, Some(post_backfill_notification));
|
||||
|
||||
// The probe notification (block 3) that we just sent is delivered next.
|
||||
let probe = notifications.next().await.transpose()?;
|
||||
assert_eq!(probe, Some(probe_notification));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ use reth_network_p2p::{
|
||||
};
|
||||
use reth_primitives_traits::{size::InMemorySize, Block, SealedHeader};
|
||||
use reth_storage_api::HeaderProvider;
|
||||
use reth_tasks::{TaskSpawner, TokioTaskExecutor};
|
||||
use reth_tasks::Runtime;
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::BinaryHeap,
|
||||
@@ -285,17 +285,9 @@ where
|
||||
C: BodiesClient<Body = B::Body> + 'static,
|
||||
Provider: HeaderProvider<Header = B::Header> + Unpin + 'static,
|
||||
{
|
||||
/// Spawns the downloader task via [`tokio::task::spawn`]
|
||||
pub fn into_task(self) -> TaskDownloader<B> {
|
||||
self.into_task_with(&TokioTaskExecutor::default())
|
||||
}
|
||||
|
||||
/// Convert the downloader into a [`TaskDownloader`] by spawning it via the given spawner.
|
||||
pub fn into_task_with<S>(self, spawner: &S) -> TaskDownloader<B>
|
||||
where
|
||||
S: TaskSpawner,
|
||||
{
|
||||
TaskDownloader::spawn_with(self, spawner)
|
||||
/// Convert the downloader into a [`TaskDownloader`] by spawning it via the given [`Runtime`].
|
||||
pub fn into_task_with(self, runtime: &Runtime) -> TaskDownloader<B> {
|
||||
TaskDownloader::spawn_with(self, runtime)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -245,8 +245,7 @@ where
|
||||
}
|
||||
|
||||
// Buffer any empty headers
|
||||
while this.pending_headers.front().is_some_and(|h| h.is_empty()) {
|
||||
let header = this.pending_headers.pop_front().unwrap();
|
||||
while let Some(header) = this.pending_headers.pop_front_if(|h| h.is_empty()) {
|
||||
this.buffer.push(BlockResponse::Empty(header));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
use alloy_primitives::BlockNumber;
|
||||
use futures::Stream;
|
||||
use futures_util::{FutureExt, StreamExt};
|
||||
use futures_util::StreamExt;
|
||||
use pin_project::pin_project;
|
||||
use reth_network_p2p::{
|
||||
bodies::downloader::{BodyDownloader, BodyDownloaderResult},
|
||||
error::DownloadResult,
|
||||
};
|
||||
use reth_primitives_traits::Block;
|
||||
use reth_tasks::{TaskSpawner, TokioTaskExecutor};
|
||||
use reth_tasks::Runtime;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
future::Future,
|
||||
@@ -32,50 +32,11 @@ pub struct TaskDownloader<B: Block> {
|
||||
}
|
||||
|
||||
impl<B: Block + 'static> TaskDownloader<B> {
|
||||
/// Spawns the given `downloader` via [`tokio::task::spawn`] returns a [`TaskDownloader`] that's
|
||||
/// connected to that task.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This method panics if called outside of a Tokio runtime
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// use reth_consensus::Consensus;
|
||||
/// use reth_downloaders::bodies::{bodies::BodiesDownloaderBuilder, task::TaskDownloader};
|
||||
/// use reth_network_p2p::bodies::client::BodiesClient;
|
||||
/// use reth_primitives_traits::{Block, InMemorySize};
|
||||
/// use reth_storage_api::HeaderProvider;
|
||||
/// use std::{fmt::Debug, sync::Arc};
|
||||
///
|
||||
/// fn t<
|
||||
/// B: Block + 'static,
|
||||
/// C: BodiesClient<Body = B::Body> + 'static,
|
||||
/// Provider: HeaderProvider<Header = B::Header> + Unpin + 'static,
|
||||
/// >(
|
||||
/// client: Arc<C>,
|
||||
/// consensus: Arc<dyn Consensus<B>>,
|
||||
/// provider: Provider,
|
||||
/// ) {
|
||||
/// let downloader =
|
||||
/// BodiesDownloaderBuilder::default().build::<B, _, _>(client, consensus, provider);
|
||||
/// let downloader = TaskDownloader::spawn(downloader);
|
||||
/// }
|
||||
/// ```
|
||||
pub fn spawn<T>(downloader: T) -> Self
|
||||
where
|
||||
T: BodyDownloader<Block = B> + 'static,
|
||||
{
|
||||
Self::spawn_with(downloader, &TokioTaskExecutor::default())
|
||||
}
|
||||
|
||||
/// Spawns the given `downloader` via the given [`TaskSpawner`] returns a [`TaskDownloader`]
|
||||
/// Spawns the given `downloader` via the given [`Runtime`] and returns a [`TaskDownloader`]
|
||||
/// that's connected to that task.
|
||||
pub fn spawn_with<T, S>(downloader: T, spawner: &S) -> Self
|
||||
pub fn spawn_with<T>(downloader: T, runtime: &Runtime) -> Self
|
||||
where
|
||||
T: BodyDownloader<Block = B> + 'static,
|
||||
S: TaskSpawner,
|
||||
{
|
||||
let (bodies_tx, bodies_rx) = mpsc::channel(BODIES_TASK_BUFFER_SIZE);
|
||||
let (to_downloader, updates_rx) = mpsc::unbounded_channel();
|
||||
@@ -86,7 +47,7 @@ impl<B: Block + 'static> TaskDownloader<B> {
|
||||
downloader,
|
||||
};
|
||||
|
||||
spawner.spawn_task(downloader.boxed());
|
||||
runtime.spawn_task(downloader);
|
||||
|
||||
Self { from_downloader: ReceiverStream::new(bodies_rx), to_downloader }
|
||||
}
|
||||
@@ -201,7 +162,8 @@ mod tests {
|
||||
Arc::new(TestConsensus::default()),
|
||||
factory,
|
||||
);
|
||||
let mut downloader = TaskDownloader::spawn(downloader);
|
||||
let runtime = Runtime::test();
|
||||
let mut downloader = TaskDownloader::spawn_with(downloader, &runtime);
|
||||
|
||||
downloader.set_download_range(0..=19).expect("failed to set download range");
|
||||
|
||||
@@ -224,7 +186,8 @@ mod tests {
|
||||
Arc::new(TestConsensus::default()),
|
||||
factory,
|
||||
);
|
||||
let mut downloader = TaskDownloader::spawn(downloader);
|
||||
let runtime = Runtime::test();
|
||||
let mut downloader = TaskDownloader::spawn_with(downloader, &runtime);
|
||||
|
||||
downloader.set_download_range(1..=0).expect("failed to set download range");
|
||||
assert_matches!(downloader.next().await, Some(Err(DownloadError::InvalidBodyRange { .. })));
|
||||
|
||||
@@ -21,7 +21,7 @@ use reth_network_p2p::{
|
||||
};
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_primitives_traits::{GotExpected, SealedHeader};
|
||||
use reth_tasks::{TaskSpawner, TokioTaskExecutor};
|
||||
use reth_tasks::Runtime;
|
||||
use std::{
|
||||
cmp::{Ordering, Reverse},
|
||||
collections::{binary_heap::PeekMut, BinaryHeap},
|
||||
@@ -660,20 +660,12 @@ where
|
||||
H: HeadersClient,
|
||||
Self: HeaderDownloader + 'static,
|
||||
{
|
||||
/// Spawns the downloader task via [`tokio::task::spawn`]
|
||||
pub fn into_task(self) -> TaskDownloader<<Self as HeaderDownloader>::Header> {
|
||||
self.into_task_with(&TokioTaskExecutor::default())
|
||||
}
|
||||
|
||||
/// Convert the downloader into a [`TaskDownloader`] by spawning it via the given `spawner`.
|
||||
pub fn into_task_with<S>(
|
||||
/// Convert the downloader into a [`TaskDownloader`] by spawning it via the given [`Runtime`].
|
||||
pub fn into_task_with(
|
||||
self,
|
||||
spawner: &S,
|
||||
) -> TaskDownloader<<Self as HeaderDownloader>::Header>
|
||||
where
|
||||
S: TaskSpawner,
|
||||
{
|
||||
TaskDownloader::spawn_with(self, spawner)
|
||||
runtime: &Runtime,
|
||||
) -> TaskDownloader<<Self as HeaderDownloader>::Header> {
|
||||
TaskDownloader::spawn_with(self, runtime)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
use alloy_primitives::Sealable;
|
||||
use futures::{FutureExt, Stream};
|
||||
use futures::Stream;
|
||||
use futures_util::StreamExt;
|
||||
use pin_project::pin_project;
|
||||
use reth_network_p2p::headers::{
|
||||
@@ -7,7 +7,7 @@ use reth_network_p2p::headers::{
|
||||
error::HeadersDownloaderResult,
|
||||
};
|
||||
use reth_primitives_traits::SealedHeader;
|
||||
use reth_tasks::{TaskSpawner, TokioTaskExecutor};
|
||||
use reth_tasks::Runtime;
|
||||
use std::{
|
||||
fmt::Debug,
|
||||
future::Future,
|
||||
@@ -33,42 +33,11 @@ pub struct TaskDownloader<H: Sealable> {
|
||||
// === impl TaskDownloader ===
|
||||
|
||||
impl<H: Sealable + Send + Sync + Unpin + 'static> TaskDownloader<H> {
|
||||
/// Spawns the given `downloader` via [`tokio::task::spawn`] and returns a [`TaskDownloader`]
|
||||
/// Spawns the given `downloader` via the given [`Runtime`] and returns a [`TaskDownloader`]
|
||||
/// that's connected to that task.
|
||||
///
|
||||
/// # Panics
|
||||
///
|
||||
/// This method panics if called outside of a Tokio runtime
|
||||
///
|
||||
/// # Example
|
||||
///
|
||||
/// ```
|
||||
/// # use std::sync::Arc;
|
||||
/// # use reth_downloaders::headers::reverse_headers::ReverseHeadersDownloader;
|
||||
/// # use reth_downloaders::headers::task::TaskDownloader;
|
||||
/// # use reth_consensus::HeaderValidator;
|
||||
/// # use reth_network_p2p::headers::client::HeadersClient;
|
||||
/// # use reth_primitives_traits::BlockHeader;
|
||||
/// # fn t<H: HeadersClient<Header: BlockHeader> + 'static>(consensus:Arc<dyn HeaderValidator<H::Header>>, client: Arc<H>) {
|
||||
/// let downloader = ReverseHeadersDownloader::<H>::builder().build(
|
||||
/// client,
|
||||
/// consensus
|
||||
/// );
|
||||
/// let downloader = TaskDownloader::spawn(downloader);
|
||||
/// # }
|
||||
pub fn spawn<T>(downloader: T) -> Self
|
||||
pub fn spawn_with<T>(downloader: T, runtime: &Runtime) -> Self
|
||||
where
|
||||
T: HeaderDownloader<Header = H> + 'static,
|
||||
{
|
||||
Self::spawn_with(downloader, &TokioTaskExecutor::default())
|
||||
}
|
||||
|
||||
/// Spawns the given `downloader` via the given [`TaskSpawner`] returns a [`TaskDownloader`]
|
||||
/// that's connected to that task.
|
||||
pub fn spawn_with<T, S>(downloader: T, spawner: &S) -> Self
|
||||
where
|
||||
T: HeaderDownloader<Header = H> + 'static,
|
||||
S: TaskSpawner,
|
||||
{
|
||||
let (headers_tx, headers_rx) = mpsc::channel(HEADERS_TASK_BUFFER_SIZE);
|
||||
let (to_downloader, updates_rx) = mpsc::unbounded_channel();
|
||||
@@ -78,7 +47,7 @@ impl<H: Sealable + Send + Sync + Unpin + 'static> TaskDownloader<H> {
|
||||
updates: UnboundedReceiverStream::new(updates_rx),
|
||||
downloader,
|
||||
};
|
||||
spawner.spawn_task(downloader.boxed());
|
||||
runtime.spawn_task(downloader);
|
||||
|
||||
Self { from_downloader: ReceiverStream::new(headers_rx), to_downloader }
|
||||
}
|
||||
@@ -209,7 +178,8 @@ mod tests {
|
||||
.request_limit(1)
|
||||
.build(Arc::clone(&client), Arc::new(TestConsensus::default()));
|
||||
|
||||
let mut downloader = TaskDownloader::spawn(downloader);
|
||||
let runtime = Runtime::test();
|
||||
let mut downloader = TaskDownloader::spawn_with(downloader, &runtime);
|
||||
downloader.update_local_head(p3.clone());
|
||||
downloader.update_sync_target(SyncTarget::Tip(p0.hash()));
|
||||
|
||||
|
||||
27
crates/net/eth-wire-types/src/block_access_lists.rs
Normal file
27
crates/net/eth-wire-types/src/block_access_lists.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
//! Implements the `GetBlockAccessLists` and `BlockAccessLists` message types.
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_rlp::{RlpDecodableWrapper, RlpEncodableWrapper};
|
||||
use reth_codecs_derive::add_arbitrary_tests;
|
||||
|
||||
/// A request for block access lists from the given block hashes.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, RlpEncodableWrapper, RlpDecodableWrapper, Default)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct GetBlockAccessLists(
|
||||
/// The block hashes to request block access lists for.
|
||||
pub Vec<B256>,
|
||||
);
|
||||
|
||||
/// Response for [`GetBlockAccessLists`] containing one BAL per requested block hash.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, RlpEncodableWrapper, RlpDecodableWrapper, Default)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct BlockAccessLists(
|
||||
/// The requested block access lists as opaque bytes. Unavailable entries are represented by
|
||||
/// empty byte slices.
|
||||
pub Vec<Bytes>,
|
||||
);
|
||||
@@ -169,7 +169,10 @@ impl NewPooledTransactionHashes {
|
||||
matches!(version, EthVersion::Eth67 | EthVersion::Eth66)
|
||||
}
|
||||
Self::Eth68(_) => {
|
||||
matches!(version, EthVersion::Eth68 | EthVersion::Eth69 | EthVersion::Eth70)
|
||||
matches!(
|
||||
version,
|
||||
EthVersion::Eth68 | EthVersion::Eth69 | EthVersion::Eth70 | EthVersion::Eth71
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -110,6 +110,11 @@ impl Capability {
|
||||
Self::eth(EthVersion::Eth70)
|
||||
}
|
||||
|
||||
/// Returns the [`EthVersion::Eth71`] capability.
|
||||
pub const fn eth_71() -> Self {
|
||||
Self::eth(EthVersion::Eth71)
|
||||
}
|
||||
|
||||
/// Whether this is eth v66 protocol.
|
||||
#[inline]
|
||||
pub fn is_eth_v66(&self) -> bool {
|
||||
@@ -140,6 +145,12 @@ impl Capability {
|
||||
self.name == "eth" && self.version == 70
|
||||
}
|
||||
|
||||
/// Whether this is eth v71.
|
||||
#[inline]
|
||||
pub fn is_eth_v71(&self) -> bool {
|
||||
self.name == "eth" && self.version == 71
|
||||
}
|
||||
|
||||
/// Whether this is any eth version.
|
||||
#[inline]
|
||||
pub fn is_eth(&self) -> bool {
|
||||
@@ -147,7 +158,8 @@ impl Capability {
|
||||
self.is_eth_v67() ||
|
||||
self.is_eth_v68() ||
|
||||
self.is_eth_v69() ||
|
||||
self.is_eth_v70()
|
||||
self.is_eth_v70() ||
|
||||
self.is_eth_v71()
|
||||
}
|
||||
}
|
||||
|
||||
@@ -167,7 +179,7 @@ impl From<EthVersion> for Capability {
|
||||
#[cfg(any(test, feature = "arbitrary"))]
|
||||
impl<'a> arbitrary::Arbitrary<'a> for Capability {
|
||||
fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
|
||||
let version = u.int_in_range(66..=70)?; // Valid eth protocol versions are 66-70
|
||||
let version = u.int_in_range(66..=71)?; // Valid eth protocol versions are 66-71
|
||||
// Only generate valid eth protocol name for now since it's the only supported protocol
|
||||
Ok(Self::new_static("eth", version))
|
||||
}
|
||||
@@ -183,6 +195,7 @@ pub struct Capabilities {
|
||||
eth_68: bool,
|
||||
eth_69: bool,
|
||||
eth_70: bool,
|
||||
eth_71: bool,
|
||||
}
|
||||
|
||||
impl Capabilities {
|
||||
@@ -194,6 +207,7 @@ impl Capabilities {
|
||||
eth_68: value.iter().any(Capability::is_eth_v68),
|
||||
eth_69: value.iter().any(Capability::is_eth_v69),
|
||||
eth_70: value.iter().any(Capability::is_eth_v70),
|
||||
eth_71: value.iter().any(Capability::is_eth_v71),
|
||||
inner: value,
|
||||
}
|
||||
}
|
||||
@@ -212,7 +226,7 @@ impl Capabilities {
|
||||
/// Whether the peer supports `eth` sub-protocol.
|
||||
#[inline]
|
||||
pub const fn supports_eth(&self) -> bool {
|
||||
self.eth_70 || self.eth_69 || self.eth_68 || self.eth_67 || self.eth_66
|
||||
self.eth_71 || self.eth_70 || self.eth_69 || self.eth_68 || self.eth_67 || self.eth_66
|
||||
}
|
||||
|
||||
/// Whether this peer supports eth v66 protocol.
|
||||
@@ -244,6 +258,12 @@ impl Capabilities {
|
||||
pub const fn supports_eth_v70(&self) -> bool {
|
||||
self.eth_70
|
||||
}
|
||||
|
||||
/// Whether this peer supports eth v71 protocol.
|
||||
#[inline]
|
||||
pub const fn supports_eth_v71(&self) -> bool {
|
||||
self.eth_71
|
||||
}
|
||||
}
|
||||
|
||||
impl From<Vec<Capability>> for Capabilities {
|
||||
@@ -268,6 +288,7 @@ impl Decodable for Capabilities {
|
||||
eth_68: inner.iter().any(Capability::is_eth_v68),
|
||||
eth_69: inner.iter().any(Capability::is_eth_v69),
|
||||
eth_70: inner.iter().any(Capability::is_eth_v70),
|
||||
eth_71: inner.iter().any(Capability::is_eth_v71),
|
||||
inner,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -38,6 +38,9 @@ pub use state::*;
|
||||
pub mod receipts;
|
||||
pub use receipts::*;
|
||||
|
||||
pub mod block_access_lists;
|
||||
pub use block_access_lists::*;
|
||||
|
||||
pub mod disconnect_reason;
|
||||
pub use disconnect_reason::*;
|
||||
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
//! Implements Ethereum wire protocol for versions 66 through 70.
|
||||
//! Implements Ethereum wire protocol for versions 66 through 71.
|
||||
//! Defines structs/enums for messages, request-response pairs, and broadcasts.
|
||||
//! Handles compatibility with [`EthVersion`].
|
||||
//!
|
||||
@@ -7,10 +7,10 @@
|
||||
//! Reference: [Ethereum Wire Protocol](https://github.com/ethereum/devp2p/blob/master/caps/eth.md).
|
||||
|
||||
use super::{
|
||||
broadcast::NewBlockHashes, BlockBodies, BlockHeaders, GetBlockBodies, GetBlockHeaders,
|
||||
GetNodeData, GetPooledTransactions, GetReceipts, GetReceipts70, NewPooledTransactionHashes66,
|
||||
NewPooledTransactionHashes68, NodeData, PooledTransactions, Receipts, Status, StatusEth69,
|
||||
Transactions,
|
||||
broadcast::NewBlockHashes, BlockAccessLists, BlockBodies, BlockHeaders, GetBlockAccessLists,
|
||||
GetBlockBodies, GetBlockHeaders, GetNodeData, GetPooledTransactions, GetReceipts,
|
||||
GetReceipts70, NewPooledTransactionHashes66, NewPooledTransactionHashes68, NodeData,
|
||||
PooledTransactions, Receipts, Status, StatusEth69, Transactions,
|
||||
};
|
||||
use crate::{
|
||||
status::StatusMessage, BlockRangeUpdate, EthNetworkPrimitives, EthVersion, NetworkPrimitives,
|
||||
@@ -168,6 +168,18 @@ impl<N: NetworkPrimitives> ProtocolMessage<N> {
|
||||
}
|
||||
EthMessage::BlockRangeUpdate(BlockRangeUpdate::decode(buf)?)
|
||||
}
|
||||
EthMessageID::GetBlockAccessLists => {
|
||||
if version < EthVersion::Eth71 {
|
||||
return Err(MessageError::Invalid(version, EthMessageID::GetBlockAccessLists))
|
||||
}
|
||||
EthMessage::GetBlockAccessLists(RequestPair::decode(buf)?)
|
||||
}
|
||||
EthMessageID::BlockAccessLists => {
|
||||
if version < EthVersion::Eth71 {
|
||||
return Err(MessageError::Invalid(version, EthMessageID::BlockAccessLists))
|
||||
}
|
||||
EthMessage::BlockAccessLists(RequestPair::decode(buf)?)
|
||||
}
|
||||
EthMessageID::Other(_) => {
|
||||
let raw_payload = Bytes::copy_from_slice(buf);
|
||||
buf.advance(raw_payload.len());
|
||||
@@ -250,6 +262,8 @@ impl<N: NetworkPrimitives> From<EthBroadcastMessage<N>> for ProtocolBroadcastMes
|
||||
///
|
||||
/// The `eth/70` (EIP-7975) keeps the eth/69 status format and introduces partial receipts.
|
||||
/// requests/responses.
|
||||
///
|
||||
/// The `eth/71` draft extends eth/70 with block access list request/response messages.
|
||||
#[derive(Clone, Debug, PartialEq, Eq)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
pub enum EthMessage<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
@@ -310,6 +324,8 @@ pub enum EthMessage<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
/// `GetReceipts` in EIP-7975 inlines the request id. The type still wraps
|
||||
/// a [`RequestPair`], but with a custom inline encoding.
|
||||
GetReceipts70(RequestPair<GetReceipts70>),
|
||||
/// Represents a `GetBlockAccessLists` request-response pair for eth/71.
|
||||
GetBlockAccessLists(RequestPair<GetBlockAccessLists>),
|
||||
/// Represents a Receipts request-response pair.
|
||||
#[cfg_attr(
|
||||
feature = "serde",
|
||||
@@ -332,6 +348,8 @@ pub enum EthMessage<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
/// request id. The type still wraps a [`RequestPair`], but with a custom
|
||||
/// inline encoding.
|
||||
Receipts70(RequestPair<Receipts70<N::Receipt>>),
|
||||
/// Represents a `BlockAccessLists` request-response pair for eth/71.
|
||||
BlockAccessLists(RequestPair<BlockAccessLists>),
|
||||
/// Represents a `BlockRangeUpdate` message broadcast to the network.
|
||||
#[cfg_attr(
|
||||
feature = "serde",
|
||||
@@ -364,6 +382,8 @@ impl<N: NetworkPrimitives> EthMessage<N> {
|
||||
Self::GetReceipts(_) | Self::GetReceipts70(_) => EthMessageID::GetReceipts,
|
||||
Self::Receipts(_) | Self::Receipts69(_) | Self::Receipts70(_) => EthMessageID::Receipts,
|
||||
Self::BlockRangeUpdate(_) => EthMessageID::BlockRangeUpdate,
|
||||
Self::GetBlockAccessLists(_) => EthMessageID::GetBlockAccessLists,
|
||||
Self::BlockAccessLists(_) => EthMessageID::BlockAccessLists,
|
||||
Self::Other(msg) => EthMessageID::Other(msg.id as u8),
|
||||
}
|
||||
}
|
||||
@@ -376,6 +396,7 @@ impl<N: NetworkPrimitives> EthMessage<N> {
|
||||
Self::GetBlockHeaders(_) |
|
||||
Self::GetReceipts(_) |
|
||||
Self::GetReceipts70(_) |
|
||||
Self::GetBlockAccessLists(_) |
|
||||
Self::GetPooledTransactions(_) |
|
||||
Self::GetNodeData(_)
|
||||
)
|
||||
@@ -389,6 +410,7 @@ impl<N: NetworkPrimitives> EthMessage<N> {
|
||||
Self::Receipts(_) |
|
||||
Self::Receipts69(_) |
|
||||
Self::Receipts70(_) |
|
||||
Self::BlockAccessLists(_) |
|
||||
Self::BlockHeaders(_) |
|
||||
Self::BlockBodies(_) |
|
||||
Self::NodeData(_)
|
||||
@@ -443,9 +465,11 @@ impl<N: NetworkPrimitives> Encodable for EthMessage<N> {
|
||||
Self::NodeData(data) => data.encode(out),
|
||||
Self::GetReceipts(request) => request.encode(out),
|
||||
Self::GetReceipts70(request) => request.encode(out),
|
||||
Self::GetBlockAccessLists(request) => request.encode(out),
|
||||
Self::Receipts(receipts) => receipts.encode(out),
|
||||
Self::Receipts69(receipt69) => receipt69.encode(out),
|
||||
Self::Receipts70(receipt70) => receipt70.encode(out),
|
||||
Self::BlockAccessLists(block_access_lists) => block_access_lists.encode(out),
|
||||
Self::BlockRangeUpdate(block_range_update) => block_range_update.encode(out),
|
||||
Self::Other(unknown) => out.put_slice(&unknown.payload),
|
||||
}
|
||||
@@ -468,9 +492,11 @@ impl<N: NetworkPrimitives> Encodable for EthMessage<N> {
|
||||
Self::NodeData(data) => data.length(),
|
||||
Self::GetReceipts(request) => request.length(),
|
||||
Self::GetReceipts70(request) => request.length(),
|
||||
Self::GetBlockAccessLists(request) => request.length(),
|
||||
Self::Receipts(receipts) => receipts.length(),
|
||||
Self::Receipts69(receipt69) => receipt69.length(),
|
||||
Self::Receipts70(receipt70) => receipt70.length(),
|
||||
Self::BlockAccessLists(block_access_lists) => block_access_lists.length(),
|
||||
Self::BlockRangeUpdate(block_range_update) => block_range_update.length(),
|
||||
Self::Other(unknown) => unknown.length(),
|
||||
}
|
||||
@@ -559,6 +585,14 @@ pub enum EthMessageID {
|
||||
///
|
||||
/// Introduced in Eth69
|
||||
BlockRangeUpdate = 0x11,
|
||||
/// Requests block access lists.
|
||||
///
|
||||
/// Introduced in Eth71
|
||||
GetBlockAccessLists = 0x12,
|
||||
/// Represents block access lists.
|
||||
///
|
||||
/// Introduced in Eth71
|
||||
BlockAccessLists = 0x13,
|
||||
/// Represents unknown message types.
|
||||
Other(u8),
|
||||
}
|
||||
@@ -583,13 +617,17 @@ impl EthMessageID {
|
||||
Self::GetReceipts => 0x0f,
|
||||
Self::Receipts => 0x10,
|
||||
Self::BlockRangeUpdate => 0x11,
|
||||
Self::GetBlockAccessLists => 0x12,
|
||||
Self::BlockAccessLists => 0x13,
|
||||
Self::Other(value) => *value, // Return the stored `u8`
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the max value for the given version.
|
||||
pub const fn max(version: EthVersion) -> u8 {
|
||||
if version as u8 >= EthVersion::Eth69 as u8 {
|
||||
if version.is_eth71() {
|
||||
Self::BlockAccessLists.to_u8()
|
||||
} else if version.is_eth69_or_newer() {
|
||||
Self::BlockRangeUpdate.to_u8()
|
||||
} else {
|
||||
Self::Receipts.to_u8()
|
||||
@@ -634,6 +672,8 @@ impl Decodable for EthMessageID {
|
||||
0x0f => Self::GetReceipts,
|
||||
0x10 => Self::Receipts,
|
||||
0x11 => Self::BlockRangeUpdate,
|
||||
0x12 => Self::GetBlockAccessLists,
|
||||
0x13 => Self::BlockAccessLists,
|
||||
unknown => Self::Other(*unknown),
|
||||
};
|
||||
buf.advance(1);
|
||||
@@ -662,6 +702,8 @@ impl TryFrom<usize> for EthMessageID {
|
||||
0x0f => Ok(Self::GetReceipts),
|
||||
0x10 => Ok(Self::Receipts),
|
||||
0x11 => Ok(Self::BlockRangeUpdate),
|
||||
0x12 => Ok(Self::GetBlockAccessLists),
|
||||
0x13 => Ok(Self::BlockAccessLists),
|
||||
_ => Err("Invalid message ID"),
|
||||
}
|
||||
}
|
||||
@@ -742,8 +784,9 @@ where
|
||||
mod tests {
|
||||
use super::MessageError;
|
||||
use crate::{
|
||||
message::RequestPair, EthMessage, EthMessageID, EthNetworkPrimitives, EthVersion,
|
||||
GetNodeData, NodeData, ProtocolMessage, RawCapabilityMessage,
|
||||
message::RequestPair, BlockAccessLists, EthMessage, EthMessageID, EthNetworkPrimitives,
|
||||
EthVersion, GetBlockAccessLists, GetNodeData, NodeData, ProtocolMessage,
|
||||
RawCapabilityMessage,
|
||||
};
|
||||
use alloy_primitives::hex;
|
||||
use alloy_rlp::{Decodable, Encodable, Error};
|
||||
@@ -784,6 +827,60 @@ mod tests {
|
||||
assert!(matches!(msg, Err(MessageError::Invalid(..))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_message_version_gating() {
|
||||
let get_block_access_lists =
|
||||
EthMessage::<EthNetworkPrimitives>::GetBlockAccessLists(RequestPair {
|
||||
request_id: 1337,
|
||||
message: GetBlockAccessLists(vec![]),
|
||||
});
|
||||
let buf = encode(ProtocolMessage {
|
||||
message_type: EthMessageID::GetBlockAccessLists,
|
||||
message: get_block_access_lists,
|
||||
});
|
||||
let msg = ProtocolMessage::<EthNetworkPrimitives>::decode_message(
|
||||
EthVersion::Eth70,
|
||||
&mut &buf[..],
|
||||
);
|
||||
assert!(matches!(
|
||||
msg,
|
||||
Err(MessageError::Invalid(EthVersion::Eth70, EthMessageID::GetBlockAccessLists))
|
||||
));
|
||||
|
||||
let block_access_lists =
|
||||
EthMessage::<EthNetworkPrimitives>::BlockAccessLists(RequestPair {
|
||||
request_id: 1337,
|
||||
message: BlockAccessLists(vec![]),
|
||||
});
|
||||
let buf = encode(ProtocolMessage {
|
||||
message_type: EthMessageID::BlockAccessLists,
|
||||
message: block_access_lists,
|
||||
});
|
||||
let msg = ProtocolMessage::<EthNetworkPrimitives>::decode_message(
|
||||
EthVersion::Eth70,
|
||||
&mut &buf[..],
|
||||
);
|
||||
assert!(matches!(
|
||||
msg,
|
||||
Err(MessageError::Invalid(EthVersion::Eth70, EthMessageID::BlockAccessLists))
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_bal_message_eth71_roundtrip() {
|
||||
let msg = ProtocolMessage::from(EthMessage::<EthNetworkPrimitives>::GetBlockAccessLists(
|
||||
RequestPair { request_id: 42, message: GetBlockAccessLists(vec![]) },
|
||||
));
|
||||
let encoded = encode(msg.clone());
|
||||
let decoded = ProtocolMessage::<EthNetworkPrimitives>::decode_message(
|
||||
EthVersion::Eth71,
|
||||
&mut &encoded[..],
|
||||
)
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(decoded, msg);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn request_pair_encode() {
|
||||
let request_pair = RequestPair { request_id: 1337, message: vec![5u8] };
|
||||
|
||||
@@ -29,6 +29,8 @@ pub enum EthVersion {
|
||||
Eth69 = 69,
|
||||
/// The `eth` protocol version 70.
|
||||
Eth70 = 70,
|
||||
/// The `eth` protocol version 71.
|
||||
Eth71 = 71,
|
||||
}
|
||||
|
||||
impl EthVersion {
|
||||
@@ -62,9 +64,19 @@ impl EthVersion {
|
||||
pub const fn is_eth70(&self) -> bool {
|
||||
matches!(self, Self::Eth70)
|
||||
}
|
||||
|
||||
/// Returns true if the version is eth/71
|
||||
pub const fn is_eth71(&self) -> bool {
|
||||
matches!(self, Self::Eth71)
|
||||
}
|
||||
|
||||
/// Returns true if the version is eth/69 or newer.
|
||||
pub const fn is_eth69_or_newer(&self) -> bool {
|
||||
matches!(self, Self::Eth69 | Self::Eth70 | Self::Eth71)
|
||||
}
|
||||
}
|
||||
|
||||
/// RLP encodes `EthVersion` as a single byte (66-69).
|
||||
/// RLP encodes `EthVersion` as a single byte (66-71).
|
||||
impl Encodable for EthVersion {
|
||||
fn encode(&self, out: &mut dyn BufMut) {
|
||||
(*self as u8).encode(out)
|
||||
@@ -76,7 +88,7 @@ impl Encodable for EthVersion {
|
||||
}
|
||||
|
||||
/// RLP decodes a single byte into `EthVersion`.
|
||||
/// Returns error if byte is not a valid version (66-69).
|
||||
/// Returns error if byte is not a valid version (66-71).
|
||||
impl Decodable for EthVersion {
|
||||
fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
|
||||
let version = u8::decode(buf)?;
|
||||
@@ -104,6 +116,7 @@ impl TryFrom<&str> for EthVersion {
|
||||
"68" => Ok(Self::Eth68),
|
||||
"69" => Ok(Self::Eth69),
|
||||
"70" => Ok(Self::Eth70),
|
||||
"71" => Ok(Self::Eth71),
|
||||
_ => Err(ParseVersionError(s.to_string())),
|
||||
}
|
||||
}
|
||||
@@ -129,6 +142,7 @@ impl TryFrom<u8> for EthVersion {
|
||||
68 => Ok(Self::Eth68),
|
||||
69 => Ok(Self::Eth69),
|
||||
70 => Ok(Self::Eth70),
|
||||
71 => Ok(Self::Eth71),
|
||||
_ => Err(ParseVersionError(u.to_string())),
|
||||
}
|
||||
}
|
||||
@@ -159,6 +173,7 @@ impl From<EthVersion> for &'static str {
|
||||
EthVersion::Eth68 => "68",
|
||||
EthVersion::Eth69 => "69",
|
||||
EthVersion::Eth70 => "70",
|
||||
EthVersion::Eth71 => "71",
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -216,6 +231,7 @@ mod tests {
|
||||
assert_eq!(EthVersion::Eth68, EthVersion::try_from("68").unwrap());
|
||||
assert_eq!(EthVersion::Eth69, EthVersion::try_from("69").unwrap());
|
||||
assert_eq!(EthVersion::Eth70, EthVersion::try_from("70").unwrap());
|
||||
assert_eq!(EthVersion::Eth71, EthVersion::try_from("71").unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -225,6 +241,7 @@ mod tests {
|
||||
assert_eq!(EthVersion::Eth68, "68".parse().unwrap());
|
||||
assert_eq!(EthVersion::Eth69, "69".parse().unwrap());
|
||||
assert_eq!(EthVersion::Eth70, "70".parse().unwrap());
|
||||
assert_eq!(EthVersion::Eth71, "71".parse().unwrap());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -235,6 +252,7 @@ mod tests {
|
||||
EthVersion::Eth68,
|
||||
EthVersion::Eth69,
|
||||
EthVersion::Eth70,
|
||||
EthVersion::Eth71,
|
||||
];
|
||||
|
||||
for version in versions {
|
||||
@@ -253,6 +271,7 @@ mod tests {
|
||||
(68_u8, Ok(EthVersion::Eth68)),
|
||||
(69_u8, Ok(EthVersion::Eth69)),
|
||||
(70_u8, Ok(EthVersion::Eth70)),
|
||||
(71_u8, Ok(EthVersion::Eth71)),
|
||||
(65_u8, Err(RlpError::Custom("invalid eth version"))),
|
||||
];
|
||||
|
||||
|
||||
@@ -294,7 +294,8 @@ mod tests {
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::Encodable;
|
||||
use reth_eth_wire_types::{
|
||||
message::RequestPair, GetAccountRangeMessage, GetBlockHeaders, HeadersDirection,
|
||||
message::RequestPair, GetAccountRangeMessage, GetBlockAccessLists, GetBlockHeaders,
|
||||
HeadersDirection,
|
||||
};
|
||||
|
||||
// Helper to create eth message and its bytes
|
||||
@@ -419,4 +420,40 @@ mod tests {
|
||||
let snap_boundary_result = inner.decode_message(snap_boundary_bytes);
|
||||
assert!(snap_boundary_result.is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_eth70_message_id_0x12_is_snap() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth70);
|
||||
let snap_msg = SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage {
|
||||
request_id: 1,
|
||||
root_hash: B256::default(),
|
||||
starting_hash: B256::default(),
|
||||
limit_hash: B256::default(),
|
||||
response_bytes: 1000,
|
||||
});
|
||||
|
||||
let encoded = inner.encode_snap_message(snap_msg);
|
||||
assert_eq!(encoded[0], EthMessageID::message_count(EthVersion::Eth70));
|
||||
|
||||
let decoded = inner.decode_message(BytesMut::from(&encoded[..])).unwrap();
|
||||
assert!(matches!(decoded, EthSnapMessage::Snap(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_eth71_message_id_0x12_is_eth() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth71);
|
||||
let eth_msg = EthMessage::<EthNetworkPrimitives>::GetBlockAccessLists(RequestPair {
|
||||
request_id: 1,
|
||||
message: GetBlockAccessLists(vec![B256::ZERO]),
|
||||
});
|
||||
let protocol_msg = ProtocolMessage::from(eth_msg.clone());
|
||||
let mut buf = Vec::new();
|
||||
protocol_msg.encode(&mut buf);
|
||||
|
||||
let decoded = inner.decode_message(BytesMut::from(&buf[..])).unwrap();
|
||||
let EthSnapMessage::Eth(decoded_eth) = decoded else {
|
||||
panic!("expected eth message");
|
||||
};
|
||||
assert_eq!(decoded_eth, eth_msg);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -84,5 +84,7 @@ mod tests {
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth67).messages(), 17);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth68).messages(), 17);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth69).messages(), 18);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth70).messages(), 18);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth71).messages(), 20);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
//! API related to listening for network events.
|
||||
|
||||
use reth_eth_wire_types::{
|
||||
message::RequestPair, BlockBodies, BlockHeaders, Capabilities, DisconnectReason, EthMessage,
|
||||
EthNetworkPrimitives, EthVersion, GetBlockBodies, GetBlockHeaders, GetNodeData,
|
||||
GetPooledTransactions, GetReceipts, GetReceipts70, NetworkPrimitives, NodeData,
|
||||
PooledTransactions, Receipts, Receipts69, Receipts70, UnifiedStatus,
|
||||
message::RequestPair, BlockAccessLists, BlockBodies, BlockHeaders, Capabilities,
|
||||
DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, GetBlockAccessLists,
|
||||
GetBlockBodies, GetBlockHeaders, GetNodeData, GetPooledTransactions, GetReceipts,
|
||||
GetReceipts70, NetworkPrimitives, NodeData, PooledTransactions, Receipts, Receipts69,
|
||||
Receipts70, UnifiedStatus,
|
||||
};
|
||||
use reth_ethereum_forks::ForkId;
|
||||
use reth_network_p2p::error::{RequestError, RequestResult};
|
||||
@@ -252,6 +253,15 @@ pub enum PeerRequest<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
/// The channel to send the response for receipts.
|
||||
response: oneshot::Sender<RequestResult<Receipts70<N::Receipt>>>,
|
||||
},
|
||||
/// Requests block access lists from the peer.
|
||||
///
|
||||
/// The response should be sent through the channel.
|
||||
GetBlockAccessLists {
|
||||
/// The request for block access lists.
|
||||
request: GetBlockAccessLists,
|
||||
/// The channel to send the response for block access lists.
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
},
|
||||
}
|
||||
|
||||
// === impl PeerRequest ===
|
||||
@@ -272,9 +282,19 @@ impl<N: NetworkPrimitives> PeerRequest<N> {
|
||||
Self::GetReceipts { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetReceipts69 { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetReceipts70 { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetBlockAccessLists { response, .. } => response.send(Err(err)).ok(),
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns true if this request is supported for the negotiated eth protocol version.
|
||||
#[inline]
|
||||
pub fn is_supported_by_eth_version(&self, version: EthVersion) -> bool {
|
||||
match self {
|
||||
Self::GetBlockAccessLists { .. } => version >= EthVersion::Eth71,
|
||||
_ => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the [`EthMessage`] for this type
|
||||
pub fn create_request_message(&self, request_id: u64) -> EthMessage<N> {
|
||||
match self {
|
||||
@@ -299,6 +319,12 @@ impl<N: NetworkPrimitives> PeerRequest<N> {
|
||||
Self::GetReceipts70 { request, .. } => {
|
||||
EthMessage::GetReceipts70(RequestPair { request_id, message: request.clone() })
|
||||
}
|
||||
Self::GetBlockAccessLists { request, .. } => {
|
||||
EthMessage::GetBlockAccessLists(RequestPair {
|
||||
request_id,
|
||||
message: request.clone(),
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -349,3 +375,18 @@ impl<R> fmt::Debug for PeerRequestSender<R> {
|
||||
f.debug_struct("PeerRequestSender").field("peer_id", &self.peer_id).finish_non_exhaustive()
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_get_block_access_lists_version_support() {
|
||||
let (tx, _rx) = oneshot::channel();
|
||||
let req: PeerRequest<EthNetworkPrimitives> =
|
||||
PeerRequest::GetBlockAccessLists { request: GetBlockAccessLists(vec![]), response: tx };
|
||||
|
||||
assert!(!req.is_supported_by_eth_version(EthVersion::Eth70));
|
||||
assert!(req.is_supported_by_eth_version(EthVersion::Eth71));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -20,7 +20,7 @@ use reth_ethereum_forks::{ForkFilter, Head};
|
||||
use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPeer};
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_storage_api::{noop::NoopProvider, BlockNumReader, BlockReader, HeaderProvider};
|
||||
use reth_tasks::{TaskSpawner, TokioTaskExecutor};
|
||||
use reth_tasks::Runtime;
|
||||
use secp256k1::SECP256K1;
|
||||
use std::{collections::HashSet, net::SocketAddr, sync::Arc};
|
||||
|
||||
@@ -76,7 +76,7 @@ pub struct NetworkConfig<C, N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
/// The default mode of the network.
|
||||
pub network_mode: NetworkMode,
|
||||
/// The executor to use for spawning tasks.
|
||||
pub executor: Box<dyn TaskSpawner>,
|
||||
pub executor: Runtime,
|
||||
/// The `Status` message to send to peers at the beginning.
|
||||
pub status: UnifiedStatus,
|
||||
/// Sets the hello message for the p2p handshake in `RLPx`
|
||||
@@ -206,7 +206,7 @@ pub struct NetworkConfigBuilder<N: NetworkPrimitives = EthNetworkPrimitives> {
|
||||
/// The default mode of the network.
|
||||
network_mode: NetworkMode,
|
||||
/// The executor to use for spawning tasks.
|
||||
executor: Option<Box<dyn TaskSpawner>>,
|
||||
executor: Option<Runtime>,
|
||||
/// Sets the hello message for the p2p handshake in `RLPx`
|
||||
hello_message: Option<HelloMessageWithProtocols>,
|
||||
/// The executor to use for spawning tasks.
|
||||
@@ -342,7 +342,7 @@ impl<N: NetworkPrimitives> NetworkConfigBuilder<N> {
|
||||
/// Sets the executor to use for spawning tasks.
|
||||
///
|
||||
/// If `None`, then [`tokio::spawn`] is used for spawning tasks.
|
||||
pub fn with_task_executor(mut self, executor: Box<dyn TaskSpawner>) -> Self {
|
||||
pub fn with_task_executor(mut self, executor: Runtime) -> Self {
|
||||
self.executor = Some(executor);
|
||||
self
|
||||
}
|
||||
@@ -691,7 +691,11 @@ impl<N: NetworkPrimitives> NetworkConfigBuilder<N> {
|
||||
chain_id,
|
||||
block_import: block_import.unwrap_or_else(|| Box::<ProofOfStakeBlockImport>::default()),
|
||||
network_mode,
|
||||
executor: executor.unwrap_or_else(|| Box::<TokioTaskExecutor>::default()),
|
||||
executor: executor.unwrap_or_else(|| match tokio::runtime::Handle::try_current() {
|
||||
Ok(handle) => Runtime::with_existing_handle(handle)
|
||||
.expect("failed to create runtime with existing handle"),
|
||||
Err(_) => Runtime::test(),
|
||||
}),
|
||||
status,
|
||||
hello_message,
|
||||
extra_protocols,
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user