Compare commits

..

4 Commits

Author SHA1 Message Date
Brian Picciano
e9cdd1fbf8 perf(engine): restore persistence defaults from ac6106
Co-Authored-By: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
2026-04-16 09:24:19 +00:00
Brian Picciano
4ebc6a18c6 perf(engine): set memory_block_buffer_target to 20
Buffer of 40 caused MDBX read transaction timeouts (300s limit)
in the deferred trie task. 20 is a safe middle ground.

- memory_block_buffer_target: 40 → 20
- persistence_threshold: 42 → 22 (triggers at gap 23, persists 3)
- backpressure_threshold: 164 → 84 (2 × (22 + 20))

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019d90cb-dc82-73be-8bbc-1a27d8705156
Co-authored-by: Amp <amp@ampcode.com>
2026-04-15 21:05:17 +00:00
Brian Picciano
7fa5608b45 perf(engine): increase memory_block_buffer_target to 40
Raises buffer from 10 to 40 blocks to filter more overwritten
hashed state and trie writes from MDBX persistence.

- memory_block_buffer_target: 10 → 40
- persistence_threshold: 12 → 42 (triggers at gap 43, persists 3)
- backpressure_threshold: 44 → 164 (2 × (42 + 40))

Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019d90cb-dc82-73be-8bbc-1a27d8705156
Co-authored-by: Amp <amp@ampcode.com>
2026-04-15 14:57:47 +00:00
Brian Picciano
ac6106e6f3 perf(engine): pass buffer blocks to save_blocks and filter overwritten hashed state/trie updates
Introduces SaveBlocksBatch to carry both blocks-to-persist and in-memory
buffer blocks through the persistence pipeline. Buffer blocks' hashed
state keys and trie node paths are collected into skip sets so that MDBX
writes which will be immediately overwritten on the next persistence
cycle are elided.

Raises default persistence thresholds:
- memory_block_buffer_target: 0 → 10
- persistence_threshold: 2 → 12 (triggers at gap 13, persists 3)
- backpressure_threshold: 16 → 44 (2 × (12 + 10))

Co-Authored-By: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Amp-Thread-ID: https://ampcode.com/threads/T-019d90cb-dc82-73be-8bbc-1a27d8705156
Co-authored-by: Amp <amp@ampcode.com>
2026-04-15 11:37:28 +00:00
227 changed files with 14794 additions and 22272 deletions

View File

@@ -1,21 +1,24 @@
#!/usr/bin/env bash
#
# Builds reth binaries for benchmarking from local source only.
# Builds (or fetches from cache) reth binaries for benchmarking.
#
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit>
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
#
# baseline — build the baseline binary at <commit> (merge-base)
# baseline — build/fetch the baseline binary at <commit> (merge-base)
# source-dir must be checked out at <commit>
# feature — build the candidate binary + reth-bench at <commit>
# feature — build/fetch the candidate binary + reth-bench at <commit>
# source-dir must be checked out at <commit>
# optional branch-sha is the PR head commit for cache key
#
# Outputs:
# baseline: <source-dir>/target/profiling/reth (or reth-bb if BENCH_BIG_BLOCKS=true)
# feature: <source-dir>/target/profiling/reth (or reth-bb), reth-bench installed to cargo bin
#
# Required: mc (MinIO client) with a configured alias
# Optional env: BENCH_BIG_BLOCKS (true/false) — build reth-bb instead of reth
set -euxo pipefail
MC="mc"
MODE="$1"
SOURCE_DIR="$2"
COMMIT="$3"
@@ -39,38 +42,103 @@ if [ "${BENCH_TRACY:-off}" != "off" ]; then
EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
fi
# Build the requested node binary with the benchmark profile.
build_node_binary() {
local features_arg=""
local workspace_arg=""
# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
else
BUILD_SUFFIX=""
fi
cd "$SOURCE_DIR"
if [ -n "$EXTRA_FEATURES" ]; then
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
features_arg="--features ${EXTRA_FEATURES}"
workspace_arg="--workspace"
# Verify a cached reth binary was built from the expected commit.
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
verify_binary() {
local binary="$1" expected_commit="$2"
local version binary_sha
version=$("$binary" --version 2>/dev/null) || return 1
binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
if [ -z "$binary_sha" ]; then
echo "Warning: could not extract commit SHA from version output"
return 1
fi
# shellcheck disable=SC2086
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg
if [ "$binary_sha" = "$expected_commit" ]; then
return 0
fi
echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
return 1
}
case "$MODE" in
baseline|main)
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
build_node_binary
BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
mkdir -p "${SOURCE_DIR}/target/profiling"
CACHE_VALID=false
if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null; then
echo "Cache hit for baseline (${COMMIT}), downloading ${NODE_BIN}..."
if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
CACHE_VALID=true
else
echo "Cached baseline binary is stale or download failed, rebuilding..."
fi
fi
if [ "$CACHE_VALID" = false ]; then
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
cd "${SOURCE_DIR}"
FEATURES_ARG=""
WORKSPACE_ARG=""
if [ -n "$EXTRA_FEATURES" ]; then
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
FEATURES_ARG="--features ${EXTRA_FEATURES}"
WORKSPACE_ARG="--workspace"
fi
# shellcheck disable=SC2086
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
cargo build --profile profiling $NODE_PKG $WORKSPACE_ARG $FEATURES_ARG
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
fi
;;
feature|branch)
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
rustup show active-toolchain || rustup default stable
build_node_binary
make -C "$SOURCE_DIR" install-reth-bench
BRANCH_SHA="${4:-$COMMIT}"
BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
CACHE_VALID=false
if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null && $MC stat --no-list "${BUCKET}/reth-bench" &>/dev/null; then
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
mkdir -p "${SOURCE_DIR}/target/profiling"
if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench && \
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" /home/ubuntu/.cargo/bin/reth-bench && \
verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
CACHE_VALID=true
else
echo "Cached feature binary is stale or download failed, rebuilding..."
fi
fi
if [ "$CACHE_VALID" = false ]; then
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
cd "${SOURCE_DIR}"
rustup show active-toolchain || rustup default stable
if [ -n "$EXTRA_FEATURES" ]; then
# Can't use `make profiling` when adding features; build explicitly
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
cargo build --profile profiling --workspace $NODE_PKG --features "${EXTRA_FEATURES}"
else
# shellcheck disable=SC2086
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
cargo build --profile profiling $NODE_PKG
fi
make install-reth-bench
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
fi
;;
*)
echo "Usage: $0 <baseline|feature> <source-dir> <commit>"
echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
exit 1
;;
esac

View File

@@ -2,7 +2,7 @@
#
# local-reth-bench.sh — Run the reth Engine API benchmark locally.
#
# Replicates the CI bench.yml workflow (build, local snapshot validation, system tuning,
# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
# interleaved B-F-F-B execution, summary, charts) without any GitHub
# Actions glue (no PR comments, no artifact upload, no Slack).
#
@@ -21,17 +21,15 @@
# Requires: the reth repo at RETH_REPO (default: ~/reth)
#
# Dependencies (install before first run):
# schelk, cpupower, taskset, stdbuf, python3, curl,
# make, uv, jq, Rust toolchain (cargo/rustup)
# Optional:
# mc for Tracy profile upload
# mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
# make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
#
# The script delegates to the existing bench-reth-*.sh scripts in the reth
# repo for the actual build, snapshot, and run steps.
set -euxo pipefail
# ── PATH ──────────────────────────────────────────────────────────────
# Ensure cargo and user-local bins (uv) are visible
# Ensure cargo and user-local bins (mc, uv) are visible
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
# ── Defaults ──────────────────────────────────────────────────────────
@@ -108,7 +106,7 @@ fi
# ── Check dependencies ───────────────────────────────────────────────
missing=()
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq cargo; do
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
@@ -240,14 +238,19 @@ echo " Baseline src : $BASELINE_SRC"
echo " Feature src : $FEATURE_SRC"
echo
# ── Step 3: Validate local snapshot ──────────────────────────────────
echo "▸ Validating local snapshot..."
# ── Step 3: Check / download snapshot ────────────────────────────────
echo "▸ Checking snapshot..."
cd "$RETH_REPO"
"${SCRIPTS_DIR}/bench-reth-snapshot.sh"
echo " Snapshot is ready."
SNAPSHOT_NEEDED=false
if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
SNAPSHOT_NEEDED=true
echo " Snapshot needs update."
else
echo " Snapshot is up-to-date."
fi
echo
# ── Step 4: Build binaries in parallel ───────────────────────────────
# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
echo "▸ Building binaries (parallel)..."
cd "$RETH_REPO"
@@ -259,11 +262,19 @@ PID_BASELINE=$!
"${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
PID_FEATURE=$!
PID_SNAPSHOT=
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
echo " Also downloading snapshot in parallel..."
"${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
PID_SNAPSHOT=$!
fi
wait $PID_BASELINE || FAIL=1
wait $PID_FEATURE || FAIL=1
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
if [ $FAIL -ne 0 ]; then
echo "Error: one or more build tasks failed"
echo "Error: one or more parallel tasks failed (builds / snapshot)"
exit 1
fi
echo " Binaries built successfully."

View File

@@ -88,16 +88,10 @@ trap cleanup EXIT
# Stop any leftover reth process in the scope, then recover schelk state.
sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
sudo schelk recover -y --kill || true
# Mount
sudo schelk mount -y || true
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
echo "::error::Failed to mount benchmark datadir at ${DATADIR}"
ls -la "$SCHELK_MOUNT" || true
ls -la "$DATADIR" || true
exit 1
fi
sudo schelk mount -y
sync
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
echo "=== Cache state after drop ==="

View File

@@ -1,56 +1,129 @@
#!/usr/bin/env bash
#
# Validates that the benchmark snapshot has already been populated into the
# local schelk volume.
# Downloads the latest snapshot into the schelk volume using
# `reth download` with progress reporting to the GitHub PR comment.
#
# Skips the download if the manifest content hasn't changed since
# the last successful download (checked via SHA-256 of the manifest).
#
# Usage: bench-reth-snapshot.sh [--check]
# --check Exit 0 if the local snapshot is ready, 10 if it is missing.
# --check Only check if a download is needed; exits 0 if up-to-date, 10 if not.
#
# Required env:
# SCHELK_MOUNT schelk mount point (e.g. /reth-bench)
# Optional env:
# BENCH_BIG_BLOCKS true when validating the big-blocks snapshot datadir
# BENCH_SNAPSHOT_NAME expected snapshot label for log/error output
# SCHELK_MOUNT schelk mount point (e.g. /reth-bench)
# BENCH_RETH_BINARY path to the reth binary
# GITHUB_TOKEN token for GitHub API calls (only for download)
# BENCH_COMMENT_ID PR comment ID to update (optional)
# BENCH_REPO owner/repo (e.g. paradigmxyz/reth)
# BENCH_JOB_URL link to the Actions job
# BENCH_ACTOR user who triggered the benchmark
# BENCH_CONFIG config summary line
set -euxo pipefail
: "${SCHELK_MOUNT:?SCHELK_MOUNT must be set}"
MC="mc"
BUCKET="minio/reth-snapshots"
# Allow overriding the snapshot name (e.g. for big-blocks mode where the
# big-blocks manifest specifies which base snapshot to use).
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable}"
MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
DATADIR_NAME="datadir"
HASH_MODE_SUFFIX=""
if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
DATADIR_NAME="datadir-big-blocks"
HASH_MODE_SUFFIX="-big-blocks"
fi
DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
HASH_FILE="$HOME/.reth-bench-snapshot-hash${HASH_MODE_SUFFIX}"
describe_snapshot() {
if [ -n "${BENCH_SNAPSHOT_NAME:-}" ]; then
printf '%s' "${BENCH_SNAPSHOT_NAME}"
elif [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
printf '%s' 'big-block weekly snapshot'
else
printf '%s' 'benchmark snapshot'
fi
# Fetch manifest and compute content hash for reliable freshness check
MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
exit 2
}
REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
snapshot_ready() {
[ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ]
}
LOCAL_HASH=""
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
EXPECTED_SNAPSHOT="$(describe_snapshot)"
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
sudo schelk mount -y || true
if snapshot_ready; then
echo "Found local ${EXPECTED_SNAPSHOT} at ${DATADIR}"
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
exit 0
fi
echo "::error::Missing local ${EXPECTED_SNAPSHOT} at ${DATADIR}. Benchmarks no longer download snapshots; pre-populate the local schelk data first."
ls -la "$SCHELK_MOUNT" || true
ls -la "$DATADIR" || true
echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
if [ "${1:-}" = "--check" ]; then
exit 10
fi
exit 1
RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
if [ ! -x "$RETH" ]; then
echo "::error::reth binary not found or not executable at $RETH"
exit 1
fi
# Resolve the MinIO HTTP endpoint from the mc alias so reth can
# fetch archives over HTTP (the manifest's embedded base_url points
# to the cluster-internal address which is unreachable from runners).
MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
if [ -z "$MINIO_ENDPOINT" ]; then
echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
exit 1
fi
BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/${SNAPSHOT_NAME}"
# Rewrite manifest's base_url with the runner-reachable endpoint
MANIFEST_TMP=$(mktemp --suffix=.json)
trap 'rm -f -- "$MANIFEST_TMP"' EXIT
echo "$MANIFEST_CONTENT" \
| jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
# Prepare mount. If a previous run left the volume mounted, recover first.
sudo schelk recover -y --kill || true
sudo schelk mount -y
sudo rm -rf "$DATADIR"
sudo mkdir -p "$DATADIR"
# reth download runs as current user (not root), needs write access
sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
update_comment() {
local status="$1"
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
local body
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
curl -sf -X PATCH \
-H "Authorization: token ${GITHUB_TOKEN}" \
-H "Accept: application/vnd.github.v3+json" \
"https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
-d "$(jq -nc --arg body "$body" '{body: $body}')" \
> /dev/null 2>&1 || true
}
update_comment "Downloading snapshot…"
# Download using reth download (manifest-path with rewritten base_url)
"$RETH" download \
--manifest-path "$MANIFEST_TMP" \
-y \
--minimal \
--datadir "$DATADIR"
update_comment "Downloading snapshot… done"
echo "Snapshot download complete"
# Sanity check: verify expected directories exist
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
ls -la "$DATADIR" || true
exit 1
fi
# Promote the new snapshot to become the schelk baseline (virgin volume).
# This copies changed blocks from scratch → virgin so that future
# `schelk recover` calls restore to this new state.
sync
sudo schelk promote -y
# Save manifest hash
echo "$REMOTE_HASH" > "$HASH_FILE"
echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"

View File

@@ -111,14 +111,6 @@ def compute_stats(combined: list[dict]) -> dict:
wall_clock_s = sum(total_latencies_ms) / 1_000
mean_total_lat_ms = sum(total_latencies_ms) / n
# Persistence wait mean (for main table)
persist_values_ms = []
for r in combined:
v = r.get("persistence_wait_us")
if v is not None:
persist_values_ms.append(v / 1_000)
mean_persist_ms = sum(persist_values_ms) / len(persist_values_ms) if persist_values_ms else 0.0
return {
"n": n,
"mean_ms": mean_lat,
@@ -129,7 +121,6 @@ def compute_stats(combined: list[dict]) -> dict:
"mean_mgas_s": mean_mgas_s,
"wall_clock_s": wall_clock_s,
"mean_total_lat_ms": mean_total_lat_ms,
"mean_persist_ms": mean_persist_ms,
}
@@ -154,7 +145,7 @@ def compute_wait_stats(combined: list[dict], field: str) -> dict:
def _paired_data(
baseline: list[dict], feature: list[dict]
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float], list[float]]:
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float]]:
"""Match blocks and return paired latencies and per-block diffs.
Returns:
@@ -162,7 +153,6 @@ def _paired_data(
lat_diffs_ms: list of feature baseline latency diffs in ms
mgas_diffs: list of feature baseline Mgas/s diffs
total_lat_diffs_ms: list of feature baseline total latency diffs in ms
persist_diffs_ms: list of feature baseline persistence wait diffs in ms
"""
baseline_by_block = {r["block_number"]: r for r in baseline}
feature_by_block = {r["block_number"]: r for r in feature}
@@ -172,7 +162,6 @@ def _paired_data(
lat_diffs_ms = []
mgas_diffs = []
total_lat_diffs_ms = []
persist_diffs_ms = []
for bn in common_blocks:
b = baseline_by_block[bn]
f = feature_by_block[bn]
@@ -190,10 +179,7 @@ def _paired_data(
total_lat_diffs_ms.append(
f["total_latency_us"] / 1_000 - b["total_latency_us"] / 1_000
)
b_persist = (b.get("persistence_wait_us") or 0) / 1_000
f_persist = (f.get("persistence_wait_us") or 0) / 1_000
persist_diffs_ms.append(f_persist - b_persist)
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms, persist_diffs_ms
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms
def compute_paired_stats(
@@ -209,15 +195,13 @@ def compute_paired_stats(
all_lat_diffs = []
all_mgas_diffs = []
all_total_lat_diffs = []
all_persist_diffs = []
blocks_per_pair = []
for baseline, feature in zip(baseline_runs, feature_runs):
pairs, lat_diffs, mgas_diffs, total_lat_diffs, persist_diffs = _paired_data(baseline, feature)
pairs, lat_diffs, mgas_diffs, total_lat_diffs = _paired_data(baseline, feature)
all_pairs.extend(pairs)
all_lat_diffs.extend(lat_diffs)
all_mgas_diffs.extend(mgas_diffs)
all_total_lat_diffs.extend(total_lat_diffs)
all_persist_diffs.extend(persist_diffs)
blocks_per_pair.append(len(pairs))
if not all_lat_diffs:
@@ -261,11 +245,6 @@ def compute_paired_stats(
total_se = std_total_diff / math.sqrt(len(all_total_lat_diffs)) if all_total_lat_diffs else 0.0
wall_clock_ci_ms = T_CRITICAL * total_se
mean_persist_diff = sum(all_persist_diffs) / len(all_persist_diffs) if all_persist_diffs else 0.0
std_persist_diff = stddev(all_persist_diffs, mean_persist_diff) if len(all_persist_diffs) > 1 else 0.0
persist_se = std_persist_diff / math.sqrt(len(all_persist_diffs)) if all_persist_diffs else 0.0
persist_ci_ms = T_CRITICAL * persist_se
return {
"n": n,
"mean_diff_ms": mean_diff,
@@ -279,7 +258,6 @@ def compute_paired_stats(
"mean_mgas_diff": mean_mgas_diff,
"mgas_ci": mgas_ci,
"wall_clock_ci_ms": wall_clock_ci_ms,
"persist_ci_ms": persist_ci_ms,
"blocks": max(blocks_per_pair),
}
@@ -358,7 +336,6 @@ def compute_changes(
("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True),
("persist_wait", "mean_persist_ms", "persist_ci_ms", "mean_persist_ms", True),
]
changes = {}
for name, stat_key, ci_key, base_key, lower_is_better in metrics:
@@ -400,8 +377,6 @@ def generate_comparison_table(
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
p99_pct = pct(run1["p99_ms"], run2["p99_ms"])
persist_pct = pct(run1["mean_persist_ms"], run2["mean_persist_ms"])
# Bootstrap CIs as % of baseline percentile
p50_ci_pct = paired["p50_ci_ms"] / run1["p50_ms"] * 100.0 if run1["p50_ms"] > 0 else 0.0
p90_ci_pct = paired["p90_ci_ms"] / run1["p90_ms"] * 100.0 if run1["p90_ms"] > 0 else 0.0
@@ -411,7 +386,6 @@ def generate_comparison_table(
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
wall_ci_pct = paired["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0
persist_ci_pct = paired["persist_ci_ms"] / run1["mean_persist_ms"] * 100.0 if run1["mean_persist_ms"] > 0 else 0.0
base_url = f"https://github.com/{repo}/commit"
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
@@ -427,7 +401,6 @@ def generate_comparison_table(
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, lower_is_better=True)} |",
f"| Persist Wait | {fmt_ms(run1['mean_persist_ms'])} | {fmt_ms(run2['mean_persist_ms'])} | {change_str(persist_pct, persist_ci_pct, lower_is_better=True)} |",
"",
]
meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}"]

View File

@@ -83,7 +83,6 @@ function metricRows(summary) {
{ label: 'P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) },
{ label: 'Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
{ label: 'Wall Clock', baseline: fmtS(b.wall_clock_s), feature: fmtS(f.wall_clock_s), change: fmtChange(c.wall_clock) },
{ label: 'Persist Wait', baseline: fmtMs(b.mean_persist_ms || 0), feature: fmtMs(f.mean_persist_ms || 0), change: fmtChange(c.persist_wait) },
];
}

View File

@@ -1,244 +0,0 @@
#!/usr/bin/env python3
"""
Fetch a Grafana dashboard and convert it to the portable import format.
Fetches the dashboard via API, replaces internal datasource/variable references
with template variables, and adds __inputs/__requires/__elements so the JSON is
importable on any Grafana instance.
Usage:
export FETCH_GRAFANA_DASHBOARD_URL=https://<NAMESPACE>.grafana.net
export FETCH_GRAFANA_DASHBOARD_TOKEN=glsa_...
python3 .github/scripts/fetch-grafana-dashboard.py <dashboard-uid> > output.json
"""
import json
import os
import sys
import urllib.request
PANEL_TYPE_NAMES = {
"bargauge": "Bar gauge",
"gauge": "Gauge",
"heatmap": "Heatmap",
"piechart": "Pie chart",
"stat": "Stat",
"table": "Table",
"timeseries": "Time series",
"barchart": "Bar chart",
"text": "Text",
"dashlist": "Dashboard list",
"logs": "Logs",
"nodeGraph": "Node Graph",
"histogram": "Histogram",
"candlestick": "Candlestick",
"state-timeline": "State timeline",
"status-history": "Status history",
"geomap": "Geomap",
"canvas": "Canvas",
"news": "News",
"xychart": "XY Chart",
"trend": "Trend",
"datagrid": "Datagrid",
"flamegraph": "Flame Graph",
"traces": "Traces",
}
def fetch_json(base_url: str, token: str, path: str) -> dict:
url = f"{base_url}{path}"
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
with urllib.request.urlopen(req) as resp:
return json.loads(resp.read())
def fetch_dashboard(base_url: str, token: str, uid: str) -> dict:
return fetch_json(base_url, token, f"/api/dashboards/uid/{uid}")
def fetch_grafana_version(base_url: str) -> str:
req = urllib.request.Request(f"{base_url}/api/health")
with urllib.request.urlopen(req) as resp:
data = json.loads(resp.read())
# version string like "13.0.0-23940615780.patch2" -> take just the semver part
version = data.get("version", "")
# strip build metadata after the first hyphen if it looks like a pre-release
parts = version.split("-")
return parts[0] if parts else version
def collect_panel_types(panels: list) -> set[str]:
types = set()
for panel in panels:
ptype = panel.get("type", "")
if ptype and ptype != "row":
types.add(ptype)
# nested panels inside collapsed rows
for sub in panel.get("panels", []):
sub_type = sub.get("type", "")
if sub_type and sub_type != "row":
types.add(sub_type)
return types
def has_expression_datasource(dashboard: dict) -> bool:
return "__expr__" in json.dumps(dashboard)
def make_exportable(dashboard: dict, grafana_version: str = "") -> dict:
dash = json.loads(json.dumps(dashboard)) # deep copy
# --- Strip internal fields ---
dash.pop("id", None)
# --- Rewrite links: point to the public repo instead of internal ---
dash["links"] = [
{
"asDropdown": False,
"icon": "external link",
"includeVars": False,
"keepTime": False,
"tags": [],
"targetBlank": True,
"title": "Source (GitHub)",
"tooltip": "View source file in repository",
"type": "link",
"url": "https://github.com/paradigmxyz/reth/tree/main/etc/grafana/dashboards",
}
]
# --- Datasource: victoriametrics -> prometheus ---
dash_str = json.dumps(dash)
dash_str = dash_str.replace("victoriametrics-metrics-datasource", "prometheus")
dash = json.loads(dash_str)
# --- Templating: instance_label constant -> ${VAR_INSTANCE_LABEL} ---
# Also strip default-value fields the API returns that are not needed for import
STRIP_VAR_DEFAULTS = {"allowCustomValue", "regexApplyTo"}
for var in dash.get("templating", {}).get("list", []):
if var.get("name") == "instance_label" and var.get("type") == "constant":
var["query"] = "${VAR_INSTANCE_LABEL}"
var["current"] = {
"value": "${VAR_INSTANCE_LABEL}",
"text": "${VAR_INSTANCE_LABEL}",
"selected": False,
}
var["options"] = [
{
"value": "${VAR_INSTANCE_LABEL}",
"text": "${VAR_INSTANCE_LABEL}",
"selected": False,
}
]
# Clear current values for query/datasource vars (not meaningful for import)
elif var.get("type") in ("query", "datasource"):
var["current"] = {}
# Remove noisy default fields
for field in STRIP_VAR_DEFAULTS:
var.pop(field, None)
# Strip falsy defaults on query/datasource vars (API returns them, export omits them)
if var.get("type") in ("query", "datasource"):
for field in ("hide", "multi", "skipUrlSync"):
if not var.get(field):
var.pop(field, None)
# --- Build __inputs ---
inputs = [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus",
},
]
if has_expression_datasource(dash):
inputs.append(
{
"name": "DS_EXPRESSION",
"label": "Expression",
"description": "",
"type": "datasource",
"pluginId": "__expr__",
}
)
inputs.append(
{
"name": "VAR_INSTANCE_LABEL",
"type": "constant",
"label": "Instance Label",
"value": "job",
"description": "",
}
)
# --- Build __requires ---
requires = []
if has_expression_datasource(dash):
requires.append({"type": "datasource", "id": "__expr__", "version": "1.0.0"})
panel_types = collect_panel_types(dash.get("panels", []))
for pt in sorted(panel_types):
requires.append(
{
"type": "panel",
"id": pt,
"name": PANEL_TYPE_NAMES.get(pt, pt),
"version": "",
}
)
requires.append(
{"type": "grafana", "id": "grafana", "name": "Grafana", "version": grafana_version}
)
requires.append(
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0",
}
)
# --- Assemble output (with __inputs/__requires/__elements first) ---
output = {
"__inputs": inputs,
"__elements": {},
"__requires": requires,
}
output.update(dash)
return output
def main():
if len(sys.argv) < 2:
print(f"Usage: {sys.argv[0]} <dashboard-uid>", file=sys.stderr)
sys.exit(1)
uid = sys.argv[1]
base_url = os.environ.get("FETCH_GRAFANA_DASHBOARD_URL", "").rstrip("/")
token = os.environ.get("FETCH_GRAFANA_DASHBOARD_TOKEN", "")
if not base_url or not token:
print(
"Error: FETCH_GRAFANA_DASHBOARD_URL and FETCH_GRAFANA_DASHBOARD_TOKEN env vars required",
file=sys.stderr,
)
sys.exit(1)
resp = fetch_dashboard(base_url, token, uid)
dashboard = resp["dashboard"]
grafana_version = fetch_grafana_version(base_url)
exported = make_exportable(dashboard, grafana_version)
print(json.dumps(exported, indent=2))
if __name__ == "__main__":
main()

View File

@@ -112,7 +112,7 @@ jobs:
- name: Alert on long-running hourly
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
uses: actions/github-script@v9
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -154,7 +154,7 @@ jobs:
- name: Alert on stale nightly
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
uses: actions/github-script@v9
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -278,7 +278,7 @@ jobs:
- name: Resolve job URL
id: job-url
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
@@ -307,6 +307,12 @@ jobs:
linux-tools-"$(uname -r)" || \
sudo apt-get install -y --no-install-recommends linux-tools-generic
# mc (MinIO client)
if ! command -v mc &>/dev/null; then
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
chmod +x "$HOME/.local/bin/mc"
fi
# uv (Python package manager)
if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
@@ -334,7 +340,7 @@ jobs:
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
missing=()
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
@@ -360,30 +366,35 @@ jobs:
echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
- name: Validate local snapshot
- name: Check if snapshot needs update
id: snapshot-check
run: .github/scripts/bench-reth-snapshot.sh
run: |
set +e
.github/scripts/bench-reth-snapshot.sh --check
rc=$?
set -e
case "$rc" in
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
*) echo "::error::Snapshot check failed (exit $rc)"
exit "$rc" ;;
esac
- name: Prepare source dirs
run: |
prepare_source_dir() {
local dir="$1"
local ref="$2"
if [ -d ../reth-baseline ]; then
git -C ../reth-baseline fetch origin "$BASELINE_REF"
else
git clone . ../reth-baseline
fi
git -C ../reth-baseline checkout "$BASELINE_REF"
if [ -d "$dir" ]; then
git -C "$dir" reset --hard HEAD
git -C "$dir" clean -fdx
git -C "$dir" fetch origin "$ref"
else
git clone . "$dir"
fi
git -C "$dir" checkout --force "$ref"
}
prepare_source_dir ../reth-baseline "$BASELINE_REF"
prepare_source_dir ../reth-feature "$FEATURE_REF"
if [ -d ../reth-feature ]; then
git -C ../reth-feature fetch origin "$FEATURE_REF"
else
git clone . ../reth-feature
fi
git -C ../reth-feature checkout "$FEATURE_REF"
- name: Build binaries
id: build
@@ -407,6 +418,15 @@ jobs:
exit 1
fi
- name: Download snapshot
id: snapshot-download
if: steps.snapshot-check.outputs.needed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BENCH_REPO: ${{ github.repository }}
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
run: .github/scripts/bench-reth-snapshot.sh
# System tuning for reproducible benchmarks
- name: System setup
run: |
@@ -660,7 +680,7 @@ jobs:
- name: Write job summary
if: success()
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const fs = require('fs');
@@ -739,7 +759,7 @@ jobs:
- name: Send Slack notification (success)
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
uses: actions/github-script@v9
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -894,7 +914,7 @@ jobs:
- name: Send Slack notification (failure)
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
uses: actions/github-script@v9
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -905,8 +925,8 @@ jobs:
if (!token || !channel) return;
const steps_status = [
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
['building binaries', '${{ steps.build.outcome }}'],
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
@@ -928,7 +948,7 @@ jobs:
},
{
type: 'section',
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n<@U0AAA8F0JEM> investigate this` },
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n@ai investigate this` },
},
{
type: 'actions',
@@ -955,11 +975,6 @@ jobs:
}),
});
- name: Clean build outputs
if: always()
run: |
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
- name: Restore system settings
if: always()
run: |

View File

@@ -14,7 +14,7 @@ on:
blocks:
description: "Number of blocks to benchmark"
required: false
default: "500"
default: "200"
type: string
big_blocks:
description: "Use big blocks mode (pre-generated merged payloads with reth-bb)"
@@ -34,7 +34,7 @@ on:
warmup:
description: "Number of warmup blocks"
required: false
default: "200"
default: "100"
type: string
baseline:
description: "Baseline git ref (default: merge-base)"
@@ -133,7 +133,7 @@ jobs:
steps:
- name: Check org membership
if: github.event_name == 'issue_comment'
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@@ -152,7 +152,7 @@ jobs:
- name: Parse arguments
id: args
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -164,9 +164,9 @@ jobs:
if (context.eventName === 'workflow_dispatch') {
actor = '${{ github.actor }}';
blocks = '${{ github.event.inputs.blocks }}' || '500';
warmup = '${{ github.event.inputs.warmup }}' || '200';
if (warmup !== '200') explicitWarmup = true;
blocks = '${{ github.event.inputs.blocks }}' || '200';
warmup = '${{ github.event.inputs.warmup }}' || '100';
if (warmup !== '100') explicitWarmup = true;
baseline = '${{ github.event.inputs.baseline }}';
feature = '${{ github.event.inputs.feature }}';
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
@@ -205,7 +205,7 @@ jobs:
const enumArgs = new Map([['bal', validBalModes], ['slack', validSlackModes]]);
const durationArgs = new Set(['wait-time']);
const stringArgs = new Set(['baseline-args', 'feature-args']);
const defaults = { blocks: '500', warmup: '200', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const defaults = { blocks: '200', warmup: '100', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const unknown = [];
const invalid = [];
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -359,7 +359,7 @@ jobs:
- name: Acknowledge request
id: ack
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -445,7 +445,7 @@ jobs:
- name: Poll queue position
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -560,7 +560,7 @@ jobs:
- name: Resolve checkout ref
id: checkout-ref
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
if (!process.env.BENCH_PR) {
@@ -586,7 +586,7 @@ jobs:
- name: Resolve job URL and update status
if: env.BENCH_COMMENT_ID
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -650,6 +650,12 @@ jobs:
linux-tools-"$(uname -r)" || \
sudo apt-get install -y --no-install-recommends linux-tools-generic
# mc (MinIO client)
if ! command -v mc &>/dev/null; then
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
chmod +x "$HOME/.local/bin/mc"
fi
# uv (Python package manager)
if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
@@ -684,7 +690,7 @@ jobs:
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
missing=()
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
@@ -696,7 +702,7 @@ jobs:
# Build binaries
- name: Resolve PR head branch
id: pr-info
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
if (process.env.BENCH_PR) {
@@ -714,7 +720,7 @@ jobs:
- name: Resolve refs
id: refs
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const { execSync } = require('child_process');
@@ -760,68 +766,79 @@ jobs:
core.setOutput('feature-ref', featureRef);
core.setOutput('feature-name', featureName);
- name: Validate local big blocks
- name: Check big-blocks freshness
if: env.BENCH_BIG_BLOCKS == 'true'
id: big-blocks-check
run: |
set -euo pipefail
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
PAYLOAD_DIR="$BIG_BLOCKS_DIR/payloads"
MANIFEST="$BIG_BLOCKS_DIR/manifest.json"
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
if [ ! -f "$MANIFEST" ]; then
echo "::error::Missing local big-blocks manifest at $MANIFEST"
MC="mc --config-dir /home/ubuntu/.mc"
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
HASH_FILE="$HOME/.reth-bench-big-blocks-hash"
echo "Fetching big-blocks manifest from $MANIFEST..."
BB_MANIFEST=$($MC cat "$MANIFEST" 2>/dev/null) || {
echo "::error::Failed to fetch big-blocks manifest from $MANIFEST"
exit 1
fi
BASE_SNAPSHOT=$(jq -r '.base_snapshot // empty' "$MANIFEST")
}
BASE_SNAPSHOT=$(echo "$BB_MANIFEST" | jq -r '.base_snapshot // empty')
if [ -z "$BASE_SNAPSHOT" ]; then
echo "::error::Big-blocks manifest missing base_snapshot field"
exit 1
fi
if [ ! -d "$PAYLOAD_DIR" ]; then
echo "::error::Missing local big-block payload directory at $PAYLOAD_DIR"
exit 1
fi
PAYLOAD_COUNT=$(find "$PAYLOAD_DIR" -name '*.json' | wc -l)
if [ "$PAYLOAD_COUNT" -eq 0 ]; then
echo "::error::No payload files found in $PAYLOAD_DIR"
exit 1
fi
echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
echo "Payload files: $PAYLOAD_COUNT"
echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
- name: Validate local snapshot
REMOTE_HASH=$(echo "$BB_MANIFEST" | sha256sum | awk '{print $1}')
LOCAL_HASH=""
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
echo "Big blocks up-to-date (hash: ${REMOTE_HASH:0:16}…)"
echo "needed=false" >> "$GITHUB_OUTPUT"
else
echo "Big blocks need update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
echo "needed=true" >> "$GITHUB_OUTPUT"
echo "remote-hash=${REMOTE_HASH}" >> "$GITHUB_OUTPUT"
fi
- name: Check if snapshot needs update
id: snapshot-check
run: .github/scripts/bench-reth-snapshot.sh
run: |
set +e
.github/scripts/bench-reth-snapshot.sh --check
rc=$?
set -e
case "$rc" in
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
*) echo "::error::Snapshot check failed (exit $rc)"
exit "$rc" ;;
esac
- name: Update status (snapshot needed)
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const s = require('./.github/scripts/bench-update-status.js');
await s({github, context, status: 'Building binaries (snapshot update pending)...'});
- name: Prepare source dirs
run: |
prepare_source_dir() {
local dir="$1"
local ref="$2"
if [ -d "$dir" ]; then
git -C "$dir" reset --hard HEAD
git -C "$dir" clean -fdx
git -C "$dir" fetch origin "$ref"
else
git clone . "$dir"
fi
git -C "$dir" checkout --force "$ref"
}
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
prepare_source_dir ../reth-baseline "$BASELINE_REF"
if [ -d ../reth-baseline ]; then
git -C ../reth-baseline fetch origin "$BASELINE_REF"
else
git clone . ../reth-baseline
fi
git -C ../reth-baseline checkout "$BASELINE_REF"
FEATURE_REF="${{ steps.refs.outputs.feature-ref }}"
prepare_source_dir ../reth-feature "$FEATURE_REF"
if [ -d ../reth-feature ]; then
git -C ../reth-feature fetch origin "$FEATURE_REF"
else
git clone . ../reth-feature
fi
git -C ../reth-feature checkout "$FEATURE_REF"
- name: Build binaries
id: build
@@ -845,6 +862,15 @@ jobs:
exit 1
fi
- name: Download snapshot
id: snapshot-download
if: steps.snapshot-check.outputs.needed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BENCH_REPO: ${{ github.repository }}
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
run: .github/scripts/bench-reth-snapshot.sh
# System tuning for reproducible benchmarks
- name: System setup
run: |
@@ -884,11 +910,8 @@ jobs:
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
done || true
# Replace any stale PM QoS holders left behind by earlier benchmark jobs.
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
# Prevent deep C-states (avoids wake-up latency jitter)
sudo bash -c 'exec 3<>/dev/cpu_dma_latency; printf "\0\0\0\0" >&3; exec -a bench-cpu-dma-latency sleep infinity' &
echo "BENCH_CPU_DMA_LATENCY_PID=$!" >> "$GITHUB_ENV"
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
# Move all IRQs to core 0 (housekeeping core)
for irq in /proc/irq/*/smp_affinity_list; do
echo 0 | sudo tee "$irq" 2>/dev/null || true
@@ -915,6 +938,45 @@ jobs:
rm -rf "$BENCH_WORK_DIR"
mkdir -p "$BENCH_WORK_DIR"
- name: Download big blocks
if: env.BENCH_BIG_BLOCKS == 'true'
run: |
set -euo pipefail
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
if [ "${{ steps.big-blocks-check.outputs.needed }}" = "false" ]; then
echo "Big blocks cached at $BIG_BLOCKS_DIR, skipping download"
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
exit 0
fi
MC="mc --config-dir /home/ubuntu/.mc"
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
# Download and parse manifest
echo "Downloading manifest from $MANIFEST..."
$MC cat "$MANIFEST" > "$BIG_BLOCKS_DIR/manifest.json"
UPLOAD_PATH=$(jq -r '.upload_path' "$BIG_BLOCKS_DIR/manifest.json")
COUNT=$(jq -r '.count' "$BIG_BLOCKS_DIR/manifest.json")
TARGET_GAS=$(jq -r '.target_gas' "$BIG_BLOCKS_DIR/manifest.json")
echo "Manifest: count=$COUNT, target_gas=$TARGET_GAS, archive=$UPLOAD_PATH"
# Download and extract archive
ARCHIVE="minio/$UPLOAD_PATH"
echo "Downloading big blocks from $ARCHIVE..."
$MC cat "$ARCHIVE" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
# Verify expected directory structure
if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
echo "::error::Big blocks archive missing expected payloads/ directory"
ls -laR "$BIG_BLOCKS_DIR"
exit 1
fi
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
# Save manifest hash for freshness check on next run
echo "${{ steps.big-blocks-check.outputs.remote-hash }}" > "$HOME/.reth-bench-big-blocks-hash"
- name: Start metrics proxy
run: |
BENCH_ID="ci-${{ github.run_id }}"
@@ -937,7 +999,7 @@ jobs:
- name: Update status (running benchmarks)
if: success() && env.BENCH_COMMENT_ID
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1199,7 +1261,7 @@ jobs:
- name: Compare & comment
if: success() && env.BENCH_COMMENT_ID
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1279,7 +1341,7 @@ jobs:
- name: Write job summary
if: success()
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const jobSummary = require('./.github/scripts/bench-job-summary.js');
@@ -1293,7 +1355,7 @@ jobs:
- name: Send Slack notification (success)
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
uses: actions/github-script@v9
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -1304,16 +1366,14 @@ jobs:
- name: Update status (failed)
if: failure() && env.BENCH_COMMENT_ID
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
const steps_status = [
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
['building binaries', '${{ steps.build.outcome }}'],
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
@@ -1340,18 +1400,16 @@ jobs:
- name: Send Slack notification (failure)
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
uses: actions/github-script@v9
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
const steps_status = [
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
['building binaries', '${{ steps.build.outcome }}'],
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
@@ -1364,7 +1422,7 @@ jobs:
- name: Update status (cancelled)
if: cancelled() && env.BENCH_COMMENT_ID
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1375,11 +1433,6 @@ jobs:
body: `cc @${process.env.BENCH_ACTOR}\n\n⚠ Benchmark cancelled. [View logs](${jobUrl})`,
});
- name: Clean build outputs
if: always()
run: |
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
- name: Restore system settings
if: always()
run: |
@@ -1392,9 +1445,5 @@ jobs:
done
# Restore amd-pstate to active (EPP) mode with powersave governor
echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
if [ -n "${BENCH_CPU_DMA_LATENCY_PID:-}" ]; then
sudo kill "$BENCH_CPU_DMA_LATENCY_PID" 2>/dev/null || true
fi
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
sudo cpupower frequency-set -g powersave 2>/dev/null || true
sudo systemctl start irqbalance cron atd 2>/dev/null || true

View File

@@ -50,7 +50,7 @@ jobs:
uses: actions/configure-pages@v6
- name: Upload artifact
uses: actions/upload-pages-artifact@v5
uses: actions/upload-pages-artifact@v4
with:
path: "./docs/vocs/docs/dist"

View File

@@ -1,62 +0,0 @@
name: Fetch Grafana Dashboard
on:
workflow_dispatch:
inputs:
dashboard_uid:
description: "Grafana dashboard UID to export"
required: true
default: "2k8BXz24x"
target_path:
description: "Target file path in the repo (e.g. etc/grafana/dashboards/overview.json)"
required: true
default: "etc/grafana/dashboards/overview.json"
jobs:
fetch:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v6
with:
python-version: "3.12"
- name: Fetch dashboard from Grafana
env:
FETCH_GRAFANA_DASHBOARD_URL: ${{ secrets.FETCH_GRAFANA_DASHBOARD_URL }}
FETCH_GRAFANA_DASHBOARD_TOKEN: ${{ secrets.FETCH_GRAFANA_DASHBOARD_TOKEN }}
run: |
python3 .github/scripts/fetch-grafana-dashboard.py "${{ inputs.dashboard_uid }}" \
> "${{ inputs.target_path }}"
- name: Check for changes
id: diff
run: |
if git diff --quiet "${{ inputs.target_path }}"; then
echo "changed=false" >> "$GITHUB_OUTPUT"
echo "No changes detected."
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Create pull request
if: steps.diff.outputs.changed == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
TARGET="${{ inputs.target_path }}"
FILENAME="$(basename "$TARGET")"
BRANCH="chore/sync-grafana-${FILENAME%.*}-$(date +%Y%m%d-%H%M%S)"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b "$BRANCH"
git add "$TARGET"
git commit -m "chore: update Grafana dashboard ${FILENAME}"
git push origin "$BRANCH"
gh pr create \
--title "chore: update Grafana dashboard ${FILENAME}" \
--body "Automated export from Grafana (dashboard UID: \`${{ inputs.dashboard_uid }}\`, target: \`${TARGET}\`)."

View File

@@ -11,22 +11,11 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Validate dashboard format
- name: Check for ${DS_PROMETHEUS} in overview.json
run: |
python3 -c "
import json, sys
with open('etc/grafana/dashboards/overview.json') as f:
d = json.load(f)
errors = []
if '__inputs' not in d:
errors.append('missing __inputs')
if '__requires' not in d:
errors.append('missing __requires')
if d.get('id') is not None:
errors.append('contains internal id field — use export-dashboard.py')
if errors:
for e in errors:
print(f'Error: {e}', file=sys.stderr)
sys.exit(1)
print('✓ overview.json is a valid exported dashboard')
"
if grep -Fn '${DS_PROMETHEUS}' etc/grafana/dashboards/overview.json; then
echo "Error: overview.json contains '\${DS_PROMETHEUS}' placeholder"
echo "Please replace it with '\${datasource}'"
exit 1
fi
echo "✓ overview.json does not contain '\${DS_PROMETHEUS}' placeholder"

View File

@@ -16,7 +16,7 @@ jobs:
fetch-depth: 0
- name: Label PRs
uses: actions/github-script@v9
uses: actions/github-script@v8
with:
script: |
const label_pr = require('./.github/scripts/label_pr.js')

View File

@@ -195,7 +195,7 @@ jobs:
fi
body=$(cat <<- "ENDBODY"
![image](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-2.png)
![image](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-prod.png)
## Testing Checklist (DELETE ME)

995
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "2.1.0"
version = "2.0.0"
edition = "2024"
rust-version = "1.93"
license = "MIT OR Apache-2.0"
@@ -129,7 +129,6 @@ members = [
"examples/custom-node-components/",
"examples/custom-payload-builder/",
"examples/custom-rlpx-subprotocol",
"examples/custom-auth-http-middleware",
"examples/custom-rpc-middleware",
"examples/db-access",
"examples/exex-subscription",
@@ -326,8 +325,8 @@ reth-cli = { path = "crates/cli/cli" }
reth-cli-commands = { path = "crates/cli/commands" }
reth-cli-runner = { path = "crates/cli/runner" }
reth-cli-util = { path = "crates/cli/util" }
reth-codecs = { version = "0.3.1", default-features = false }
reth-codecs-derive = "0.3.1"
reth-codecs = { version = "0.3.0", default-features = false }
reth-codecs-derive = "0.3.0"
reth-config = { path = "crates/config", default-features = false }
reth-consensus = { path = "crates/consensus/consensus", default-features = false }
reth-consensus-common = { path = "crates/consensus/common", default-features = false }
@@ -395,7 +394,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
reth-payload-primitives = { path = "crates/payload/primitives" }
reth-payload-validator = { path = "crates/payload/validator" }
reth-payload-util = { path = "crates/payload/util" }
reth-primitives-traits = { version = "0.3.1", default-features = false }
reth-primitives-traits = { version = "0.3.0", default-features = false }
reth-provider = { path = "crates/storage/provider" }
reth-prune = { path = "crates/prune/prune" }
reth-prune-types = { path = "crates/prune/types", default-features = false }
@@ -411,7 +410,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal
reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
reth-rpc-traits = { version = "0.3.1", default-features = false }
reth-rpc-traits = { version = "0.3.0", default-features = false }
reth-stages = { path = "crates/stages/stages" }
reth-stages-api = { path = "crates/stages/api" }
reth-stages-types = { path = "crates/stages/types", default-features = false }
@@ -430,17 +429,17 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
reth-trie-db = { path = "crates/trie/db" }
reth-trie-parallel = { path = "crates/trie/parallel" }
reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
reth-zstd-compressors = { version = "0.3.1", default-features = false }
reth-zstd-compressors = { version = "0.3.0", default-features = false }
# revm
revm = { version = "=37.0.0", default-features = false }
revm-bytecode = { version = "=10.0.0", default-features = false }
revm-database = { version = "=13.0.0", default-features = false }
revm-state = { version = "=11.0.0", default-features = false }
revm-primitives = { version = "=23.0.0", default-features = false }
revm-interpreter = { version = "=35.0.0", default-features = false }
revm-database-interface = { version = "=11.0.0", default-features = false }
revm-inspectors = "=0.39.0"
revm = { version = "37.0.0", default-features = false }
revm-bytecode = { version = "10.0.0", default-features = false }
revm-database = { version = "13.0.0", default-features = false }
revm-state = { version = "11.0.0", default-features = false }
revm-primitives = { version = "23.0.0", default-features = false }
revm-interpreter = { version = "35.0.0", default-features = false }
revm-database-interface = { version = "11.0.0", default-features = false }
revm-inspectors = "0.38.0"
# eth
alloy-dyn-abi = "1.5.6"
@@ -449,40 +448,40 @@ alloy-sol-types = { version = "1.5.6", default-features = false }
alloy-chains = { version = "0.2.33", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.4", default-features = false }
alloy-evm = { version = "0.33.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.32.0", default-features = false }
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-hardforks = "0.4.7"
alloy-consensus = { version = "2.0.1", default-features = false }
alloy-contract = { version = "2.0.1", default-features = false }
alloy-eips = { version = "2.0.1", default-features = false }
alloy-genesis = { version = "2.0.1", default-features = false }
alloy-json-rpc = { version = "2.0.1", default-features = false }
alloy-network = { version = "2.0.1", default-features = false }
alloy-network-primitives = { version = "2.0.1", default-features = false }
alloy-provider = { version = "2.0.1", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "2.0.1", default-features = false }
alloy-rpc-client = { version = "2.0.1", default-features = false }
alloy-rpc-types = { version = "2.0.1", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "2.0.1", default-features = false }
alloy-rpc-types-anvil = { version = "2.0.1", default-features = false }
alloy-rpc-types-beacon = { version = "2.0.1", default-features = false }
alloy-rpc-types-debug = { version = "2.0.1", default-features = false }
alloy-rpc-types-engine = { version = "2.0.1", default-features = false }
alloy-rpc-types-eth = { version = "2.0.1", default-features = false }
alloy-rpc-types-mev = { version = "2.0.1", default-features = false }
alloy-rpc-types-trace = { version = "2.0.1", default-features = false }
alloy-rpc-types-txpool = { version = "2.0.1", default-features = false }
alloy-serde = { version = "2.0.1", default-features = false }
alloy-signer = { version = "2.0.1", default-features = false }
alloy-signer-local = { version = "2.0.1", default-features = false }
alloy-transport = { version = "2.0.1" }
alloy-transport-http = { version = "2.0.1", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "2.0.1", default-features = false }
alloy-transport-ws = { version = "2.0.1", default-features = false }
alloy-consensus = { version = "2.0.0", default-features = false }
alloy-contract = { version = "2.0.0", default-features = false }
alloy-eips = { version = "2.0.0", default-features = false }
alloy-genesis = { version = "2.0.0", default-features = false }
alloy-json-rpc = { version = "2.0.0", default-features = false }
alloy-network = { version = "2.0.0", default-features = false }
alloy-network-primitives = { version = "2.0.0", default-features = false }
alloy-provider = { version = "2.0.0", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "2.0.0", default-features = false }
alloy-rpc-client = { version = "2.0.0", default-features = false }
alloy-rpc-types = { version = "2.0.0", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "2.0.0", default-features = false }
alloy-rpc-types-anvil = { version = "2.0.0", default-features = false }
alloy-rpc-types-beacon = { version = "2.0.0", default-features = false }
alloy-rpc-types-debug = { version = "2.0.0", default-features = false }
alloy-rpc-types-engine = { version = "2.0.0", default-features = false }
alloy-rpc-types-eth = { version = "2.0.0", default-features = false }
alloy-rpc-types-mev = { version = "2.0.0", default-features = false }
alloy-rpc-types-trace = { version = "2.0.0", default-features = false }
alloy-rpc-types-txpool = { version = "2.0.0", default-features = false }
alloy-serde = { version = "2.0.0", default-features = false }
alloy-signer = { version = "2.0.0", default-features = false }
alloy-signer-local = { version = "2.0.0", default-features = false }
alloy-transport = { version = "2.0.0" }
alloy-transport-http = { version = "2.0.0", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "2.0.0", default-features = false }
alloy-transport-ws = { version = "2.0.0", default-features = false }
# misc
either = { version = "1.15.0", default-features = false }
@@ -507,7 +506,6 @@ eyre = "0.6"
fdlimit = "0.3.0"
fixed-map = { version = "0.9", default-features = false }
humantime = "2.1"
imbl = "7"
humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
@@ -700,24 +698,3 @@ vergen-git2 = "9.1.0"
# networking
ipnet = "2.11"
[patch.crates-io]
revm = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-context = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-database = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-state = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "a2c7a41977b468d016a339f560acb76e002766f3" }
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "da7633f6bc9554f5a6e60773ef21b8e9d6e0cca6" }
reth-codecs = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
reth-codecs-derive = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
reth-rpc-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
reth-zstd-compressors = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }

View File

@@ -69,7 +69,6 @@ default = [
"jemalloc",
"reth-cli-util/jemalloc",
"asm-keccak",
"keccak-cache-global",
"min-debug-logs",
]
@@ -90,12 +89,6 @@ asm-keccak = [
"revm-primitives/asm-keccak",
]
keccak-cache-global = [
"reth-node-core/keccak-cache-global",
"reth-node-ethereum/keccak-cache-global",
"alloy-primitives/keccak-cache-global",
]
min-debug-logs = [
"tracing/release_max_level_debug",
"reth-ethereum-cli/min-debug-logs",

View File

@@ -238,7 +238,6 @@ where
withdrawals: prev_segment.ctx.withdrawals.clone(),
extra_data: prev_segment.ctx.extra_data.clone(),
tx_count_hint: prev_segment.ctx.tx_count_hint,
slot_number: prev_segment.ctx.slot_number,
};
// Clone the next segment's data before we consume inner.
@@ -253,7 +252,6 @@ where
withdrawals: new_segment.ctx.withdrawals.clone(),
extra_data: new_segment.ctx.extra_data.clone(),
tx_count_hint: new_segment.ctx.tx_count_hint,
slot_number: new_segment.ctx.slot_number,
};
plan.next_segment += 1;
@@ -366,7 +364,6 @@ where
withdrawals: seg0.ctx.withdrawals.clone(),
extra_data: seg0.ctx.extra_data.clone(),
tx_count_hint: seg0.ctx.tx_count_hint,
slot_number: seg0.ctx.slot_number,
};
let inner = self.inner_mut();
@@ -425,7 +422,6 @@ where
withdrawals: last_seg.ctx.withdrawals.clone(),
extra_data: last_seg.ctx.extra_data.clone(),
tx_count_hint: last_seg.ctx.tx_count_hint,
slot_number: last_seg.ctx.slot_number,
};
self.inner_mut().ctx = last_ctx;
}

View File

@@ -266,7 +266,6 @@ where
ommers: &[],
withdrawals: ctx.withdrawals.map(|w| std::borrow::Cow::Owned(w.into_owned())),
extra_data: ctx.extra_data,
slot_number: ctx.slot_number,
};
BigBlockSegment { start_tx, evm_env, ctx }
})

View File

@@ -176,7 +176,6 @@ impl BbAddOns {
BasicEngineApiBuilder::default(),
BasicEngineValidatorBuilder::default(),
Default::default(),
Default::default(),
)
}
}

View File

@@ -21,8 +21,6 @@ pub(crate) struct BenchContext {
pub(crate) auth_provider: RootProvider<AnyNetwork>,
/// The block provider is used for block queries.
pub(crate) block_provider: RootProvider<AnyNetwork>,
/// The local regular RPC provider is used for non-authenticated node RPCs like `testing_*`.
pub(crate) local_rpc_provider: RootProvider<AnyNetwork>,
/// The benchmark mode, which defines whether the benchmark should run for a closed or open
/// range of blocks.
pub(crate) benchmark_mode: BenchMode,
@@ -85,11 +83,6 @@ impl BenchContext {
let client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(client);
let local_rpc_url = Url::parse(&bench_args.local_rpc_url)?;
info!(target: "reth-bench", "Connecting to local regular RPC at {} for testing namespace calls", local_rpc_url);
let local_rpc_provider =
RootProvider::<AnyNetwork>::new(ClientBuilder::default().http(local_rpc_url));
// Computes the block range for the benchmark.
//
// - If `--advance` is provided, fetches the latest block from the engine and sets:
@@ -166,7 +159,6 @@ impl BenchContext {
Ok(Self {
auth_provider,
block_provider,
local_rpc_provider,
benchmark_mode,
next_block,
use_reth_namespace,

View File

@@ -10,7 +10,7 @@ use alloy_eips::{
eip1559::BaseFeeParams,
eip7840::BlobParams,
eip7928::{AccountChanges, BlockAccessList, SlotChanges},
Typed2718,
BlockNumberOrTag, Typed2718,
};
use alloy_primitives::{Bloom, Bytes, B256};
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
@@ -32,8 +32,6 @@ use serde::{Deserialize, Serialize};
use std::{collections::HashMap, future::Future};
use tracing::{info, warn};
use crate::bench::helpers::fetch_block_access_list;
/// A single transaction with its gas used and raw encoded bytes.
#[derive(Debug, Clone)]
pub struct RawTransaction {
@@ -671,6 +669,20 @@ impl Command {
}
}
async fn fetch_block_access_list(
provider: &RootProvider<AnyNetwork>,
block_number: u64,
) -> eyre::Result<BlockAccessList> {
provider
.client()
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
.await
.map_err(Into::into)
.and_then(|block_access_list: Option<BlockAccessList>| {
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
})
}
fn merge_block_access_list(
merged: &mut BlockAccessList,
incoming: BlockAccessList,

View File

@@ -1,7 +1,5 @@
//! Common helpers for reth-bench commands.
use alloy_eips::{eip7928::BlockAccessList, BlockNumberOrTag};
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use eyre::Result;
use std::{
io::{BufReader, Read},
@@ -71,21 +69,6 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
}
}
/// Fetches the block access list for a given block number using the provided provider.
pub(crate) async fn fetch_block_access_list(
provider: &RootProvider<AnyNetwork>,
block_number: u64,
) -> eyre::Result<BlockAccessList> {
provider
.client()
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
.await
.map_err(Into::into)
.and_then(|block_access_list: Option<BlockAccessList>| {
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
})
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -4,7 +4,7 @@
use crate::{
bench::{
context::BenchContext,
helpers::{fetch_block_access_list, parse_duration},
helpers::parse_duration,
metrics_scraper::MetricsScraper,
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
@@ -14,24 +14,14 @@ use crate::{
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
},
};
use alloy_consensus::TxEnvelope;
use alloy_eips::Encodable2718;
use alloy_primitives::B256;
use alloy_provider::{
ext::DebugApi,
network::{AnyNetwork, AnyRpcBlock},
Provider, RootProvider,
};
use alloy_rpc_types_engine::{
ExecutionData, ExecutionPayloadEnvelopeV5, ForkchoiceState, PayloadAttributes,
};
use alloy_provider::{ext::DebugApi, Provider};
use alloy_rpc_types_engine::ForkchoiceState;
use clap::Parser;
use eyre::{bail, ensure, Context, OptionExt};
use eyre::{Context, OptionExt};
use futures::{stream, StreamExt, TryStreamExt};
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use reth_rpc_api::{RethNewPayloadInput, TestingBuildBlockRequestV1};
use std::time::{Duration, Instant};
use tracing::{debug, info, warn};
@@ -42,22 +32,6 @@ pub struct Command {
#[arg(long, value_name = "RPC_URL", verbatim_doc_comment)]
rpc_url: String,
/// Build a separate fork with `testing_buildBlockV1` and alternate forkchoice updates between
/// the canonical chain and that fork on every block while the fork grows up to the configured
/// depth.
///
/// This requires enabling the hidden `testing` RPC module on the target node,
/// for example with `reth node --http --http.api eth,testing`.
#[arg(
long,
value_name = "DEPTH",
num_args = 0..=1,
default_missing_value = "8",
value_parser = parse_reorg_depth,
verbatim_doc_comment
)]
reorg: Option<usize>,
/// How long to wait after a forkchoice update before sending the next payload.
///
/// Accepts a duration string (e.g. `100ms`, `2s`) or a bare integer treated as
@@ -101,87 +75,22 @@ pub struct Command {
)]
rpc_block_buffer_size: usize,
/// Weather to enable bal by default or not.
#[arg(long, default_value = "false", verbatim_doc_comment)]
enable_bal: bool,
#[command(flatten)]
benchmark: BenchmarkArgs,
}
#[derive(Debug)]
struct PreparedBuiltBlock {
block_hash: B256,
params: serde_json::Value,
}
#[derive(Debug)]
struct QueuedForkBlock {
block_number: u64,
prepared: PreparedBuiltBlock,
}
#[derive(Debug)]
struct ReorgState {
depth: usize,
fork_length: usize,
branch_point_hash: Option<B256>,
fork_parent_hash: Option<B256>,
}
impl ReorgState {
const fn new(depth: usize) -> Self {
Self { depth, fork_length: 0, branch_point_hash: None, fork_parent_hash: None }
}
const fn push_fork_head(&mut self, canonical_parent_hash: B256, fork_head_hash: B256) {
if self.fork_length == 0 {
self.branch_point_hash = Some(canonical_parent_hash);
}
self.fork_length += 1;
self.fork_parent_hash = Some(fork_head_hash);
}
fn forkchoice_state(&self, fork_head_hash: B256) -> eyre::Result<ForkchoiceState> {
let branch_point_hash = self.branch_point_hash.ok_or_eyre("missing reorg branch point")?;
Ok(ForkchoiceState {
head_block_hash: fork_head_hash,
safe_block_hash: branch_point_hash,
finalized_block_hash: branch_point_hash,
})
}
const fn reset(&mut self) {
self.fork_length = 0;
self.branch_point_hash = None;
self.fork_parent_hash = None;
}
}
impl Command {
/// Execute `benchmark new-payload-fcu` command
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
if self.reorg.is_some() && self.benchmark.rlp_blocks {
bail!("--reorg cannot be combined with --rlp-blocks")
}
if self.reorg.is_some() && self.enable_bal {
bail!("--reorg cannot be combined with --enable-bal")
}
// Log mode configuration
if let Some(duration) = self.wait_time {
info!(target: "reth-bench", "Using wait-time mode with {}ms minimum interval between blocks", duration.as_millis());
}
if let Some(depth) = self.reorg {
info!(target: "reth-bench", depth, "Using testing_buildBlockV1 reorg mode");
}
let BenchContext {
benchmark_mode,
block_provider,
auth_provider,
local_rpc_provider,
next_block,
use_reth_namespace,
rlp_blocks,
@@ -269,8 +178,7 @@ impl Command {
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
let mut reorg_state = self.reorg.map(ReorgState::new);
let mut queued_fork_block = None;
while let Some((block, head, safe, finalized, rlp)) = {
let wait_start = Instant::now();
let result = blocks.try_next().await?;
@@ -280,39 +188,27 @@ impl Command {
let gas_used = block.header.gas_used;
let gas_limit = block.header.gas_limit;
let block_number = block.header.number;
let canonical_parent_hash = block.header.parent_hash;
let transaction_count = block.transactions.len() as u64;
let deferred_branch_start_block = reorg_state
.as_ref()
.filter(|state| state.fork_length == 0 && queued_fork_block.is_none())
.map(|_| block.clone());
let canonical_forkchoice_state = ForkchoiceState {
debug!(target: "reth-bench", ?block_number, "Sending payload");
let forkchoice_state = ForkchoiceState {
head_block_hash: head,
safe_block_hash: safe,
finalized_block_hash: finalized,
};
let bal = if rlp.is_none() &&
(block.header.block_access_list_hash.is_some() || self.enable_bal)
{
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
} else {
None
};
let (version, params) = block_to_new_payload(
block,
rlp,
use_reth_namespace,
wait_for_persistence,
no_wait_for_caches,
bal,
)?;
debug!(target: "reth-bench", ?block_number, "Sending payload");
let start = Instant::now();
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
let np_latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
let new_payload_result = NewPayloadResult {
@@ -333,12 +229,17 @@ impl Command {
};
let fcu_start = Instant::now();
call_forkchoice_updated_with_reth(&auth_provider, version, canonical_forkchoice_state)
.await?;
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
let fcu_latency = fcu_start.elapsed();
let total_latency =
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
let total_latency = if server_timings.is_some() {
// When using server-side latency for newPayload, derive total from the
// independently measured components to avoid mixing server-side and
// client-side (network-inclusive) timings.
np_latency + fcu_latency
} else {
start.elapsed()
};
let combined_result = CombinedResult {
block_number,
gas_limit,
@@ -348,88 +249,6 @@ impl Command {
total_latency,
};
if let Some(reorg_state) = reorg_state.as_mut() {
if queued_fork_block.is_none() && reorg_state.fork_length == 0 {
// A branch start uses a canonical parent, so it can be built lazily here
// instead of being queued ahead of time.
let block = deferred_branch_start_block
.as_ref()
.ok_or_eyre("missing deferred fork block for reorg branch start")?;
queued_fork_block = Some(QueuedForkBlock {
block_number,
prepared: prepare_built_block(
&local_rpc_provider,
block,
canonical_parent_hash,
no_wait_for_caches,
)
.await?,
});
}
let queued = queued_fork_block
.take()
.ok_or_eyre("missing queued fork block for reorg replay")?;
ensure!(
queued.block_number == block_number,
"queued fork block {} does not match source block {}",
queued.block_number,
block_number
);
let prepared = queued.prepared;
call_new_payload_with_reth(&auth_provider, None, prepared.params).await?;
reorg_state.push_fork_head(canonical_parent_hash, prepared.block_hash);
let forkchoice_state = reorg_state.forkchoice_state(prepared.block_hash)?;
info!(
target: "reth-bench",
block_number,
branch_point = %forkchoice_state.safe_block_hash,
fork_head = %prepared.block_hash,
fork_depth = reorg_state.fork_length,
max_reorg_depth = reorg_state.depth,
"Switching forkchoice to reorg branch"
);
let fcu_start = Instant::now();
call_forkchoice_updated_with_reth(&auth_provider, None, forkchoice_state).await?;
let _fork_fcu_latency = fcu_start.elapsed();
let next_fork_block_number = block_number + 1;
if reorg_state.fork_length < reorg_state.depth {
queued_fork_block = queue_fork_block(
&block_provider,
&local_rpc_provider,
&benchmark_mode,
next_fork_block_number,
Some(prepared.block_hash),
no_wait_for_caches,
)
.await?;
} else {
info!(
target: "reth-bench",
block_number,
reorg_depth = reorg_state.depth,
"Resetting reorg branch after reaching max depth"
);
// `testing_buildBlockV1` resolves the parent from canonical state, so switch
// back to the source chain before reseeding the next queued fork block.
call_forkchoice_updated_with_reth(
&auth_provider,
version,
canonical_forkchoice_state,
)
.await?;
reorg_state.reset();
queued_fork_block = None;
}
}
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
// We want to measure engine throughput, not RPC fetch latency.
blocks_processed += 1;
@@ -486,155 +305,3 @@ impl Command {
Ok(())
}
}
async fn prepare_built_block(
block_provider: &RootProvider<AnyNetwork>,
block: &AnyRpcBlock,
parent_block_hash: B256,
no_wait_for_caches: bool,
) -> eyre::Result<PreparedBuiltBlock> {
const MAX_BUILD_ATTEMPTS: usize = 10;
const BUILD_RETRY_INTERVAL: Duration = Duration::from_millis(100);
let request = build_block_request(block, parent_block_hash)?;
let built_payload: ExecutionPayloadEnvelopeV5 = {
let mut attempts_remaining = MAX_BUILD_ATTEMPTS;
loop {
match block_provider.client().request("testing_buildBlockV1", [request.clone()]).await {
Ok(payload) => break payload,
Err(err) if attempts_remaining > 1 && is_retryable_build_block_error(&err) => {
warn!(
target: "reth-bench",
block_number = block.header.number,
%parent_block_hash,
attempts_remaining,
error = %err,
"Retrying testing_buildBlockV1 after transient fork build failure"
);
attempts_remaining -= 1;
tokio::time::sleep(BUILD_RETRY_INTERVAL).await;
}
Err(err) => {
return Err(err).wrap_err_with(|| {
format!(
"Failed to build block {} via testing_buildBlockV1",
block.header.number
)
})
}
}
}
};
let payload = &built_payload.execution_payload.payload_inner.payload_inner;
let block_hash = payload.block_hash;
let (payload, sidecar) = built_payload
.into_payload_and_sidecar(block.header.parent_beacon_block_root.unwrap_or_default());
// Fork payloads are built immediately before the next `testing_buildBlockV1` call. Leaving
// reth's default persistence wait enabled here gives the regular RPC side a consistent base
// state for the next synthetic fork block build.
let params = serde_json::to_value((
RethNewPayloadInput::ExecutionData(ExecutionData { payload, sidecar }),
None::<bool>,
no_wait_for_caches.then_some(false),
))?;
Ok(PreparedBuiltBlock { block_hash, params })
}
#[allow(clippy::too_many_arguments)]
async fn queue_fork_block(
block_provider: &RootProvider<AnyNetwork>,
local_rpc_provider: &RootProvider<AnyNetwork>,
benchmark_mode: &crate::bench_mode::BenchMode,
block_number: u64,
parent_block_hash: Option<B256>,
no_wait_for_caches: bool,
) -> eyre::Result<Option<QueuedForkBlock>> {
if !benchmark_mode.contains(block_number) {
return Ok(None)
}
let future_block = block_provider
.get_block_by_number(alloy_eips::BlockNumberOrTag::Number(block_number))
.full()
.await
.wrap_err_with(|| format!("Failed to fetch block by number {block_number}"))?
.ok_or_eyre("Block not found")?;
let parent_block_hash = parent_block_hash.unwrap_or(future_block.header.parent_hash);
Ok(Some(QueuedForkBlock {
block_number,
prepared: prepare_built_block(
local_rpc_provider,
&future_block,
parent_block_hash,
no_wait_for_caches,
)
.await?,
}))
}
fn is_retryable_build_block_error(err: &alloy_transport::TransportError) -> bool {
let message = err.to_string();
message.contains("block not found: hash") ||
message.contains("block hash not found for block number")
}
fn build_block_request(
block: &AnyRpcBlock,
parent_block_hash: B256,
) -> eyre::Result<TestingBuildBlockRequestV1> {
let mut transactions = block
.clone()
.try_into_transactions()
.map_err(|_| eyre::eyre!("Block transactions must be fetched in full for --reorg"))?
.into_iter()
.map(|tx| {
let tx: TxEnvelope =
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type in RPC block"))?;
if tx.is_eip4844() {
return Ok(None)
}
Ok(Some(tx.encoded_2718().into()))
})
.filter_map(|tx| tx.transpose())
.collect::<eyre::Result<Vec<_>>>()?;
// `testing_buildBlockV1` only takes raw transaction bytes, so we exclude blob transactions
// from the synthetic fork blocks rather than trying to reconstruct their sidecars.
// Keep only 90% of the remaining transactions so the alternate branch produces a materially
// different post-state instead of only differing by header data.
let keep = transactions.len().saturating_mul(9) / 10;
transactions.truncate(keep);
let rpc_block = block.clone().into_inner();
Ok(TestingBuildBlockRequestV1 {
parent_block_hash,
payload_attributes: PayloadAttributes {
timestamp: block.header.timestamp,
prev_randao: block.header.mix_hash.unwrap_or_default(),
suggested_fee_recipient: block.header.beneficiary,
withdrawals: rpc_block.withdrawals.map(|withdrawals| withdrawals.into_inner()),
parent_beacon_block_root: block.header.parent_beacon_block_root,
slot_number: block.header.slot_number,
},
transactions,
extra_data: Some(block.header.extra_data.clone()),
})
}
fn parse_reorg_depth(value: &str) -> Result<usize, String> {
let depth = value
.trim()
.parse::<usize>()
.map_err(|_| format!("invalid reorg depth {value:?}, expected a positive integer"))?;
if depth == 0 {
return Err("reorg depth must be greater than 0".to_string())
}
Ok(depth)
}

View File

@@ -3,7 +3,6 @@
use crate::{
bench::{
context::BenchContext,
helpers::fetch_block_access_list,
metrics_scraper::MetricsScraper,
output::{
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
@@ -54,7 +53,6 @@ impl Command {
rlp_blocks,
wait_for_persistence,
no_wait_for_caches,
..
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
@@ -71,9 +69,7 @@ impl Command {
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
let block_provider_clone = block_provider.clone();
tokio::task::spawn(async move {
let block_provider = block_provider_clone;
while benchmark_mode.contains(next_block) {
let block_res = block_provider
.get_block_by_number(next_block.into())
@@ -127,19 +123,12 @@ impl Command {
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
} else {
None
};
let (version, params) = block_to_new_payload(
block,
rlp,
use_reth_namespace,
wait_for_persistence,
no_wait_for_caches,
bal,
)?;
let start = Instant::now();

View File

@@ -18,7 +18,7 @@ use alloy_primitives::B256;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV6,
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV4,
ExecutionPayloadSidecar, ExecutionPayloadV4, ForkchoiceState, JwtSecret, PraguePayloadFields,
};
use clap::Parser;
@@ -315,7 +315,7 @@ impl Command {
let requests =
execution_data.sidecar.requests().cloned().unwrap_or_default().to_vec();
(
Some(EngineApiMessageVersion::V6),
Some(EngineApiMessageVersion::V4),
serde_json::to_value((
execution_data.payload.clone(),
Vec::<B256>::new(),
@@ -423,7 +423,7 @@ impl Command {
/// Load and parse all payload files from the directory.
///
/// Tries to load each file as a [`BigBlockPayload`] first (which includes `env_switches`),
/// falling back to [`ExecutionPayloadEnvelopeV6`] for backwards compatibility.
/// falling back to [`ExecutionPayloadEnvelopeV4`] for backwards compatibility.
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
let mut payloads = Vec::new();
@@ -450,11 +450,12 @@ impl Command {
let name_str = name.to_string_lossy();
let index = if let Some(rest) = name_str.strip_prefix("payload_block_") {
rest.strip_suffix(".json")?.parse::<u64>().ok()?
} else {
let rest = name_str.strip_prefix("big_block_")?;
} else if let Some(rest) = name_str.strip_prefix("big_block_") {
// "big_block_FROM_to_TO.json" — use FROM as the index
let rest = rest.strip_suffix(".json")?;
rest.split("_to_").next()?.parse::<u64>().ok()?
} else {
return None;
};
Some((index, e.path()))
})
@@ -480,10 +481,10 @@ impl Command {
{
(big_block.execution_data, big_block.big_block_data, big_block.block_access_list)
} else {
let envelope: ExecutionPayloadEnvelopeV6 = serde_json::from_str(&content)
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let execution_data = ExecutionData {
payload: envelope.execution_payload.clone().into(),
payload: envelope.envelope_inner.execution_payload.clone().into(),
sidecar: ExecutionPayloadSidecar::v4(
CancunPayloadFields {
versioned_hashes: Vec::new(),

View File

@@ -1,8 +1,6 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{
ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4,
};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
@@ -23,7 +21,6 @@ pub(super) struct InvalidationConfig {
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
pub(super) slot_number: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
@@ -38,8 +35,6 @@ pub(super) struct InvalidationConfig {
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
pub(super) invalidate_block_access_list: bool,
pub(super) invalidate_slot_number: bool,
}
impl InvalidationConfig {
@@ -221,30 +216,4 @@ impl InvalidationConfig {
changes
}
/// Applies invalidations to a V4 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v4(&self, payload: &mut ExecutionPayloadV4) -> Vec<String> {
let mut changes = self.apply_to_payload_v3(&mut payload.payload_inner);
// Explicit override for slot_number
if let Some(slot_number) = self.slot_number {
payload.slot_number = slot_number;
changes.push(format!("slot_number = {slot_number}"));
}
// Handle block access list invalidation (V4+)
if self.invalidate_block_access_list {
let fake_bal = Bytes::from_static(&[0x01, 0x02, 0x03]);
payload.block_access_list = fake_bal.clone();
changes.push(format!("block_access_list = {fake_bal} (auto-invalidated)"));
}
// Handle slot number invalidation (V4+)
if self.invalidate_slot_number {
payload.slot_number = u64::MAX;
changes.push("slot_number = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -1,18 +1,12 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use alloy_rpc_client::ClientBuilder;
use invalidation::InvalidationConfig;
use crate::bench::helpers::fetch_block_access_list;
use super::helpers::{load_jwt_secret, read_input};
use alloy_consensus::TxEnvelope;
use alloy_primitives::{Address, Bytes, B256};
use alloy_provider::{
network::{AnyNetwork, AnyRpcBlock},
RootProvider,
};
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
@@ -111,9 +105,6 @@ pub struct Command {
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
/// Override the slot number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
slot_number: Option<u64>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
@@ -167,14 +158,6 @@ pub struct Command {
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
/// Invalidate the block access list by setting it to a random value (EIP-7928).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_access_list: bool,
/// Invalidate the slot number by setting it to an random value.(EIP-7843).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_slot_number: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
@@ -216,7 +199,6 @@ impl Command {
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
slot_number: self.slot_number,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
@@ -229,8 +211,6 @@ impl Command {
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
invalidate_block_access_list: self.invalidate_block_access_list,
invalidate_slot_number: self.invalidate_slot_number,
}
}
@@ -254,21 +234,15 @@ impl Command {
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let use_v5 = block.header.block_access_list_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = if use_v5 {
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
} else {
ExecutionPayload::from_block_slow(&block).0
};
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
ExecutionPayload::V4(p) => config.apply_to_payload_v4(p),
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
@@ -338,13 +312,7 @@ impl Command {
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v5 {
"engine_newPayloadV5"
} else if use_v4 {
"engine_newPayloadV4"
} else {
"engine_newPayloadV3"
};
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
@@ -385,17 +353,4 @@ impl Command {
Ok(())
}
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
let rpc_url = self
.rpc_url
.as_deref()
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
let client = ClientBuilder::default()
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let provider = RootProvider::<AnyNetwork>::new(client);
let bal = fetch_block_access_list(&provider, block_number).await?;
Ok(alloy_rlp::encode(bal).into())
}
}

View File

@@ -1,11 +1,6 @@
use super::helpers::{fetch_block_access_list, load_jwt_secret, read_input};
use super::helpers::{load_jwt_secret, read_input};
use alloy_consensus::TxEnvelope;
use alloy_primitives::Bytes;
use alloy_provider::{
network::{AnyNetwork, AnyRpcBlock},
RootProvider,
};
use alloy_rpc_client::ClientBuilder;
use alloy_provider::network::AnyRpcBlock;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
@@ -74,9 +69,6 @@ impl Command {
})?
.into_consensus();
let use_v4 = block.header.requests_hash.is_some();
let use_v5 = block.header.block_access_list_hash.is_some();
// Extract parent beacon block root
let parent_beacon_block_root = block.header.parent_beacon_block_root;
@@ -84,14 +76,10 @@ impl Command {
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
// V5 payloads must carry the full RLP-encoded block access list, not just the hash stored
// in the header.
let execution_payload = if use_v5 {
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
} else {
ExecutionPayload::from_block_slow(&block).0
};
// Convert to execution payload
let execution_payload = ExecutionPayload::from_block_slow(&block).0;
let use_v4 = block.header.requests_hash.is_some();
// Create JSON request data
let json_request = if use_v4 {
@@ -114,13 +102,7 @@ impl Command {
Mode::Execute => {
// Create cast command
let mut command = std::process::Command::new("cast");
let method = if use_v5 {
"engine_newPayloadV5"
} else if use_v4 {
"engine_newPayloadV4"
} else {
"engine_newPayloadV3"
};
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
@@ -164,17 +146,4 @@ impl Command {
Ok(())
}
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
let rpc_url = self
.rpc_url
.as_deref()
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
let client = ClientBuilder::default()
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let provider = RootProvider::<AnyNetwork>::new(client);
let bal = fetch_block_access_list(&provider, block_number).await?;
Ok(alloy_rlp::encode(bal).into())
}
}

View File

@@ -3,7 +3,6 @@
//! before sending additional calls.
use alloy_consensus::TxEnvelope;
use alloy_eips::eip7928::BlockAccessList;
use alloy_primitives::Bytes;
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
use alloy_rpc_types_engine::{
@@ -44,14 +43,6 @@ pub trait EngineApiValidWaitExt<N>: Send + Sync {
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated>;
/// Calls `engine_forkChoiceUpdatedV4` with the given [`ForkchoiceState`] and optional
/// [`PayloadAttributes`], and waits until the response is VALID.
async fn fork_choice_updated_v4_wait(
&self,
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated>;
}
#[async_trait::async_trait]
@@ -171,40 +162,6 @@ where
Ok(status)
}
async fn fork_choice_updated_v4_wait(
&self,
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
target: "reth-bench",
method = "engine_forkchoiceUpdatedV3",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
while !status.is_valid() {
if status.is_invalid() {
error!(
target: "reth-bench",
?status,
?fork_choice_state,
?payload_attributes,
"Invalid forkchoiceUpdatedV4 message",
);
panic!("Invalid forkchoiceUpdatedV4: {status:?}");
}
status =
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
}
Ok(status)
}
}
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
@@ -219,7 +176,6 @@ pub(crate) fn block_to_new_payload(
reth_new_payload: bool,
wait_for_persistence: WaitForPersistence,
no_wait_for_caches: bool,
bal: Option<BlockAccessList>,
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
let block_number = block.header.number;
let wait_for_persistence = wait_for_persistence.rpc_value(block_number);
@@ -242,11 +198,7 @@ pub(crate) fn block_to_new_payload(
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
})?
.into_consensus();
let block_access_list = alloy_rlp::encode(bal.unwrap_or_default());
let (payload, sidecar) =
ExecutionPayload::from_block_slow_with_bal(&block, block_access_list.into());
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
let (version, params, execution_data) = payload_to_new_payload(payload, sidecar, None)?;
if reth_new_payload {
@@ -275,22 +227,6 @@ pub(crate) fn payload_to_new_payload(
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
let (version, params) = match payload {
ExecutionPayload::V4(payload) => {
let cancun = sidecar
.cancun()
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
let version = target_version.unwrap_or(EngineApiMessageVersion::V6);
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
(
version,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
requests,
))?,
)
}
ExecutionPayload::V3(payload) => {
let cancun = sidecar
.cancun()
@@ -330,6 +266,22 @@ pub(crate) fn payload_to_new_payload(
ExecutionPayload::V1(payload) => {
(EngineApiMessageVersion::V1, serde_json::to_value((payload,))?)
}
ExecutionPayload::V4(payload) => {
let cancun = sidecar
.cancun()
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
(
version,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
requests,
))?,
)
}
};
Ok((version, params, execution_data))
@@ -434,10 +386,10 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 |
EngineApiMessageVersion::V6 => {
provider.fork_choice_updated_v4_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V2 => {

View File

@@ -150,22 +150,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
// commands can proceed.
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
reth_fs_util::create_dir_all(&rocksdb_path)?;
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level);
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
builder = builder.with_block_cache_size(cache_size);
}
builder.build()?
} else {
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write());
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
builder = builder.with_block_cache_size(cache_size);
}
builder.build()?
.build()?
} else {
RocksDBProvider::builder(data_dir.rocksdb())
.with_default_tables()
.with_database_log_level(self.db.log_level)
.with_read_only(!access.is_read_write())
.build()?
};
let provider_factory =

View File

@@ -1,4 +1,4 @@
use alloy_primitives::{hex, Address, BlockHash, B256};
use alloy_primitives::{hex, BlockHash};
use clap::Parser;
use reth_db::{
static_file::{
@@ -10,20 +10,16 @@ use reth_db::{
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRO},
database::Database,
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
table::{Compress, Decompress, DupSort, Table},
tables,
transaction::DbTx,
RawKey, RawTable, TableViewer,
RawKey, RawTable, Receipts, TableViewer, Transactions,
};
use reth_db_common::DbTool;
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{
providers::ProviderNodeTypes, ChangeSetReader, RocksDBProviderFactory,
StaticFileProviderFactory,
};
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
@@ -77,55 +73,6 @@ enum Subcommand {
#[arg(long)]
raw: bool,
},
/// Gets the content of a RocksDB table for the given key
///
/// For history tables (accounts-history, storages-history), you can pass a plain address
/// instead of a full JSON ShardedKey. Use --block to query a specific block number
/// (seeks to the shard containing that block), or --all-shards to list all shards for
/// the address.
///
/// Examples:
/// reth db get rocksdb accounts-history 0xdBBE3D8c2d2b22A2611c5A94A9a12C2fCD49Eb29
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --block 1000000
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --all-shards
/// reth db get rocksdb storages-history 0xdBBE...Eb29 --storage-key 0x0000...0003
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: RocksDbTable,
/// The key to get content for. For history tables, this can be a plain address.
#[arg(value_parser = maybe_json_value_parser)]
key: String,
/// Target block number for history tables. Seeks to the shard containing this block.
/// Defaults to the latest shard if not specified.
#[arg(long)]
block: Option<u64>,
/// Storage key for storages-history table lookups.
#[arg(long)]
storage_key: Option<String>,
/// List all shards for the given key (history tables only).
#[arg(long)]
all_shards: bool,
/// Output bytes instead of human-readable decoded value
#[arg(long)]
raw: bool,
},
}
/// RocksDB tables that can be queried.
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl Command {
@@ -135,9 +82,6 @@ impl Command {
Subcommand::Mdbx { table, key, subkey, end_key, end_subkey, raw } => {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::Rocksdb { table, key, block, storage_key, all_shards, raw } => {
get_rocksdb(tool, table, &key, block, storage_key.as_deref(), all_shards, raw)?;
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
@@ -264,12 +208,15 @@ impl Command {
);
}
StaticFileSegment::Transactions => {
let transaction = TxTy::<N>::decompress(content[0].as_slice())?;
let transaction = <<Transactions as Table>::Value>::decompress(
content[0].as_slice(),
)?;
println!("{}", serde_json::to_string_pretty(&transaction)?);
}
StaticFileSegment::Receipts => {
let receipt =
ReceiptTy::<N>::decompress(content[0].as_slice())?;
let receipt = <<Receipts as Table>::Value>::decompress(
content[0].as_slice(),
)?;
println!("{}", serde_json::to_string_pretty(&receipt)?);
}
StaticFileSegment::TransactionSenders => {
@@ -299,208 +246,6 @@ impl Command {
}
}
/// Gets a value from a RocksDB table by key.
fn get_rocksdb<N: ProviderNodeTypes>(
tool: &DbTool<N>,
table: RocksDbTable,
key: &str,
block: Option<u64>,
storage_key: Option<&str>,
all_shards: bool,
raw: bool,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
match table {
RocksDbTable::TransactionHashNumbers => {
if block.is_some() || all_shards || storage_key.is_some() {
return Err(eyre::eyre!(
"--block, --all-shards, and --storage-key are only supported for history tables"
));
}
get_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, key, raw)
}
RocksDbTable::AccountsHistory => {
if storage_key.is_some() {
return Err(eyre::eyre!("--storage-key is only supported for storages-history"));
}
get_rocksdb_account_history(&rocksdb, key, block, all_shards, raw)
}
RocksDbTable::StoragesHistory => {
get_rocksdb_storage_history(&rocksdb, key, storage_key, block, all_shards, raw)
}
}
}
/// Try to parse a key string as a plain address, falling back to JSON `ShardedKey` parsing.
fn parse_address(key: &str) -> eyre::Result<Address> {
// Strip surrounding quotes that `maybe_json_value_parser` may have added
let stripped = key.trim_matches('"');
stripped.parse::<Address>().map_err(|e| eyre::eyre!("failed to parse address: {e}"))
}
/// Gets account history from RocksDB with ergonomic key parsing.
///
/// Accepts a plain address and uses seek to find the relevant shard.
fn get_rocksdb_account_history(
rocksdb: &reth_provider::providers::RocksDBProvider,
key: &str,
block: Option<u64>,
all_shards: bool,
raw: bool,
) -> eyre::Result<()> {
// Try parsing as a plain address first, fall back to full JSON ShardedKey
match parse_address(key) {
Ok(address) => {
let block_number = block.unwrap_or(u64::MAX);
let seek_key = ShardedKey::new(address, block_number);
if all_shards {
// Iterate all shards: seek from (address, 0) until address changes
let start = ShardedKey::new(address, 0);
let iter = rocksdb.iter_from::<tables::AccountsHistory>(start)?;
for result in iter {
let (k, v) = result?;
if k.key != address {
break;
}
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"highest_block_number": k.highest_block_number,
"value": v,
}))?
);
}
} else {
// Seek to the first shard with highest_block_number >= target
let mut iter = rocksdb.iter_from::<tables::AccountsHistory>(seek_key)?;
match iter.next() {
Some(Ok((k, v))) if k.key == address => {
if raw {
let raw_val = rocksdb.get_raw::<tables::AccountsHistory>(k)?;
if let Some(bytes) = raw_val {
println!("{}", hex::encode_prefixed(&bytes));
}
} else {
println!("{}", serde_json::to_string_pretty(&v)?);
}
}
_ => {
error!(target: "reth::cli", "No content for the given table key.");
}
}
}
Ok(())
}
Err(_) => {
// Fall back to full JSON key parsing (e.g.
// `{"key":"0x...","highest_block_number":...}`)
if all_shards || block.is_some() {
return Err(eyre::eyre!(
"--block and --all-shards require a plain address, not a JSON key"
));
}
get_rocksdb_table::<tables::AccountsHistory>(rocksdb, key, raw)
}
}
}
/// Gets storage history from RocksDB with ergonomic key parsing.
///
/// Accepts a plain address + optional `--storage-key` and uses seek.
fn get_rocksdb_storage_history(
rocksdb: &reth_provider::providers::RocksDBProvider,
key: &str,
storage_key: Option<&str>,
block: Option<u64>,
all_shards: bool,
raw: bool,
) -> eyre::Result<()> {
match parse_address(key) {
Ok(address) => {
let storage_key = storage_key
.map(|s| s.trim_matches('"').parse::<B256>())
.transpose()
.map_err(|e| eyre::eyre!("failed to parse storage key: {e}"))?
.unwrap_or_default();
let block_number = block.unwrap_or(u64::MAX);
let seek_key = StorageShardedKey::new(address, storage_key, block_number);
if all_shards {
let start = StorageShardedKey::new(address, storage_key, 0);
let iter = rocksdb.iter_from::<tables::StoragesHistory>(start)?;
for result in iter {
let (k, v) = result?;
if k.address != address || k.sharded_key.key != storage_key {
break;
}
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"highest_block_number": k.sharded_key.highest_block_number,
"value": v,
}))?
);
}
} else {
let mut iter = rocksdb.iter_from::<tables::StoragesHistory>(seek_key)?;
match iter.next() {
Some(Ok((k, v)))
if k.address == address && k.sharded_key.key == storage_key =>
{
if raw {
let raw_val = rocksdb.get_raw::<tables::StoragesHistory>(k)?;
if let Some(bytes) = raw_val {
println!("{}", hex::encode_prefixed(&bytes));
}
} else {
println!("{}", serde_json::to_string_pretty(&v)?);
}
}
_ => {
error!(target: "reth::cli", "No content for the given table key.");
}
}
}
Ok(())
}
Err(_) => {
if all_shards || block.is_some() || storage_key.is_some() {
return Err(eyre::eyre!(
"--block, --all-shards, and --storage-key require a plain address, not a JSON key"
));
}
get_rocksdb_table::<tables::StoragesHistory>(rocksdb, key, raw)
}
}
}
/// Gets a value from a specific RocksDB table by exact key and prints it.
fn get_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
key_str: &str,
raw: bool,
) -> eyre::Result<()> {
let key = table_key::<T>(key_str)?;
if raw {
let content = rocksdb.get_raw::<T>(key)?;
match content {
Some(bytes) => println!("{}", hex::encode_prefixed(&bytes)),
None => error!(target: "reth::cli", "No content for the given table key."),
}
} else {
let content = rocksdb.get::<T>(key)?;
match content {
Some(value) => println!("{}", serde_json::to_string_pretty(&value)?),
None => error!(target: "reth::cli", "No content for the given table key."),
}
}
Ok(())
}
/// Get an instance of key for given table
pub(crate) fn table_key<T: Table>(key: &str) -> Result<T::Key, eyre::Error> {
serde_json::from_str(key).map_err(|e| eyre::eyre!(e))

View File

@@ -1,361 +0,0 @@
//! `reth db migrate-v2` command for migrating v1 storage layout to v2.
//!
//! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to
//! static files, clears recomputable tables (senders, indices, trie, plain
//! state), compacts MDBX, then runs the pipeline to rebuild them.
use crate::common::CliNodeTypes;
use clap::Parser;
use reth_db::{
mdbx::{self, ffi},
models::StorageBeforeTx,
DatabaseEnv,
};
use reth_db_api::{
cursor::DbCursorRO,
database::Database,
table::Table,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::{
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider,
MetadataWriter, ProviderFactory, PruneCheckpointReader, StageCheckpointWriter,
StaticFileProviderFactory, StaticFileWriter, StorageSettings,
};
use reth_prune_types::PruneSegment;
use reth_stages_types::{StageCheckpoint, StageId};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StageCheckpointReader;
use tracing::info;
/// `reth db migrate-v2` command
#[derive(Debug, Parser)]
pub struct Command;
impl Command {
/// Execute the full v1 → v2 migration:
///
/// 1. Migrate changesets + receipts to static files
/// 2. Flip `StorageSettings` to v2
/// 3. Clear recomputable MDBX tables + reset stage checkpoints
/// 4. Compact MDBX
pub async fn execute<N: CliNodeTypes>(
self,
provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()>
where
N::Primitives: reth_primitives_traits::NodePrimitives<
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
>,
{
// === Phase 0: Preflight ===
info!(target: "reth::cli", "Starting v1 → v2 storage migration");
let provider = provider_factory.provider()?;
let current_settings = provider.storage_settings()?;
if current_settings.is_some_and(|s| s.is_v2()) {
info!(target: "reth::cli", "Storage is already v2, nothing to do");
return Ok(());
}
let tip =
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number).unwrap_or(0);
info!(target: "reth::cli", tip, "Chain tip block number");
let sf_provider = provider_factory.static_file_provider();
for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets]
{
if sf_provider.get_highest_static_file_block(segment).is_some() {
eyre::bail!(
"Static file segment {segment:?} already contains data. \
Cannot migrate — target must be empty."
);
}
}
drop(provider);
// === Phase 1: Migrate changesets → static files ===
Self::migrate_account_changesets(&provider_factory, tip)?;
Self::migrate_storage_changesets(&provider_factory, tip)?;
// === Phase 2: Migrate receipts → static files ===
Self::migrate_receipts::<NodeTypesWithDBAdapter<N, DatabaseEnv>>(&provider_factory, tip)?;
// === Phase 3: Flip metadata to v2 ===
info!(target: "reth::cli", "Writing StorageSettings v2 metadata");
{
let provider_rw = provider_factory.database_provider_rw()?;
provider_rw.write_storage_settings(StorageSettings::v2())?;
provider_rw.commit()?;
}
info!(target: "reth::cli", "Storage settings updated to v2");
// === Phase 4: Clear recomputable tables ===
Self::clear_recomputable_tables(&provider_factory)?;
// === Phase 5: Compact MDBX (before pipeline, so it runs on a smaller DB) ===
let db_path = provider_factory.db_ref().path();
Self::compact_mdbx(provider_factory.db_ref())?;
// Drop to release DB handle for swap
drop(provider_factory);
let compact_path = db_path.with_file_name("db_compact");
Self::swap_compacted_db(&db_path, &compact_path)?;
// === Phase 6: Reopen DB and run pipeline ===
// The caller will reopen the environment and run the pipeline.
// We return here — the pipeline step is handled in mod.rs after
// reopening the database with the compacted copy.
info!(target: "reth::cli", "Migration complete. You should now restart the node and let it run the pipeline to rebuild the remaining data.");
Ok(())
}
fn migrate_account_changesets<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
tip: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Migrating AccountChangeSets → static files");
let provider = factory.provider()?.disable_long_read_transaction_safety();
let sf_provider = factory.static_file_provider();
let mut cursor = provider.tx_ref().cursor_read::<tables::AccountChangeSets>()?;
let first_block = provider
.get_prune_checkpoint(PruneSegment::AccountHistory)?
.and_then(|cp| cp.block_number)
.map_or(0, |b| b + 1);
let mut writer =
sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?;
let mut count = 0u64;
let mut walker = cursor.walk(Some(first_block))?.peekable();
for block in first_block..=tip {
let mut entries = Vec::new();
while let Some(Ok((block_number, _))) = walker.peek() {
if *block_number != block {
break;
}
let (_, entry) = walker.next().expect("peeked")?;
entries.push(entry);
}
count += entries.len() as u64;
writer.append_account_changeset(entries, block)?;
}
writer.commit()?;
info!(target: "reth::cli", count, "AccountChangeSets migrated");
Ok(())
}
fn migrate_storage_changesets<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
tip: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Migrating StorageChangeSets → static files");
let provider = factory.provider()?.disable_long_read_transaction_safety();
let sf_provider = factory.static_file_provider();
let mut cursor = provider.tx_ref().cursor_read::<tables::StorageChangeSets>()?;
let first_block = provider
.get_prune_checkpoint(PruneSegment::StorageHistory)?
.and_then(|cp| cp.block_number)
.map_or(0, |b| b + 1);
let mut writer =
sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?;
let mut count = 0u64;
let mut walker = cursor.walk(Some(Default::default()))?.peekable();
for block in first_block..=tip {
let mut entries = Vec::new();
while let Some(Ok((key, _))) = walker.peek() {
if key.block_number() != block {
break;
}
let (key, entry) = walker.next().expect("peeked")?;
entries.push(StorageBeforeTx {
address: key.address(),
key: entry.key,
value: entry.value,
});
}
count += entries.len() as u64;
writer.append_storage_changeset(entries, block)?;
}
writer.commit()?;
info!(target: "reth::cli", count, "StorageChangeSets migrated");
Ok(())
}
fn migrate_receipts<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
tip: u64,
) -> eyre::Result<()>
where
N::Primitives: reth_primitives_traits::NodePrimitives<
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
>,
{
let provider = factory.provider()?;
if !provider.prune_modes_ref().receipts_log_filter.is_empty() {
info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX");
return Ok(());
}
drop(provider);
let sf_provider = factory.static_file_provider();
let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts);
if existing.is_some_and(|b| b >= tip) {
info!(target: "reth::cli", "Receipts already in static files, skipping");
return Ok(());
}
info!(target: "reth::cli", "Migrating Receipts → static files");
let provider = factory.provider()?.disable_long_read_transaction_safety();
let prune_start = provider
.get_prune_checkpoint(PruneSegment::Receipts)?
.and_then(|cp| cp.block_number)
.map_or(0, |b| b + 1);
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
let block_range = first_block..=tip;
let segment = reth_static_file::segments::Receipts;
reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?;
sf_provider.commit()?;
info!(target: "reth::cli", "Receipts migrated");
Ok(())
}
/// Clears tables that can be recomputed by the pipeline and resets their
/// stage checkpoints.
fn clear_recomputable_tables<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Clearing recomputable MDBX tables");
let db = factory.db_ref();
macro_rules! clear_table {
($table:ty) => {{
let tx = db.tx_mut()?;
tx.clear::<$table>()?;
tx.commit()?;
info!(target: "reth::cli", table = <$table as Table>::NAME, "Cleared");
}};
}
// Migrated changeset tables (now in static files)
clear_table!(tables::AccountChangeSets);
clear_table!(tables::StorageChangeSets);
// Senders — rebuilt by SenderRecovery
clear_table!(tables::TransactionSenders);
// Indices — rebuilt by TransactionLookup / IndexAccountHistory / IndexStorageHistory
clear_table!(tables::TransactionHashNumbers);
clear_table!(tables::AccountsHistory);
clear_table!(tables::StoragesHistory);
// Plain state — superseded by hashed state in v2
clear_table!(tables::PlainAccountState);
clear_table!(tables::PlainStorageState);
// Trie — rebuilt by MerkleExecute
clear_table!(tables::AccountsTrie);
clear_table!(tables::StoragesTrie);
// Reset stage checkpoints so the pipeline rebuilds everything
info!(target: "reth::cli", "Resetting stage checkpoints");
let provider_rw = factory.database_provider_rw()?;
for stage in [
StageId::SenderRecovery,
StageId::TransactionLookup,
StageId::IndexAccountHistory,
StageId::IndexStorageHistory,
StageId::MerkleExecute,
StageId::MerkleUnwind,
] {
provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(0))?;
info!(target: "reth::cli", %stage, "Checkpoint reset to 0");
}
provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?;
provider_rw.commit()?;
info!(target: "reth::cli", "Recomputable tables cleared");
Ok(())
}
/// Creates a compacted copy of the MDBX database.
fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
let db_path = db.path();
let compact_path = db_path.with_file_name("db_compact");
reth_fs_util::create_dir_all(&compact_path)?;
info!(target: "reth::cli", ?db_path, ?compact_path, "Compacting MDBX database");
let compact_dest = compact_path.join("mdbx.dat");
let dest_cstr = std::ffi::CString::new(
compact_dest.to_str().ok_or_else(|| eyre::eyre!("compact path must be valid UTF-8"))?,
)?;
let flags = ffi::MDBX_CP_COMPACT | ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
});
if rc != 0 {
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
});
}
info!(target: "reth::cli", "MDBX compaction complete");
Ok(())
}
/// Swaps the original MDBX database with a compacted copy.
fn swap_compacted_db(
db_path: &std::path::Path,
compact_path: &std::path::Path,
) -> eyre::Result<()> {
let backup_path = db_path.with_file_name("db_pre_compact");
info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database");
std::fs::rename(db_path, &backup_path)?;
if let Err(e) = std::fs::rename(compact_path, db_path) {
let _ = std::fs::rename(&backup_path, db_path);
return Err(e.into());
}
std::fs::remove_dir_all(&backup_path)?;
info!(target: "reth::cli", "Database compaction swap complete");
Ok(())
}
}

View File

@@ -16,7 +16,6 @@ mod copy;
mod diff;
mod get;
mod list;
mod migrate_v2;
mod prune_checkpoints;
mod repair_trie;
mod settings;
@@ -78,9 +77,6 @@ pub enum Subcommands {
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
State(state::Command),
/// Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
#[command(name = "migrate-v2")]
MigrateV2(migrate_v2::Command),
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
@@ -235,13 +231,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::MigrateV2(command) => {
let Environment { provider_factory, .. } =
self.env.init::<N>(AccessRights::RW, ctx.task_executor.clone())?;
// Migrate changesets+receipts, clear tables, compact MDBX
command.execute::<N>(provider_factory).await?;
}
}
Ok(())

View File

@@ -1,368 +0,0 @@
use super::{
extract::{extract_archive_raw, streaming_download_and_extract, CompressionFormat},
fetch::ArchiveFetcher,
manifest::SnapshotArchive,
planning::{PlannedArchive, PlannedDownloads},
progress::{
spawn_progress_display, ArchiveDownloadProgress, ArchiveExtractionProgress,
ArchiveVerificationProgress, DownloadRequestLimiter, SharedProgress,
},
session::{ArchiveProcessContext, DownloadSession},
verify::OutputVerifier,
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
};
use eyre::Result;
use futures::stream::{self, StreamExt};
use reth_cli_util::cancellation::CancellationToken;
use reth_fs_util as fs;
use std::{
path::Path,
sync::{atomic::Ordering, Arc},
time::Duration,
};
use tokio::task;
use tracing::{debug, info, warn};
const DOWNLOAD_CACHE_DIR: &str = ".download-cache";
/// Runs all planned modular archive downloads for one command invocation.
pub(crate) async fn run_modular_downloads(
planned_downloads: PlannedDownloads,
target_dir: &Path,
download_concurrency: usize,
cancel_token: CancellationToken,
) -> Result<()> {
let download_cache_dir = target_dir.join(DOWNLOAD_CACHE_DIR);
fs::create_dir_all(&download_cache_dir)?;
let shared = SharedProgress::new(
planned_downloads.total_download_size,
planned_downloads.total_output_size,
planned_downloads.total_archives() as u64,
cancel_token.clone(),
);
let session = DownloadSession::new(
Some(Arc::clone(&shared)),
Some(DownloadRequestLimiter::new(download_concurrency)),
cancel_token,
);
let ctx =
ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session);
ModularDownloadJob::new(ctx, download_concurrency).run(planned_downloads).await
}
/// Schedules modular archive work for one run of `reth download`.
struct ModularDownloadJob {
/// Shared paths and session state for each archive in this job.
ctx: ArchiveProcessContext,
/// Maximum number of archives processed at once.
archive_concurrency: usize,
}
impl ModularDownloadJob {
/// Creates the modular download job for one command run.
const fn new(ctx: ArchiveProcessContext, archive_concurrency: usize) -> Self {
Self { ctx, archive_concurrency }
}
/// Runs all planned archives and waits for the shared progress task to finish.
async fn run(self, planned_downloads: PlannedDownloads) -> Result<()> {
let shared = Arc::clone(
self.ctx.session().progress().expect("modular downloads always use shared progress"),
);
let progress_handle = spawn_progress_display(Arc::clone(&shared));
let ctx = self.ctx.clone();
let results: Vec<Result<()>> = stream::iter(planned_downloads.archives)
.map(move |archive| {
let ctx = ctx.clone();
async move { Self::process_archive(ctx, archive).await }
})
.buffer_unordered(self.archive_concurrency)
.collect()
.await;
shared.done.store(true, Ordering::Relaxed);
let _ = progress_handle.await;
for result in results {
result?;
}
Ok(())
}
/// Runs one archive on the blocking pool so fetch and extraction stay off the async executor.
async fn process_archive(ctx: ArchiveProcessContext, archive: PlannedArchive) -> Result<()> {
task::spawn_blocking(move || ArchiveProcessor::new(archive, ctx).run()).await??;
Ok(())
}
}
/// Explicit retry states for one modular archive.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ArchiveAttemptState {
/// Start or restart one full archive attempt.
RunAttempt,
/// Check whether the extracted outputs verify.
VerifyOutputs,
/// Wait and decide whether another full attempt should run.
RetryAttempt,
/// Finish successfully.
Complete,
/// Stop with an error after retries are exhausted.
Fail,
}
/// Processes one modular archive from reuse check through extraction and verification.
struct ArchiveProcessor {
/// The concrete archive and component being processed.
archive: PlannedArchive,
/// Shared paths and session state for this archive attempt.
ctx: ArchiveProcessContext,
}
impl ArchiveProcessor {
/// Creates a processor for one archive and the shared download context.
fn new(archive: PlannedArchive, ctx: ArchiveProcessContext) -> Self {
Self { archive, ctx }
}
/// Runs the archive retry state machine until outputs are verified or retries are exhausted.
fn run(self) -> Result<()> {
let archive = self.archive();
if self.try_reuse_outputs()? {
info!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, "Skipping already verified plain files");
return Ok(());
}
let mode = ArchiveMode::new(&self.ctx)?;
let format = CompressionFormat::from_url(&archive.file_name)?;
let mut attempt = 1;
let mut last_error: Option<eyre::Error> = None;
let mut state = ArchiveAttemptState::RunAttempt;
loop {
match state {
ArchiveAttemptState::RunAttempt => {
self.cleanup_outputs();
if attempt > 1 {
info!(target: "reth::cli",
file = %archive.file_name,
component = %self.archive.component,
attempt,
max = MAX_DOWNLOAD_RETRIES,
"Retrying archive from scratch"
);
}
match self.run_attempt(mode, format) {
Ok(()) => state = ArchiveAttemptState::VerifyOutputs,
Err(error) if mode.retries_fetch_errors() => {
warn!(target: "reth::cli",
file = %archive.file_name,
component = %self.archive.component,
attempt,
err = %format_args!("{error:#}"),
"Archive attempt failed, retrying from scratch"
);
last_error = Some(error);
state = ArchiveAttemptState::RetryAttempt;
}
Err(error) => return Err(error),
}
}
ArchiveAttemptState::VerifyOutputs => {
if self.verify_outputs_with_progress()? {
state = ArchiveAttemptState::Complete;
} else {
warn!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, attempt, "Archive extracted, but output verification failed, retrying");
state = ArchiveAttemptState::RetryAttempt;
}
}
ArchiveAttemptState::RetryAttempt => {
if attempt >= MAX_DOWNLOAD_RETRIES {
state = ArchiveAttemptState::Fail;
} else {
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
attempt += 1;
state = ArchiveAttemptState::RunAttempt;
}
}
ArchiveAttemptState::Complete => return Ok(()),
ArchiveAttemptState::Fail => {
if let Some(error) = last_error {
return Err(error.wrap_err(format!(
"Failed after {} attempts for {}",
MAX_DOWNLOAD_RETRIES, archive.file_name
)));
}
eyre::bail!(
"Failed integrity validation after {} attempts for {}",
MAX_DOWNLOAD_RETRIES,
archive.file_name
)
}
}
}
}
/// Returns the concrete archive being fetched or verified.
fn archive(&self) -> &SnapshotArchive {
&self.archive.archive
}
/// Returns the verifier for this archive's output files.
fn output_verifier(&self) -> OutputVerifier<'_> {
OutputVerifier::new(self.ctx.target_dir())
}
/// Returns `true` if this archive can be reused from existing verified outputs.
/// Returns `false` if a fresh archive attempt is still needed.
fn try_reuse_outputs(&self) -> Result<bool> {
if self.verify_outputs()? {
self.mark_complete();
return Ok(true);
}
Ok(false)
}
/// Removes any partial outputs before a fresh archive attempt.
fn cleanup_outputs(&self) {
self.output_verifier().cleanup(&self.archive().output_files);
}
/// Returns `true` if all declared plain outputs verify.
/// Returns `false` if any output is missing or does not match.
fn verify_outputs(&self) -> Result<bool> {
self.output_verifier().verify(&self.archive().output_files)
}
/// Records archive completion in shared progress once outputs verify.
fn mark_complete(&self) {
self.ctx.session().record_reused_archive(self.archive().size, self.archive().output_size());
}
/// Executes one archive attempt according to the selected cache-vs-stream mode.
fn run_attempt(&self, mode: ArchiveMode, format: CompressionFormat) -> Result<()> {
mode.execute(self, format)
}
/// Downloads the archive into the cache, then extracts from the cached file.
fn run_cached_attempt(&self, format: CompressionFormat) -> Result<()> {
let cache_dir =
self.ctx.cache_dir().ok_or_else(|| eyre::eyre!("Missing download cache directory"))?;
let fetcher =
ArchiveFetcher::new(self.archive().url.clone(), cache_dir, self.ctx.session().clone());
if self.archive.ty == super::manifest::SnapshotComponentType::State {
debug!(target: "reth::cli", url = %self.archive().url, "Downloading state snapshot archive");
}
let download_result = {
let mut download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
let result = fetcher.download(Some(&mut download_progress));
if let Ok(ref downloaded) = result &&
download_progress.has_tracked_bytes()
{
download_progress.complete(downloaded.size);
}
result
};
let downloaded = match download_result {
Ok(downloaded) => downloaded,
Err(error) => {
fetcher.cleanup_downloaded_files();
return Err(error);
}
};
info!(target: "reth::cli",
file = %self.archive().file_name,
component = %self.archive.component,
size = %super::progress::DownloadProgress::format_size(downloaded.size),
"Archive download complete"
);
let extract_result = self.extract_cached_archive(&downloaded.path, format);
fetcher.cleanup_downloaded_files();
extract_result
}
/// Streams the archive directly into extraction without keeping a cached copy.
fn run_streaming_attempt(&self, format: CompressionFormat) -> Result<()> {
let _download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
streaming_download_and_extract(
&self.archive().url,
format,
self.ctx.target_dir(),
self.ctx.session(),
)
}
/// Extracts a cached archive file while updating shared extraction activity.
fn extract_cached_archive(&self, archive_path: &Path, format: CompressionFormat) -> Result<()> {
let mut extraction_progress = ArchiveExtractionProgress::new(self.ctx.session().progress());
let file = fs::open(archive_path)?;
let result = extract_archive_raw(
file,
format,
self.ctx.target_dir(),
Some(&mut extraction_progress),
);
extraction_progress.finish();
result
}
/// Returns `true` if all declared plain outputs verify while updating shared verification
/// progress.
fn verify_outputs_with_progress(&self) -> Result<bool> {
let mut verification_progress =
ArchiveVerificationProgress::new(self.ctx.session().progress());
let verified = self
.output_verifier()
.verify_with_progress(&self.archive().output_files, Some(&mut verification_progress))?;
if verified {
verification_progress.complete(self.archive().output_size());
}
Ok(verified)
}
}
/// Chooses whether an archive attempt uses the cache or streams directly.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ArchiveMode {
/// Download the archive to the cache, then extract it.
Cached,
/// Stream the archive directly into extraction.
Streaming,
}
impl ArchiveMode {
/// Picks the archive mode from the process context.
fn new(ctx: &ArchiveProcessContext) -> Result<Self> {
if ctx.cache_dir().is_some() {
ctx.session().require_request_limiter()?;
return Ok(Self::Cached)
}
Ok(Self::Streaming)
}
/// Returns `true` when fetch failures should retry the whole archive attempt.
const fn retries_fetch_errors(&self) -> bool {
matches!(self, Self::Cached)
}
/// Runs the selected archive mode for a single attempt.
fn execute(&self, processor: &ArchiveProcessor, format: CompressionFormat) -> Result<()> {
match self {
Self::Cached => processor.run_cached_attempt(format),
Self::Streaming => processor.run_streaming_attempt(format),
}
}
}

View File

@@ -248,7 +248,6 @@ fn selection_to_prune_mode(
ComponentSelection::Distance(d) => {
Some(PruneMode::Distance(min_distance.map_or(d, |min| d.max(min))))
}
ComponentSelection::Since(block) => Some(PruneMode::Before(block)),
ComponentSelection::None => Some(min_distance.map_or(PruneMode::Full, PruneMode::Distance)),
}
}
@@ -270,7 +269,6 @@ pub(crate) fn describe_prune_config(config: &Config) -> Vec<String> {
.collect()
}
/// Formats one prune mode for the generated config summary.
fn format_mode(mode: &PruneMode) -> String {
match mode {
PruneMode::Full => "\"full\"".to_string(),
@@ -455,36 +453,6 @@ mod tests {
assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Distance(10_064)));
}
#[test]
fn selections_since_maps_to_before_prune_mode() {
let mut selections = BTreeMap::new();
selections.insert(SnapshotComponentType::State, ComponentSelection::All);
selections.insert(SnapshotComponentType::Headers, ComponentSelection::All);
selections
.insert(SnapshotComponentType::Transactions, ComponentSelection::Since(15_537_394));
selections.insert(SnapshotComponentType::Receipts, ComponentSelection::Since(15_537_394));
selections.insert(
SnapshotComponentType::AccountChangesets,
ComponentSelection::Since(15_537_394),
);
selections.insert(
SnapshotComponentType::StorageChangesets,
ComponentSelection::Since(15_537_394),
);
let config = config_for_selections(
&selections,
&empty_manifest(),
None,
None::<&reth_chainspec::ChainSpec>,
);
assert_eq!(config.prune.segments.bodies_history, Some(PruneMode::Before(15_537_394)));
assert_eq!(config.prune.segments.receipts, Some(PruneMode::Before(15_537_394)));
assert_eq!(config.prune.segments.account_history, Some(PruneMode::Before(15_537_394)));
assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Before(15_537_394)));
}
#[test]
fn full_preset_matches_default_full_prune_config() {
let mut selections = BTreeMap::new();

View File

@@ -1,490 +0,0 @@
use super::{
fetch::{ArchiveFetcher, DownloadedArchive},
progress::{
ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress,
DownloadRequestLimiter, ProgressReader, SharedProgress, SharedProgressReader,
},
session::DownloadSession,
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
};
use eyre::{Result, WrapErr};
use lz4::Decoder;
use reqwest::blocking::Client as BlockingClient;
use reth_cli_util::cancellation::CancellationToken;
use reth_fs_util as fs;
use std::{
io::Read,
path::{Component, Path, PathBuf},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
thread,
time::{Duration, Instant},
};
use tar::Archive;
use tokio::task;
use tracing::{info, warn};
use url::Url;
use zstd::stream::read::Decoder as ZstdDecoder;
const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
const STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE: u64 = 64 * 1024 * 1024;
const EXTRACTION_PROGRESS_POLL_INTERVAL: Duration = Duration::from_millis(100);
/// Supported compression formats for snapshots
#[derive(Debug, Clone, Copy)]
pub(crate) enum CompressionFormat {
/// LZ4-compressed tar archive.
Lz4,
/// Zstandard-compressed tar archive.
Zstd,
}
impl CompressionFormat {
/// Detect compression format from file extension
pub(crate) fn from_url(url: &str) -> Result<Self> {
let path =
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
if path.ends_with(EXTENSION_TAR_LZ4) {
Ok(Self::Lz4)
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
Ok(Self::Zstd)
} else {
Err(eyre::eyre!(
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
path
))
}
}
}
/// Extracts a compressed tar archive to the target directory with progress tracking.
fn extract_archive<R: Read>(
reader: R,
total_size: u64,
format: CompressionFormat,
target_dir: &Path,
cancel_token: CancellationToken,
) -> Result<()> {
let progress_reader = ProgressReader::new(reader, total_size, cancel_token);
match format {
CompressionFormat::Lz4 => {
let decoder = Decoder::new(progress_reader)?;
Archive::new(decoder).unpack(target_dir)?;
}
CompressionFormat::Zstd => {
let decoder = ZstdDecoder::new(progress_reader)?;
Archive::new(decoder).unpack(target_dir)?;
}
}
println!();
Ok(())
}
/// Extracts a compressed tar archive without progress tracking.
pub(crate) fn extract_archive_raw<R: Read>(
reader: R,
format: CompressionFormat,
target_dir: &Path,
progress: Option<&mut ArchiveExtractionProgress>,
) -> Result<()> {
match format {
CompressionFormat::Lz4 => {
unpack_archive(Archive::new(Decoder::new(reader)?), target_dir, progress)?;
}
CompressionFormat::Zstd => {
unpack_archive(Archive::new(ZstdDecoder::new(reader)?), target_dir, progress)?;
}
}
Ok(())
}
fn unpack_archive<R: Read>(
mut archive: Archive<R>,
target_dir: &Path,
mut progress: Option<&mut ArchiveExtractionProgress>,
) -> Result<()> {
let entries = archive.entries().wrap_err_with(|| {
format!("failed to read archive entries for `{}`", target_dir.display())
})?;
for entry in entries {
let mut entry = entry.wrap_err_with(|| {
format!("failed to read archive entry for `{}`", target_dir.display())
})?;
extract_entry_with_progress(&mut entry, target_dir, progress.as_deref_mut())?;
}
Ok(())
}
fn extract_entry_with_progress<R: Read>(
entry: &mut tar::Entry<'_, R>,
target_dir: &Path,
progress: Option<&mut ArchiveExtractionProgress>,
) -> Result<()> {
let size = entry.header().entry_size().unwrap_or(0);
let entry_type = entry.header().entry_type();
if !entry_type.is_file() || size == 0 {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
return Ok(())
}
if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
if let Some(progress) = progress {
progress.record_extracted(size);
}
return Ok(())
}
let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
return Ok(())
};
let Some(entry_path) = entry_destination_path(entry, target_dir)? else {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
return Ok(())
};
let stop = Arc::new(AtomicBool::new(false));
let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop));
let unpack_result = entry
.unpack_in(target_dir)
.wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display()));
stop.store(true, Ordering::Relaxed);
let monitor_result = monitor.join();
unpack_result?;
monitor_result.map_err(|_| eyre::eyre!("extraction progress monitor panicked"))?;
Ok(())
}
fn entry_destination_path<R: Read>(
entry: &tar::Entry<'_, R>,
target_dir: &Path,
) -> Result<Option<PathBuf>> {
let mut file_dst = target_dir.to_path_buf();
let path = entry.path().wrap_err("invalid path in archive entry")?;
for part in path.components() {
match part {
Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
Component::ParentDir => return Ok(None),
Component::Normal(part) => file_dst.push(part),
}
}
if file_dst == target_dir {
return Ok(None)
}
Ok(Some(file_dst))
}
fn spawn_extraction_progress_monitor(
entry_path: PathBuf,
progress: ArchiveExtractionProgressHandle,
stop: Arc<AtomicBool>,
) -> thread::JoinHandle<()> {
thread::spawn(move || {
let mut extracted = 0_u64;
loop {
record_extracted_file_bytes(&entry_path, &progress, &mut extracted);
if stop.load(Ordering::Relaxed) {
break;
}
thread::sleep(EXTRACTION_PROGRESS_POLL_INTERVAL);
}
})
}
fn record_extracted_file_bytes(
entry_path: &Path,
progress: &ArchiveExtractionProgressHandle,
extracted: &mut u64,
) {
let Ok(meta) = fs::metadata(entry_path) else { return };
let len = meta.len();
if len > *extracted {
progress.record_extracted(len - *extracted);
*extracted = len;
}
}
/// Extracts a snapshot from a local file.
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let file = std::fs::File::open(path)?;
let total_size = file.metadata()?.len();
info!(target: "reth::cli",
file = %path.display(),
size = %DownloadProgress::format_size(total_size),
"Extracting local archive"
);
let start = Instant::now();
extract_archive(file, total_size, format, target_dir, CancellationToken::new())?;
info!(target: "reth::cli",
file = %path.display(),
elapsed = %DownloadProgress::format_duration(start.elapsed()),
"Local extraction complete"
);
Ok(())
}
/// Streams a remote archive directly into the extractor without writing to disk.
///
/// On failure, retries from scratch up to [`MAX_DOWNLOAD_RETRIES`] times.
pub(crate) fn streaming_download_and_extract(
url: &str,
format: CompressionFormat,
target_dir: &Path,
session: &DownloadSession,
) -> Result<()> {
let shared = session.progress();
let quiet = session.progress().is_some();
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
if attempt > 1 {
info!(target: "reth::cli",
url = %url,
attempt,
max = MAX_DOWNLOAD_RETRIES,
"Retrying streaming download from scratch"
);
}
let client = BlockingClient::builder().connect_timeout(Duration::from_secs(30)).build()?;
let _request_permit = session
.request_limiter()
.map(|limiter| limiter.acquire(session.progress(), session.cancel_token()))
.transpose()?;
let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(error) => {
let err = eyre::Error::from(error);
if attempt < MAX_DOWNLOAD_RETRIES {
warn!(target: "reth::cli",
url = %url,
attempt,
max = MAX_DOWNLOAD_RETRIES,
err = %err,
"Streaming request failed, retrying"
);
}
last_error = Some(err);
if attempt < MAX_DOWNLOAD_RETRIES {
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
if !quiet && let Some(size) = response.content_length() {
info!(target: "reth::cli",
url = %url,
size = %DownloadProgress::format_size(size),
"Streaming archive"
);
}
let result = if let Some(progress) = shared {
let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) };
extract_archive_raw(reader, format, target_dir, None)
} else {
let total_size = response.content_length().unwrap_or(0);
extract_archive(
response,
total_size,
format,
target_dir,
session.cancel_token().clone(),
)
};
match result {
Ok(()) => return Ok(()),
Err(error) => {
if attempt < MAX_DOWNLOAD_RETRIES {
warn!(target: "reth::cli",
url = %url,
attempt,
max = MAX_DOWNLOAD_RETRIES,
err = %error,
"Streaming extraction failed, retrying"
);
}
last_error = Some(error);
if attempt < MAX_DOWNLOAD_RETRIES {
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
}
}
}
Err(last_error.unwrap_or_else(|| {
eyre::eyre!("Streaming download failed after {MAX_DOWNLOAD_RETRIES} attempts")
}))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
fn download_and_extract(
url: &str,
format: CompressionFormat,
target_dir: &Path,
session: DownloadSession,
) -> Result<()> {
let quiet = session.progress().is_some();
let fetcher = ArchiveFetcher::new(url.to_string(), target_dir, session.clone());
let DownloadedArchive { path: downloaded_path, size: total_size } = fetcher.download(None)?;
let file_name =
downloaded_path.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
if !quiet {
info!(target: "reth::cli",
file = %file_name,
size = %DownloadProgress::format_size(total_size),
"Extracting archive"
);
}
let file = fs::open(&downloaded_path)?;
if quiet {
extract_archive_raw(file, format, target_dir, None)?;
} else {
extract_archive(file, total_size, format, target_dir, session.cancel_token().clone())?;
info!(target: "reth::cli",
file = %file_name,
"Extraction complete"
);
}
fetcher.cleanup_downloaded_files();
session.record_archive_output_complete(total_size);
Ok(())
}
/// Downloads and extracts a snapshot, blocking until finished.
///
/// Supports `file://` URLs for local files and HTTP(S) URLs for remote downloads.
/// When `resumable` is true, downloads to a `.part` file first with HTTP Range resume
/// support. Otherwise streams directly into the extractor.
fn blocking_download_and_extract(
url: &str,
target_dir: &Path,
shared: Option<Arc<SharedProgress>>,
resumable: bool,
request_limiter: Option<Arc<DownloadRequestLimiter>>,
cancel_token: CancellationToken,
) -> Result<()> {
let format = CompressionFormat::from_url(url)?;
if let Ok(parsed_url) = Url::parse(url) &&
parsed_url.scheme() == "file"
{
let session = DownloadSession::new(shared, request_limiter, cancel_token);
let file_path = parsed_url
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
let result = extract_from_file(&file_path, format, target_dir);
if result.is_ok() {
session.record_archive_output_complete(file_path.metadata()?.len());
}
result
} else if let Some(request_limiter) = request_limiter {
download_and_extract(
url,
format,
target_dir,
DownloadSession::new(shared, Some(request_limiter), cancel_token),
)
} else if resumable {
let session =
DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token);
download_and_extract(url, format, target_dir, session)
} else {
let session = DownloadSession::new(shared, None, cancel_token);
let result = streaming_download_and_extract(url, format, target_dir, &session);
if result.is_ok() {
session.record_archive_output_complete(0);
}
result
}
}
/// Downloads and extracts a snapshot archive asynchronously.
///
/// When `shared` is provided, download progress is reported to the shared
/// counter for aggregated display. Otherwise uses a local progress bar.
/// When `resumable` is true, uses two-phase download with `.part` files.
pub(crate) async fn stream_and_extract(
url: &str,
target_dir: &Path,
shared: Option<Arc<SharedProgress>>,
resumable: bool,
request_limiter: Option<Arc<DownloadRequestLimiter>>,
cancel_token: CancellationToken,
) -> Result<()> {
let target_dir = target_dir.to_path_buf();
let url = url.to_string();
task::spawn_blocking(move || {
blocking_download_and_extract(
&url,
&target_dir,
shared,
resumable,
request_limiter,
cancel_token,
)
})
.await??;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_compression_format_detection() {
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -10,10 +10,6 @@ use std::{
};
use tracing::info;
fn is_zero(value: &u64) -> bool {
*value == 0
}
/// A snapshot manifest describes available components for a snapshot at a given block height.
///
/// Each component is either a single archive (state) or a set of chunked archives (static file
@@ -66,12 +62,6 @@ pub struct SingleArchive {
pub file: String,
/// Compressed archive size in bytes.
pub size: u64,
/// Total extracted plain-output size in bytes.
///
/// Older manifests may omit this, in which case downloaders should derive it from
/// `output_files`.
#[serde(default, skip_serializing_if = "is_zero")]
pub decompressed_size: u64,
/// Optional BLAKE3 checksum of the compressed archive.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub blake3: Option<String>,
@@ -93,12 +83,6 @@ pub struct ChunkedArchive {
/// Computed during manifest generation. Older manifests may omit this.
#[serde(default)]
pub chunk_sizes: Vec<u64>,
/// Extracted plain-output size of each chunk in bytes, ordered from first to last.
///
/// Older manifests may omit this, in which case downloaders should derive it from
/// `chunk_output_files`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub chunk_decompressed_sizes: Vec<u64>,
/// Expected extracted plain files per chunk, ordered from first to last.
///
/// This is the authoritative integrity source for the modular download path.
@@ -117,9 +101,9 @@ pub struct OutputFileChecksum {
pub blake3: String,
}
/// A concrete snapshot archive with its download and verification metadata.
/// A single archive with concrete URL and optional integrity metadata.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct SnapshotArchive {
pub struct ArchiveDescriptor {
pub url: String,
pub file_name: String,
pub size: u64,
@@ -127,13 +111,6 @@ pub struct SnapshotArchive {
pub output_files: Vec<OutputFileChecksum>,
}
impl SnapshotArchive {
/// Returns the total extracted plain-output size for this archive.
pub fn output_size(&self) -> u64 {
self.output_files.iter().map(|file| file.size).sum()
}
}
/// How much of a component to download.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ComponentSelection {
@@ -142,9 +119,6 @@ pub enum ComponentSelection {
/// Download only the most recent chunks covering at least `distance` blocks.
/// Maps to `PruneMode::Distance(distance)` in the generated config.
Distance(u64),
/// Download chunks starting at the specified block number.
/// Maps to `PruneMode::Before(block)` in the generated config.
Since(u64),
/// Don't download this component at all.
/// Maps to `PruneMode::Full` for tx-based segments, or a minimal distance for others.
None,
@@ -155,7 +129,6 @@ impl std::fmt::Display for ComponentSelection {
match self {
Self::All => write!(f, "All"),
Self::Distance(d) => write!(f, "Last {d} blocks"),
Self::Since(block) => write!(f, "Since block {block}"),
Self::None => write!(f, "None"),
}
}
@@ -338,19 +311,19 @@ impl SnapshotManifest {
}
}
/// Returns concrete snapshot archives for a component, optionally limited to distance.
pub fn snapshot_archives_for_distance(
/// Returns concrete archive descriptors for a component, optionally limited to distance.
pub fn archive_descriptors_for_distance(
&self,
ty: SnapshotComponentType,
distance: Option<u64>,
) -> Vec<SnapshotArchive> {
) -> Vec<ArchiveDescriptor> {
let Some(component) = self.component(ty) else {
return vec![];
};
match component {
ComponentManifest::Single(single) => {
vec![SnapshotArchive {
vec![ArchiveDescriptor {
url: format!("{}/{}", self.base_url_or_empty(), single.file),
file_name: single.file.clone(),
size: single.size,
@@ -380,7 +353,7 @@ impl SnapshotManifest {
let output_files =
chunked.chunk_output_files.get(i as usize).cloned().unwrap_or_default();
SnapshotArchive {
ArchiveDescriptor {
url: format!("{}/{}", self.base_url_or_empty(), file_name),
file_name,
size,
@@ -421,36 +394,6 @@ impl SnapshotManifest {
}
}
/// Returns the exact extracted plain-output size for a component given a distance selection.
pub fn output_size_for_distance(
&self,
ty: SnapshotComponentType,
distance: Option<u64>,
) -> u64 {
let Some(component) = self.component(ty) else {
return 0;
};
match component {
ComponentManifest::Single(single) => single.output_size(),
ComponentManifest::Chunked(chunked) => {
let num_chunks = chunked.num_chunks();
let start_chunk = match distance {
Some(dist) => {
let needed = dist.min(chunked.total_blocks);
let needed_chunks = needed.div_ceil(chunked.blocks_per_file);
num_chunks.saturating_sub(needed_chunks)
}
None => 0,
};
(start_chunk..num_chunks)
.map(|index| chunked.chunk_output_size(index as usize))
.sum()
}
}
}
/// Returns the number of chunks that would be downloaded for a given distance.
pub fn chunks_for_distance(&self, ty: SnapshotComponentType, distance: Option<u64>) -> u64 {
let Some(ComponentManifest::Chunked(chunked)) = self.component(ty) else {
@@ -474,14 +417,6 @@ impl ComponentManifest {
Self::Chunked(c) => c.chunk_sizes.iter().sum(),
}
}
/// Returns the total extracted plain-output size for this component.
pub fn total_output_size(&self) -> u64 {
match self {
Self::Single(single) => single.output_size(),
Self::Chunked(chunked) => chunked.total_output_size(),
}
}
}
impl ChunkedArchive {
@@ -489,39 +424,6 @@ impl ChunkedArchive {
pub fn num_chunks(&self) -> u64 {
self.total_blocks.div_ceil(self.blocks_per_file)
}
/// Returns the extracted plain-output size for one chunk.
pub fn chunk_output_size(&self, index: usize) -> u64 {
self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| {
self.chunk_output_files
.get(index)
.map(|files| files.iter().map(|file| file.size).sum())
.unwrap_or(0)
})
}
/// Returns the total extracted plain-output size across all chunks.
pub fn total_output_size(&self) -> u64 {
if !self.chunk_decompressed_sizes.is_empty() {
self.chunk_decompressed_sizes.iter().sum()
} else {
self.chunk_output_files
.iter()
.map(|files| files.iter().map(|file| file.size).sum::<u64>())
.sum()
}
}
}
impl SingleArchive {
/// Returns the total extracted plain-output size for this archive.
pub fn output_size(&self) -> u64 {
if self.decompressed_size != 0 {
self.decompressed_size
} else {
self.output_files.iter().map(|file| file.size).sum()
}
}
}
/// Fetch a snapshot manifest from a URL.
@@ -610,10 +512,6 @@ pub fn generate_manifest(
blocks_per_file,
total_blocks: block,
chunk_sizes,
chunk_decompressed_sizes: chunk_output_files
.iter()
.map(|files| files.iter().map(|file| file.size).sum())
.collect(),
chunk_output_files,
}),
);
@@ -630,7 +528,6 @@ pub fn generate_manifest(
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: state_size,
decompressed_size: state_output_files.iter().map(|file| file.size).sum(),
blake3: None,
output_files: state_output_files,
}),
@@ -645,7 +542,6 @@ pub fn generate_manifest(
ComponentManifest::Single(SingleArchive {
file: "rocksdb_indices.tar.zst".to_string(),
size: rocksdb_size,
decompressed_size: rocksdb_output_files.iter().map(|file| file.size).sum(),
blake3: None,
output_files: rocksdb_output_files,
}),
@@ -914,7 +810,6 @@ mod tests {
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 100,
decompressed_size: 0,
blake3: None,
output_files: vec![],
}),
@@ -925,7 +820,6 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 1_500_000,
chunk_sizes: vec![80_000, 100_000, 120_000],
chunk_decompressed_sizes: vec![],
chunk_output_files: vec![vec![], vec![], vec![]],
}),
);
@@ -935,7 +829,6 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 1_500_000,
chunk_sizes: vec![40_000, 50_000, 60_000],
chunk_decompressed_sizes: vec![],
chunk_output_files: vec![vec![], vec![], vec![]],
}),
);
@@ -986,7 +879,6 @@ mod tests {
ComponentManifest::Single(SingleArchive {
file: "rocksdb_indices.tar.zst".to_string(),
size: 777,
decompressed_size: 0,
blake3: None,
output_files: vec![],
}),
@@ -1044,7 +936,6 @@ mod tests {
fn component_selection_display() {
assert_eq!(ComponentSelection::All.to_string(), "All");
assert_eq!(ComponentSelection::Distance(10_064).to_string(), "Last 10064 blocks");
assert_eq!(ComponentSelection::Since(15_537_394).to_string(), "Since block 15537394");
assert_eq!(ComponentSelection::None.to_string(), "None");
}
@@ -1059,7 +950,6 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 24_396_822,
chunk_sizes: vec![100; 49], // 49 chunks
chunk_decompressed_sizes: vec![],
chunk_output_files: vec![vec![]; 49],
}),
);
@@ -1102,68 +992,6 @@ mod tests {
assert_eq!(m.size_for_distance(SnapshotComponentType::Receipts, None), 0);
}
#[test]
fn output_size_for_distance_uses_manifest_or_output_files() {
let m = test_manifest();
assert_eq!(m.output_size_for_distance(SnapshotComponentType::Transactions, None), 0);
let mut components = BTreeMap::new();
components.insert(
"state".to_string(),
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 100,
decompressed_size: 1_000,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "db/mdbx.dat".to_string(),
size: 1_000,
blake3: "h0".to_string(),
}],
}),
);
components.insert(
"transactions".to_string(),
ComponentManifest::Chunked(ChunkedArchive {
blocks_per_file: 500_000,
total_blocks: 1_000_000,
chunk_sizes: vec![80_000, 120_000],
chunk_decompressed_sizes: vec![111, 222],
chunk_output_files: vec![
vec![OutputFileChecksum {
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
size: 111,
blake3: "h0".to_string(),
}],
vec![OutputFileChecksum {
path: "static_files/static_file_transactions_500000_999999.bin".to_string(),
size: 222,
blake3: "h1".to_string(),
}],
],
}),
);
let manifest = SnapshotManifest {
block: 1_000_000,
chain_id: 1,
storage_version: 2,
timestamp: 0,
base_url: Some("https://example.com".to_string()),
reth_version: None,
components,
};
assert_eq!(manifest.output_size_for_distance(SnapshotComponentType::State, None), 1_000);
assert_eq!(
manifest.output_size_for_distance(SnapshotComponentType::Transactions, None),
333
);
assert_eq!(
manifest.output_size_for_distance(SnapshotComponentType::Transactions, Some(500_000)),
222
);
}
#[test]
fn archive_descriptors_include_checksum_metadata() {
let mut components = BTreeMap::new();
@@ -1172,7 +1000,6 @@ mod tests {
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 100,
decompressed_size: 1_000,
blake3: Some("abc123".to_string()),
output_files: vec![OutputFileChecksum {
path: "db/mdbx.dat".to_string(),
@@ -1187,7 +1014,6 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 1_000_000,
chunk_sizes: vec![80_000, 120_000],
chunk_decompressed_sizes: vec![111, 222],
chunk_output_files: vec![
vec![OutputFileChecksum {
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
@@ -1213,13 +1039,13 @@ mod tests {
components,
};
let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None);
let state = m.archive_descriptors_for_distance(SnapshotComponentType::State, None);
assert_eq!(state.len(), 1);
assert_eq!(state[0].file_name, "state.tar.zst");
assert_eq!(state[0].blake3.as_deref(), Some("abc123"));
assert_eq!(state[0].output_files.len(), 1);
let tx = m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, None);
let tx = m.archive_descriptors_for_distance(SnapshotComponentType::Transactions, None);
assert_eq!(tx.len(), 2);
assert_eq!(tx[0].blake3, None);
assert_eq!(tx[1].blake3, None);
@@ -1242,7 +1068,6 @@ mod tests {
panic!("state should be a single archive")
};
assert_eq!(state.file, "state.tar.zst");
assert!(state.decompressed_size > 0);
assert!(!state.output_files.is_empty());
assert_eq!(state.output_files[0].path, "db/mdbx.dat");
assert!(output.path().join("state.tar.zst").exists());
@@ -1267,7 +1092,6 @@ mod tests {
panic!("rocksdb indices should be a single archive")
};
assert_eq!(rocksdb.file, "rocksdb_indices.tar.zst");
assert!(rocksdb.decompressed_size > 0);
assert!(!rocksdb.output_files.is_empty());
assert_eq!(rocksdb.output_files[0].path, "rocksdb/CURRENT");
assert!(output.path().join("rocksdb_indices.tar.zst").exists());

View File

@@ -45,7 +45,6 @@ pub struct SnapshotManifestCommand {
}
impl SnapshotManifestCommand {
/// Packages snapshot archives and writes the manifest file.
pub fn execute(self) -> Result<()> {
let block = match self.block {
Some(block) => block,
@@ -89,7 +88,6 @@ impl SnapshotManifestCommand {
}
}
/// Infers the snapshot block from the source datadir.
fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
if let Ok(block) = infer_snapshot_block_from_db(source_datadir) {
return Ok(block);
@@ -104,7 +102,6 @@ fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
Ok(block)
}
/// Reads the snapshot block from the source database Finish stage checkpoint.
fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64> {
let candidates = [source_datadir.join("db"), source_datadir.to_path_buf()];
@@ -129,7 +126,6 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64>
)
}
/// Infers the snapshot block from the highest header static-file range.
fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result<u64> {
let max_end = header_ranges(source_datadir)?
.into_iter()
@@ -139,7 +135,6 @@ fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result
Ok(max_end)
}
/// Infers the static-file block span from header file ranges.
fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
let mut inferred = None;
for (start, end) in header_ranges(source_datadir)? {
@@ -166,7 +161,6 @@ fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
})
}
/// Collects header static-file ranges from the source datadir.
fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
let static_files_dir = source_datadir.join("static_files");
let static_files_dir =
@@ -189,7 +183,6 @@ fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
Ok(ranges)
}
/// Parses the block range from a header static-file name.
fn parse_headers_range(file_name: &str) -> Option<(u64, u64)> {
let remainder = file_name.strip_prefix("static_file_headers_")?;
let (start, end_with_suffix) = remainder.split_once('_')?;

File diff suppressed because it is too large Load Diff

View File

@@ -1,322 +0,0 @@
use super::{manifest::*, verify::OutputVerifier};
use eyre::Result;
use std::{collections::BTreeMap, path::Path};
use tracing::info;
/// One archive selected from the manifest, along with its component name.
#[derive(Debug, Clone)]
pub(crate) struct PlannedArchive {
/// Snapshot component type this archive belongs to.
pub(crate) ty: SnapshotComponentType,
/// User-facing component name used in logs.
pub(crate) component: String,
/// Concrete snapshot archive metadata resolved from the manifest.
pub(crate) archive: SnapshotArchive,
}
/// The archive list for a modular snapshot download.
#[derive(Debug)]
pub(crate) struct PlannedDownloads {
/// Concrete archives that still need reuse checks or processing.
pub(crate) archives: Vec<PlannedArchive>,
/// Total compressed download size of all planned archives.
pub(crate) total_download_size: u64,
/// Total extracted plain-output size of all planned archives.
pub(crate) total_output_size: u64,
}
impl PlannedDownloads {
/// Returns the number of concrete archives queued for this snapshot selection.
pub(crate) const fn total_archives(&self) -> usize {
self.archives.len()
}
}
/// Returns the sort priority used to schedule archives.
pub(crate) const fn archive_priority_rank(ty: SnapshotComponentType) -> u8 {
match ty {
SnapshotComponentType::State => 0,
SnapshotComponentType::RocksdbIndices => 1,
_ => 2,
}
}
/// Startup summary showing how much of the selected work can be reused.
#[derive(Debug, Default, Clone, Copy)]
pub(crate) struct DownloadStartupSummary {
/// Archives whose declared outputs already verify on disk.
pub(crate) reusable: usize,
/// Archives that still need to be downloaded or retried.
pub(crate) needs_download: usize,
}
/// Checks selected archives against existing output files before work begins.
pub(crate) fn summarize_download_startup(
all_downloads: &[PlannedArchive],
target_dir: &Path,
) -> Result<DownloadStartupSummary> {
let mut summary = DownloadStartupSummary::default();
let verifier = OutputVerifier::new(target_dir);
for planned in all_downloads {
if verifier.verify(&planned.archive.output_files)? {
summary.reusable += 1;
} else {
summary.needs_download += 1;
}
}
Ok(summary)
}
/// Converts a selection into the manifest distance form used for archive lookup.
fn selection_archive_distance(
selection: &ComponentSelection,
snapshot_block: u64,
) -> Option<Option<u64>> {
match selection {
ComponentSelection::All => Some(None),
ComponentSelection::Distance(distance) => Some(Some(*distance)),
ComponentSelection::Since(block) => Some(Some(snapshot_block.saturating_sub(*block) + 1)),
ComponentSelection::None => None,
}
}
/// Sorts planned archives into a stable processing order.
fn sort_planned_archives(all_downloads: &mut [PlannedArchive]) {
all_downloads.sort_by(|a, b| {
archive_priority_rank(a.ty)
.cmp(&archive_priority_rank(b.ty))
.then_with(|| a.component.cmp(&b.component))
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
});
}
/// Expands component selections into the archives that need to be processed.
pub(crate) fn collect_planned_archives(
manifest: &SnapshotManifest,
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
) -> Result<PlannedDownloads> {
let mut archives = Vec::new();
let mut total_download_size = 0;
let mut total_output_size = 0;
for (ty, selection) in selections {
let Some(distance) = selection_archive_distance(selection, manifest.block) else {
continue;
};
total_download_size += manifest.size_for_distance(*ty, distance);
total_output_size += manifest.output_size_for_distance(*ty, distance);
let snapshot_archives = manifest.snapshot_archives_for_distance(*ty, distance);
let component = ty.display_name().to_string();
if !snapshot_archives.is_empty() {
info!(target: "reth::cli",
component = %component,
archives = snapshot_archives.len(),
selection = %selection,
"Queued component for download"
);
}
for archive in snapshot_archives {
if archive.output_files.is_empty() {
eyre::bail!(
"Invalid modular manifest: {} is missing plain output checksum metadata",
archive.file_name
);
}
archives.push(PlannedArchive { ty: *ty, component: component.clone(), archive });
}
}
sort_planned_archives(&mut archives);
Ok(PlannedDownloads { archives, total_download_size, total_output_size })
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn summarize_download_startup_counts_reusable_and_needs_download() {
let dir = tempdir().unwrap();
let target_dir = dir.path();
let ok_file = target_dir.join("ok.bin");
std::fs::write(&ok_file, vec![1_u8; 4]).unwrap();
let ok_hash = blake3::hash(&[1_u8; 4]).to_hex().to_string();
let planned = vec![
PlannedArchive {
ty: SnapshotComponentType::State,
component: "State".to_string(),
archive: SnapshotArchive {
url: "https://example.com/ok.tar.zst".to_string(),
file_name: "ok.tar.zst".to_string(),
size: 10,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "ok.bin".to_string(),
size: 4,
blake3: ok_hash,
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::Headers,
component: "Headers".to_string(),
archive: SnapshotArchive {
url: "https://example.com/missing.tar.zst".to_string(),
file_name: "missing.tar.zst".to_string(),
size: 10,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "missing.bin".to_string(),
size: 1,
blake3: "deadbeef".to_string(),
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::Transactions,
component: "Transactions".to_string(),
archive: SnapshotArchive {
url: "https://example.com/bad-size.tar.zst".to_string(),
file_name: "bad-size.tar.zst".to_string(),
size: 10,
blake3: None,
output_files: vec![],
},
},
];
let summary = summarize_download_startup(&planned, target_dir).unwrap();
assert_eq!(summary.reusable, 1);
assert_eq!(summary.needs_download, 2);
}
#[test]
fn archive_priority_prefers_state_then_rocksdb() {
let mut planned = [
PlannedArchive {
ty: SnapshotComponentType::Transactions,
component: "Transactions".to_string(),
archive: SnapshotArchive {
url: "u3".to_string(),
file_name: "t.tar.zst".to_string(),
size: 1,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "a".to_string(),
size: 1,
blake3: "x".to_string(),
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::RocksdbIndices,
component: "RocksDB Indices".to_string(),
archive: SnapshotArchive {
url: "u2".to_string(),
file_name: "rocksdb_indices.tar.zst".to_string(),
size: 1,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "b".to_string(),
size: 1,
blake3: "y".to_string(),
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::State,
component: "State (mdbx)".to_string(),
archive: SnapshotArchive {
url: "u1".to_string(),
file_name: "state.tar.zst".to_string(),
size: 1,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "c".to_string(),
size: 1,
blake3: "z".to_string(),
}],
},
},
];
planned.sort_by(|a, b| {
archive_priority_rank(a.ty)
.cmp(&archive_priority_rank(b.ty))
.then_with(|| a.component.cmp(&b.component))
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
});
assert_eq!(planned[0].ty, SnapshotComponentType::State);
assert_eq!(planned[1].ty, SnapshotComponentType::RocksdbIndices);
assert_eq!(planned[2].ty, SnapshotComponentType::Transactions);
}
#[test]
fn collect_planned_archives_tracks_download_and_output_totals() {
let mut components = BTreeMap::new();
components.insert(
"state".to_string(),
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 10,
decompressed_size: 100,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "db/mdbx.dat".to_string(),
size: 100,
blake3: "h0".to_string(),
}],
}),
);
components.insert(
"transactions".to_string(),
ComponentManifest::Chunked(ChunkedArchive {
blocks_per_file: 500_000,
total_blocks: 1_000_000,
chunk_sizes: vec![20, 30],
chunk_decompressed_sizes: vec![200, 300],
chunk_output_files: vec![
vec![OutputFileChecksum {
path: "static_files/tx-0".to_string(),
size: 200,
blake3: "h1".to_string(),
}],
vec![OutputFileChecksum {
path: "static_files/tx-1".to_string(),
size: 300,
blake3: "h2".to_string(),
}],
],
}),
);
let manifest = SnapshotManifest {
block: 1_000_000,
chain_id: 1,
storage_version: 2,
timestamp: 0,
base_url: Some("https://example.com".to_string()),
reth_version: None,
components,
};
let selections = BTreeMap::from([
(SnapshotComponentType::State, ComponentSelection::All),
(SnapshotComponentType::Transactions, ComponentSelection::Distance(500_000)),
]);
let planned = collect_planned_archives(&manifest, &selections).unwrap();
assert_eq!(planned.total_download_size, 40);
assert_eq!(planned.total_output_size, 400);
assert_eq!(planned.archives.len(), 2);
}
}

View File

@@ -1,844 +0,0 @@
use eyre::Result;
use reth_cli_util::cancellation::CancellationToken;
use std::{
io::{self, Read, Write},
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, Condvar, Mutex,
},
time::{Duration, Instant},
};
use tracing::info;
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
/// Tracks download progress and throttles display updates to every 100ms.
pub(crate) struct DownloadProgress {
/// Bytes copied so far for this single download.
pub(crate) downloaded: u64,
/// Total bytes expected for this single download.
total_size: u64,
/// Time when the progress line was last printed.
last_displayed: Instant,
/// Time when this progress tracker started.
started_at: Instant,
}
impl DownloadProgress {
/// Creates new progress tracker with given total size
pub(crate) fn new(total_size: u64) -> Self {
let now = Instant::now();
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
}
/// Converts bytes to human readable format (B, KB, MB, GB)
pub(crate) fn format_size(size: u64) -> String {
let mut size = size as f64;
let mut unit_index = 0;
while size >= 1024.0 && unit_index < BYTE_UNITS.len() - 1 {
size /= 1024.0;
unit_index += 1;
}
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
}
/// Format duration as human readable string
pub(crate) fn format_duration(duration: Duration) -> String {
let secs = duration.as_secs();
if secs < 60 {
format!("{secs}s")
} else if secs < 3600 {
format!("{}m {}s", secs / 60, secs % 60)
} else {
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
}
}
/// Updates progress bar (for single-archive legacy downloads)
pub(crate) fn update(&mut self, chunk_size: u64) -> Result<()> {
self.downloaded += chunk_size;
if self.last_displayed.elapsed() >= Duration::from_millis(100) {
let formatted_downloaded = Self::format_size(self.downloaded);
let formatted_total = Self::format_size(self.total_size);
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
let elapsed = self.started_at.elapsed();
let eta = if self.downloaded > 0 {
let remaining = self.total_size.saturating_sub(self.downloaded);
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
if speed > 0.0 {
Duration::from_secs_f64(remaining as f64 / speed)
} else {
Duration::ZERO
}
} else {
Duration::ZERO
};
let eta_str = Self::format_duration(eta);
print!(
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
);
io::stdout().flush()?;
self.last_displayed = Instant::now();
}
Ok(())
}
}
#[derive(Debug, Clone, Copy)]
struct PhaseStart {
started_at: Instant,
baseline_bytes: u64,
}
/// Shared progress counters for parallel downloads.
pub(crate) struct SharedProgress {
/// Raw HTTP bytes fetched during this session, including retries.
pub(crate) session_fetched_bytes: AtomicU64,
/// Compressed bytes from archives that have fully downloaded.
pub(crate) completed_download_bytes: AtomicU64,
/// Compressed bytes written for currently active archive download attempts.
pub(crate) active_download_bytes: AtomicU64,
/// Total compressed bytes expected across all planned archives.
pub(crate) total_download_bytes: u64,
/// Plain-output bytes from archives that have fully verified.
pub(crate) completed_output_bytes: AtomicU64,
/// Plain-output bytes unpacked by currently active extractions.
pub(crate) active_extracted_output_bytes: AtomicU64,
/// Plain-output bytes hashed by currently active verifications.
pub(crate) active_verified_output_bytes: AtomicU64,
/// Total plain-output bytes expected across all planned archives.
pub(crate) total_output_bytes: u64,
/// Total number of planned archives.
pub(crate) total_archives: u64,
/// Time when the modular download job started.
pub(crate) started_at: Instant,
/// Time and baseline when the current extraction phase started.
extraction_phase: Mutex<Option<PhaseStart>>,
/// Time and baseline when the current verification phase started.
verification_phase: Mutex<Option<PhaseStart>>,
/// Number of archives that have fully finished.
pub(crate) archives_done: AtomicU64,
/// Number of archives currently in the fetch phase.
pub(crate) active_downloads: AtomicU64,
/// Number of in-flight HTTP requests.
pub(crate) active_download_requests: AtomicU64,
/// Number of archives currently extracting.
pub(crate) active_extractions: AtomicU64,
/// Number of archives currently verifying extracted outputs.
pub(crate) active_verifications: AtomicU64,
/// Signals the background progress task to exit.
pub(crate) done: AtomicBool,
/// Cancellation token shared by the whole command.
cancel_token: CancellationToken,
}
impl SharedProgress {
/// Creates the shared progress state for a modular download job.
pub(crate) fn new(
total_download_bytes: u64,
total_output_bytes: u64,
total_archives: u64,
cancel_token: CancellationToken,
) -> Arc<Self> {
Arc::new(Self {
session_fetched_bytes: AtomicU64::new(0),
completed_download_bytes: AtomicU64::new(0),
active_download_bytes: AtomicU64::new(0),
total_download_bytes,
completed_output_bytes: AtomicU64::new(0),
active_extracted_output_bytes: AtomicU64::new(0),
active_verified_output_bytes: AtomicU64::new(0),
total_output_bytes,
total_archives,
started_at: Instant::now(),
extraction_phase: Mutex::new(None),
verification_phase: Mutex::new(None),
archives_done: AtomicU64::new(0),
active_downloads: AtomicU64::new(0),
active_download_requests: AtomicU64::new(0),
active_extractions: AtomicU64::new(0),
active_verifications: AtomicU64::new(0),
done: AtomicBool::new(false),
cancel_token,
})
}
/// Returns whether the whole command has been cancelled.
pub(crate) fn is_cancelled(&self) -> bool {
self.cancel_token.is_cancelled()
}
/// Adds raw session traffic bytes without affecting logical progress.
pub(crate) fn record_session_fetched_bytes(&self, bytes: u64) {
self.session_fetched_bytes.fetch_add(bytes, Ordering::Relaxed);
}
pub(crate) fn add_active_download_bytes(&self, bytes: u64) {
self.active_download_bytes.fetch_add(bytes, Ordering::Relaxed);
}
pub(crate) fn sub_active_download_bytes(&self, bytes: u64) {
sub_bytes(&self.active_download_bytes, bytes);
}
fn add_active_extracted_output_bytes(&self, bytes: u64) {
self.active_extracted_output_bytes.fetch_add(bytes, Ordering::Relaxed);
}
fn sub_active_extracted_output_bytes(&self, bytes: u64) {
sub_bytes(&self.active_extracted_output_bytes, bytes);
}
fn add_active_verified_output_bytes(&self, bytes: u64) {
self.active_verified_output_bytes.fetch_add(bytes, Ordering::Relaxed);
}
fn sub_active_verified_output_bytes(&self, bytes: u64) {
sub_bytes(&self.active_verified_output_bytes, bytes);
}
/// Records an archive whose outputs were already present locally.
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
self.completed_download_bytes.fetch_add(download_bytes, Ordering::Relaxed);
self.completed_output_bytes.fetch_add(output_bytes, Ordering::Relaxed);
self.archives_done.fetch_add(1, Ordering::Relaxed);
}
/// Records an archive whose compressed download completed successfully.
pub(crate) fn record_archive_download_complete(&self, bytes: u64) {
self.completed_download_bytes.fetch_add(bytes, Ordering::Relaxed);
}
/// Records an archive whose extracted outputs have fully verified.
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
self.completed_output_bytes.fetch_add(bytes, Ordering::Relaxed);
self.archives_done.fetch_add(1, Ordering::Relaxed);
}
/// Returns logical compressed download progress.
pub(crate) fn logical_downloaded_bytes(&self) -> u64 {
(self.completed_download_bytes.load(Ordering::Relaxed) +
self.active_download_bytes.load(Ordering::Relaxed))
.min(self.total_download_bytes)
}
/// Returns verified plain-output bytes.
pub(crate) fn verified_output_bytes(&self) -> u64 {
self.completed_output_bytes.load(Ordering::Relaxed).min(self.total_output_bytes)
}
/// Returns plain-output bytes currently represented by extraction progress.
pub(crate) fn extracting_output_bytes(&self) -> u64 {
(self.completed_output_bytes.load(Ordering::Relaxed) +
self.active_extracted_output_bytes.load(Ordering::Relaxed))
.min(self.total_output_bytes)
}
/// Returns plain-output bytes currently represented by verification progress.
pub(crate) fn verifying_output_bytes(&self) -> u64 {
(self.completed_output_bytes.load(Ordering::Relaxed) +
self.active_verified_output_bytes.load(Ordering::Relaxed))
.min(self.total_output_bytes)
}
fn restart_phase(slot: &Mutex<Option<PhaseStart>>, baseline_bytes: u64) {
*slot.lock().unwrap() = Some(PhaseStart { started_at: Instant::now(), baseline_bytes });
}
fn phase_eta(
slot: &Mutex<Option<PhaseStart>>,
current_bytes: u64,
total_bytes: u64,
) -> Option<Duration> {
let phase = *slot.lock().unwrap();
let phase = phase?;
let done = current_bytes.saturating_sub(phase.baseline_bytes);
let total = total_bytes.saturating_sub(phase.baseline_bytes);
eta_from_progress(phase.started_at.elapsed(), done, total)
}
fn extraction_eta(&self, current_bytes: u64) -> Option<Duration> {
Self::phase_eta(&self.extraction_phase, current_bytes, self.total_output_bytes)
}
fn verification_eta(&self, current_bytes: u64) -> Option<Duration> {
Self::phase_eta(&self.verification_phase, current_bytes, self.total_output_bytes)
}
/// Marks one archive as actively downloading.
pub(crate) fn download_started(&self) {
self.active_downloads.fetch_add(1, Ordering::Relaxed);
}
/// Marks one archive download as finished.
pub(crate) fn download_finished(&self) {
sub_bytes(&self.active_downloads, 1);
}
/// Marks one HTTP request as in flight.
pub(crate) fn request_started(&self) {
self.active_download_requests.fetch_add(1, Ordering::Relaxed);
}
/// Marks one HTTP request as finished.
pub(crate) fn request_finished(&self) {
sub_bytes(&self.active_download_requests, 1);
}
/// Marks one archive as actively extracting.
pub(crate) fn extraction_started(&self) {
if self.active_extractions.fetch_add(1, Ordering::Relaxed) == 0 {
Self::restart_phase(
&self.extraction_phase,
self.completed_output_bytes.load(Ordering::Relaxed),
);
}
}
/// Marks one archive extraction as finished.
pub(crate) fn extraction_finished(&self) {
sub_bytes(&self.active_extractions, 1);
}
/// Marks one archive as actively verifying outputs.
pub(crate) fn verification_started(&self) {
if self.active_verifications.fetch_add(1, Ordering::Relaxed) == 0 {
Self::restart_phase(
&self.verification_phase,
self.completed_output_bytes.load(Ordering::Relaxed),
);
}
}
/// Marks one archive verification as finished.
pub(crate) fn verification_finished(&self) {
sub_bytes(&self.active_verifications, 1);
}
}
fn sub_bytes(counter: &AtomicU64, bytes: u64) {
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
Some(current.saturating_sub(bytes))
});
}
fn eta_from_progress(elapsed: Duration, done: u64, total: u64) -> Option<Duration> {
if done == 0 || done >= total {
return None;
}
let secs = elapsed.as_secs_f64();
if secs <= 0.0 {
return None;
}
let speed = done as f64 / secs;
if speed <= 0.0 {
return None;
}
Some(Duration::from_secs_f64((total - done) as f64 / speed))
}
fn format_percent(done: u64, total: u64) -> String {
if total == 0 {
return "100.0%".to_string();
}
format!("{:.1}%", (done as f64 / total as f64) * 100.0)
}
fn format_eta(eta: Option<Duration>) -> String {
eta.map(DownloadProgress::format_duration).unwrap_or_else(|| "unknown".to_string())
}
/// Global request limit for the blocking downloader.
///
/// This uses `Mutex + Condvar` because the segmented path runs blocking reqwest
/// clients on OS threads.
pub(crate) struct DownloadRequestLimiter {
/// Maximum number of in-flight HTTP requests.
limit: usize,
/// Current number of acquired request slots.
active: Mutex<usize>,
/// Wakes blocked threads when a slot is released.
notify: Condvar,
}
impl DownloadRequestLimiter {
/// Creates the shared request limiter.
pub(crate) fn new(limit: usize) -> Arc<Self> {
Arc::new(Self { limit: limit.max(1), active: Mutex::new(0), notify: Condvar::new() })
}
/// Returns the configured request limit.
pub(crate) fn max_concurrency(&self) -> usize {
self.limit
}
pub(crate) fn acquire<'a>(
&'a self,
progress: Option<&'a Arc<SharedProgress>>,
cancel_token: &CancellationToken,
) -> Result<DownloadRequestPermit<'a>> {
let mut active = self.active.lock().unwrap();
loop {
if cancel_token.is_cancelled() {
return Err(eyre::eyre!("Download cancelled"));
}
if *active < self.limit {
*active += 1;
if let Some(progress) = progress {
progress.request_started();
}
return Ok(DownloadRequestPermit { limiter: self, progress });
}
// Wake periodically so cancellation can interrupt waiters even if
// no request finishes.
let (next_active, _) =
self.notify.wait_timeout(active, Duration::from_millis(100)).unwrap();
active = next_active;
}
}
}
/// RAII permit for one in-flight HTTP request.
///
/// Dropping the permit releases a slot in the shared request limit and updates
/// the live progress counters.
pub(crate) struct DownloadRequestPermit<'a> {
/// Limiter that owns the request slot.
limiter: &'a DownloadRequestLimiter,
/// Shared progress counters updated when the permit drops.
progress: Option<&'a Arc<SharedProgress>>,
}
impl Drop for DownloadRequestPermit<'_> {
/// Releases the request slot and updates shared progress counters.
fn drop(&mut self) {
let mut active = self.limiter.active.lock().unwrap();
*active = active.saturating_sub(1);
drop(active);
self.limiter.notify.notify_one();
if let Some(progress) = self.progress {
progress.request_finished();
}
}
}
/// Tracks one active archive download attempt.
pub(crate) struct ArchiveDownloadProgress<'a> {
progress: Option<&'a Arc<SharedProgress>>,
downloaded: u64,
completed: bool,
}
impl<'a> ArchiveDownloadProgress<'a> {
/// Starts tracking one archive download attempt.
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
if let Some(progress) = progress {
progress.download_started();
}
Self { progress, downloaded: 0, completed: false }
}
/// Adds logical compressed bytes written by this attempt.
pub(crate) fn record_downloaded(&mut self, bytes: u64) {
self.downloaded += bytes;
if let Some(progress) = self.progress {
progress.add_active_download_bytes(bytes);
}
}
/// Returns whether this tracker has recorded any logical bytes itself.
pub(crate) fn has_tracked_bytes(&self) -> bool {
self.downloaded > 0
}
/// Moves this archive from active download bytes into completed download bytes.
pub(crate) fn complete(&mut self, total_bytes: u64) {
if self.completed {
return;
}
if let Some(progress) = self.progress {
progress.sub_active_download_bytes(self.downloaded);
progress.record_archive_download_complete(total_bytes);
}
self.downloaded = 0;
self.completed = true;
}
}
impl Drop for ArchiveDownloadProgress<'_> {
fn drop(&mut self) {
if let Some(progress) = self.progress {
progress.sub_active_download_bytes(self.downloaded);
progress.download_finished();
}
}
}
/// Tracks one active archive extraction attempt.
pub(crate) struct ArchiveExtractionProgress {
progress: Option<Arc<SharedProgress>>,
extracted: Arc<AtomicU64>,
finished: bool,
}
/// Cloneable handle for reporting extracted bytes from background monitoring.
#[derive(Clone)]
pub(crate) struct ArchiveExtractionProgressHandle {
progress: Arc<SharedProgress>,
extracted: Arc<AtomicU64>,
}
impl ArchiveExtractionProgress {
/// Starts tracking one archive extraction attempt.
pub(crate) fn new(progress: Option<&Arc<SharedProgress>>) -> Self {
if let Some(progress) = progress {
progress.extraction_started();
}
Self {
progress: progress.cloned(),
extracted: Arc::new(AtomicU64::new(0)),
finished: false,
}
}
/// Returns a cloneable handle that can report extraction progress from another thread.
pub(crate) fn handle(&self) -> Option<ArchiveExtractionProgressHandle> {
Some(ArchiveExtractionProgressHandle {
progress: Arc::clone(self.progress.as_ref()?),
extracted: Arc::clone(&self.extracted),
})
}
/// Adds plain-output bytes extracted by this attempt.
pub(crate) fn record_extracted(&mut self, bytes: u64) {
if let Some(handle) = self.handle() {
handle.record_extracted(bytes);
}
}
/// Ends extraction tracking before verification begins.
pub(crate) fn finish(&mut self) {
if self.finished {
return;
}
if let Some(progress) = &self.progress {
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
}
self.finished = true;
}
}
impl Drop for ArchiveExtractionProgress {
fn drop(&mut self) {
if let Some(progress) = &self.progress {
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
progress.extraction_finished();
}
}
}
impl ArchiveExtractionProgressHandle {
/// Adds plain-output bytes extracted by this attempt.
pub(crate) fn record_extracted(&self, bytes: u64) {
self.extracted.fetch_add(bytes, Ordering::Relaxed);
self.progress.add_active_extracted_output_bytes(bytes);
}
}
/// Tracks one active archive verification attempt.
pub(crate) struct ArchiveVerificationProgress<'a> {
progress: Option<&'a Arc<SharedProgress>>,
verified: u64,
completed: bool,
}
impl<'a> ArchiveVerificationProgress<'a> {
/// Starts tracking one archive verification attempt.
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
if let Some(progress) = progress {
progress.verification_started();
}
Self { progress, verified: 0, completed: false }
}
/// Adds plain-output bytes hashed by this verification attempt.
pub(crate) fn record_verified(&mut self, bytes: u64) {
self.verified += bytes;
if let Some(progress) = self.progress {
progress.add_active_verified_output_bytes(bytes);
}
}
/// Moves this archive from active verification bytes into completed output bytes.
pub(crate) fn complete(&mut self, total_bytes: u64) {
if self.completed {
return;
}
if let Some(progress) = self.progress {
progress.sub_active_verified_output_bytes(self.verified);
progress.record_archive_output_complete(total_bytes);
}
self.verified = 0;
self.completed = true;
}
}
impl Drop for ArchiveVerificationProgress<'_> {
fn drop(&mut self) {
if let Some(progress) = self.progress {
progress.sub_active_verified_output_bytes(self.verified);
progress.verification_finished();
}
}
}
/// Adapter to track progress while reading (used for extraction in legacy path)
pub(crate) struct ProgressReader<R> {
/// Wrapped reader that provides archive bytes.
reader: R,
/// Per-download progress tracker for legacy paths.
progress: DownloadProgress,
/// Cancellation token checked between reads.
cancel_token: CancellationToken,
}
impl<R: Read> ProgressReader<R> {
/// Wraps a reader with per-download progress tracking.
pub(crate) fn new(reader: R, total_size: u64, cancel_token: CancellationToken) -> Self {
Self { reader, progress: DownloadProgress::new(total_size), cancel_token }
}
}
impl<R: Read> Read for ProgressReader<R> {
/// Reads bytes, checks cancellation, and updates the local progress bar.
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.cancel_token.is_cancelled() {
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
}
let bytes = self.reader.read(buf)?;
if bytes > 0 &&
let Err(error) = self.progress.update(bytes as u64)
{
return Err(io::Error::other(error));
}
Ok(bytes)
}
}
/// Wrapper that bumps a shared atomic counter while writing data.
/// Used for parallel downloads where a single display task shows aggregated progress.
pub(crate) struct SharedProgressWriter<'a, W> {
/// Wrapped writer receiving downloaded bytes.
pub(crate) inner: W,
/// Shared counters updated as bytes are written.
pub(crate) progress: Arc<SharedProgress>,
/// Optional callback for logical bytes written by the current archive attempt.
pub(crate) on_written: Option<&'a mut dyn FnMut(u64)>,
}
impl<W: Write> Write for SharedProgressWriter<'_, W> {
/// Writes bytes and records them in shared progress.
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.progress.is_cancelled() {
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
}
let n = self.inner.write(buf)?;
self.progress.record_session_fetched_bytes(n as u64);
if let Some(on_written) = self.on_written.as_deref_mut() {
on_written(n as u64);
}
Ok(n)
}
/// Flushes the wrapped writer.
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Wrapper that bumps a shared atomic counter while reading data.
/// Used for streaming downloads where a single display task shows aggregated progress.
pub(crate) struct SharedProgressReader<R> {
/// Wrapped reader producing streamed bytes.
pub(crate) inner: R,
/// Shared counters updated as bytes are read.
pub(crate) progress: Arc<SharedProgress>,
}
impl<R: Read> Read for SharedProgressReader<R> {
/// Reads bytes and records them in shared progress.
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.progress.is_cancelled() {
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
}
let n = self.inner.read(buf)?;
self.progress.record_session_fetched_bytes(n as u64);
Ok(n)
}
}
/// Spawns a background task that prints aggregated download progress.
/// Returns a handle; drop it (or call `.abort()`) to stop.
pub(crate) fn spawn_progress_display(progress: Arc<SharedProgress>) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(3));
interval.tick().await;
loop {
interval.tick().await;
if progress.done.load(Ordering::Relaxed) {
break;
}
let download_total = progress.total_download_bytes;
let output_total = progress.total_output_bytes;
if download_total == 0 && output_total == 0 {
continue;
}
let done = progress.archives_done.load(Ordering::Relaxed);
let all = progress.total_archives;
let active_downloads = progress.active_downloads.load(Ordering::Relaxed);
let active_requests = progress.active_download_requests.load(Ordering::Relaxed);
let active_extractions = progress.active_extractions.load(Ordering::Relaxed);
let active_verifications = progress.active_verifications.load(Ordering::Relaxed);
let downloaded = progress.logical_downloaded_bytes();
let extracted = progress.extracting_output_bytes();
let verified = progress.verifying_output_bytes();
let elapsed = DownloadProgress::format_duration(progress.started_at.elapsed());
let download_total_display = DownloadProgress::format_size(download_total);
let output_total_display = DownloadProgress::format_size(output_total);
let downloaded_display = DownloadProgress::format_size(downloaded);
let extracted_display = DownloadProgress::format_size(extracted);
let active_download_phase = active_downloads > 0 || active_requests > 0;
if active_download_phase {
info!(target: "reth::cli",
archives = format_args!("{done}/{all}"),
progress = %format_percent(downloaded, download_total),
elapsed = %elapsed,
eta = %format_eta(eta_from_progress(progress.started_at.elapsed(), downloaded, download_total)),
bytes = format_args!("{downloaded_display}/{download_total_display}"),
"Downloading snapshot archives"
);
} else if active_extractions > 0 {
info!(target: "reth::cli",
archives = format_args!("{done}/{all}"),
progress = %format_percent(extracted, output_total),
elapsed = %elapsed,
eta = %format_eta(progress.extraction_eta(extracted)),
bytes = format_args!("{extracted_display}/{output_total_display}"),
"Extracting snapshot archives"
);
} else if active_verifications > 0 {
info!(target: "reth::cli",
archives = format_args!("{done}/{all}"),
progress = %format_percent(verified, output_total),
elapsed = %elapsed,
eta = %format_eta(progress.verification_eta(verified)),
bytes = format_args!("{}/{output_total_display}", DownloadProgress::format_size(verified)),
"Verifying snapshot archives"
);
} else {
continue;
}
}
let completed = progress.verified_output_bytes();
let completed_display = DownloadProgress::format_size(completed);
let output_total = DownloadProgress::format_size(progress.total_output_bytes);
info!(target: "reth::cli",
archives = format_args!("{}/{}", progress.total_archives, progress.total_archives),
progress = "100.0%",
elapsed = %DownloadProgress::format_duration(progress.started_at.elapsed()),
eta = "0s",
bytes = format_args!("{completed_display}/{output_total}"),
"Snapshot archive processing complete"
);
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::Ordering;
#[test]
fn shared_progress_separates_session_fetch_from_logical_progress() {
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
progress.record_session_fetched_bytes(10);
progress.record_session_fetched_bytes(10);
progress.record_archive_download_complete(10);
progress.record_archive_output_complete(20);
assert_eq!(progress.session_fetched_bytes.load(Ordering::Relaxed), 20);
assert_eq!(progress.logical_downloaded_bytes(), 10);
assert_eq!(progress.verified_output_bytes(), 20);
assert_eq!(progress.archives_done.load(Ordering::Relaxed), 1);
}
#[test]
fn archive_download_progress_rolls_back_unfinished_attempts() {
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
{
let mut download = ArchiveDownloadProgress::new(Some(&progress));
download.record_downloaded(4);
assert_eq!(progress.logical_downloaded_bytes(), 4);
}
assert_eq!(progress.logical_downloaded_bytes(), 0);
assert_eq!(progress.active_downloads.load(Ordering::Relaxed), 0);
}
#[test]
fn extraction_phase_baseline_restarts_after_idle() {
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
progress.extraction_started();
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.completed_output_bytes.store(25, Ordering::Relaxed);
progress.extraction_started();
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.extraction_finished();
progress.extraction_finished();
progress.extraction_started();
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 25);
}
#[test]
fn verification_phase_baseline_restarts_after_idle() {
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
progress.verification_started();
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.completed_output_bytes.store(40, Ordering::Relaxed);
progress.verification_started();
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.verification_finished();
progress.verification_finished();
progress.verification_started();
assert_eq!(
progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes,
40
);
}
}

View File

@@ -1,100 +0,0 @@
use super::progress::{DownloadRequestLimiter, SharedProgress};
use eyre::Result;
use reth_cli_util::cancellation::CancellationToken;
use std::{
path::{Path, PathBuf},
sync::Arc,
};
/// Shared state for one run of `reth download`.
#[derive(Clone)]
pub(crate) struct DownloadSession {
/// Shared progress counters for this command, when enabled.
progress: Option<Arc<SharedProgress>>,
/// Shared limit for concurrent HTTP requests, when enabled.
request_limiter: Option<Arc<DownloadRequestLimiter>>,
/// Cancellation token shared by the whole command.
cancel_token: CancellationToken,
}
impl DownloadSession {
/// Stores the shared progress, request limiter, and cancellation token.
pub(crate) fn new(
progress: Option<Arc<SharedProgress>>,
request_limiter: Option<Arc<DownloadRequestLimiter>>,
cancel_token: CancellationToken,
) -> Self {
Self { progress, request_limiter, cancel_token }
}
/// Returns the shared progress tracker, if this flow uses one.
pub(crate) fn progress(&self) -> Option<&Arc<SharedProgress>> {
self.progress.as_ref()
}
/// Returns the shared HTTP request limiter, if this flow uses one.
pub(crate) fn request_limiter(&self) -> Option<&Arc<DownloadRequestLimiter>> {
self.request_limiter.as_ref()
}
/// Returns the request limiter or errors if the caller needs one.
pub(crate) fn require_request_limiter(&self) -> Result<&Arc<DownloadRequestLimiter>> {
self.request_limiter().ok_or_else(|| eyre::eyre!("Missing download request limiter"))
}
/// Returns the cancellation token for this command.
pub(crate) fn cancel_token(&self) -> &CancellationToken {
&self.cancel_token
}
/// Records one archive whose outputs were already reusable on disk.
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
if let Some(progress) = self.progress() {
progress.record_reused_archive(download_bytes, output_bytes);
}
}
/// Records one archive whose extracted outputs fully verified.
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
if let Some(progress) = self.progress() {
progress.record_archive_output_complete(bytes);
}
}
}
/// Paths used while processing one archive, plus the shared download session.
#[derive(Clone)]
pub(crate) struct ArchiveProcessContext {
/// Directory where extracted output files are written.
target_dir: PathBuf,
/// Directory used for cached archive downloads, when enabled.
cache_dir: Option<PathBuf>,
/// Shared command-scoped download state.
session: DownloadSession,
}
impl ArchiveProcessContext {
/// Creates the context used while processing modular archives.
pub(crate) fn new(
target_dir: PathBuf,
cache_dir: Option<PathBuf>,
session: DownloadSession,
) -> Self {
Self { target_dir, cache_dir, session }
}
/// Returns the directory where extracted outputs should be written.
pub(crate) fn target_dir(&self) -> &Path {
&self.target_dir
}
/// Returns the cache directory for two-phase downloads, if enabled.
pub(crate) fn cache_dir(&self) -> Option<&Path> {
self.cache_dir.as_deref()
}
/// Returns the shared download session.
pub(crate) fn session(&self) -> &DownloadSession {
&self.session
}
}

View File

@@ -1,230 +0,0 @@
use super::{manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults};
use eyre::{Result, WrapErr};
use reqwest::Client;
use reth_fs_util as fs;
use std::path::{Path, PathBuf};
use tracing::info;
use url::Url;
/// An entry from the snapshot discovery API listing.
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct SnapshotApiEntry {
#[serde(deserialize_with = "deserialize_string_or_u64")]
chain_id: u64,
#[serde(deserialize_with = "deserialize_string_or_u64")]
block: u64,
#[serde(default)]
date: Option<String>,
#[serde(default)]
profile: Option<String>,
metadata_url: String,
#[serde(default)]
size: u64,
}
impl SnapshotApiEntry {
/// Returns whether this discovery entry points to a modular manifest.
fn is_modular(&self) -> bool {
self.metadata_url.ends_with("manifest.json")
}
}
/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
///
/// Queries the configured snapshot API and returns the manifest URL for the most
/// recent modular snapshot matching the requested chain.
pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
let defaults = DownloadDefaults::get_global();
let api_url = &*defaults.snapshot_api_url;
info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
let entries = fetch_snapshot_api_entries(chain_id).await?;
let entry =
entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
eyre::eyre!(
"No modular snapshot manifest found for chain \
{chain_id} at {api_url}\n\n\
You can provide a manifest URL directly with --manifest-url, or\n\
use a direct snapshot URL with -u from:\n\
\t- {}\n\n\
Use --list to see all available snapshots.",
api_url.trim_end_matches("/api/snapshots"),
)
})?;
info!(target: "reth::cli",
block = entry.block,
url = %entry.metadata_url,
"Found latest snapshot manifest"
);
Ok(entry.metadata_url.clone())
}
/// Deserializes a JSON value that may be either a number or a string-encoded number.
fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let value = serde_json::Value::deserialize(deserializer)?;
match &value {
serde_json::Value::Number(n) => {
n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
}
serde_json::Value::String(s) => {
s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
}
_ => Err(serde::de::Error::custom("expected number or string")),
}
}
/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
let entries: Vec<SnapshotApiEntry> = Client::new()
.get(api_url)
.send()
.await
.and_then(|r| r.error_for_status())
.wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
.json()
.await?;
Ok(entries.into_iter().filter(|entry| entry.chain_id == chain_id).collect())
}
/// Prints a formatted table of available modular snapshots.
pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
println!(
"Available snapshots for chain {chain_id} ({}):\n",
api_url.trim_end_matches("/api/snapshots"),
);
println!("{:<12} {:>10} {:<10} {:>10} MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
println!("{}", "-".repeat(100));
for entry in &modular {
let date = entry.date.as_deref().unwrap_or("-");
let profile = entry.profile.as_deref().unwrap_or("-");
let size = if entry.size > 0 {
DownloadProgress::format_size(entry.size)
} else {
"-".to_string()
};
println!(
"{date:<12} {:>10} {profile:<10} {size:>10} {}",
entry.block, entry.metadata_url
);
}
if modular.is_empty() {
println!(" (no modular snapshots found)");
}
println!(
"\nTo download a specific snapshot, copy its manifest URL and run:\n \
reth download --manifest-url <URL>"
);
}
/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path.
pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
if let Ok(parsed) = Url::parse(source) {
return match parsed.scheme() {
"http" | "https" => {
let response = Client::new()
.get(source)
.send()
.await
.and_then(|r| r.error_for_status())
.wrap_err_with(|| {
let sources = DownloadDefaults::get_global()
.available_snapshots
.iter()
.map(|snapshot| format!("\t- {snapshot}"))
.collect::<Vec<_>>()
.join("\n");
format!(
"Failed to fetch snapshot manifest from {source}\n\n\
The manifest endpoint may not be available for this snapshot source.\n\
You can use a direct snapshot URL instead:\n\n\
\treth download -u <snapshot-url>\n\n\
Available snapshot sources:\n{sources}"
)
})?;
Ok(response.json().await?)
}
"file" => {
let path = parsed
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
let content = fs::read_to_string(path)?;
Ok(serde_json::from_str(&content)?)
}
_ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
};
}
let content = fs::read_to_string(source)?;
Ok(serde_json::from_str(&content)?)
}
/// Resolves the base URL used to join relative archive paths in a manifest.
pub(crate) fn resolve_manifest_base_url(
manifest: &SnapshotManifest,
source: &str,
) -> Result<String> {
if let Some(base_url) = manifest.base_url.as_deref() &&
!base_url.is_empty()
{
return Ok(base_url.trim_end_matches('/').to_string());
}
if let Ok(mut url) = Url::parse(source) {
if url.scheme() == "file" {
let mut path = url
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
path.pop();
let mut base = Url::from_directory_path(path)
.map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
.to_string();
if base.ends_with('/') {
base.pop();
}
return Ok(base);
}
{
let mut segments = url
.path_segments_mut()
.map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
segments.pop_if_empty();
segments.pop();
}
return Ok(url.as_str().trim_end_matches('/').to_string());
}
let path = Path::new(source);
let manifest_dir = if path.is_absolute() {
path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
} else {
let joined = std::env::current_dir()?.join(path);
joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
};
let mut base = Url::from_directory_path(&manifest_dir)
.map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
.to_string();
if base.ends_with('/') {
base.pop();
}
Ok(base)
}

View File

@@ -262,7 +262,6 @@ impl SelectorApp {
ComponentSelection::None => return 0,
ComponentSelection::All => None,
ComponentSelection::Distance(d) => Some(d),
ComponentSelection::Since(block) => Some(self.manifest.block - block + 1),
};
self.groups[group_idx]
.types
@@ -345,7 +344,6 @@ fn format_selection(sel: &ComponentSelection) -> String {
match sel {
ComponentSelection::All => "All".to_string(),
ComponentSelection::Distance(d) => format!("Last {d} blocks"),
ComponentSelection::Since(block) => format!("Since block {block}"),
ComponentSelection::None => "None".to_string(),
}
}

View File

@@ -1,84 +0,0 @@
use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress};
use blake3::Hasher;
use eyre::Result;
use reth_fs_util as fs;
use std::{io::Read, path::Path};
/// Verifies and cleans up extracted output files in one target directory.
pub(crate) struct OutputVerifier<'a> {
/// Directory containing the output files declared by the manifest.
target_dir: &'a Path,
}
impl<'a> OutputVerifier<'a> {
/// Creates a verifier for one extraction target directory.
pub(crate) const fn new(target_dir: &'a Path) -> Self {
Self { target_dir }
}
/// Returns `true` only when every declared output file exists and matches size and BLAKE3.
/// Returns `false` if any file is missing, mismatched, or no outputs were declared.
pub(crate) fn verify(&self, output_files: &[OutputFileChecksum]) -> Result<bool> {
self.verify_with_progress(output_files, None)
}
/// Returns `true` only when every declared output file exists and matches size and BLAKE3,
/// updating the optional verification progress as file bytes are hashed.
pub(crate) fn verify_with_progress(
&self,
output_files: &[OutputFileChecksum],
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
) -> Result<bool> {
if output_files.is_empty() {
return Ok(false);
}
for expected in output_files {
let output_path = self.target_dir.join(&expected.path);
let meta = match fs::metadata(&output_path) {
Ok(meta) => meta,
Err(_) => return Ok(false),
};
if meta.len() != expected.size {
return Ok(false);
}
let actual = Self::file_blake3_hex(&output_path, progress.as_deref_mut())?;
if !actual.eq_ignore_ascii_case(&expected.blake3) {
return Ok(false);
}
}
Ok(true)
}
/// Removes any declared output files so a fresh archive attempt can restart cleanly.
pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) {
for output in output_files {
let _ = fs::remove_file(self.target_dir.join(&output.path));
}
}
/// Computes the hex-encoded BLAKE3 checksum for one plain output file.
fn file_blake3_hex(
path: &Path,
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
) -> Result<String> {
let mut file = fs::open(path)?;
let mut hasher = Hasher::new();
let mut buf = [0_u8; 64 * 1024];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
if let Some(progress) = progress.as_deref_mut() {
progress.record_verified(n as u64);
}
}
Ok(hasher.finalize().to_hex().to_string())
}
}

View File

@@ -78,10 +78,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
self.env.init::<N>(AccessRights::RW, runtime)?;
let static_file_provider = provider_factory.static_file_provider();
let provider_rw = provider_factory.database_provider_rw()?;
if self.without_evm {
let provider_rw = provider_factory.database_provider_rw()?;
// ensure header, total difficulty and header hash are provided
let header = self.header.ok_or_else(|| eyre::eyre!("Header file must be provided"))?;
let header = without_evm::read_header_from_file::<
@@ -107,22 +106,23 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
// SAFETY: it's safe to commit static files, since in the event of a crash, they
// will be unwound according to database checkpoints.
//
// Necessary to commit, so the header is accessible to init_from_state_dump
// Necessary to commit, so the header is accessible to provider_rw and
// init_state_dump
static_file_provider.commit()?;
} else if last_block_number > 0 && last_block_number < header.number() {
return Err(eyre::eyre!(
"Data directory should be empty when calling init-state with --without-evm."
));
}
provider_rw.commit()?;
}
info!(target: "reth::cli", "Initiating state dump");
let reader = BufReader::new(reth_fs_util::open(self.state)?);
let hash = init_from_state_dump(reader, &provider_factory, config.stages.etl)?;
let hash = init_from_state_dump(reader, &provider_rw, config.stages.etl)?;
provider_rw.commit()?;
info!(target: "reth::cli", hash = ?hash, "Genesis block written");
Ok(())

View File

@@ -50,13 +50,8 @@ where
info!(target: "reth::cli", new_tip = ?header.num_hash(), "Setting up dummy EVM chain before importing state.");
let static_file_provider = provider_rw.static_file_provider();
// Write EVM dummy data up to `header - 1` block. Skip when the supplied
// header is at block 0: `header.number() - 1` would underflow in u64 to
// `u64::MAX`, sending `append_dummy_chain` into a 1..=u64::MAX loop that
// exhausts memory before failing.
if header.number() > 0 {
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
}
// Write EVM dummy data up to `header - 1` block
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
info!(target: "reth::cli", "Appending first valid block.");
@@ -196,13 +191,7 @@ mod tests {
use alloy_primitives::{address, b256};
use reth_db_common::init::init_genesis;
use reth_provider::{test_utils::create_test_provider_factory, DatabaseProviderFactory};
use std::{
io::Write,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
},
};
use std::io::Write;
use tempfile::NamedTempFile;
#[test]
@@ -275,45 +264,4 @@ mod tests {
assert_eq!(actual_next_height, expected_next_height);
}
/// Regression: a header at block 0 used to send `append_dummy_chain` into
/// a `1..=u64::MAX` loop because `header.number() - 1` underflowed in
/// u64. The guard `if header.number() > 0` skips the dummy-chain step
/// when there is no pre-genesis range to backfill, so `header_factory`
/// is never invoked.
#[test]
fn test_setup_without_evm_skips_dummy_chain_for_genesis_header() {
let header = Header { number: 0, ..Default::default() };
let header_hash = header.hash_slow();
let provider_factory = create_test_provider_factory();
init_genesis(&provider_factory).unwrap();
let provider_rw = provider_factory.database_provider_rw().unwrap();
let factory_calls = Arc::new(AtomicU64::new(0));
let factory_calls_inner = Arc::clone(&factory_calls);
// The Result of `setup_without_evm` itself is not asserted: with
// `number == 0` plus a genesis already written by `init_genesis`,
// the subsequent `append_first_block` may legitimately fail. The
// bug under test is the OOM in the dummy-chain loop, observable
// through the factory-call counter below.
let _ = setup_without_evm(
&provider_rw,
SealedHeader::new(header, header_hash),
move |number| {
// Bound calls so a regression cannot exhaust the test
// runner's memory; the only correct value here is 0.
let n = factory_calls_inner.fetch_add(1, Ordering::Relaxed);
assert!(n < 8, "header_factory must not be invoked for a genesis-block header");
Header { number, ..Default::default() }
},
);
assert_eq!(
factory_calls.load(Ordering::Relaxed),
0,
"append_dummy_chain must be skipped when header.number() == 0"
);
}
}

View File

@@ -188,7 +188,7 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
)
}
config.peers.trusted_nodes_only |= self.network.trusted_only;
config.peers.trusted_nodes_only = self.network.trusted_only;
let default_secret_key_path = data_dir.p2p_secret();
let p2p_secret_key = self.network.secret_key(default_secret_key_path)?;

View File

@@ -5,7 +5,6 @@ use crate::common::{
EnvironmentArgs,
};
use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt};
use alloy_primitives::{Address, B256, U256};
use clap::Parser;
use eyre::WrapErr;
use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
@@ -13,19 +12,14 @@ use reth_cli::chainspec::ChainSpecParser;
use reth_cli_util::cancellation::CancellationToken;
use reth_consensus::FullConsensus;
use reth_evm::{execute::Executor, ConfigureEvm};
use reth_primitives_traits::{format_gas_throughput, Account, BlockBody, GotExpected};
use reth_primitives_traits::{format_gas_throughput, BlockBody, GotExpected};
use reth_provider::{
BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider,
StaticFileProviderFactory, TransactionVariant,
};
use reth_revm::{
database::StateProviderDatabase,
db::{states::reverts::AccountInfoRevert, BundleState},
};
use reth_revm::database::StateProviderDatabase;
use reth_stages::stages::calculate_gas_used_from_headers;
use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader};
use std::{
collections::HashMap,
sync::{
atomic::{AtomicU64, Ordering},
Arc,
@@ -74,18 +68,13 @@ impl<C: ChainSpecParser> Command<C> {
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>> Command<C> {
/// Execute `re-execute` command
pub async fn execute<N>(
mut self,
self,
components: impl CliComponentsBuilder<N>,
runtime: reth_tasks::Runtime,
) -> eyre::Result<()>
where
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
{
// Default to 4GB RocksDB block cache for re-execute unless explicitly set.
if self.env.db.rocksdb_block_cache_size.is_none() {
self.env.db.rocksdb_block_cache_size = Some(4 << 30);
}
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO, runtime)?;
let components = components(provider_factory.chain_spec());
@@ -119,6 +108,15 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
min_block..=max_block,
)?;
let db_at = {
let provider_factory = provider_factory.clone();
move |block_number: u64| {
StateProviderDatabase(
provider_factory.history_by_block_number(block_number).unwrap(),
)
}
};
let skip_invalid_blocks = self.skip_invalid_blocks;
let blocks_per_chunk = self.blocks_per_chunk;
let (stats_tx, mut stats_rx) = mpsc::unbounded_channel();
@@ -134,23 +132,13 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let provider_factory = provider_factory.clone();
let evm_config = components.evm_config().clone();
let consensus = components.consensus().clone();
let db_at = db_at.clone();
let stats_tx = stats_tx.clone();
let info_tx = info_tx.clone();
let cancellation = cancellation.clone();
let next_block = Arc::clone(&next_block);
tasks.spawn_blocking(move || {
let executor_lifetime = Duration::from_secs(600);
let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety();
let db_at = {
|block_number: u64| {
StateProviderDatabase(
provider
.history_by_block_number(block_number)
.unwrap(),
)
}
};
let executor_lifetime = Duration::from_secs(120);
loop {
if cancellation.is_cancelled() {
@@ -257,31 +245,14 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let _ = stats_tx.send(block.gas_used());
// Reset DB once in a while to avoid OOM or read tx timeouts
if executor.size_hint() > 5_000_000 ||
if executor.size_hint() > 1_000_000 ||
executor_created.elapsed() > executor_lifetime
{
let last_block = block.number();
let old_executor = std::mem::replace(
&mut executor,
evm_config.batch_executor(db_at(last_block)),
);
let bundle = old_executor.into_state().take_bundle();
verify_bundle_against_changesets(
&provider,
&bundle,
last_block,
)?;
executor =
evm_config.batch_executor(db_at(block.number()));
executor_created = Instant::now();
}
}
// Full verification at chunk end for remaining unverified blocks
let bundle = executor.into_state().take_bundle();
verify_bundle_against_changesets(
&provider,
&bundle,
chunk_end - 1,
)?;
}
eyre::Ok(())
@@ -362,98 +333,3 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
Ok(())
}
}
/// Verifies reverts against database changesets.
///
/// For each block, reverts must match changeset entries exactly. No extra slots/accounts
/// in reverts for non-destroyed accounts. Destroyed accounts may have extra changeset slots
/// (from DB storage wipe) absent from reverts.
fn verify_bundle_against_changesets<P>(
provider: &P,
bundle: &BundleState,
last_block: u64,
) -> eyre::Result<()>
where
P: ChangeSetReader + StorageChangeSetReader,
{
// Verify reverts against changesets per block
for (i, block_reverts) in bundle.reverts.iter().rev().enumerate() {
let block_number = last_block - i as u64;
let mut cs_accounts: HashMap<Address, Option<Account>> = provider
.account_block_changeset(block_number)?
.into_iter()
.map(|cs| (cs.address, cs.info))
.collect();
let mut cs_storage: HashMap<Address, HashMap<B256, U256>> = HashMap::new();
for (bna, entry) in provider.storage_changeset(block_number)? {
cs_storage.entry(bna.address()).or_default().insert(entry.key, entry.value);
}
for (addr, revert) in block_reverts {
// Verify account info
match &revert.account {
AccountInfoRevert::DoNothing => {
eyre::ensure!(
!cs_accounts.contains_key(addr),
"Block {block_number}: account {addr} in changeset but revert is DoNothing",
);
}
AccountInfoRevert::DeleteIt => {
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
eyre::eyre!("Block {block_number}: account {addr} revert is DeleteIt but not in changeset")
})?;
eyre::ensure!(
cs_info.is_none(),
"Block {block_number}: account {addr} revert is DeleteIt but changeset has {cs_info:?}",
);
}
AccountInfoRevert::RevertTo(info) => {
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
eyre::eyre!("Block {block_number}: account {addr} revert is RevertTo but not in changeset")
})?;
let revert_acct = Some(Account::from(info));
eyre::ensure!(
revert_acct == cs_info,
"Block {block_number}: account {addr} info mismatch: revert={revert_acct:?} cs={cs_info:?}",
);
}
}
// Verify storage slots — remove matched changeset entries as we go
let mut cs_slots = cs_storage.get_mut(addr);
for (slot_key, revert_slot) in &revert.storage {
let b256_key = B256::from(*slot_key);
match cs_slots.as_mut().and_then(|s| s.remove(&b256_key)) {
Some(cs_value) => eyre::ensure!(
revert_slot.to_previous_value() == cs_value,
"Block {block_number}: {addr} slot {b256_key} mismatch: \
revert={} cs={cs_value}",
revert_slot.to_previous_value(),
),
None => eyre::ensure!(
revert.wipe_storage,
"Block {block_number}: {addr} slot {b256_key} in reverts but not in changeset",
),
}
}
// Any remaining cs_storage slots for this address must be from a destroyed account
if let Some(remaining) = cs_slots.filter(|s| !s.is_empty()) {
eyre::ensure!(
revert.wipe_storage,
"Block {block_number}: {addr} has {} unmatched storage slots in changeset",
remaining.len(),
);
}
}
// Any remaining cs_accounts entries had no corresponding revert
if let Some(addr) = cs_accounts.keys().next() {
eyre::bail!("Block {block_number}: account {addr} in changeset but not in reverts");
}
}
Ok(())
}

View File

@@ -6,7 +6,7 @@ use reth_db_api::{
};
use reth_db_common::DbTool;
use reth_evm::ConfigureEvm;
use reth_node_api::{HeaderTy, TxTy};
use reth_node_api::HeaderTy;
use reth_node_core::dirs::{ChainPath, DataDirPath};
use reth_provider::{
providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
@@ -88,7 +88,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
)
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::BlockOmmers<HeaderTy<N>>, _>(
tx.import_table_with_range::<tables::BlockOmmers, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from),
to,
@@ -110,7 +110,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
})??;
output_db.update(|tx| {
tx.import_table_with_range::<tables::Transactions<TxTy<N>>, _>(
tx.import_table_with_range::<tables::Transactions, _>(
&db_tool.provider_factory.db_ref().tx()?,
Some(from_tx),
to_tx,

View File

@@ -29,10 +29,7 @@ use execution::dump_execution_stage;
mod merkle;
use merkle::dump_merkle_stage;
/// `reth dump-stage` command.
///
/// Note: mutates the source datadir (unwinds hashing/merkle/execution before copying tables).
/// Stop the node and back up the datadir first.
/// `reth dump-stage` command
#[derive(Debug, Parser)]
pub struct Command<C: ChainSpecParser> {
#[command(flatten)]
@@ -103,9 +100,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
Comp: CliNodeComponents<N>,
F: FnOnce(Arc<C::ChainSpec>) -> Comp,
{
// `unwind_and_copy` opens a RW provider on the source datadir, so open RW here.
let Environment { provider_factory, .. } =
self.env.init::<N>(AccessRights::RW, runtime.clone())?;
self.env.init::<N>(AccessRights::RO, runtime.clone())?;
let tool = DbTool::new(provider_factory)?;
let components = components(tool.chain());
let evm_config = components.evm_config().clone();

View File

@@ -210,7 +210,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let consensus = Arc::new(components.consensus().clone());
let mut config = config;
config.peers.trusted_nodes_only |= self.network.trusted_only;
config.peers.trusted_nodes_only = self.network.trusted_only;
config.peers.trusted_nodes.extend(self.network.trusted_peers.clone());
let network_secret_path = self

View File

@@ -30,16 +30,10 @@
extern crate alloc;
use alloc::{
boxed::Box,
fmt::Debug,
string::{String, ToString},
sync::Arc,
vec::Vec,
};
use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
use alloy_consensus::Header;
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
use core::{error::Error, fmt::Display};
use core::error::Error;
/// Pre-computed receipt root and logs bloom.
///
@@ -110,18 +104,6 @@ pub trait Consensus<B: Block>: HeaderValidator<B::Header> {
/// Note: validating blocks does not include other validations of the Consensus
fn validate_block_pre_execution(&self, block: &SealedBlock<B>) -> Result<(), ConsensusError>;
/// Returns `true` if the given consensus error is transient and may resolve on its own.
///
/// On fast chains, clock skew between nodes can cause a valid block's timestamp to
/// appear briefly in the future. Caching such blocks as permanently invalid would
/// prevent them from being re-validated once the local clock catches up.
///
/// Transient errors will not cause the block hash to be cached as permanently invalid,
/// allowing the block to be re-validated later.
fn is_transient_error(&self, _error: &ConsensusError) -> bool {
false
}
/// Validate a block disregarding world state using an optional pre-computed transaction root.
///
/// If `transaction_root` is provided, the implementation should use the pre-computed
@@ -474,49 +456,19 @@ pub enum ConsensusError {
/// EIP-7825: Transaction gas limit exceeds maximum allowed
#[error(transparent)]
TransactionGasLimitTooHigh(Box<TxGasLimitTooHighErr>),
/// Any additional consensus error, for example L2-specific errors.
/// Other, likely an injected L2 error.
#[error("{0}")]
Other(String),
/// Other unspecified error.
#[error(transparent)]
Other(#[from] Arc<dyn Error + Send + Sync>),
Custom(#[from] Arc<dyn Error + Send + Sync>),
}
impl ConsensusError {
/// Returns a new [`ConsensusError::Other`] instance with the given error.
pub fn other<E>(error: E) -> Self
where
E: Error + Send + Sync + 'static,
{
Self::Other(Arc::new(error))
}
/// Returns a new [`ConsensusError::Other`] instance with the given message.
pub fn msg(msg: impl Display) -> Self {
Self::other(MessageError(msg.to_string()))
}
/// Returns `true` if the error is a state root error.
pub const fn is_state_root_error(&self) -> bool {
matches!(self, Self::BodyStateRootDiff(_))
}
/// Returns the arbitrary error if it is [`ConsensusError::Other`].
pub fn as_other(&self) -> Option<&(dyn Error + Send + Sync + 'static)> {
match self {
Self::Other(err) => Some(err.as_ref()),
_ => None,
}
}
/// Returns a reference to the [`ConsensusError::Other`] value if it is of that type.
/// Returns `None` otherwise.
pub fn downcast_other_ref<T: Error + 'static>(&self) -> Option<&T> {
let other = self.as_other()?;
other.downcast_ref()
}
/// Returns `true` if this type is a [`ConsensusError::Other`] of that error type.
pub fn is_other<T: Error + 'static>(&self) -> bool {
self.as_other().map(|err| err.is::<T>()).unwrap_or(false)
}
}
impl From<InvalidTransactionError> for ConsensusError {
@@ -548,10 +500,6 @@ pub struct TxGasLimitTooHighErr {
pub max_allowed: u64,
}
#[derive(Debug, thiserror::Error)]
#[error("{0}")]
struct MessageError(String);
#[cfg(test)]
mod tests {
use super::*;
@@ -561,31 +509,24 @@ mod tests {
struct CustomL2Error;
#[test]
fn test_other_error_conversion() {
let consensus_err = ConsensusError::other(CustomL2Error);
assert!(matches!(consensus_err, ConsensusError::Other(_)));
fn test_custom_error_conversion() {
// Test conversion from custom error to ConsensusError
let custom_err = CustomL2Error;
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
let consensus_err: ConsensusError = arc_err.into();
// Verify it's the Custom variant
assert!(matches!(consensus_err, ConsensusError::Custom(_)));
}
#[test]
fn test_other_error_display() {
let consensus_err = ConsensusError::other(CustomL2Error);
fn test_custom_error_display() {
let custom_err = CustomL2Error;
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
let consensus_err: ConsensusError = arc_err.into();
// Verify the error message is preserved through transparent attribute
let error_message = format!("{}", consensus_err);
assert_eq!(error_message, "Custom L2 consensus error");
}
#[test]
fn test_other_error_downcast() {
let consensus_err = ConsensusError::other(CustomL2Error);
assert!(consensus_err.is_other::<CustomL2Error>());
assert!(consensus_err.downcast_other_ref::<CustomL2Error>().is_some());
}
#[test]
fn test_other_msg() {
let consensus_err = ConsensusError::msg("consensus message");
assert_eq!(consensus_err.to_string(), "consensus message");
assert!(consensus_err.downcast_other_ref::<MessageError>().is_some());
}
}

View File

@@ -19,7 +19,6 @@ use reth_trie::{
MultiProofTargets, StorageMultiProof, StorageProof, TrieInput,
};
use std::{
fmt,
sync::{
atomic::{AtomicU64, AtomicUsize, Ordering},
Arc,
@@ -148,29 +147,6 @@ pub enum CachedStatus<T> {
Cached(T),
}
/// The source that is using the execution cache.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum CachedStateMetricsSource {
/// Engine (validation).
Engine,
/// Payload builder.
Builder,
/// Tests.
#[cfg(any(test, feature = "test-utils"))]
Test,
}
impl fmt::Display for CachedStateMetricsSource {
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
match self {
Self::Engine => f.write_str("engine"),
Self::Builder => f.write_str("builder"),
#[cfg(any(test, feature = "test-utils"))]
Self::Test => f.write_str("test"),
}
}
}
/// Metrics for the cached state provider, showing hits / misses for each cache
#[derive(Metrics, Clone)]
#[metrics(scope = "sync.caching")]
@@ -246,10 +222,9 @@ impl CachedStateMetrics {
self.account_cache_collisions.set(0);
}
/// Returns a new zeroed-out instance of [`CachedStateMetrics`] with a `source` label
/// to distinguish between different callers (e.g., engine vs builder).
pub fn zeroed(source: CachedStateMetricsSource) -> Self {
let zeroed = Self::new_with_labels(&[("source", source.to_string())]);
/// Returns a new zeroed-out instance of [`CachedStateMetrics`].
pub fn zeroed() -> Self {
let zeroed = Self::default();
zeroed.reset();
zeroed
}
@@ -944,15 +919,27 @@ pub struct SavedCache {
/// The caches used for the provider.
caches: ExecutionCache,
/// Metrics for the cached state provider (includes size/capacity/collisions from fixed-cache)
metrics: CachedStateMetrics,
/// A guard to track in-flight usage of this cache.
/// The cache is considered available if the strong count is 1.
usage_guard: Arc<()>,
/// Whether to skip cache metrics recording (can be expensive with large cached state).
disable_cache_metrics: bool,
}
impl SavedCache {
/// Creates a new instance with the internals
pub fn new(hash: B256, caches: ExecutionCache) -> Self {
Self { hash, caches, usage_guard: Arc::new(()) }
pub fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
}
/// Sets whether to disable cache metrics recording.
pub const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
self.disable_cache_metrics = disable;
self
}
/// Returns the hash for this cache
@@ -960,6 +947,11 @@ impl SavedCache {
self.hash
}
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
pub fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
(self.caches, self.metrics, self.disable_cache_metrics)
}
/// Returns true if the cache is available for use (no other tasks are currently using it).
pub fn is_available(&self) -> bool {
Arc::strong_count(&self.usage_guard) == 1
@@ -975,11 +967,20 @@ impl SavedCache {
&self.caches
}
/// Returns the metrics associated with this cache.
pub const fn metrics(&self) -> &CachedStateMetrics {
&self.metrics
}
/// Updates the cache metrics (size/capacity/collisions) from the stats handlers.
pub fn update_metrics(&self, metrics: Option<&CachedStateMetrics>) {
if let Some(metrics) = metrics {
self.caches.update_metrics(metrics);
///
/// Note: This can be expensive with large cached state. Use
/// `with_disable_cache_metrics(true)` to skip.
pub fn update_metrics(&self) {
if self.disable_cache_metrics {
return
}
self.caches.update_metrics(&self.metrics);
}
/// Clears all caches, resetting them to empty state,
@@ -1016,11 +1017,8 @@ mod tests {
provider.extend_accounts(vec![(address, account)]);
let caches = ExecutionCache::new(1000);
let state_provider = CachedStateProvider::new(
provider,
caches,
CachedStateMetrics::zeroed(CachedStateMetricsSource::Test),
);
let state_provider =
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
let res = state_provider.storage(address, storage_key);
assert!(res.is_ok());
@@ -1039,11 +1037,8 @@ mod tests {
provider.extend_accounts(vec![(address, account)]);
let caches = ExecutionCache::new(1000);
let state_provider = CachedStateProvider::new(
provider,
caches,
CachedStateMetrics::zeroed(CachedStateMetricsSource::Test),
);
let state_provider =
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
let res = state_provider.storage(address, storage_key);
assert!(res.is_ok());
@@ -1080,7 +1075,7 @@ mod tests {
#[test]
fn test_saved_cache_is_available() {
let execution_cache = ExecutionCache::new(1000);
let cache = SavedCache::new(B256::ZERO, execution_cache);
let cache = SavedCache::new(B256::ZERO, execution_cache, CachedStateMetrics::zeroed());
assert!(cache.is_available(), "Cache should be available initially");
@@ -1092,7 +1087,8 @@ mod tests {
#[test]
fn test_saved_cache_multiple_references() {
let execution_cache = ExecutionCache::new(1000);
let cache = SavedCache::new(B256::from([2u8; 32]), execution_cache);
let cache =
SavedCache::new(B256::from([2u8; 32]), execution_cache, CachedStateMetrics::zeroed());
let guard1 = cache.clone_guard_for_test();
let guard2 = cache.clone_guard_for_test();

View File

@@ -165,7 +165,11 @@ mod tests {
let hash = B256::from([1u8; 32]);
cache.update_with_guard(|slot| {
*slot = Some(SavedCache::new(hash, ExecutionCache::new(1_000)))
*slot = Some(SavedCache::new(
hash,
ExecutionCache::new(1_000),
CachedStateMetrics::zeroed(),
))
});
let first = cache.get_cache_for(hash);
@@ -181,7 +185,11 @@ mod tests {
let hash = B256::from([2u8; 32]);
cache.update_with_guard(|slot| {
*slot = Some(SavedCache::new(hash, ExecutionCache::new(1_000)))
*slot = Some(SavedCache::new(
hash,
ExecutionCache::new(1_000),
CachedStateMetrics::zeroed(),
))
});
let checked_out = cache.get_cache_for(hash);
@@ -199,7 +207,11 @@ mod tests {
let hash_b = B256::from([0xBB; 32]);
cache.update_with_guard(|slot| {
*slot = Some(SavedCache::new(hash_a, ExecutionCache::new(1_000)))
*slot = Some(SavedCache::new(
hash_a,
ExecutionCache::new(1_000),
CachedStateMetrics::zeroed(),
))
});
let checked_out = cache.get_cache_for(hash_b);

View File

@@ -141,11 +141,6 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
last_header: SealedHeaderFor<<T::BuiltPayload as BuiltPayload>::Primitives>,
/// Stores latest mined blocks.
last_block_hashes: VecDeque<B256>,
/// Optional sleep duration between initiating payload building and resolving.
///
/// When set, the miner sleeps after `fork_choice_updated` before calling
/// `resolve_kind`, giving the payload job time for multiple rebuild attempts.
payload_wait_time: Option<Duration>,
}
impl<T, B, Pool> LocalMiner<T, B, Pool>
@@ -175,16 +170,9 @@ where
payload_builder,
last_block_hashes: VecDeque::from([last_header.hash()]),
last_header,
payload_wait_time: None,
}
}
/// Sets the payload wait time, if any.
pub const fn with_payload_wait_time_opt(mut self, wait_time: Option<Duration>) -> Self {
self.payload_wait_time = wait_time;
self
}
/// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads.
pub async fn run(mut self) {
let mut fcu_interval = tokio::time::interval(Duration::from_secs(1));
@@ -250,10 +238,6 @@ where
let payload_id = res.payload_id.ok_or_eyre("No payload id")?;
if let Some(wait_time) = self.payload_wait_time {
tokio::time::sleep(wait_time).await;
}
let Some(Ok(payload)) =
self.payload_builder.resolve_kind(payload_id, PayloadKind::WaitForPending).await
else {

View File

@@ -57,7 +57,7 @@ where
.chain_spec
.is_cancun_active_at_timestamp(timestamp)
.then(B256::random),
slot_number: self.chain_spec.is_amsterdam_active_at_timestamp(timestamp).then_some(0),
slot_number: None,
}
}
}

View File

@@ -4,20 +4,17 @@ use alloy_eips::merge::EPOCH_SLOTS;
use core::time::Duration;
/// Triggers persistence when the number of canonical blocks in memory exceeds this threshold.
pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2;
pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 12;
/// Maximum canonical-minus-persisted gap before engine API processing is stalled.
pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16;
pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 44;
/// How close to the canonical head we persist blocks.
pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0;
pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 10;
/// The size of proof targets chunk to spawn in one multiproof calculation.
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 5;
/// Default number of cache hits before an invalid header entry is evicted and reprocessed.
pub const DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
/// Gas threshold below which the small block chunk size is used.
pub const SMALL_BLOCK_GAS_THRESHOLD: u64 = 20_000_000;
@@ -105,11 +102,6 @@ pub struct TreeConfig {
block_buffer_limit: u32,
/// Number of invalid headers to keep in cache.
max_invalid_header_cache_length: u32,
/// Number of cache hits before an invalid header entry is evicted and reprocessed.
///
/// Setting this to `0` effectively disables the cache because entries are evicted on the
/// first lookup.
invalid_header_hit_eviction_threshold: u8,
/// Maximum number of blocks to execute sequentially in a batch.
///
/// This is used as a cutoff to prevent long-running sequential block execution when we receive
@@ -178,23 +170,6 @@ pub struct TreeConfig {
share_execution_cache_with_payload_builder: bool,
/// Whether to share sparse trie with the payload builder.
share_sparse_trie_with_payload_builder: bool,
/// Whether to suppress persistence cycles while building a payload.
///
/// When enabled, persistence is deferred from the moment an FCU with payload attributes
/// arrives until the next FCU without attributes. This avoids persistence I/O competing
/// with block building on latency-sensitive chains.
suppress_persistence_during_build: bool,
/// Whether to disable BAL (Block Access List, EIP-7928) based parallel execution.
/// When disabled, falls back to transaction-based prewarming even when a BAL is available.
disable_bal_parallel_execution: bool,
/// Whether to disable BAL-driven parallel state root computation.
/// When disabled, the BAL hashed post state is not sent to the multiproof task for
/// early parallel state root computation.
disable_bal_parallel_state_root: bool,
/// Whether to disable BAL (Block Access List) batched IO during prewarming.
/// When disabled, falls back to individual per-slot storage reads instead of
/// batched cursor reads via `storage_range`.
disable_bal_batch_io: bool,
/// Maximum random jitter applied before each proof computation (trie-debug only).
/// When set, each proof worker sleeps for a random duration up to this value
/// before starting a proof calculation.
@@ -214,7 +189,6 @@ impl Default for TreeConfig {
persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT,
max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH,
invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
max_execute_block_batch_size: DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE,
legacy_state_root: false,
always_compare_trie_updates: false,
@@ -238,10 +212,6 @@ impl Default for TreeConfig {
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
share_execution_cache_with_payload_builder: false,
share_sparse_trie_with_payload_builder: false,
suppress_persistence_during_build: false,
disable_bal_parallel_execution: false,
disable_bal_parallel_state_root: false,
disable_bal_batch_io: false,
#[cfg(feature = "trie-debug")]
proof_jitter: None,
}
@@ -257,7 +227,6 @@ impl TreeConfig {
persistence_backpressure_threshold: u64,
block_buffer_limit: u32,
max_invalid_header_cache_length: u32,
invalid_header_hit_eviction_threshold: u8,
max_execute_block_batch_size: usize,
legacy_state_root: bool,
always_compare_trie_updates: bool,
@@ -291,7 +260,6 @@ impl TreeConfig {
persistence_backpressure_threshold,
block_buffer_limit,
max_invalid_header_cache_length,
invalid_header_hit_eviction_threshold,
max_execute_block_batch_size,
legacy_state_root,
always_compare_trie_updates,
@@ -315,10 +283,6 @@ impl TreeConfig {
state_root_task_timeout,
share_execution_cache_with_payload_builder,
share_sparse_trie_with_payload_builder,
suppress_persistence_during_build: false,
disable_bal_parallel_execution: false,
disable_bal_parallel_state_root: false,
disable_bal_batch_io: false,
#[cfg(feature = "trie-debug")]
proof_jitter: None,
}
@@ -349,14 +313,6 @@ impl TreeConfig {
self.max_invalid_header_cache_length
}
/// Return the invalid header cache hit eviction threshold.
///
/// Setting this to `0` effectively disables the cache because entries are evicted on the
/// first lookup.
pub const fn invalid_header_hit_eviction_threshold(&self) -> u8 {
self.invalid_header_hit_eviction_threshold
}
/// Return the maximum execute block batch size.
pub const fn max_execute_block_batch_size(&self) -> usize {
self.max_execute_block_batch_size
@@ -487,15 +443,6 @@ impl TreeConfig {
self
}
/// Setter for the invalid header cache hit eviction threshold.
pub const fn with_invalid_header_hit_eviction_threshold(
mut self,
invalid_header_hit_eviction_threshold: u8,
) -> Self {
self.invalid_header_hit_eviction_threshold = invalid_header_hit_eviction_threshold;
self
}
/// Setter for maximum execute block batch size.
pub const fn with_max_execute_block_batch_size(
mut self,
@@ -699,56 +646,6 @@ impl TreeConfig {
self
}
/// Returns whether persistence is suppressed during payload building.
pub const fn suppress_persistence_during_build(&self) -> bool {
self.suppress_persistence_during_build
}
/// Setter for whether to suppress persistence during payload building.
pub const fn with_suppress_persistence_during_build(mut self, value: bool) -> Self {
self.suppress_persistence_during_build = value;
self
}
/// Returns whether BAL-based parallel execution is disabled.
pub const fn disable_bal_parallel_execution(&self) -> bool {
self.disable_bal_parallel_execution
}
/// Setter for whether to disable BAL-based parallel execution.
pub const fn without_bal_parallel_execution(
mut self,
disable_bal_parallel_execution: bool,
) -> Self {
self.disable_bal_parallel_execution = disable_bal_parallel_execution;
self
}
/// Returns whether BAL-driven parallel state root computation is disabled.
pub const fn disable_bal_parallel_state_root(&self) -> bool {
self.disable_bal_parallel_state_root
}
/// Setter for whether to disable BAL-driven parallel state root computation.
pub const fn without_bal_parallel_state_root(
mut self,
disable_bal_parallel_state_root: bool,
) -> Self {
self.disable_bal_parallel_state_root = disable_bal_parallel_state_root;
self
}
/// Returns whether BAL batched IO is disabled.
pub const fn disable_bal_batch_io(&self) -> bool {
self.disable_bal_batch_io
}
/// Setter for whether to disable BAL batched IO.
pub const fn without_bal_batch_io(mut self, disable_bal_batch_io: bool) -> Self {
self.disable_bal_batch_io = disable_bal_batch_io;
self
}
/// Returns the proof jitter duration, if configured (trie-debug only).
#[cfg(feature = "trie-debug")]
pub const fn proof_jitter(&self) -> Option<Duration> {

View File

@@ -102,8 +102,8 @@ where
self.sync_metrics_tx.send(MetricEvent::SyncHeight { height: new_tip_num });
let _ = sender.send(PersistenceResult { last_block, commit_duration: None });
}
PersistenceAction::SaveBlocks(blocks, sender) => {
let result = self.on_save_blocks(blocks)?;
PersistenceAction::SaveBlocks(batch, sender) => {
let result = self.on_save_blocks(batch)?;
let result_number = result.last_block.map(|b| b.number);
let _ = sender.send(result);
@@ -144,14 +144,18 @@ where
Ok(new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num }))
}
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = blocks.len()))]
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = batch.blocks.len()))]
fn on_save_blocks(
&mut self,
blocks: Vec<ExecutedBlock<N::Primitives>>,
batch: SaveBlocksBatch<N::Primitives>,
) -> Result<PersistenceResult, PersistenceError> {
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
let block_count = blocks.len();
let SaveBlocksBatch { blocks, persist_count } = batch;
let (blocks_to_persist, buffer_blocks) = blocks.split_at(persist_count);
let blocks_to_persist = blocks_to_persist.to_vec();
let first_block = blocks_to_persist.first().map(|b| b.recovered_block.num_hash());
let last_block = blocks_to_persist.last().map(|b| b.recovered_block.num_hash());
let block_count = blocks_to_persist.len();
let pending_finalized = self.pending_finalized_block.take();
let pending_safe = self.pending_safe_block.take();
@@ -162,7 +166,7 @@ where
if let Some(last) = last_block {
let provider_rw = self.provider.database_provider_rw()?;
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
provider_rw.save_blocks(blocks_to_persist, SaveBlocksMode::Full, buffer_blocks)?;
if let Some(finalized) = pending_finalized {
provider_rw.save_finalized_block_number(finalized.min(last.number))?;
@@ -216,6 +220,28 @@ pub enum PersistenceError {
ProviderError(#[from] ProviderError),
}
/// A batch of blocks passed to the persistence service.
///
/// Contains all canonical blocks collected during a persistence cycle. Only the first
/// `persist_count` blocks are written to disk; the remaining blocks are included for
/// downstream consumers but are not persisted yet.
#[derive(Debug, Clone, PartialEq)]
pub struct SaveBlocksBatch<N: NodePrimitives = EthPrimitives> {
/// All blocks in the batch, ordered oldest to newest.
pub blocks: Vec<ExecutedBlock<N>>,
/// Number of leading blocks to actually persist. The blocks at indices
/// `[persist_count..]` are passed through but not written to storage.
pub persist_count: usize,
}
impl<N: NodePrimitives> SaveBlocksBatch<N> {
/// Creates a new batch where all blocks should be persisted.
pub fn persist_all(blocks: Vec<ExecutedBlock<N>>) -> Self {
let persist_count = blocks.len();
Self { blocks, persist_count }
}
}
/// A signal to the persistence service that part of the tree state can be persisted.
#[derive(Debug)]
pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
@@ -224,7 +250,7 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
///
/// First, header, transaction, and receipt-related data should be written to static files.
/// Then the execution history-related data will be written to the database.
SaveBlocks(Vec<ExecutedBlock<N>>, CrossbeamSender<PersistenceResult>),
SaveBlocks(SaveBlocksBatch<N>, CrossbeamSender<PersistenceResult>),
/// Removes block data above the given block number from the database.
///
@@ -308,10 +334,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
/// If there are no blocks to persist, then `None` is sent in the sender.
pub fn save_blocks(
&self,
blocks: Vec<ExecutedBlock<T>>,
batch: SaveBlocksBatch<T>,
tx: CrossbeamSender<PersistenceResult>,
) -> Result<(), SendError<PersistenceAction<T>>> {
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
self.send_action(PersistenceAction::SaveBlocks(batch, tx))
}
/// Queues the finalized block number to be persisted on disk.
@@ -407,7 +433,7 @@ mod tests {
let blocks = vec![];
let (tx, rx) = crossbeam_channel::bounded(1);
handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
let result = rx.recv().unwrap();
assert!(result.last_block.is_none());
@@ -426,7 +452,7 @@ mod tests {
let blocks = vec![executed];
let (tx, rx) = crossbeam_channel::bounded(1);
handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out");
@@ -443,7 +469,7 @@ mod tests {
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
let result = rx.recv().unwrap();
assert_eq!(last_hash, result.last_block.unwrap().hash);
}
@@ -460,7 +486,7 @@ mod tests {
let last_hash = blocks.last().unwrap().recovered_block().hash();
let (tx, rx) = crossbeam_channel::bounded(1);
handle.save_blocks(blocks, tx).unwrap();
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
let result = rx.recv().unwrap();
assert_eq!(last_hash, result.last_block.unwrap().hash);
@@ -555,7 +581,7 @@ mod tests {
{
let provider_rw = provider_factory.database_provider_rw().unwrap();
provider_rw.save_blocks(blocks_a, SaveBlocksMode::Full).unwrap();
provider_rw.save_blocks(blocks_a, SaveBlocksMode::Full, &[]).unwrap();
provider_rw.commit().unwrap();
}
@@ -612,7 +638,7 @@ mod tests {
provider_rw.commit().unwrap();
let provider_rw = pf.database_provider_rw().unwrap();
provider_rw.save_blocks(vec![block_b2], SaveBlocksMode::Full).unwrap();
provider_rw.save_blocks(vec![block_b2], SaveBlocksMode::Full, &[]).unwrap();
provider_rw.commit().unwrap();
});

View File

@@ -8,28 +8,25 @@ use schnellru::{ByLength, LruMap};
use std::fmt::Debug;
use tracing::warn;
/// The max hit counter for invalid headers in the cache before it is forcefully evicted.
///
/// In other words, if a header is referenced more than this number of times, it will be evicted to
/// allow for reprocessing.
const INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
/// Keeps track of invalid headers.
#[derive(Debug)]
pub struct InvalidHeaderCache {
/// This maps a header hash to a reference to its invalid ancestor.
headers: LruMap<B256, HeaderEntry>,
/// Number of cache hits before an invalid header entry is evicted and reprocessed.
hit_eviction_threshold: u8,
/// Metrics for the cache.
metrics: InvalidHeaderCacheMetrics,
}
impl InvalidHeaderCache {
/// Invalid header cache constructor.
///
/// Setting `hit_eviction_threshold` to `0` effectively disables the cache because entries are
/// evicted on the first lookup.
pub fn new(max_length: u32, hit_eviction_threshold: u8) -> Self {
Self {
headers: LruMap::new(ByLength::new(max_length)),
hit_eviction_threshold,
metrics: Default::default(),
}
pub fn new(max_length: u32) -> Self {
Self { headers: LruMap::new(ByLength::new(max_length)), metrics: Default::default() }
}
fn insert_entry(&mut self, hash: B256, header: BlockWithParent) {
@@ -44,7 +41,7 @@ impl InvalidHeaderCache {
{
let entry = self.headers.get(hash)?;
entry.hit_count += 1;
if entry.hit_count < self.hit_eviction_threshold {
if entry.hit_count < INVALID_HEADER_HIT_EVICTION_THRESHOLD {
return Some(entry.header)
}
}
@@ -113,28 +110,17 @@ mod tests {
#[test]
fn test_hit_eviction() {
let hit_eviction_threshold = 3;
let mut cache = InvalidHeaderCache::new(10, hit_eviction_threshold);
let mut cache = InvalidHeaderCache::new(10);
let header = Header::default();
let header = SealedHeader::seal_slow(header);
cache.insert(header.block_with_parent());
assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, 0);
for hit in 1..hit_eviction_threshold {
for hit in 1..INVALID_HEADER_HIT_EVICTION_THRESHOLD {
assert!(cache.get(&header.hash()).is_some());
assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, hit);
}
assert!(cache.get(&header.hash()).is_none());
}
#[test]
fn test_zero_hit_eviction_threshold_effectively_disables_cache() {
let mut cache = InvalidHeaderCache::new(10, 0);
let header = SealedHeader::seal_slow(Header::default());
cache.insert(header.block_with_parent());
assert!(cache.get(&header.hash()).is_none());
assert_eq!(cache.headers.len(), 0);
}
}

View File

@@ -2,7 +2,7 @@ use crate::{
backfill::{BackfillAction, BackfillSyncState},
chain::FromOrchestrator,
engine::{DownloadRequest, EngineApiEvent, EngineApiKind, EngineApiRequest, FromEngine},
persistence::PersistenceHandle,
persistence::{PersistenceHandle, SaveBlocksBatch},
tree::{error::InsertPayloadError, payload_validator::TreeCtx},
};
use alloy_consensus::BlockHeader;
@@ -11,7 +11,7 @@ use alloy_primitives::B256;
use alloy_rpc_types_engine::{
ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError,
};
use error::{InsertBlockError, InsertBlockFatalError, InsertBlockValidationError};
use error::{InsertBlockError, InsertBlockFatalError};
use reth_chain_state::{
CanonicalInMemoryState, ComputedTrieData, ExecutedBlock, ExecutionTimingStats,
MemoryOverlayStateProvider, NewCanonicalChain,
@@ -71,8 +71,7 @@ pub use payload_validator::{BasicEngineValidator, EngineValidator};
pub use persistence_state::PersistenceState;
pub use reth_engine_primitives::TreeConfig;
pub use reth_execution_cache::{
CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider, ExecutionCache,
PayloadExecutionCache, SavedCache,
CachedStateMetrics, CachedStateProvider, ExecutionCache, PayloadExecutionCache, SavedCache,
};
pub mod state;
@@ -151,15 +150,11 @@ impl<N: NodePrimitives> EngineApiTreeState<N> {
fn new(
block_buffer_limit: u32,
max_invalid_header_cache_length: u32,
invalid_header_hit_eviction_threshold: u8,
canonical_block: BlockNumHash,
engine_kind: EngineApiKind,
) -> Self {
Self {
invalid_headers: InvalidHeaderCache::new(
max_invalid_header_cache_length,
invalid_header_hit_eviction_threshold,
),
invalid_headers: InvalidHeaderCache::new(max_invalid_header_cache_length),
buffer: BlockBuffer::new(block_buffer_limit),
tree_state: TreeState::new(canonical_block, engine_kind),
forkchoice_state_tracker: ForkchoiceStateTracker::default(),
@@ -309,9 +304,6 @@ where
/// Stored here (not in `ExecutedBlock`) to avoid leaking observability concerns into the block
/// type. Entries are removed when blocks are persisted or invalidated.
execution_timing_stats: HashMap<B256, Box<ExecutionTimingStats>>,
/// Set when an FCU with payload attributes is received, cleared on the next FCU without.
/// Suppresses persistence cycles during payload building.
building_payload: bool,
/// Task runtime for spawning blocking work on named, reusable threads.
runtime: reth_tasks::Runtime,
}
@@ -403,7 +395,6 @@ where
evm_config,
changeset_cache,
execution_timing_stats: HashMap::new(),
building_payload: false,
runtime,
}
}
@@ -440,7 +431,6 @@ where
let state = EngineApiTreeState::new(
config.block_buffer_limit(),
config.max_invalid_header_cache_length(),
config.invalid_header_hit_eviction_threshold(),
header.num_hash(),
kind,
);
@@ -1121,8 +1111,6 @@ where
) -> ProviderResult<TreeOutcome<OnForkChoiceUpdated>> {
trace!(target: "engine::tree", ?attrs, "invoked forkchoice update");
self.building_payload = attrs.is_some() && self.config.suppress_persistence_during_build();
// Record metrics
self.record_forkchoice_metrics();
@@ -1361,22 +1349,25 @@ where
/// Helper method to save blocks and set the persistence state. This ensures we keep track of
/// the current persistence action while we're saving blocks.
fn persist_blocks(&mut self, blocks_to_persist: Vec<ExecutedBlock<N>>) {
if blocks_to_persist.is_empty() {
fn persist_blocks(&mut self, batch: SaveBlocksBatch<N>) {
if batch.blocks.is_empty() {
debug!(target: "engine::tree", "Returned empty set of blocks to persist");
return
}
// NOTE: checked non-empty above
let highest_num_hash = blocks_to_persist
// The highest block to be persisted determines the persistence state tracking.
// Use persist_count to find the highest block that will actually be written to disk.
let highest_num_hash = batch
.blocks
.iter()
.take(batch.persist_count)
.max_by_key(|block| block.recovered_block().number())
.map(|b| b.recovered_block().num_hash())
.expect("Checked non-empty persisting blocks");
debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(), "Persisting blocks");
debug!(target: "engine::tree", count=batch.blocks.len(), persist_count=batch.persist_count, blocks = ?batch.blocks.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(), "Persisting blocks");
let (tx, rx) = crossbeam_channel::bounded(1);
let _ = self.persistence.save_blocks(blocks_to_persist, tx);
let _ = self.persistence.save_blocks(batch, tx);
self.persistence_state.start_save(highest_num_hash, rx);
}
@@ -1390,9 +1381,8 @@ where
if let Some(new_tip_num) = self.find_disk_reorg()? {
self.remove_blocks(new_tip_num)
} else if self.should_persist() {
let blocks_to_persist =
self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?;
self.persist_blocks(blocks_to_persist);
let batch = self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?;
self.persist_blocks(batch);
}
}
@@ -1423,15 +1413,15 @@ where
self.on_persistence_complete(result, start_time)?;
}
let blocks_to_persist = self.get_canonical_blocks_to_persist(PersistTarget::Head)?;
let batch = self.get_canonical_blocks_to_persist(PersistTarget::Head)?;
if blocks_to_persist.is_empty() {
if batch.blocks.is_empty() {
debug!(target: "engine::tree", "persistence complete, signaling termination");
return Ok(())
}
debug!(target: "engine::tree", count = blocks_to_persist.len(), "persisting remaining blocks before shutdown");
self.persist_blocks(blocks_to_persist);
debug!(target: "engine::tree", count = batch.blocks.len(), "persisting remaining blocks before shutdown");
self.persist_blocks(batch);
}
}
@@ -2016,13 +2006,9 @@ where
}
/// Returns true if the canonical chain length minus the last persisted
/// block is greater than or equal to the persistence threshold,
/// backfill is not running, and no payload is currently being built.
/// block is greater than or equal to the persistence threshold and
/// backfill is not running.
pub const fn should_persist(&self) -> bool {
if self.building_payload {
return false
}
if !self.backfill_sync_state.is_idle() {
// can't persist if backfill is running
return false
@@ -2034,16 +2020,20 @@ where
}
/// Returns a batch of consecutive canonical blocks to persist in the range
/// `(last_persisted_number .. target]`. The expected order is oldest -> newest.
/// `(last_persisted_number .. canonical_head]`. The expected order is oldest -> newest.
///
/// All blocks above `last_persisted_number` are included in the batch, but only
/// those up to the persistence target (determined by [`PersistTarget`]) are marked
/// for actual persistence via [`SaveBlocksBatch::persist_count`].
fn get_canonical_blocks_to_persist(
&self,
target: PersistTarget,
) -> Result<Vec<ExecutedBlock<N>>, AdvancePersistenceError> {
) -> Result<SaveBlocksBatch<N>, AdvancePersistenceError> {
// We will calculate the state root using the database, so we need to be sure there are no
// changes
debug_assert!(!self.persistence_state.in_progress());
let mut blocks_to_persist = Vec::new();
let mut all_blocks = Vec::new();
let mut current_hash = self.state.tree_state.canonical_block_hash();
let last_persisted_number = self.persistence_state.last_persisted_block.number;
let canonical_head_number = self.state.tree_state.canonical_block_number();
@@ -2068,17 +2058,18 @@ where
break;
}
if block.recovered_block().number() <= target_number {
blocks_to_persist.push(block.clone());
}
all_blocks.push(block.clone());
current_hash = block.recovered_block().parent_hash();
}
// Reverse the order so that the oldest block comes first
blocks_to_persist.reverse();
all_blocks.reverse();
Ok(blocks_to_persist)
// Only blocks up to target_number are persisted; the rest are buffer blocks
let persist_count =
all_blocks.iter().filter(|b| b.recovered_block().number() <= target_number).count();
Ok(SaveBlocksBatch { blocks: all_blocks, persist_count })
}
/// This clears the blocks from the in-memory tree state that have been persisted to the
@@ -3024,22 +3015,8 @@ where
);
let latest_valid_hash = self.latest_valid_hash_for_invalid_payload(block.parent_hash())?;
// keep track of the invalid header unless the consensus impl considers it transient
let is_transient = match &validation_err {
InsertBlockValidationError::Consensus(err) => self.consensus.is_transient_error(err),
_ => false,
};
if is_transient {
warn!(
target: "engine::tree",
invalid_hash=%block.hash(),
invalid_number=block.number(),
%validation_err,
"Skipping invalid header cache insert for transient validation error",
);
} else {
self.state.invalid_headers.insert(block.block_with_parent());
}
// keep track of the invalid header
self.state.invalid_headers.insert(block.block_with_parent());
self.emit_event(EngineApiEvent::BeaconConsensus(ConsensusEngineEvent::InvalidBlock(
Box::new(block),
)));

View File

@@ -4,10 +4,10 @@ use super::precompile_cache::PrecompileCacheMap;
use crate::tree::{
payload_processor::prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
sparse_trie::SparseTrieCacheTask,
CacheWaitDurations, CachedStateMetrics, CachedStateMetricsSource, ExecutionCache,
PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig, WaitForCaches,
CacheWaitDurations, CachedStateMetrics, ExecutionCache, PayloadExecutionCache, SavedCache,
StateProviderBuilder, TreeConfig, WaitForCaches,
};
use alloy_eip7928::bal::DecodedBal;
use alloy_eip7928::BlockAccessList;
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal};
use alloy_primitives::B256;
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
@@ -96,8 +96,6 @@ where
executor: Runtime,
/// The most recent cache used for execution.
execution_cache: PayloadExecutionCache,
/// Metrics for the execution cache.
cache_metrics: Option<CachedStateMetrics>,
/// Metrics for trie operations
trie_metrics: MultiProofTaskMetrics,
/// Cross-block cache size in bytes.
@@ -122,12 +120,8 @@ where
sparse_trie_max_hot_accounts: usize,
/// Whether sparse trie cache pruning is fully disabled.
disable_sparse_trie_cache_pruning: bool,
/// Whether to disable BAL-based parallel execution (falls back to tx-based prewarming).
disable_bal_parallel_execution: bool,
/// Whether to disable BAL-driven parallel state root computation.
disable_bal_parallel_state_root: bool,
/// Whether BAL batched IO is disabled.
disable_bal_batch_io: bool,
/// Whether to disable cache metrics recording.
disable_cache_metrics: bool,
}
impl<N, Evm> PayloadProcessor<Evm>
@@ -161,11 +155,7 @@ where
sparse_trie_max_hot_slots: config.sparse_trie_max_hot_slots(),
sparse_trie_max_hot_accounts: config.sparse_trie_max_hot_accounts(),
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
cache_metrics: (!config.disable_cache_metrics())
.then(|| CachedStateMetrics::zeroed(CachedStateMetricsSource::Engine)),
disable_bal_parallel_execution: config.disable_bal_parallel_execution(),
disable_bal_parallel_state_root: config.disable_bal_parallel_state_root(),
disable_bal_batch_io: config.disable_bal_batch_io(),
disable_cache_metrics: config.disable_cache_metrics(),
}
}
}
@@ -250,6 +240,7 @@ where
provider_builder: StateProviderBuilder<N, P>,
multiproof_provider_factory: F,
config: &TreeConfig,
bal: Option<Arc<BlockAccessList>>,
) -> IteratorPayloadHandle<Evm, I, N>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -272,12 +263,13 @@ where
halve_workers,
config,
);
let install_state_hook = env.decoded_bal.is_none();
let install_state_hook = bal.is_none();
let prewarm_handle = self.spawn_caching_with(
env,
prewarm_rx,
provider_builder,
Some(state_root_handle.updates_tx().clone()),
bal,
);
PayloadHandle {
@@ -298,13 +290,14 @@ where
env: ExecutionEnv<Evm>,
transactions: I,
provider_builder: StateProviderBuilder<N, P>,
bal: Option<Arc<BlockAccessList>>,
) -> IteratorPayloadHandle<Evm, I, N>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
{
let (prewarm_rx, execution_rx) =
self.spawn_tx_iterator(transactions, env.transaction_count);
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None);
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal);
PayloadHandle {
state_root_handle: None,
install_state_hook: false,
@@ -462,7 +455,7 @@ where
level = "debug",
target = "engine::tree::payload_processor",
skip_all,
fields(bal=%env.decoded_bal.is_some())
fields(bal=%bal.is_some())
)]
fn spawn_caching_with<P>(
&self,
@@ -470,6 +463,7 @@ where
transactions: mpsc::Receiver<(usize, impl ExecutableTxFor<Evm> + Clone + Send + 'static)>,
provider_builder: StateProviderBuilder<N, P>,
to_sparse_trie_task: Option<CrossbeamSender<StateRootMessage>>,
bal: Option<Arc<BlockAccessList>>,
) -> CacheTaskHandle<N::Receipt>
where
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
@@ -480,7 +474,7 @@ where
let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash));
let executed_tx_index = Arc::new(AtomicUsize::new(0));
let maybe_decoded_bal = env.decoded_bal.clone();
// configure prewarming
let prewarm_ctx = PrewarmContext {
env,
@@ -488,13 +482,10 @@ where
saved_cache: saved_cache.clone(),
provider: provider_builder,
metrics: PrewarmMetrics::default(),
cache_metrics: self.cache_metrics.clone(),
terminate_execution: Arc::new(AtomicBool::new(false)),
executed_tx_index: Arc::clone(&executed_tx_index),
precompile_cache_disabled: self.precompile_cache_disabled,
precompile_cache_map: self.precompile_cache_map.clone(),
disable_bal_parallel_state_root: self.disable_bal_parallel_state_root,
disable_bal_batch_io: self.disable_bal_batch_io,
};
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
@@ -503,16 +494,14 @@ where
prewarm_ctx,
to_sparse_trie_task,
);
{
let to_prewarm_task = to_prewarm_task.clone();
let disable_bal_parallel_execution = self.disable_bal_parallel_execution;
self.executor.spawn_blocking_named("prewarm", move || {
let mode = if skip_prewarm {
PrewarmMode::Skipped
} else if let Some(decoded_bal) =
maybe_decoded_bal.filter(|_| !disable_bal_parallel_execution)
{
PrewarmMode::BlockAccessList(decoded_bal)
} else if let Some(bal) = bal {
PrewarmMode::BlockAccessList(bal)
} else {
PrewarmMode::Transactions(transactions)
};
@@ -520,12 +509,7 @@ where
});
}
CacheTaskHandle {
saved_cache,
to_prewarm_task: Some(to_prewarm_task),
executed_tx_index,
cache_metrics: self.cache_metrics.clone(),
}
CacheTaskHandle { saved_cache, to_prewarm_task: Some(to_prewarm_task), executed_tx_index }
}
/// Returns the cache for the given parent hash.
@@ -541,10 +525,10 @@ where
debug!("creating new execution cache on cache miss");
let start = Instant::now();
let cache = ExecutionCache::new(self.cross_block_cache_size);
if let Some(metrics) = &self.cache_metrics {
metrics.record_cache_creation(start.elapsed());
}
SavedCache::new(parent_hash, cache)
let metrics = CachedStateMetrics::zeroed();
metrics.record_cache_creation(start.elapsed());
SavedCache::new(parent_hash, cache, metrics)
.with_disable_cache_metrics(self.disable_cache_metrics)
}
}
@@ -606,7 +590,6 @@ where
proof_worker_handle,
trie_metrics.clone(),
sparse_state_trie,
parent_state_root,
chunk_size,
);
@@ -692,7 +675,7 @@ where
block_with_parent: BlockWithParent,
bundle_state: &BundleState,
) {
let cache_metrics = self.cache_metrics.clone();
let disable_cache_metrics = self.disable_cache_metrics;
self.execution_cache.update_with_guard(|cached| {
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
debug!(
@@ -704,19 +687,25 @@ where
}
// Take existing cache (if any) or create fresh caches
let caches = match cached.take() {
Some(existing) => existing.cache().clone(),
None => ExecutionCache::new(self.cross_block_cache_size),
let (caches, cache_metrics, _) = match cached.take() {
Some(existing) => existing.split(),
None => (
ExecutionCache::new(self.cross_block_cache_size),
CachedStateMetrics::zeroed(),
false,
),
};
// Insert the block's bundle state into cache
let new_cache = SavedCache::new(block_with_parent.block.hash, caches);
let new_cache =
SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
if new_cache.cache().insert_state(bundle_state).is_err() {
*cached = None;
debug!(target: "engine::caching", "cleared execution cache on update error");
return
}
new_cache.update_metrics(cache_metrics.as_ref());
new_cache.update_metrics();
// Replace with the updated cache
*cached = Some(new_cache);
@@ -810,9 +799,9 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.cache().clone())
}
/// Returns engine cache metrics if a cache exists for prewarming.
/// Returns a clone of the cache metrics used by prewarming
pub fn cache_metrics(&self) -> Option<CachedStateMetrics> {
self.prewarm_handle.cache_metrics.clone()
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.metrics().clone())
}
/// Returns a reference to the shared executed transaction index counter.
@@ -863,8 +852,6 @@ pub struct CacheTaskHandle<R> {
/// Shared counter tracking the next transaction index to be executed by the main execution
/// loop. Prewarm workers skip transactions below this index.
executed_tx_index: Arc<AtomicUsize>,
/// Metrics for the execution cache.
cache_metrics: Option<CachedStateMetrics>,
}
impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
@@ -933,9 +920,6 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
/// Withdrawals included in the block.
/// Used to generate prefetch targets for withdrawal addresses.
pub withdrawals: Option<Vec<Withdrawal>>,
/// Optional decoded BAL for the block.
/// Used to validate and optimize execution.
pub decoded_bal: Option<Arc<DecodedBal>>,
}
impl<Evm: ConfigureEvm> ExecutionEnv<Evm>
@@ -953,7 +937,6 @@ where
transaction_count: 0,
gas_used: 0,
withdrawals: None,
decoded_bal: None,
}
}
}
@@ -963,7 +946,8 @@ mod tests {
use crate::tree::{
payload_processor::{evm_state_to_hashed_post_state, ExecutionEnv, PayloadProcessor},
precompile_cache::PrecompileCacheMap,
ExecutionCache, PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig,
CachedStateMetrics, ExecutionCache, PayloadExecutionCache, SavedCache,
StateProviderBuilder, TreeConfig,
};
use alloy_eips::eip1898::{BlockNumHash, BlockWithParent};
use alloy_evm::block::StateChangeSource;
@@ -975,7 +959,7 @@ mod tests {
use reth_evm_ethereum::EthEvmConfig;
use reth_primitives_traits::{Account, Recovered, StorageEntry};
use reth_provider::{
providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory},
providers::{BlockchainProvider, OverlayStateProviderFactory},
test_utils::create_test_provider_factory_with_chain_spec,
ChainSpecProvider, HashingWriter,
};
@@ -989,7 +973,7 @@ mod tests {
fn make_saved_cache(hash: B256) -> SavedCache {
let execution_cache = ExecutionCache::new(1_000);
SavedCache::new(hash, execution_cache)
SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed())
}
#[test]
@@ -1159,16 +1143,19 @@ mod tests {
}
}
let mut account = revm_state::Account::default();
account.info = AccountInfo {
balance: U256::from(rng.random::<u64>()),
nonce: rng.random::<u64>(),
code_hash: KECCAK_EMPTY,
code: Some(Default::default()),
account_id: None,
let account = revm_state::Account {
info: AccountInfo {
balance: U256::from(rng.random::<u64>()),
nonce: rng.random::<u64>(),
code_hash: KECCAK_EMPTY,
code: Some(Default::default()),
account_id: None,
},
original_info: Box::new(AccountInfo::default()),
storage,
status: AccountStatus::Touched,
transaction_id: 0,
};
account.storage = storage;
account.status = AccountStatus::Touched;
state_update.insert(address, account);
}
@@ -1247,11 +1234,9 @@ mod tests {
std::convert::identity,
),
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
OverlayStateProviderFactory::new(
provider_factory,
OverlayBuilder::new(ChangesetCache::new()),
),
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
&TreeConfig::default(),
None, // No BAL for test
);
let mut state_hook = handle.state_hook().expect("state hook is None");

View File

@@ -14,11 +14,10 @@
use crate::tree::{
payload_processor::multiproof::StateRootMessage,
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
CachedStateMetrics, CachedStateProvider, ExecutionEnv, PayloadExecutionCache, SavedCache,
StateProviderBuilder,
CachedStateProvider, ExecutionEnv, PayloadExecutionCache, SavedCache, StateProviderBuilder,
};
use alloy_consensus::transaction::TxHashRef;
use alloy_eip7928::bal::DecodedBal;
use alloy_eip7928::BlockAccessList;
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{keccak256, StorageKey, B256};
use crossbeam_channel::Sender as CrossbeamSender;
@@ -48,7 +47,7 @@ pub enum PrewarmMode<Tx> {
/// Prewarm by executing transactions from a stream, each paired with its block index.
Transactions(Receiver<(usize, Tx)>),
/// Prewarm by prefetching slots from a Block Access List.
BlockAccessList(Arc<DecodedBal>),
BlockAccessList(Arc<BlockAccessList>),
/// Transaction prewarming is skipped (e.g. small blocks where the overhead exceeds the
/// benefit). No workers are spawned.
Skipped,
@@ -277,20 +276,19 @@ where
) {
let start = Instant::now();
let Self {
execution_cache,
ctx: PrewarmContext { env, metrics, cache_metrics, saved_cache, .. },
..
} = self;
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
self;
let hash = env.hash;
if let Some(saved_cache) = saved_cache {
debug!(target: "engine::caching", parent_hash=?hash, "Updating execution cache");
// Perform all cache operations atomically under the lock
execution_cache.update_with_guard(|cached| {
// consumes the `SavedCache` held by the prewarming task, which releases its usage
// guard
let caches = saved_cache.cache().clone();
let new_cache = SavedCache::new(hash, caches);
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
let new_cache = SavedCache::new(hash, caches, cache_metrics)
.with_disable_cache_metrics(disable_cache_metrics);
// Insert state into cache while holding the lock
// Access the BundleState through the shared ExecutionOutcome
@@ -301,7 +299,7 @@ where
return;
}
new_cache.update_metrics(cache_metrics.as_ref());
new_cache.update_metrics();
if valid_block_rx.recv().is_ok() {
// Replace the shared cache with the new one; the previous cache (if any) is
@@ -331,10 +329,9 @@ where
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
fn run_bal_prewarm(
&self,
decoded_bal: Arc<DecodedBal>,
bal: Arc<BlockAccessList>,
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
) {
let bal = decoded_bal.as_bal();
if bal.is_empty() {
if let Some(to_sparse_trie_task) = self.to_sparse_trie_task.as_ref() {
let _ = to_sparse_trie_task.send(StateRootMessage::FinishedStateUpdates);
@@ -356,8 +353,8 @@ where
let parent_span = Span::current();
let prefetch_parent_span = parent_span.clone();
let stream_parent_span = parent_span;
let prefetch_bal = Arc::clone(&decoded_bal);
let stream_bal = Arc::clone(&decoded_bal);
let prefetch_bal = Arc::clone(&bal);
let stream_bal = Arc::clone(&bal);
let (prefetch_tx, prefetch_rx) = oneshot::channel();
let (stream_tx, stream_rx) = oneshot::channel();
@@ -368,12 +365,12 @@ where
target: "engine::tree::payload_processor::prewarm",
parent: &prefetch_parent_span,
"bal_prefetch_storage",
bal_accounts = prefetch_bal.as_bal().len(),
bal_accounts = prefetch_bal.len(),
);
let provider_parent_span = branch_span.clone();
let _span = branch_span.entered();
prefetch_bal.as_bal().par_iter().for_each_init(
prefetch_bal.par_iter().for_each_init(
|| {
(
prefetch_ctx.clone(),
@@ -401,12 +398,12 @@ where
target: "engine::tree::payload_processor::prewarm",
parent: &stream_parent_span,
"bal_hashed_state_stream",
bal_accounts = stream_bal.as_bal().len(),
bal_accounts = stream_bal.len(),
);
let provider_parent_span = branch_span.clone();
let _span = branch_span.entered();
stream_bal.as_bal().par_iter().for_each_init(
stream_bal.par_iter().for_each_init(
|| (ctx.clone(), None::<Box<dyn AccountReader>>, provider_parent_span.clone()),
|(ctx, provider, parent_span), account_changes| {
ctx.send_bal_hashed_state(
@@ -524,9 +521,6 @@ where
pub provider: StateProviderBuilder<N, P>,
/// The metrics for the prewarm task.
pub metrics: PrewarmMetrics,
/// Metrics for the execution cache.
/// Metrics for the execution cache. `None` disables metrics recording.
pub cache_metrics: Option<CachedStateMetrics>,
/// An atomic bool that tells prewarm tasks to not start any more execution.
pub terminate_execution: Arc<AtomicBool>,
/// Shared counter tracking the next transaction index to be executed by the main execution
@@ -537,10 +531,6 @@ where
pub precompile_cache_disabled: bool,
/// The precompile cache map.
pub precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
/// Whether to disable BAL-driven parallel state root computation.
pub disable_bal_parallel_state_root: bool,
/// Whether BAL batched IO is disabled.
pub disable_bal_batch_io: bool,
}
/// Per-thread EVM state initialised by [`PrewarmContext::evm_for_ctx`] and stored in
@@ -572,11 +562,9 @@ where
// Use the caches to create a new provider with caching
if let Some(saved_cache) = &self.saved_cache {
let caches = saved_cache.cache().clone();
state_provider = Box::new(CachedStateProvider::new_prewarm(
state_provider,
caches,
self.cache_metrics.clone().unwrap_or_default(),
));
let cache_metrics = saved_cache.metrics().clone();
state_provider =
Box::new(CachedStateProvider::new_prewarm(state_provider, caches, cache_metrics));
}
let state_provider = StateProviderDatabase::new(state_provider);
@@ -636,9 +624,6 @@ where
account_changes: &alloy_eip7928::AccountChanges,
to_sparse_trie_task: &CrossbeamSender<StateRootMessage>,
) {
if self.disable_bal_parallel_state_root {
return;
}
let address = account_changes.address;
let mut hashed_address = None;
@@ -680,11 +665,8 @@ where
};
let boxed: Box<dyn AccountReader> = if let Some(saved) = &self.saved_cache {
let caches = saved.cache().clone();
Box::new(CachedStateProvider::new_prewarm(
inner,
caches,
self.cache_metrics.clone().unwrap_or_default(),
))
let cache_metrics = saved.metrics().clone();
Box::new(CachedStateProvider::new_prewarm(inner, caches, cache_metrics))
} else {
Box::new(inner)
};
@@ -779,11 +761,8 @@ where
let saved_cache =
self.saved_cache.as_ref().expect("BAL prewarm should only run with cache");
let caches = saved_cache.cache().clone();
slot.insert(CachedStateProvider::new_prewarm(
built,
caches,
self.cache_metrics.clone().unwrap_or_default(),
))
let cache_metrics = saved_cache.metrics().clone();
slot.insert(CachedStateProvider::new_prewarm(built, caches, cache_metrics))
}
};

View File

@@ -27,9 +27,8 @@ use reth_trie_parallel::{
root::ParallelStateRootError,
};
use reth_trie_sparse::{
errors::{SparseStateTrieErrorKind, SparseTrieErrorKind, SparseTrieResult},
ConfigurableSparseTrie, DeferredDrops, LeafUpdate, RevealableSparseTrie, SparseStateTrie,
SparseTrie,
errors::SparseTrieResult, ConfigurableSparseTrie, DeferredDrops, LeafUpdate,
RevealableSparseTrie, SparseStateTrie, SparseTrie,
};
use revm_primitives::{hash_map::Entry, B256Map};
use tracing::{debug, debug_span, error, instrument, trace_span};
@@ -47,8 +46,6 @@ pub(super) struct SparseTrieCacheTask<A = ConfigurableSparseTrie, S = Configurab
updates: CrossbeamReceiver<SparseTrieTaskMessage>,
/// `SparseStateTrie` used for computing the state root.
trie: SparseStateTrie<A, S>,
/// The parent block's state root.
parent_state_root: B256,
/// Handle to the proof worker pools (storage and account).
proof_worker_handle: ProofWorkerHandle,
@@ -123,7 +120,6 @@ where
proof_worker_handle: ProofWorkerHandle,
metrics: MultiProofTaskMetrics,
trie: SparseStateTrie<A, S>,
parent_state_root: B256,
chunk_size: usize,
) -> Self {
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
@@ -142,7 +138,6 @@ where
updates: hashed_state_rx,
proof_worker_handle,
trie,
parent_state_root,
chunk_size,
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
account_updates: Default::default(),
@@ -364,25 +359,10 @@ where
debug!(target: "engine::root", "All proofs processed, ending calculation");
let start = Instant::now();
let (state_root, trie_updates) = match self.trie.root_with_updates() {
Ok(result) => result,
Err(err)
if matches!(
err.kind(),
SparseStateTrieErrorKind::Sparse(SparseTrieErrorKind::Blind)
) =>
{
// A still-blind account trie means this block never changed state, so preserve
// the cached parent root instead of fetching and revealing
// the unchanged root node.
(self.parent_state_root, TrieUpdates::default())
}
Err(err) => {
return Err(ParallelStateRootError::Other(format!(
"could not calculate state root: {err:?}"
)))
}
};
let (state_root, trie_updates) =
self.trie.root_with_updates(&self.proof_worker_handle).map_err(|e| {
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
})?;
#[cfg(feature = "trie-debug")]
let debug_recorders = self.trie.take_debug_recorders();
@@ -893,12 +873,6 @@ enum SparseTrieTaskMessage {
mod tests {
use super::*;
use alloy_primitives::{keccak256, Address, B256, U256};
use reth_provider::{
providers::{OverlayBuilder, OverlayStateProviderFactory},
test_utils::create_test_provider_factory,
};
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::proof_task::ProofTaskCtx;
use reth_trie_sparse::ArenaParallelSparseTrie;
#[test]
@@ -979,45 +953,4 @@ mod tests {
assert_eq!(decoded.storage_root, storage_root);
assert_eq!(account_rlp_buf, encoded);
}
#[test]
fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() {
let runtime = reth_tasks::Runtime::test();
let provider_factory = create_test_provider_factory();
let overlay_factory = OverlayStateProviderFactory::new(
provider_factory,
OverlayBuilder::new(ChangesetCache::new()),
);
let proof_worker_handle =
ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false);
let default_trie = RevealableSparseTrie::blind_from(ConfigurableSparseTrie::Arena(
ArenaParallelSparseTrie::default(),
));
let trie = SparseStateTrie::default()
.with_accounts_trie(default_trie.clone())
.with_default_storage_trie(default_trie)
.with_updates(true);
let parent_state_root = B256::from([0x55; 32]);
let (updates_tx, updates_rx) = crossbeam_channel::unbounded();
let mut task = SparseTrieCacheTask::new_with_trie(
&runtime,
updates_rx,
proof_worker_handle,
MultiProofTaskMetrics::default(),
trie,
parent_state_root,
1,
);
updates_tx.send(StateRootMessage::FinishedStateUpdates).unwrap();
drop(updates_tx);
let outcome = task.run().expect("state root computation should succeed");
assert_eq!(outcome.state_root, parent_state_root);
assert!(outcome.trie_updates.is_empty());
assert!(task.trie.state_trie_ref().is_none(), "blind trie should not be revealed");
}
}

View File

@@ -48,10 +48,7 @@ use crate::tree::{
PayloadHandle, StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches,
};
use alloy_consensus::transaction::{Either, TxHashRef};
use alloy_eip7928::{
bal::{Bal, DecodedBal},
BlockAccessList,
};
use alloy_eip7928::{bal::Bal, BlockAccessList};
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash};
use alloy_evm::Evm;
use alloy_primitives::{map::B256Set, B256};
@@ -80,14 +77,13 @@ use reth_primitives_traits::{
RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable,
};
use reth_provider::{
providers::{OverlayBuilder, OverlayStateProviderFactory},
BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory,
DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader,
StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader,
StorageChangeSetReader, StorageSettingsCache,
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
};
use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State};
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState};
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot};
use reth_trie_db::ChangesetCache;
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
use revm_primitives::{Address, KECCAK_EMPTY};
@@ -491,12 +487,6 @@ where
.in_scope(|| self.evm_env_for(&input))
.map_err(NewPayloadError::other)?;
// Extract the decoded BAL, if valid and available.
let decoded_bal = ensure_ok!(input
.try_decoded_access_list()
.map_err(|err| { Box::<dyn std::error::Error + Send + Sync>::from(err) }))
.map(Arc::new);
let env = ExecutionEnv {
evm_env,
hash: input.hash(),
@@ -505,7 +495,6 @@ where
transaction_count: input.transaction_count(),
gas_used: input.gas_used(),
withdrawals: input.withdrawals().map(|w| w.to_vec()),
decoded_bal,
};
// Plan the strategy used for state root computation.
@@ -520,26 +509,33 @@ where
// Get an iterator over the transactions in the payload
let txs = self.tx_iterator_for(&input)?;
// Extract the BAL, if valid and available
let block_access_list = ensure_ok!(input
.block_access_list()
.transpose()
// Eventually gets converted to a `InsertBlockErrorKind::Other`
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
.map(Arc::new);
// Create lazy overlay from ancestors - this doesn't block, allowing execution to start
// before the trie data is ready. The overlay will be computed on first access.
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state());
// Create overlay factory for payload processor (StateRootTask path needs it for
// multiproofs)
let provider_factory = self.provider.clone();
let overlay_builder = OverlayBuilder::new(self.changeset_cache.clone())
.with_block_hash(Some(anchor_hash))
.with_lazy_overlay(lazy_overlay);
let overlay_factory =
OverlayStateProviderFactory::new(provider_factory.clone(), overlay_builder.clone());
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
.with_block_hash(Some(anchor_hash))
.with_lazy_overlay(lazy_overlay);
// Spawn the appropriate processor based on strategy
let mut handle = ensure_ok!(self.spawn_payload_processor(
env.clone(),
txs,
provider_builder.clone(),
provider_builder,
overlay_factory.clone(),
strategy,
block_access_list,
));
// Create optional cache stats for detailed block logging
@@ -668,7 +664,7 @@ where
let task_result = ensure_ok_post_block!(
self.await_state_root_with_timeout(
&mut handle,
provider_builder.clone(),
overlay_factory.clone(),
&hashed_state,
),
block
@@ -692,9 +688,7 @@ where
// Compare trie updates with serial computation if configured
if self.config.always_compare_trie_updates() {
let _has_diff = self.compare_trie_updates_with_serial(
provider_builder.clone(),
provider_factory,
overlay_builder,
overlay_factory.clone(),
&hashed_state,
trie_updates.as_ref().clone(),
);
@@ -733,11 +727,7 @@ where
}
StateRootStrategy::Parallel => {
debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm");
match self.compute_state_root_parallel(
provider_factory,
overlay_builder,
&hashed_state,
) {
match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) {
Ok(result) => {
let elapsed = root_time.elapsed();
info!(
@@ -773,9 +763,7 @@ where
}
let (root, updates) = ensure_ok_post_block!(
provider_builder
.build()
.and_then(|provider| Self::compute_state_root_serial(provider, &hashed_state)),
Self::compute_state_root_serial(overlay_factory.clone(), &hashed_state),
block
);
@@ -1099,8 +1087,7 @@ where
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
fn compute_state_root_parallel(
&self,
provider_factory: P,
overlay_builder: OverlayBuilder,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &LazyHashedPostState,
) -> Result<(B256, TrieUpdates), ParallelStateRootError> {
let hashed_state = hashed_state.get();
@@ -1108,24 +1095,34 @@ where
// need to use the prefix sets which were generated from it to indicate to the
// ParallelStateRoot which parts of the trie need to be recomputed.
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
let overlay_factory = OverlayStateProviderFactory::new(
provider_factory,
overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()),
);
let overlay_factory =
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone())
.incremental_root_with_updates()
}
/// Compute state root for the given hashed post state in serial.
///
/// Uses the same provider construction path as main execution and computes the state root and
/// trie updates for this block directly via
/// [`reth_provider::StateRootProvider::state_root_with_updates`].
/// Uses an overlay factory which provides the state of the parent block, along with the
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
/// trie updates for this block.
fn compute_state_root_serial(
state_provider: StateProviderBox,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &LazyHashedPostState,
) -> ProviderResult<(B256, TrieUpdates)> {
state_provider.state_root_with_updates(hashed_state.get().clone())
let hashed_state = hashed_state.get();
// The `hashed_state` argument will be taken into account as part of the overlay, but we
// need to use the prefix sets which were generated from it to indicate to the
// StateRoot which parts of the trie need to be recomputed.
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
let overlay_factory =
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
let provider = overlay_factory.database_provider_ro()?;
Ok(StateRoot::new(&provider, &provider)
.with_prefix_sets(prefix_sets)
.root_with_updates()?)
}
/// Awaits the state root from the background task, with an optional timeout fallback.
@@ -1150,7 +1147,7 @@ where
fn await_state_root_with_timeout<Tx, Err, R: Send + Sync + 'static>(
&self,
handle: &mut PayloadHandle<Tx, Err, R>,
state_provider_builder: StateProviderBuilder<N, P>,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &LazyHashedPostState,
) -> ProviderResult<Result<StateRootComputeOutcome, ParallelStateRootError>> {
let Some(timeout) = self.config.state_root_task_timeout() else {
@@ -1175,11 +1172,10 @@ where
let (seq_tx, seq_rx) =
std::sync::mpsc::channel::<ProviderResult<(B256, TrieUpdates)>>();
let seq_overlay = overlay_factory;
let seq_hashed_state = hashed_state.clone();
self.payload_processor.executor().spawn_blocking_named("serial-root", move || {
let result = state_provider_builder.build().and_then(|provider| {
Self::compute_state_root_serial(provider, &seq_hashed_state)
});
let result = Self::compute_state_root_serial(seq_overlay, &seq_hashed_state);
let _ = seq_tx.send(result);
});
@@ -1243,18 +1239,13 @@ where
/// updates.
fn compare_trie_updates_with_serial(
&self,
state_provider_builder: StateProviderBuilder<N, P>,
provider_factory: P,
overlay_builder: OverlayBuilder,
overlay_factory: OverlayStateProviderFactory<P>,
hashed_state: &LazyHashedPostState,
task_trie_updates: TrieUpdates,
) -> bool {
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
match state_provider_builder
.build()
.and_then(|provider| Self::compute_state_root_serial(provider, hashed_state))
{
match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) {
Ok((serial_root, serial_trie_updates)) => {
debug!(
target: "engine::tree::payload_validator",
@@ -1263,8 +1254,6 @@ where
);
// Get a database provider to use as trie cursor factory
let overlay_factory =
OverlayStateProviderFactory::new(provider_factory, overlay_builder);
match overlay_factory.database_provider_ro() {
Ok(provider) => {
match super::trie_updates::compare_trie_updates(
@@ -1450,6 +1439,7 @@ where
provider_builder: StateProviderBuilder<N, P>,
overlay_factory: OverlayStateProviderFactory<P>,
strategy: StateRootStrategy,
block_access_list: Option<Arc<BlockAccessList>>,
) -> Result<
PayloadHandle<
impl ExecutableTxFor<Evm> + use<N, P, Evm, V, T>,
@@ -1469,6 +1459,7 @@ where
provider_builder,
overlay_factory,
&self.config,
block_access_list,
);
// record prewarming initialization duration
@@ -1481,8 +1472,12 @@ where
}
StateRootStrategy::Parallel | StateRootStrategy::Synchronous => {
let start = Instant::now();
let handle =
self.payload_processor.spawn_cache_exclusive(env, txs, provider_builder);
let handle = self.payload_processor.spawn_cache_exclusive(
env,
txs,
provider_builder,
block_access_list,
);
// Record prewarming initialization duration
self.metrics
@@ -2031,12 +2026,10 @@ where
state: &EngineApiTreeState<N>,
) -> Option<StateRootHandle> {
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state);
let overlay_factory = OverlayStateProviderFactory::new(
self.provider.clone(),
OverlayBuilder::new(self.changeset_cache.clone())
let overlay_factory =
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
.with_block_hash(Some(anchor_hash))
.with_lazy_overlay(lazy_overlay),
);
.with_lazy_overlay(lazy_overlay);
Some(self.payload_processor.spawn_state_root(
overlay_factory,
@@ -2117,17 +2110,6 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
}
}
/// Returns the decoded block access list, if present and successfully decoded.
pub fn try_decoded_access_list(&self) -> Result<Option<DecodedBal>, alloy_rlp::Error> {
match self {
Self::Payload(payload) => payload
.block_access_list()
.map(|block_access_list| DecodedBal::from_rlp_bytes(block_access_list.clone()))
.transpose(),
Self::Block(_) => Ok(None),
}
}
/// Returns the number of transactions in the payload or block.
pub fn transaction_count(&self) -> usize
where
@@ -2160,15 +2142,4 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
Self::Block(block) => block.gas_used(),
}
}
/// Returns the gas limit used by the block.
pub fn gas_limit(&self) -> u64
where
T::ExecutionData: ExecutionPayload,
{
match self {
Self::Payload(payload) => payload.gas_limit(),
Self::Block(block) => block.gas_limit(),
}
}
}

View File

@@ -10,7 +10,6 @@ use reth_primitives_traits::dashmap::DashMap;
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
use revm_primitives::Address;
use std::{hash::Hash, sync::Arc};
use tracing::error;
/// Default max cache size for [`PrecompileCache`]
const MAX_CACHE_SIZE: u32 = 10_000;
@@ -88,14 +87,8 @@ impl<S> CacheEntry<S> {
self.output.gas_used
}
/// Converts the cache entry to a precompile result. Accepts state gas reservoir as input.
///
/// All cached precompiles are not expected to access/created state and thus reservoir is always
/// kept as is.
fn to_precompile_result(&self, reservoir: u64) -> PrecompileResult {
let mut output = self.output.clone();
output.reservoir = reservoir;
Ok(output)
fn to_precompile_result(&self) -> PrecompileResult {
Ok(self.output.clone())
}
}
@@ -182,34 +175,22 @@ where
input.gas >= entry.gas_used()
{
self.increment_by_one_precompile_cache_hits();
return entry.to_precompile_result(input.reservoir);
return entry.to_precompile_result()
}
let calldata = input.data;
let reservoir = input.reservoir;
let result = self.precompile.call(input);
match &result {
// Only successful outputs are cacheable. Non-success statuses and errors must execute
// again instead of poisoning the cache for subsequent calls.
Ok(output) if output.is_success() => {
// Sanity-check precompile output to ensure that it does not affect state gas in any
// way.
//
// This does not fully protect us from caching stateful precompiles but might make
// it obvious when the node is misconfigured.
if output.reservoir != reservoir {
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile decremented reservoir, skipping cache insertion");
} else if output.state_gas_used != 0 {
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile used state gas, skipping cache insertion");
} else {
let size = self.cache.insert(
Bytes::copy_from_slice(calldata),
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
);
self.set_precompile_cache_size_metric(size as f64);
self.increment_by_one_precompile_cache_misses();
}
let size = self.cache.insert(
Bytes::copy_from_slice(calldata),
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
);
self.set_precompile_cache_size_metric(size as f64);
self.increment_by_one_precompile_cache_misses();
}
_ => {
self.increment_by_one_precompile_errors();
@@ -265,7 +246,6 @@ mod tests {
gas_used: 0,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: Bytes::default(),
})
})
@@ -279,7 +259,6 @@ mod tests {
gas_used: 50,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
};
@@ -313,7 +292,6 @@ mod tests {
gas_used: 5000,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
})
}
@@ -330,7 +308,6 @@ mod tests {
gas_used: 7000,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
})
}

View File

@@ -1,6 +1,6 @@
use super::*;
use crate::{
persistence::PersistenceAction,
persistence::{PersistenceAction, SaveBlocksBatch},
tree::{
payload_validator::{BasicEngineValidator, TreeCtx, ValidationOutcome},
persistence_state::CurrentPersistenceAction,
@@ -184,18 +184,11 @@ impl TestHarness {
let payload_validator = MockEngineValidator;
let (from_tree_tx, from_tree_rx) = unbounded_channel();
let tree_config =
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true);
let header = chain_spec.genesis_header().clone();
let header = SealedHeader::seal_slow(header);
let engine_api_tree_state = EngineApiTreeState::new(
10,
10,
tree_config.invalid_header_hit_eviction_threshold(),
header.num_hash(),
EngineApiKind::Ethereum,
);
let engine_api_tree_state =
EngineApiTreeState::new(10, 10, header.num_hash(), EngineApiKind::Ethereum);
let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None);
let (to_payload_service, _payload_command_rx) = unbounded_channel();
@@ -224,7 +217,8 @@ impl TestHarness {
persistence_handle,
PersistenceState { last_persisted_block: BlockNumHash::default(), rx: None },
payload_builder,
tree_config,
// always assume enough parallelism for tests
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
EngineApiKind::Ethereum,
evm_config,
changeset_cache,
@@ -554,12 +548,14 @@ async fn test_tree_persist_blocks() {
let received_action =
test_harness.action_rx.recv().expect("Failed to receive save blocks action");
if let PersistenceAction::SaveBlocks(saved_blocks, _) = received_action {
if let PersistenceAction::SaveBlocks(batch, _) = received_action {
// all blocks are included in the batch
assert_eq!(batch.blocks.len(), blocks.len());
assert_eq!(batch.blocks, blocks);
// only blocks.len() - tree_config.memory_block_buffer_target() will be
// persisted
let expected_persist_len = blocks.len() - tree_config.memory_block_buffer_target() as usize;
assert_eq!(saved_blocks.len(), expected_persist_len);
assert_eq!(saved_blocks, blocks[..expected_persist_len]);
assert_eq!(batch.persist_count, expected_persist_len);
} else {
panic!("unexpected action received {received_action:?}");
}
@@ -824,10 +820,12 @@ async fn test_tree_state_on_new_head_reorg() {
// get rid of the prev action
let received_action = test_harness.action_rx.recv().unwrap();
let PersistenceAction::SaveBlocks(saved_blocks, sender) = received_action else {
let PersistenceAction::SaveBlocks(batch, sender) = received_action else {
panic!("received wrong action");
};
assert_eq!(saved_blocks, vec![blocks[0].clone(), blocks[1].clone()]);
// all blocks above last_persisted are included, persist_count covers those up to the target
assert_eq!(batch.persist_count, 2);
assert_eq!(&batch.blocks[..batch.persist_count], &[blocks[0].clone(), blocks[1].clone()]);
// send the response so we can advance again
sender
@@ -985,8 +983,10 @@ async fn test_get_canonical_blocks_to_persist() {
.try_into()
.unwrap();
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
for (i, item) in blocks_to_persist.iter().enumerate().take(expected_blocks_to_persist_length) {
assert_eq!(blocks_to_persist.persist_count, expected_blocks_to_persist_length);
for (i, item) in
blocks_to_persist.blocks.iter().enumerate().take(expected_blocks_to_persist_length)
{
assert_eq!(item.recovered_block().number, last_persisted_block_number + i as u64 + 1);
}
@@ -999,13 +999,16 @@ async fn test_get_canonical_blocks_to_persist() {
let blocks_to_persist =
test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap();
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
assert_eq!(blocks_to_persist.persist_count, expected_blocks_to_persist_length);
// check that the fork block is not included in the blocks to persist
assert!(!blocks_to_persist.iter().any(|b| b.recovered_block().hash() == fork_block_hash));
assert!(!blocks_to_persist
.blocks
.iter()
.any(|b| b.recovered_block().hash() == fork_block_hash));
// check that the original block 4 is still included
assert!(blocks_to_persist.iter().any(|b| b.recovered_block().number == 4 &&
assert!(blocks_to_persist.blocks.iter().any(|b| b.recovered_block().number == 4 &&
b.recovered_block().hash() == blocks[4].recovered_block().hash()));
// check that if we advance persistence, the persistence action is the correct value
@@ -1013,7 +1016,11 @@ async fn test_get_canonical_blocks_to_persist() {
assert_eq!(
test_harness.tree.persistence_state.current_action().cloned(),
Some(CurrentPersistenceAction::SavingBlocks {
highest: blocks_to_persist.last().unwrap().recovered_block().num_hash()
highest: blocks_to_persist.blocks[..blocks_to_persist.persist_count]
.last()
.unwrap()
.recovered_block()
.num_hash()
})
);
}
@@ -2112,15 +2119,16 @@ mod forkchoice_updated_tests {
break;
}
if let Ok(PersistenceAction::SaveBlocks(saved_blocks, sender)) =
if let Ok(PersistenceAction::SaveBlocks(batch, sender)) =
action_rx.recv_timeout(std::time::Duration::from_millis(100))
{
if let Some(last) = saved_blocks.last() {
let persisted = &batch.blocks[..batch.persist_count];
if let Some(last) = persisted.last() {
last_persisted_number = last.recovered_block().number;
}
sender
.send(PersistenceResult {
last_block: saved_blocks.last().map(|b| b.recovered_block().num_hash()),
last_block: persisted.last().map(|b| b.recovered_block().num_hash()),
commit_duration: Some(Duration::ZERO),
})
.unwrap();

View File

@@ -2,8 +2,8 @@
//! and injecting them into era1 files with `Era1Writer`.
use crate::calculate_td_by_number;
use alloy_consensus::{BlockHeader, Sealable, TxReceipt};
use alloy_primitives::{BlockNumber, U256};
use alloy_consensus::BlockHeader;
use alloy_primitives::{BlockNumber, B256, U256};
use eyre::{eyre, Result};
use reth_era::{
common::file_ops::{EraFileId, StreamWriter},
@@ -13,7 +13,7 @@ use reth_era::{
types::{
execution::{
Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts,
HeaderRecord, TotalDifficulty, MAX_BLOCKS_PER_ERA1,
TotalDifficulty, MAX_BLOCKS_PER_ERA1,
},
group::{BlockIndex, Era1Id},
},
@@ -139,21 +139,17 @@ where
let headers = provider.headers_range(start_block..=end_block)?;
// Pre-compute accumulator from headers to determine filename
let mut precompute_td = total_difficulty;
let header_records: Vec<HeaderRecord> = headers
.iter()
.map(|h| {
precompute_td += h.difficulty();
HeaderRecord { block_hash: h.hash_slow(), total_difficulty: precompute_td }
// Extract first 4 bytes of last block's state root as historical identifier
let historical_root = headers
.last()
.map(|header| {
let state_root = header.state_root();
[state_root[0], state_root[1], state_root[2], state_root[3]]
})
.collect();
let accumulator = Accumulator::from_header_records(&header_records)
.map_err(|e| eyre!("Failed to compute accumulator: {e}"))?;
let file_hash: [u8; 4] = accumulator.root[..4].try_into().unwrap();
.unwrap_or([0u8; 4]);
let era1_id =
Era1Id::new(&config.network, start_block, block_count as u32).with_hash(file_hash);
let era1_id = Era1Id::new(&config.network, start_block, block_count as u32)
.with_hash(historical_root);
let era1_id = if config.max_blocks_per_file == MAX_BLOCKS_PER_ERA1 as u64 {
era1_id
@@ -170,6 +166,7 @@ where
let mut offsets = Vec::<i64>::with_capacity(block_count);
let mut position = VERSION_ENTRY_SIZE as i64;
let mut blocks_written = 0;
let mut final_header_data = Vec::new();
for (i, header) in headers.into_iter().enumerate() {
let expected_block_number = start_block + i as u64;
@@ -181,6 +178,11 @@ where
&mut total_difficulty,
)?;
// Save last block's header data for accumulator
if expected_block_number == end_block {
final_header_data = compressed_header.data.clone();
}
let difficulty = TotalDifficulty::new(total_difficulty);
let header_size = compressed_header.data.len() + ENTRY_HEADER_SIZE;
@@ -216,12 +218,10 @@ where
}
}
if blocks_written > 0 {
// Convert absolute offsets to relative (measured from block-index entry start)
let accumulator_entry_size = (ENTRY_HEADER_SIZE + 32) as i64;
let block_index_position = position + accumulator_entry_size;
let relative_offsets: Vec<i64> =
offsets.iter().map(|&abs| abs - block_index_position).collect();
let block_index = BlockIndex::new(start_block, relative_offsets);
let accumulator_hash =
B256::from_slice(&final_header_data[0..32.min(final_header_data.len())]);
let accumulator = Accumulator::new(accumulator_hash);
let block_index = BlockIndex::new(start_block, offsets);
writer.write_accumulator(&accumulator)?;
writer.write_block_index(&block_index)?;
@@ -310,9 +310,7 @@ where
let compressed_header = CompressedHeader::from_header(&header)?;
let compressed_body = CompressedBody::from_body(&body)?;
let receipts_with_bloom: Vec<_> =
receipts.iter().map(|r| TxReceipt::with_bloom_ref(r)).collect();
let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts_with_bloom)
let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts)
.map_err(|e| eyre!("Failed to compress receipts: {}", e))?;
Ok((compressed_header, compressed_body, compressed_receipts))

View File

@@ -24,7 +24,6 @@ snap.workspace = true
# ssz encoding and decoding
ethereum_ssz.workspace = true
ethereum_ssz_derive.workspace = true
sha2.workspace = true
[dev-dependencies]
eyre.workspace = true

View File

@@ -76,7 +76,6 @@ use crate::{
use alloy_consensus::{Block, BlockBody, Header};
use alloy_primitives::{B256, U256};
use alloy_rlp::{Decodable, Encodable};
use sha2::{Digest, Sha256};
use snap::{read::FrameDecoder, write::FrameEncoder};
use std::{
io::{Read, Write},
@@ -494,73 +493,6 @@ impl Accumulator {
Ok(Self { root: B256::from(root) })
}
/// Compute the accumulator from a list of header records.
///
/// Implements `hash_tree_root(List[HeaderRecord, 8192])` per the ERA1 spec:
/// - Each leaf is `sha256(block_hash || total_difficulty_le_bytes32)`
/// - Leaves are padded to `MAX_BLOCKS_PER_ERA1` (8192) with zero hashes
/// - Binary Merkle tree is computed bottom-up
/// - Final root is `sha256(merkle_root || le_bytes32(actual_count))`
///
/// Returns `Err` if `records` exceeds [`MAX_BLOCKS_PER_ERA1`].
pub fn from_header_records(records: &[HeaderRecord]) -> Result<Self, E2sError> {
let capacity = MAX_BLOCKS_PER_ERA1;
if records.len() > capacity {
return Err(E2sError::Ssz(format!(
"Too many header records: got {}, max {}",
records.len(),
capacity
)));
}
// Compute leaf hash for each header record
let mut leaves = Vec::with_capacity(capacity);
for record in records {
let mut data = [0u8; 64];
data[..32].copy_from_slice(record.block_hash.as_slice());
data[32..].copy_from_slice(&record.total_difficulty.to_le_bytes::<32>());
leaves.push(<[u8; 32]>::from(Sha256::digest(data)));
}
// Pad to capacity with zero hashes
leaves.resize(capacity, [0u8; 32]);
// Binary Merkle tree bottom-up (capacity is always a power of two)
while leaves.len() > 1 {
let mut next_level = Vec::with_capacity(leaves.len() / 2);
for pair in leaves.chunks_exact(2) {
let mut data = [0u8; 64];
data[..32].copy_from_slice(&pair[0]);
data[32..].copy_from_slice(&pair[1]);
next_level.push(<[u8; 32]>::from(Sha256::digest(data)));
}
leaves = next_level;
}
let merkle_root = leaves[0];
// mix_in_length: sha256(merkle_root || le_bytes32(actual_length))
let mut mix = [0u8; 64];
mix[..32].copy_from_slice(&merkle_root);
let length = records.len() as u64;
mix[32..40].copy_from_slice(&length.to_le_bytes());
// remaining bytes stay zero (uint256 LE padding)
Ok(Self { root: B256::from(<[u8; 32]>::from(Sha256::digest(mix))) })
}
}
/// A header record used to compute the ERA1 accumulator.
///
/// Per the ERA1 spec: `header-record := { block-hash: Bytes32, total-difficulty: Uint256 }`
#[derive(Debug, Clone)]
pub struct HeaderRecord {
/// The block hash (keccak256 of RLP-encoded header)
pub block_hash: B256,
/// The cumulative total difficulty at this block
pub total_difficulty: U256,
}
/// A block tuple in an Era1 file, containing all components for a single block
@@ -759,44 +691,6 @@ mod tests {
}
}
#[test]
fn test_accumulator_from_header_records_known_vectors() {
// Known-answer vectors computed from the SSZ spec:
// hash_tree_root(List[HeaderRecord, 8192])
let expected_empty: B256 =
"4a8c3a07c8d23adc5bac61157555c3c784d53d9bc110c1370809bd23cd93777d".parse().unwrap();
let expected_single_zero: B256 =
"81fd641249670887a731386e756a7a1538dc781b1b0bf016889045d350812817".parse().unwrap();
let expected_single_nonzero: B256 =
"ada35c48d81117f4fd588554cd4c4752356336e84cb41106dea1ceb4cfac8799".parse().unwrap();
// Empty list
let acc_empty = Accumulator::from_header_records(&[]).unwrap();
assert_eq!(acc_empty.root, expected_empty);
// Single record with zero values
let records = vec![HeaderRecord { block_hash: B256::ZERO, total_difficulty: U256::ZERO }];
let acc = Accumulator::from_header_records(&records).unwrap();
assert_eq!(acc.root, expected_single_zero);
// Single record with non-zero values
let records2 = vec![HeaderRecord {
block_hash: B256::from([1u8; 32]),
total_difficulty: U256::from(100u64),
}];
let acc2 = Accumulator::from_header_records(&records2).unwrap();
assert_eq!(acc2.root, expected_single_nonzero);
}
#[test]
fn test_accumulator_rejects_oversized_input() {
let records = vec![
HeaderRecord { block_hash: B256::ZERO, total_difficulty: U256::ZERO };
MAX_BLOCKS_PER_ERA1 + 1
];
assert!(Accumulator::from_header_records(&records).is_err());
}
#[test]
fn test_receipt_list_compression() {
let receipts = create_test_receipts();

View File

@@ -102,8 +102,8 @@ pub struct Era1Id {
/// Number of blocks in the file
pub block_count: u32,
/// Optional hash identifier for this file.
/// First 4 bytes of the accumulator root hash.
/// Optional hash identifier for this file
/// First 4 bytes of the last historical root in the last state in the era file
pub hash: Option<[u8; 4]>,
/// Whether to include era count in filename

View File

@@ -491,6 +491,7 @@ mod tests {
fn parse_env_filter_directives() {
let temp_dir = tempfile::tempdir().unwrap();
unsafe { std::env::set_var("RUST_LOG", "info,evm=debug") };
let reth = Cli::try_parse_args_from([
"reth",
"init",

View File

@@ -109,8 +109,13 @@ where
result: &BlockExecutionResult<N::Receipt>,
receipt_root_bloom: Option<ReceiptRootBloom>,
) -> Result<(), ConsensusError> {
let res =
validate_block_post_execution(block, &self.chain_spec, result, receipt_root_bloom);
let res = validate_block_post_execution(
block,
&self.chain_spec,
&result.receipts,
&result.requests,
receipt_root_bloom,
);
if self.skip_requests_hash_check &&
let Err(ConsensusError::BodyRequestsHashDiff(_)) = &res

View File

@@ -1,10 +1,9 @@
use alloc::vec::Vec;
use alloy_consensus::{proofs::calculate_receipt_root, BlockHeader, TxReceipt};
use alloy_eips::Encodable2718;
use alloy_eips::{eip7685::Requests, Encodable2718};
use alloy_primitives::{Bloom, Bytes, B256};
use reth_chainspec::EthereumHardforks;
use reth_consensus::ConsensusError;
use reth_execution_types::BlockExecutionResult;
use reth_primitives_traits::{
receipt::gas_spent_by_transactions, Block, GotExpected, Receipt, RecoveredBlock,
};
@@ -19,7 +18,8 @@ use reth_primitives_traits::{
pub fn validate_block_post_execution<B, R, ChainSpec>(
block: &RecoveredBlock<B>,
chain_spec: &ChainSpec,
result: &BlockExecutionResult<R>,
receipts: &[R],
requests: &Requests,
receipt_root_bloom: Option<(B256, Bloom)>,
) -> Result<(), ConsensusError>
where
@@ -28,10 +28,12 @@ where
ChainSpec: EthereumHardforks,
{
// Check if gas used matches the value set in header.
if block.header().gas_used() != result.gas_used {
let cumulative_gas_used =
receipts.last().map(|receipt| receipt.cumulative_gas_used()).unwrap_or(0);
if block.header().gas_used() != cumulative_gas_used {
return Err(ConsensusError::BlockGasUsed {
gas: GotExpected { got: result.gas_used, expected: block.header().gas_used() },
gas_spent_by_tx: gas_spent_by_transactions(&result.receipts),
gas: GotExpected { got: cumulative_gas_used, expected: block.header().gas_used() },
gas_spent_by_tx: gas_spent_by_transactions(receipts),
})
}
@@ -40,7 +42,7 @@ where
// transaction This was replaced with is_success flag.
// See more about EIP here: https://eips.ethereum.org/EIPS/eip-658
if chain_spec.is_byzantium_active_at_block(block.header().number()) {
let res = if let Some((receipts_root, logs_bloom)) = receipt_root_bloom {
let result = if let Some((receipts_root, logs_bloom)) = receipt_root_bloom {
compare_receipts_root_and_logs_bloom(
receipts_root,
logs_bloom,
@@ -48,16 +50,11 @@ where
block.header().logs_bloom(),
)
} else {
verify_receipts(
block.header().receipts_root(),
block.header().logs_bloom(),
&result.receipts,
)
verify_receipts(block.header().receipts_root(), block.header().logs_bloom(), receipts)
};
if let Err(error) = res {
let receipts = result
.receipts
if let Err(error) = result {
let receipts = receipts
.iter()
.map(|r| Bytes::from(r.with_bloom_ref().encoded_2718()))
.collect::<Vec<_>>();
@@ -71,7 +68,7 @@ where
let Some(header_requests_hash) = block.header().requests_hash() else {
return Err(ConsensusError::RequestsHashMissing)
};
let requests_hash = result.requests.requests_hash();
let requests_hash = requests.requests_hash();
if requests_hash != header_requests_hash {
return Err(ConsensusError::BodyRequestsHashDiff(
GotExpected::new(requests_hash, header_requests_hash).into(),

View File

@@ -195,7 +195,6 @@ where
ommers: &block.body().ommers,
withdrawals: block.body().withdrawals.as_ref().map(|w| Cow::Borrowed(w.as_slice())),
extra_data: block.header().extra_data.clone(),
slot_number: block.header().slot_number,
})
}
@@ -211,7 +210,6 @@ where
ommers: &[],
withdrawals: attributes.withdrawals.map(|w| Cow::Owned(w.into_inner())),
extra_data: attributes.extra_data,
slot_number: attributes.slot_number,
})
}
}
@@ -276,7 +274,7 @@ where
gas_limit: payload.payload.gas_limit(),
basefee: payload.payload.saturated_base_fee_per_gas(),
blob_excess_gas_and_price,
slot_num: payload.payload.as_v4().map(|v4| v4.slot_number).unwrap_or_default(),
slot_num: 0,
};
Ok(EvmEnv { cfg_env, block_env })
@@ -293,7 +291,6 @@ where
ommers: &[],
withdrawals: payload.payload.withdrawals().map(|w| Cow::Borrowed(w.as_slice())),
extra_data: payload.payload.as_v1().extra_data.clone(),
slot_number: payload.payload.as_v4().map(|v4| v4.slot_number),
})
}

View File

@@ -37,6 +37,7 @@ reth-rpc-eth-types.workspace = true
reth-engine-local.workspace = true
reth-engine-primitives = { workspace = true, features = ["std"] }
reth-payload-primitives.workspace = true
# ethereum
alloy-eips.workspace = true
alloy-network.workspace = true
@@ -48,7 +49,7 @@ tokio.workspace = true
# revm with required ethereum features
# Note: this must be kept to ensure all features are properly enabled/forwarded
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit", "p256-aws-lc-rs"] }
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] }
# misc
eyre.workspace = true

View File

@@ -26,10 +26,9 @@ use reth_node_builder::{
},
node::{FullNodeTypes, NodeTypes},
rpc::{
BasicEngineApiBuilder, BasicEngineValidatorBuilder, Either, EngineApiBuilder,
EngineValidatorAddOn, EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity,
PayloadValidatorBuilder, RethAuthHttpMiddleware, RethRpcAddOns, RethRpcMiddleware,
RpcAddOns, RpcHandle, Stack,
BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn,
EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity, PayloadValidatorBuilder,
RethRpcAddOns, RpcAddOns, RpcHandle,
},
BuilderContext, DebugNode, Node, NodeAdapter,
};
@@ -40,7 +39,7 @@ use reth_rpc::{
TestingApi, ValidationApi,
};
use reth_rpc_api::servers::{BlockSubmissionValidationApiServer, TestingApiServer};
use reth_rpc_builder::config::RethRpcServerConfig;
use reth_rpc_builder::{config::RethRpcServerConfig, middleware::RethRpcMiddleware};
use reth_rpc_eth_api::{
helpers::{
config::{EthConfigApiServer, EthConfigHandler},
@@ -166,21 +165,17 @@ pub struct EthereumAddOns<
EB = BasicEngineApiBuilder<PVB>,
EVB = BasicEngineValidatorBuilder<PVB>,
RpcMiddleware = Identity,
AuthHttpMiddleware = Identity,
> {
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>,
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
where
N: FullNodeComponents,
EthB: EthApiBuilder<N>,
{
/// Creates a new instance from the inner `RpcAddOns`.
pub const fn new(
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
) -> Self {
pub const fn new(inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>) -> Self {
Self { inner }
}
}
@@ -204,13 +199,11 @@ where
BasicEngineApiBuilder::default(),
BasicEngineValidatorBuilder::default(),
Default::default(),
Identity::new(),
))
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
where
N: FullNodeComponents,
EthB: EthApiBuilder<N>,
@@ -219,7 +212,7 @@ where
pub fn with_engine_api<T>(
self,
engine_api_builder: T,
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware, AuthHttpMiddleware>
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware>
where
T: Send,
{
@@ -231,7 +224,7 @@ where
pub fn with_payload_validator<V, T>(
self,
payload_validator_builder: T,
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware, AuthHttpMiddleware> {
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware> {
let Self { inner } = self;
EthereumAddOns::new(inner.with_payload_validator(payload_validator_builder))
}
@@ -240,7 +233,7 @@ where
pub fn with_rpc_middleware<T>(
self,
rpc_middleware: T,
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T, AuthHttpMiddleware>
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T>
where
T: Send,
{
@@ -248,45 +241,6 @@ where
EthereumAddOns::new(inner.with_rpc_middleware(rpc_middleware))
}
/// Configures the HTTP transport middleware for the auth / Engine API server.
pub fn with_auth_http_middleware<T>(
self,
auth_http_middleware: T,
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, T>
where
T: Send,
{
let Self { inner } = self;
EthereumAddOns::new(inner.with_auth_http_middleware(auth_http_middleware))
}
/// Stacks an additional HTTP transport middleware layer for the auth / Engine API server.
pub fn layer_auth_http_middleware<T>(
self,
layer: T,
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, Stack<AuthHttpMiddleware, T>> {
let Self { inner } = self;
EthereumAddOns::new(inner.layer_auth_http_middleware(layer))
}
/// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server.
#[expect(clippy::type_complexity)]
pub fn option_layer_auth_http_middleware<T>(
self,
layer: Option<T>,
) -> EthereumAddOns<
N,
EthB,
PVB,
EB,
EVB,
RpcMiddleware,
Stack<AuthHttpMiddleware, Either<T, Identity>>,
> {
let Self { inner } = self;
EthereumAddOns::new(inner.option_layer_auth_http_middleware(layer))
}
/// Sets the tokio runtime for the RPC servers.
///
/// Caution: This runtime must not be created from within asynchronous context.
@@ -296,8 +250,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> NodeAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> NodeAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
where
N: FullNodeComponents<
Types: NodeTypes<
@@ -314,7 +268,6 @@ where
EthApiError: FromEvmError<N::Evm>,
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
RpcMiddleware: RethRpcMiddleware,
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
{
type Handle = RpcHandle<N, EthB::EthApi>;
@@ -370,8 +323,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> RethRpcAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> RethRpcAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
where
N: FullNodeComponents<
Types: NodeTypes<
@@ -388,7 +341,6 @@ where
EthApiError: FromEvmError<N::Evm>,
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
RpcMiddleware: RethRpcMiddleware,
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
{
type EthApi = EthB::EthApi;
@@ -397,8 +349,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> EngineValidatorAddOn<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EngineValidatorAddOn<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
where
N: FullNodeComponents<
Types: NodeTypes<
@@ -415,7 +367,6 @@ where
EthApiError: FromEvmError<N::Evm>,
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
RpcMiddleware: Send,
AuthHttpMiddleware: Send,
{
type ValidatorBuilder = EVB;

View File

@@ -25,7 +25,7 @@ use reth_evm::{
ConfigureEvm, Evm, NextBlockEnvAttributes,
};
use reth_evm_ethereum::EthEvmConfig;
use reth_execution_cache::{CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider};
use reth_execution_cache::CachedStateProvider;
use reth_payload_builder::{BlobSidecars, EthBuiltPayload};
use reth_payload_builder_primitives::PayloadBuilderError;
use reth_payload_primitives::PayloadAttributes;
@@ -172,19 +172,12 @@ where
state_provider = Box::new(CachedStateProvider::new(
state_provider,
execution_cache.cache().clone(),
// It's ok to recreate the cache every time, because it's cheap to do so for a vanilla
// Ethereum builder every 12s.
CachedStateMetrics::zeroed(CachedStateMetricsSource::Builder),
execution_cache.metrics().clone(),
));
}
let state = StateProviderDatabase::new(state_provider.as_ref());
let chain_spec = client.chain_spec();
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(attributes.timestamp());
let mut db = State::builder()
.with_database(cached_reads.as_db_mut(state))
.with_bundle_update()
.with_bal_builder_if(is_amsterdam)
.build();
let mut db =
State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build();
let mut builder = evm_config
.builder_for_next_block(
@@ -203,6 +196,8 @@ where
)
.map_err(PayloadBuilderError::other)?;
let chain_spec = client.chain_spec();
debug!(target: "payload_builder", id=%payload_id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload");
let mut cumulative_gas_used = 0;
let block_gas_limit: u64 = builder.evm_mut().block().gas_limit();
@@ -362,25 +357,6 @@ where
}
continue
}
// EIP-7778: the executor tracks gas_before_refund while the payload builder's
// pre-check uses gas_after_refund. Near-full blocks can pass the pre-check but
// fail the executor's check. Skip the tx and continue building.
Err(BlockExecutionError::Validation(
BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas {
transaction_gas_limit,
block_available_gas,
},
)) => {
trace!(target: "payload_builder", %transaction_gas_limit, %block_available_gas, ?tx_hash, "skipping transaction exceeding block gas limit");
best_txs.mark_invalid(
&pool_tx,
&InvalidPoolTransactionError::ExceedsGasLimit(
transaction_gas_limit,
block_available_gas,
),
);
continue
}
// this is an error that we should treat as fatal for this attempt
Err(err) => return Err(PayloadBuilderError::evm(err)),
};

View File

@@ -225,7 +225,7 @@ impl Discv5 {
bootstrap_lookup_interval,
bootstrap_lookup_countdown,
metrics.clone(),
Arc::downgrade(&discv5),
discv5.clone(),
);
Ok((
@@ -248,7 +248,7 @@ impl Discv5 {
discv5::Event::SocketUpdated(_) | discv5::Event::TalkRequest(_) |
// `Discovered` not unique discovered peers
discv5::Event::Discovered(_) => None,
discv5::Event::NodeInserted { .. } => {
discv5::Event::NodeInserted { replaced: _, .. } => {
// node has been inserted into kbuckets
@@ -573,19 +573,14 @@ pub fn spawn_populate_kbuckets_bg(
bootstrap_lookup_interval: u64,
bootstrap_lookup_countdown: u64,
metrics: Discv5Metrics,
discv5: std::sync::Weak<discv5::Discv5>,
discv5: Arc<discv5::Discv5>,
) {
let local_node_id = discv5.local_enr().node_id();
let lookup_interval = Duration::from_secs(lookup_interval);
let metrics = metrics.discovered_peers;
let mut kbucket_index = MAX_KBUCKET_INDEX;
let pulse_lookup_interval = Duration::from_secs(bootstrap_lookup_interval);
task::spawn(async move {
let Some(discv5_handle) = discv5.upgrade() else {
return;
};
let local_node_id = discv5_handle.local_enr().node_id();
drop(discv5_handle);
// make many fast lookup queries at bootstrap, trying to fill kbuckets at furthest
// log2distance from local node
for i in (0..bootstrap_lookup_countdown).rev() {
@@ -598,12 +593,7 @@ pub fn spawn_populate_kbuckets_bg(
"starting bootstrap boost lookup query"
);
{
let Some(discv5_handle) = discv5.upgrade() else {
return;
};
lookup(target, &discv5_handle, &metrics).await;
}
lookup(target, &discv5, &metrics).await;
tokio::time::sleep(pulse_lookup_interval).await;
}
@@ -620,12 +610,7 @@ pub fn spawn_populate_kbuckets_bg(
"starting periodic lookup query"
);
{
let Some(discv5_handle) = discv5.upgrade() else {
return;
};
lookup(target, &discv5_handle, &metrics).await;
}
lookup(target, &discv5, &metrics).await;
if kbucket_index > DEFAULT_MIN_TARGET_KBUCKET_INDEX {
// try to populate bucket one step closer
@@ -713,10 +698,8 @@ mod test {
use ::enr::{CombinedKey, EnrKey};
use rand_08::thread_rng;
use reth_chainspec::MAINNET;
use std::{
net::UdpSocket,
time::{Duration, Instant},
};
use reth_tracing::init_test_tracing;
use std::env;
use tracing::trace;
fn discv5_noop() -> Discv5 {
@@ -755,61 +738,6 @@ mod test {
Discv5::start(&secret_key, discv5_config).await.expect("should build discv5")
}
async fn start_discovery_node_with_key(
secret_key: &SecretKey,
udp_port_discv5: u16,
) -> Result<(Discv5, mpsc::Receiver<discv5::Event>), Error> {
let discv5_addr: SocketAddr = format!("127.0.0.1:{udp_port_discv5}").parse().unwrap();
let rlpx_addr: SocketAddr = "127.0.0.1:30303".parse().unwrap();
let discv5_listen_config = ListenConfig::from(discv5_addr);
let discv5_config = Config::builder(rlpx_addr)
.discv5_config(discv5::ConfigBuilder::new(discv5_listen_config).build())
.build();
Discv5::start(secret_key, discv5_config).await
}
fn unused_udp_port() -> u16 {
UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
}
async fn wait_for_udp_port_release(port: u16, timeout: Duration) {
let deadline = Instant::now() + timeout;
loop {
match UdpSocket::bind(("127.0.0.1", port)) {
Ok(socket) => {
drop(socket);
return;
}
Err(err) if Instant::now() < deadline => {
trace!(target: "net::discv5::test", %port, %err, "waiting for discv5 port release");
tokio::time::sleep(Duration::from_millis(10)).await;
}
Err(err) => panic!("discv5 did not release port {port} before timeout: {err}"),
}
}
}
#[tokio::test(flavor = "multi_thread")]
async fn discv5_releases_port_on_drop() {
reth_tracing::init_test_tracing();
let secret_key = SecretKey::new(&mut thread_rng());
let port = unused_udp_port();
let (node, updates) =
start_discovery_node_with_key(&secret_key, port).await.expect("should start discv5");
drop(updates);
drop(node);
wait_for_udp_port_release(port, Duration::from_secs(1)).await;
let restarted = start_discovery_node_with_key(&secret_key, port).await;
assert!(restarted.is_ok(), "discv5 failed to rebind dropped port: {restarted:?}");
}
#[tokio::test(flavor = "multi_thread")]
async fn discv5() {
reth_tracing::init_test_tracing();
@@ -1009,6 +937,11 @@ mod test {
#[test]
fn get_fork_id_with_different_network_stack_ids() {
unsafe {
env::set_var("RUST_LOG", "net::discv5=trace");
}
init_test_tracing();
let fork_id = MAINNET.latest_fork_id();
let sk = SecretKey::new(&mut thread_rng());

View File

@@ -20,7 +20,6 @@ reth-primitives-traits.workspace = true
# ethereum
alloy-chains = { workspace = true, features = ["rlp"] }
alloy-eip7928 = { workspace = true, features = ["rlp"], optional = true }
alloy-eips.workspace = true
alloy-primitives = { workspace = true, features = ["map"] }
alloy-rlp = { workspace = true, features = ["derive"] }
@@ -41,7 +40,6 @@ alloy-hardforks.workspace = true
reth-ethereum-primitives = { workspace = true, features = ["arbitrary"] }
alloy-primitives = { workspace = true, features = ["arbitrary", "rand"] }
alloy-consensus = { workspace = true, features = ["arbitrary"] }
alloy-eip7928 = { workspace = true, features = ["arbitrary", "rlp"] }
alloy-eips = { workspace = true, features = ["arbitrary"] }
alloy-genesis.workspace = true
alloy-chains = { workspace = true, features = ["arbitrary"] }
@@ -55,7 +53,6 @@ default = ["std"]
std = [
"alloy-chains/std",
"alloy-consensus/std",
"alloy-eip7928?/std",
"alloy-eips/std",
"alloy-genesis/std",
"alloy-primitives/std",
@@ -71,8 +68,6 @@ std = [
arbitrary = [
"reth-ethereum-primitives/arbitrary",
"alloy-chains/arbitrary",
"dep:alloy-eip7928",
"alloy-eip7928/arbitrary",
"dep:arbitrary",
"dep:proptest",
"dep:proptest-arbitrary-interop",
@@ -93,5 +88,4 @@ serde = [
"reth-primitives-traits/serde",
"reth-ethereum-primitives/serde",
"alloy-hardforks/serde",
"alloy-eip7928?/serde",
]

View File

@@ -2,7 +2,7 @@
use alloc::vec::Vec;
use alloy_primitives::{Bytes, B256};
use alloy_rlp::{BufMut, Decodable, Encodable, Header, RlpDecodableWrapper, RlpEncodableWrapper};
use alloy_rlp::{RlpDecodableWrapper, RlpEncodableWrapper};
use reth_codecs_derive::add_arbitrary_tests;
/// A request for block access lists from the given block hashes.
@@ -16,209 +16,12 @@ pub struct GetBlockAccessLists(
);
/// Response for [`GetBlockAccessLists`] containing one BAL per requested block hash.
///
/// The inner `Bytes` values store raw BAL RLP payloads and are encoded as nested RLP items, not
/// as RLP byte strings.
#[derive(Clone, Debug, PartialEq, Eq, Default)]
#[derive(Clone, Debug, PartialEq, Eq, RlpEncodableWrapper, RlpDecodableWrapper, Default)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
#[add_arbitrary_tests(rlp)]
pub struct BlockAccessLists(
/// The requested block access lists as raw RLP blobs. Per EIP-8159, unavailable entries are
/// represented by an RLP-encoded empty list (`0xc0`).
/// The requested block access lists as opaque bytes. Unavailable entries are represented by
/// empty byte slices.
pub Vec<Bytes>,
);
impl Encodable for BlockAccessLists {
fn encode(&self, out: &mut dyn BufMut) {
let payload_length = self.0.iter().map(|bytes| bytes.len()).sum();
Header { list: true, payload_length }.encode(out);
for bal in &self.0 {
out.put_slice(bal);
}
}
fn length(&self) -> usize {
let payload_length = self.0.iter().map(|bytes| bytes.len()).sum();
Header { list: true, payload_length }.length_with_payload()
}
}
impl Decodable for BlockAccessLists {
fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
let header = Header::decode(buf)?;
if !header.list {
return Err(alloy_rlp::Error::UnexpectedString)
}
let (mut payload, rest) = buf.split_at(header.payload_length);
*buf = rest;
let mut bals = Vec::new();
while !payload.is_empty() {
let item_start = payload;
let item_header = Header::decode(&mut payload)?;
if !item_header.list {
return Err(alloy_rlp::Error::UnexpectedString)
}
let item_length = item_header.length_with_payload();
bals.push(Bytes::copy_from_slice(&item_start[..item_length]));
payload = &payload[item_header.payload_length..];
}
Ok(Self(bals))
}
}
#[cfg(any(test, feature = "arbitrary"))]
impl<'a> arbitrary::Arbitrary<'a> for BlockAccessLists {
fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
let entries = Vec::<Vec<alloy_eip7928::AccountChanges>>::arbitrary(u)?
.into_iter()
.map(|entry| {
let mut out = Vec::new();
alloy_rlp::encode_list(&entry, &mut out);
Bytes::from(out)
})
.collect();
Ok(Self(entries))
}
}
#[cfg(test)]
mod tests {
use super::*;
use alloy_eip7928::{
AccountChanges, BalanceChange, CodeChange, NonceChange, SlotChanges, StorageChange,
};
use alloy_primitives::{Address, U256};
use alloy_rlp::EMPTY_LIST_CODE;
fn elaborate_account_changes(seed: u8) -> Vec<AccountChanges> {
vec![
AccountChanges {
address: Address::from([seed; 20]),
storage_changes: vec![SlotChanges::new(
U256::from_be_bytes([seed.wrapping_add(1); 32]),
vec![
StorageChange::new(1, U256::from_be_bytes([seed.wrapping_add(2); 32])),
StorageChange::new(2, U256::from_be_bytes([seed.wrapping_add(3); 32])),
],
)],
storage_reads: vec![
U256::from_be_bytes([seed.wrapping_add(4); 32]),
U256::from_be_bytes([seed.wrapping_add(5); 32]),
],
balance_changes: vec![
BalanceChange::new(1, U256::from(1_000 + seed as u64)),
BalanceChange::new(2, U256::from(2_000 + seed as u64)),
],
nonce_changes: vec![
NonceChange::new(1, seed as u64),
NonceChange::new(2, seed as u64 + 1),
],
code_changes: vec![CodeChange::new(
1,
Bytes::from(vec![0x60, seed, 0x61, seed.wrapping_add(1), 0x56]),
)],
},
AccountChanges {
address: Address::from([seed.wrapping_add(9); 20]),
storage_changes: Vec::new(),
storage_reads: vec![U256::from_be_bytes([seed.wrapping_add(10); 32])],
balance_changes: vec![BalanceChange::new(3, U256::from(3_000 + seed as u64))],
nonce_changes: vec![NonceChange::new(3, seed as u64 + 2)],
code_changes: vec![CodeChange::new(2, Bytes::from(vec![0x5f, 0x5f, 0xf3]))],
},
]
}
fn elaborate_bal_entry(seed: u8) -> Bytes {
let account_changes = elaborate_account_changes(seed);
let mut out = Vec::new();
alloy_rlp::encode_list(&account_changes, &mut out);
Bytes::from(out)
}
#[test]
fn empty_bal_entry_encodes_as_empty_list() {
let encoded =
alloy_rlp::encode(BlockAccessLists(vec![Bytes::from_static(&[EMPTY_LIST_CODE])]));
assert_eq!(encoded, vec![0xc1, EMPTY_LIST_CODE]);
}
#[test]
fn block_access_lists_roundtrip_preserves_raw_bal_items() {
let original = BlockAccessLists(vec![
Bytes::from_static(&[EMPTY_LIST_CODE]),
Bytes::from_static(&[0xc1, EMPTY_LIST_CODE]),
Bytes::from_static(&[0xc2, EMPTY_LIST_CODE, EMPTY_LIST_CODE]),
]);
let encoded = alloy_rlp::encode(&original);
let decoded = alloy_rlp::decode_exact::<BlockAccessLists>(&encoded).unwrap();
assert_eq!(decoded, original);
}
#[test]
fn empty_response_roundtrips() {
let original = BlockAccessLists(Vec::new());
let encoded = alloy_rlp::encode(&original);
let decoded = alloy_rlp::decode_exact::<BlockAccessLists>(&encoded).unwrap();
assert_eq!(decoded, original);
}
#[test]
fn rejects_non_list_bal_entries() {
let err = alloy_rlp::decode_exact::<BlockAccessLists>(&[0xc1, 0x01]).unwrap_err();
assert!(matches!(err, alloy_rlp::Error::UnexpectedString));
}
#[test]
fn elaborate_bal_entry_roundtrips_into_account_changes() {
let expected = elaborate_account_changes(0x11);
let decoded =
alloy_rlp::decode_exact::<Vec<AccountChanges>>(&elaborate_bal_entry(0x11)).unwrap();
assert_eq!(decoded, expected);
}
#[test]
fn elaborate_block_access_lists_roundtrip_preserves_complex_bal_contents() {
let original = BlockAccessLists(vec![
elaborate_bal_entry(0x11),
Bytes::from_static(&[EMPTY_LIST_CODE]),
elaborate_bal_entry(0x77),
]);
let encoded = alloy_rlp::encode(&original);
let decoded = alloy_rlp::decode_exact::<BlockAccessLists>(&encoded).unwrap();
assert_eq!(decoded, original);
assert_eq!(
alloy_rlp::decode_exact::<Vec<AccountChanges>>(&decoded.0[0]).unwrap(),
elaborate_account_changes(0x11)
);
assert_eq!(alloy_rlp::decode_exact::<Vec<AccountChanges>>(&decoded.0[1]).unwrap(), vec![]);
assert_eq!(
alloy_rlp::decode_exact::<Vec<AccountChanges>>(&decoded.0[2]).unwrap(),
elaborate_account_changes(0x77)
);
}
#[test]
fn elaborate_block_access_lists_embed_raw_bal_payloads_without_reencoding() {
let first = elaborate_bal_entry(0x21);
let second = elaborate_bal_entry(0x42);
let encoded = alloy_rlp::encode(BlockAccessLists(vec![first.clone(), second.clone()]));
let header = alloy_rlp::Header::decode(&mut &encoded[..]).unwrap();
let payload = &encoded[header.length()..];
let expected_payload = [first.as_ref(), second.as_ref()].concat();
assert!(header.list);
assert_eq!(payload, expected_payload.as_slice());
}
}

View File

@@ -1,6 +1,6 @@
//! All capability related types
use crate::{EthMessageID, EthVersion, SnapVersion};
use crate::{EthMessageID, EthVersion};
use alloc::{borrow::Cow, string::String, vec::Vec};
use alloy_primitives::bytes::Bytes;
use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
@@ -85,11 +85,6 @@ impl Capability {
Self::new_static("eth", version as usize)
}
/// Returns the corresponding snap capability for the given version.
pub const fn snap(version: SnapVersion) -> Self {
Self::new_static("snap", version as usize)
}
/// Returns the [`EthVersion::Eth66`] capability.
pub const fn eth_66() -> Self {
Self::eth(EthVersion::Eth66)
@@ -120,16 +115,6 @@ impl Capability {
Self::eth(EthVersion::Eth71)
}
/// Returns the `snap/1` capability.
pub const fn snap_1() -> Self {
Self::snap(SnapVersion::V1)
}
/// Returns the `snap/2` capability.
pub const fn snap_2() -> Self {
Self::snap(SnapVersion::V2)
}
/// Whether this is eth v66 protocol.
#[inline]
pub fn is_eth_v66(&self) -> bool {

View File

@@ -1,5 +1,6 @@
//! `RLPx` disconnect reason sent to/received from peer
use alloc::vec;
use alloy_primitives::bytes::{Buf, BufMut};
use alloy_rlp::{Decodable, Encodable, Header};
use derive_more::Display;
@@ -83,10 +84,10 @@ impl Encodable for DisconnectReason {
/// The [`Encodable`] implementation for [`DisconnectReason`] encodes the disconnect reason in
/// a single-element RLP list.
fn encode(&self, out: &mut dyn BufMut) {
alloy_rlp::encode_list(&[*self as u8], out);
vec![*self as u8].encode(out);
}
fn length(&self) -> usize {
alloy_rlp::list_length(&[*self as u8])
vec![*self as u8].length()
}
}

View File

@@ -3,41 +3,13 @@
//! facilitating the exchange of Ethereum state snapshots between peers
//! Reference: [Ethereum Snapshot Protocol](https://github.com/ethereum/devp2p/blob/master/caps/snap.md#protocol-messages)
//!
//! This module currently includes snap/1 plus preparatory snap/2 message definitions.
//! Current version: snap/1
use crate::BlockAccessLists;
use alloc::vec::Vec;
use alloy_primitives::{Bytes, B256};
use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
use reth_codecs_derive::add_arbitrary_tests;
/// Supported SNAP protocol versions.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)]
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
#[repr(u8)]
pub enum SnapVersion {
/// The original snapshot protocol.
#[default]
V1 = 1,
/// BAL-based healing as proposed by EIP-8189.
V2 = 2,
}
impl SnapVersion {
/// Returns the number of messages supported by this version.
pub const fn message_count(self) -> u8 {
match self {
Self::V1 => 8,
Self::V2 => 10,
}
}
/// Returns the highest supported message id for this version.
pub const fn max_message_id(self) -> u8 {
self.message_count() - 1
}
}
/// Message IDs for the snap sync protocol
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum SnapMessageId {
@@ -55,21 +27,9 @@ pub enum SnapMessageId {
/// Response for the number of requested contract codes.
ByteCodes = 0x05,
/// Request of the number of state (either account or storage) Merkle trie nodes by path.
///
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
GetTrieNodes = 0x06,
/// Response for the number of requested state trie nodes.
///
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
TrieNodes = 0x07,
/// Request BALs for a list of block hashes.
///
/// Only valid for `snap/2`.
GetBlockAccessLists = 0x08,
/// Response containing BALs for the requested block hashes.
///
/// Only valid for `snap/2`.
BlockAccessLists = 0x09,
}
/// Request for a range of accounts from the state trie.
@@ -227,30 +187,6 @@ pub struct TrieNodesMessage {
pub nodes: Vec<Bytes>,
}
/// Request BALs for the given block hashes.
#[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)]
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
#[add_arbitrary_tests(rlp)]
pub struct GetBlockAccessListsMessage {
/// Request ID to match up responses with.
pub request_id: u64,
/// Block hashes to retrieve BALs for.
pub block_hashes: Vec<B256>,
/// Soft limit at which to stop returning data (in bytes).
pub response_bytes: u64,
}
/// Response containing one BAL per requested block hash.
#[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)]
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
#[add_arbitrary_tests(rlp)]
pub struct BlockAccessListsMessage {
/// ID of the request this is a response for.
pub request_id: u64,
/// Raw BAL payloads in request order.
pub block_access_lists: BlockAccessLists,
}
/// Represents all types of messages in the snap sync protocol.
#[derive(Debug, Clone, PartialEq, Eq)]
pub enum SnapProtocolMessage {
@@ -267,21 +203,9 @@ pub enum SnapProtocolMessage {
/// Response with contract codes - see [`ByteCodesMessage`]
ByteCodes(ByteCodesMessage),
/// Request for trie nodes - see [`GetTrieNodesMessage`]
///
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
GetTrieNodes(GetTrieNodesMessage),
/// Response with trie nodes - see [`TrieNodesMessage`]
///
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
TrieNodes(TrieNodesMessage),
/// Request for block access lists - see [`GetBlockAccessListsMessage`]
///
/// Only valid for `snap/2`.
GetBlockAccessLists(GetBlockAccessListsMessage),
/// Response with block access lists - see [`BlockAccessListsMessage`]
///
/// Only valid for `snap/2`.
BlockAccessLists(BlockAccessListsMessage),
}
impl SnapProtocolMessage {
@@ -298,8 +222,6 @@ impl SnapProtocolMessage {
Self::ByteCodes(_) => SnapMessageId::ByteCodes,
Self::GetTrieNodes(_) => SnapMessageId::GetTrieNodes,
Self::TrieNodes(_) => SnapMessageId::TrieNodes,
Self::GetBlockAccessLists(_) => SnapMessageId::GetBlockAccessLists,
Self::BlockAccessLists(_) => SnapMessageId::BlockAccessLists,
}
}
@@ -319,8 +241,6 @@ impl SnapProtocolMessage {
Self::ByteCodes(msg) => msg.encode(&mut buf),
Self::GetTrieNodes(msg) => msg.encode(&mut buf),
Self::TrieNodes(msg) => msg.encode(&mut buf),
Self::GetBlockAccessLists(msg) => msg.encode(&mut buf),
Self::BlockAccessLists(msg) => msg.encode(&mut buf),
}
Bytes::from(buf)
@@ -394,20 +314,6 @@ impl SnapProtocolMessage {
TrieNodes,
TrieNodesMessage
);
decode_snap_message_variant!(
message_id,
buf,
SnapMessageId::GetBlockAccessLists,
GetBlockAccessLists,
GetBlockAccessListsMessage
);
decode_snap_message_variant!(
message_id,
buf,
SnapMessageId::BlockAccessLists,
BlockAccessLists,
BlockAccessListsMessage
);
Err(alloy_rlp::Error::Custom("Unknown message ID"))
}
@@ -438,9 +344,6 @@ mod tests {
#[test]
fn test_all_message_roundtrips() {
assert_eq!(SnapVersion::V1.message_count(), 8);
assert_eq!(SnapVersion::V2.message_count(), 10);
test_roundtrip(SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage {
request_id: 42,
root_hash: b256_from_u64(123),
@@ -501,20 +404,6 @@ mod tests {
request_id: 42,
nodes: vec![Bytes::from(vec![1, 2, 3])],
}));
test_roundtrip(SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage {
request_id: 42,
block_hashes: vec![b256_from_u64(123), b256_from_u64(456)],
response_bytes: 4096,
}));
test_roundtrip(SnapProtocolMessage::BlockAccessLists(BlockAccessListsMessage {
request_id: 42,
block_access_lists: BlockAccessLists(vec![
Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]),
Bytes::from_static(&[0xc1, alloy_rlp::EMPTY_LIST_CODE]),
]),
}));
}
#[test]

View File

@@ -7,7 +7,7 @@ use super::message::MAX_MESSAGE_SIZE;
use crate::{
message::{EthBroadcastMessage, ProtocolBroadcastMessage},
EthMessage, EthMessageID, EthNetworkPrimitives, EthVersion, NetworkPrimitives, ProtocolMessage,
RawCapabilityMessage, SnapProtocolMessage, SnapVersion,
RawCapabilityMessage, SnapMessageId, SnapProtocolMessage,
};
use alloy_rlp::{Bytes, BytesMut, Encodable};
use core::fmt::Debug;
@@ -74,18 +74,6 @@ where
Self { eth_snap: EthSnapStreamInner::new(eth_version), inner: stream }
}
/// Create a new eth and snap protocol stream with an explicit snap version.
pub const fn new_with_snap_version(
stream: S,
eth_version: EthVersion,
snap_version: SnapVersion,
) -> Self {
Self {
eth_snap: EthSnapStreamInner::new_with_snap_version(eth_version, snap_version),
inner: stream,
}
}
/// Create a new eth and snap protocol stream with a custom max message size.
pub const fn with_max_message_size(
stream: S,
@@ -98,35 +86,12 @@ where
}
}
/// Create a new eth and snap protocol stream with a custom max message size and snap version.
pub const fn with_max_message_size_and_snap_version(
stream: S,
eth_version: EthVersion,
snap_version: SnapVersion,
max_message_size: usize,
) -> Self {
Self {
eth_snap: EthSnapStreamInner::with_max_message_size_and_snap_version(
eth_version,
snap_version,
max_message_size,
),
inner: stream,
}
}
/// Returns the eth version
#[inline]
pub const fn eth_version(&self) -> EthVersion {
self.eth_snap.eth_version()
}
/// Returns the snap version.
#[inline]
pub const fn snap_version(&self) -> SnapVersion {
self.eth_snap.snap_version()
}
/// Returns the underlying stream
#[inline]
pub const fn inner(&self) -> &S {
@@ -228,13 +193,13 @@ where
}
}
/// Stream handling combined eth and snap protocol logic.
/// Stream handling combined eth and snap protocol logic
/// Snap version is not critical to specify yet,
/// Only one version, snap/1, does exist.
#[derive(Debug, Clone)]
struct EthSnapStreamInner<N> {
/// Eth protocol version
eth_version: EthVersion,
/// Snap protocol version
snap_version: SnapVersion,
/// Maximum allowed ETH/Snap message size.
max_message_size: usize,
/// Type marker
@@ -247,26 +212,12 @@ where
{
/// Create a new eth and snap protocol stream
const fn new(eth_version: EthVersion) -> Self {
Self::new_with_snap_version(eth_version, SnapVersion::V1)
}
/// Create a new eth and snap protocol stream with an explicit snap version.
const fn new_with_snap_version(eth_version: EthVersion, snap_version: SnapVersion) -> Self {
Self::with_max_message_size_and_snap_version(eth_version, snap_version, MAX_MESSAGE_SIZE)
Self::with_max_message_size(eth_version, MAX_MESSAGE_SIZE)
}
/// Create a new eth and snap protocol stream with a custom max message size.
const fn with_max_message_size(eth_version: EthVersion, max_message_size: usize) -> Self {
Self::with_max_message_size_and_snap_version(eth_version, SnapVersion::V1, max_message_size)
}
/// Create a new eth and snap protocol stream with a custom max message size and snap version.
const fn with_max_message_size_and_snap_version(
eth_version: EthVersion,
snap_version: SnapVersion,
max_message_size: usize,
) -> Self {
Self { eth_version, snap_version, max_message_size, _pd: PhantomData }
Self { eth_version, max_message_size, _pd: PhantomData }
}
#[inline]
@@ -274,11 +225,6 @@ where
self.eth_version
}
#[inline]
const fn snap_version(&self) -> SnapVersion {
self.snap_version
}
/// Decode a message from the stream
fn decode_message(&self, bytes: BytesMut) -> Result<EthSnapMessage<N>, EthSnapStreamError> {
if bytes.len() > self.max_message_size {
@@ -310,9 +256,8 @@ where
}
}
} else if message_id > EthMessageID::max(self.eth_version) &&
message_id <
EthMessageID::message_count(self.eth_version) +
self.snap_version.message_count()
message_id <=
EthMessageID::message_count(self.eth_version) + SnapMessageId::TrieNodes as u8
{
// Checks for multiplexed snap message IDs :
// - message_id > EthMessageID::max() : ensures it's not an eth message
@@ -368,8 +313,8 @@ mod tests {
use alloy_primitives::B256;
use alloy_rlp::Encodable;
use reth_eth_wire_types::{
message::RequestPair, BlockAccessLists, BlockAccessListsMessage, GetAccountRangeMessage,
GetBlockAccessLists, GetBlockAccessListsMessage, GetBlockHeaders, HeadersDirection,
message::RequestPair, GetAccountRangeMessage, GetBlockAccessLists, GetBlockHeaders,
HeadersDirection,
};
// Helper to create eth message and its bytes
@@ -407,22 +352,6 @@ mod tests {
(snap_msg, BytesMut::from(&encoded[..]))
}
fn create_snap2_message() -> (SnapProtocolMessage, BytesMut) {
let snap_msg = SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage {
request_id: 1,
block_hashes: vec![B256::default()],
response_bytes: 1000,
});
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
EthVersion::Eth67,
SnapVersion::V2,
);
let encoded = inner.encode_snap_message(snap_msg.clone());
(snap_msg, BytesMut::from(&encoded[..]))
}
#[test]
fn test_eth_message_roundtrip() {
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
@@ -483,25 +412,6 @@ mod tests {
}
}
#[test]
fn test_snap2_protocol() {
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
EthVersion::Eth67,
SnapVersion::V2,
);
let (snap_msg, snap_bytes) = create_snap2_message();
let encoded_bytes = inner.encode_snap_message(snap_msg.clone());
assert!(!encoded_bytes.is_empty());
let decoded_result = inner.decode_message(snap_bytes.clone());
assert!(matches!(decoded_result, Ok(EthSnapMessage::Snap(_))));
if let Ok(EthSnapMessage::Snap(decoded_msg)) = inner.decode_message(snap_bytes) {
assert_eq!(decoded_msg, snap_msg);
}
}
#[test]
fn test_message_id_boundaries() {
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
@@ -565,24 +475,4 @@ mod tests {
};
assert_eq!(decoded_eth, eth_msg);
}
#[test]
fn test_snap1_rejects_snap2_message_ids() {
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
let snap2_msg = SnapProtocolMessage::BlockAccessLists(BlockAccessListsMessage {
request_id: 1,
block_access_lists: BlockAccessLists(vec![alloy_primitives::Bytes::from_static(&[
alloy_rlp::EMPTY_LIST_CODE,
])]),
});
let encoded = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
EthVersion::Eth67,
SnapVersion::V2,
)
.encode_snap_message(snap2_msg);
let decoded = inner.decode_message(BytesMut::from(&encoded[..]));
assert!(matches!(decoded, Err(EthSnapStreamError::UnknownMessageId(_))));
}
}

View File

@@ -7,7 +7,7 @@
use crate::{
errors::{EthHandshakeError, EthStreamError},
handshake::EthereumEthHandshake,
message::{EthBroadcastMessage, EthMessageID, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
message::{EthBroadcastMessage, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
p2pstream::HANDSHAKE_TIMEOUT,
CanDisconnect, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, ProtocolMessage,
UnifiedStatus,
@@ -108,9 +108,6 @@ pub struct EthStreamInner<N> {
version: EthVersion,
/// Maximum allowed ETH message size.
max_message_size: usize,
/// When true, `NewBlock` (0x07) and `NewBlockHashes` (0x01) messages are rejected before RLP
/// decoding to avoid any memory impact for non-PoW networks.
reject_block_announcements: bool,
_pd: std::marker::PhantomData<N>,
}
@@ -125,12 +122,7 @@ where
/// Creates a new [`EthStreamInner`] with the given eth version and message size limit.
pub const fn with_max_message_size(version: EthVersion, max_message_size: usize) -> Self {
Self {
version,
max_message_size,
reject_block_announcements: false,
_pd: std::marker::PhantomData,
}
Self { version, max_message_size, _pd: std::marker::PhantomData }
}
/// Returns the eth version
@@ -139,25 +131,12 @@ where
self.version
}
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
/// RLP decoding.
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
self.reject_block_announcements = reject;
}
/// Decodes incoming bytes into an [`EthMessage`].
pub fn decode_message(&self, bytes: BytesMut) -> Result<EthMessage<N>, EthStreamError> {
if bytes.len() > self.max_message_size {
return Err(EthStreamError::MessageTooBig(bytes.len()));
}
if self.reject_block_announcements &&
let Some(&id) = bytes.first() &&
(id == EthMessageID::NewBlock.to_u8() || id == EthMessageID::NewBlockHashes.to_u8())
{
return Err(EthStreamError::UnsupportedMessage { message_id: id });
}
let msg = match ProtocolMessage::decode_message(self.version, &mut bytes.as_ref()) {
Ok(m) => m,
Err(err) => {
@@ -229,12 +208,6 @@ impl<S, N: NetworkPrimitives> EthStream<S, N> {
self.eth.version()
}
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
/// RLP decoding.
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
self.eth.set_reject_block_announcements(reject);
}
/// Returns the underlying stream.
#[inline]
pub const fn inner(&self) -> &S {

View File

@@ -1,6 +1,6 @@
//! A Protocol defines a P2P subprotocol in an `RLPx` connection
use crate::{Capability, EthMessageID, EthVersion, SnapVersion};
use crate::{Capability, EthMessageID, EthVersion};
/// Type that represents a [Capability] and the number of messages it uses.
///
@@ -30,13 +30,6 @@ impl Protocol {
Self::new(cap, messages)
}
/// Returns the corresponding snap capability for the given version.
pub const fn snap(version: SnapVersion) -> Self {
let cap = Capability::snap(version);
let messages = version.message_count();
Self::new(cap, messages)
}
/// Returns the [`EthVersion::Eth66`] capability.
pub const fn eth_66() -> Self {
Self::eth(EthVersion::Eth66)
@@ -52,16 +45,6 @@ impl Protocol {
Self::eth(EthVersion::Eth68)
}
/// Returns the `snap/1` capability.
pub const fn snap_1() -> Self {
Self::snap(SnapVersion::V1)
}
/// Returns the `snap/2` capability.
pub const fn snap_2() -> Self {
Self::snap(SnapVersion::V2)
}
/// Consumes the type and returns a tuple of the [Capability] and number of messages.
#[inline]
pub(crate) fn split(self) -> (Capability, u8) {
@@ -103,7 +86,5 @@ mod tests {
assert_eq!(Protocol::eth(EthVersion::Eth69).messages(), 18);
assert_eq!(Protocol::eth(EthVersion::Eth70).messages(), 18);
assert_eq!(Protocol::eth(EthVersion::Eth71).messages(), 20);
assert_eq!(Protocol::snap(SnapVersion::V1).messages(), 8);
assert_eq!(Protocol::snap(SnapVersion::V2).messages(), 10);
}
}

View File

@@ -13,7 +13,6 @@ use crate::{
};
use reth_eth_wire::{EthNetworkPrimitives, NetworkPrimitives};
use reth_network_api::test_utils::PeersHandleProvider;
use reth_storage_api::BalProvider;
use reth_transaction_pool::TransactionPool;
use tokio::sync::mpsc;
@@ -64,10 +63,7 @@ impl<Tx, Eth, N: NetworkPrimitives> NetworkBuilder<Tx, Eth, N> {
pub fn request_handler<Client>(
self,
client: Client,
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N>
where
Client: BalProvider,
{
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N> {
let Self { mut network, transactions, .. } = self;
let (tx, rx) = mpsc::channel(ETH_REQUEST_CHANNEL_CAPACITY);
network.set_eth_request_handler(tx);

View File

@@ -20,9 +20,7 @@ use reth_eth_wire_types::message::MAX_MESSAGE_SIZE;
use reth_ethereum_forks::{ForkFilter, Head};
use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPeer};
use reth_network_types::{PeersConfig, SessionsConfig};
use reth_storage_api::{
noop::NoopProvider, BalProvider, BlockNumReader, BlockReader, HeaderProvider,
};
use reth_storage_api::{noop::NoopProvider, BlockNumReader, BlockReader, HeaderProvider};
use reth_tasks::Runtime;
use secp256k1::SECP256K1;
use std::{collections::HashSet, net::SocketAddr, sync::Arc};
@@ -159,8 +157,7 @@ where
impl<C, N> NetworkConfig<C, N>
where
N: NetworkPrimitives,
C: BalProvider
+ BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
C: BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
+ HeaderProvider
+ Clone
+ Unpin

View File

@@ -18,7 +18,7 @@ use reth_network_api::test_utils::PeersHandle;
use reth_network_p2p::error::RequestResult;
use reth_network_peers::PeerId;
use reth_primitives_traits::Block;
use reth_storage_api::{BalProvider, BlockReader, GetBlockAccessListLimit, HeaderProvider};
use reth_storage_api::{BlockReader, HeaderProvider};
use std::{
future::Future,
pin::Pin,
@@ -282,6 +282,19 @@ where
let _ = response.send(Ok(Receipts70 { last_block_incomplete, receipts }));
}
/// Handles [`GetBlockAccessLists`] queries.
///
/// For now this returns one empty BAL per requested hash.
fn on_block_access_lists_request(
&self,
_peer_id: PeerId,
request: GetBlockAccessLists,
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
) {
let access_lists = request.0.into_iter().map(|_| Bytes::new()).collect();
let _ = response.send(Ok(BlockAccessLists(access_lists)));
}
#[inline]
fn get_receipts_response<T, F>(&self, request: GetReceipts, transform_fn: F) -> Vec<Vec<T>>
where
@@ -311,55 +324,13 @@ where
}
}
impl<C, N> EthRequestHandler<C, N>
where
N: NetworkPrimitives,
C: BalProvider,
{
/// Handles [`GetBlockAccessLists`] queries.
///
/// EIP-8159 defines the final `BlockAccessLists` response semantics:
/// <https://eips.ethereum.org/EIPS/eip-8159>
fn on_block_access_lists_request(
&self,
_peer_id: PeerId,
request: GetBlockAccessLists,
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
) {
let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit(SOFT_RESPONSE_LIMIT);
let access_lists = self
.client
.bal_store()
.get_by_hashes_with_limit(&request.0, limit)
.unwrap_or_else(|_| empty_block_access_lists_with_limit(request.0.len(), limit));
let _ = response.send(Ok(BlockAccessLists(access_lists)));
}
}
/// Builds the error fallback response while still enforcing the BAL response soft limit.
fn empty_block_access_lists_with_limit(count: usize, limit: GetBlockAccessListLimit) -> Vec<Bytes> {
let mut out = Vec::with_capacity(count);
let mut size = 0;
for _ in 0..count {
let bal = Bytes::from_static(&[0xc0]);
size += bal.len();
out.push(bal);
if limit.exceeds(size) {
break
}
}
out
}
/// An endless future.
///
/// This should be spawned or used as part of `tokio::select!`.
impl<C, N> Future for EthRequestHandler<C, N>
where
N: NetworkPrimitives,
C: BalProvider
+ BlockReader<Block = N::Block, Receipt = N::Receipt>
C: BlockReader<Block = N::Block, Receipt = N::Receipt>
+ HeaderProvider<Header = N::BlockHeader>
+ Unpin,
{

View File

@@ -6,7 +6,7 @@ use futures::{future, future::Either};
use reth_eth_wire::{BlockAccessLists, EthNetworkPrimitives, NetworkPrimitives};
use reth_network_api::test_utils::PeersHandle;
use reth_network_p2p::{
block_access_lists::client::{BalRequirement, BlockAccessListsClient},
block_access_lists::client::BlockAccessListsClient,
bodies::client::{BodiesClient, BodiesFut},
download::DownloadClient,
error::{PeerRequestResult, RequestError},
@@ -135,29 +135,11 @@ impl<N: NetworkPrimitives> BlockAccessListsClient for FetchClient<N> {
&self,
hashes: Vec<B256>,
priority: Priority,
) -> Self::Output {
self.get_block_access_lists_with_priority_and_requirement(
hashes,
priority,
BalRequirement::Mandatory,
)
}
fn get_block_access_lists_with_priority_and_requirement(
&self,
hashes: Vec<B256>,
priority: Priority,
requirement: BalRequirement,
) -> Self::Output {
let (response, rx) = oneshot::channel();
if self
.request_tx
.send(DownloadRequest::GetBlockAccessLists {
request: hashes,
response,
priority,
requirement,
})
.send(DownloadRequest::GetBlockAccessLists { request: hashes, response, priority })
.is_ok()
{
Box::pin(FlattenedResponse::from(rx))

View File

@@ -13,7 +13,6 @@ use reth_eth_wire::{
};
use reth_network_api::test_utils::PeersHandle;
use reth_network_p2p::{
block_access_lists::client::BalRequirement,
error::{EthResponseValidator, PeerRequestResult, RequestError, RequestResult},
headers::client::HeadersRequest,
priority::Priority,
@@ -160,10 +159,15 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
/// full history available
fn next_best_peer(&self, requirement: BestPeerRequirements) -> Option<PeerId> {
// filter out peers that aren't idle or don't meet the requirement
let mut idle = self
.peers
.iter()
.filter(|(_, peer)| peer.state.is_idle() && peer.satisfies(&requirement));
let mut idle = self.peers.iter().filter(|(_, peer)| {
peer.state.is_idle() &&
match &requirement {
BestPeerRequirements::EthVersion(ver) => {
peer.capabilities.supports_eth_at_least(ver)
}
_ => true,
}
});
let mut best_peer = idle.next()?;
@@ -191,14 +195,6 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
Some(*best_peer.0)
}
/// Returns whether any connected peer can serve BAL requests.
fn has_eth71_peer(&self) -> bool {
self.peers.values().any(|peer| {
!matches!(peer.state, PeerState::Closing) &&
peer.capabilities.supports_eth_at_least(&EthVersion::Eth71)
})
}
/// Returns the next action to return
fn poll_action(&mut self) -> PollAction {
// we only check and not pop here since we don't know yet whether a peer is available.
@@ -212,15 +208,9 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
let request = self.queued_requests.pop_front().expect("not empty");
let Some(peer_id) = self.next_best_peer(request.best_peer_requirements()) else {
// Optional BAL requests can lose their eth/71 peer while queued; complete them
// instead of waiting for future peer churn.
if request.is_optional_bal() && !self.has_eth71_peer() {
request.send_err_response(RequestError::UnsupportedCapability);
} else {
// no peer matches this request's requirements; requeue at the back so other
// queued requests get a chance on the next poll instead of head-of-line blocking.
self.queued_requests.push_back(request);
}
// no peer matches this request's requirements; requeue at the back so other
// queued requests get a chance on the next poll instead of head-of-line blocking.
self.queued_requests.push_back(request);
return PollAction::NoPeersAvailable
};
@@ -242,30 +232,21 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
loop {
// poll incoming requests
match self.download_requests_rx.poll_next_unpin(cx) {
Poll::Ready(Some(request)) => {
// Optional BAL requests should not wait for future peer churn if no
// connected peer can serve them right now.
if request.is_optional_bal() && !self.has_eth71_peer() {
request.send_err_response(RequestError::UnsupportedCapability);
continue
Poll::Ready(Some(request)) => match request.get_priority() {
Priority::High => {
// find the first normal request and queue before, add this request to
// the back of the high-priority queue
let pos = self
.queued_requests
.iter()
.position(|req| req.is_normal_priority())
.unwrap_or(0);
self.queued_requests.insert(pos, request);
}
match request.get_priority() {
Priority::High => {
// find first normal request and queue before it; add this request
// to the back of the high-priority queue
let pos = self
.queued_requests
.iter()
.position(|req| req.is_normal_priority())
.unwrap_or(0);
self.queued_requests.insert(pos, request);
}
Priority::Normal => {
self.queued_requests.push_back(request);
}
Priority::Normal => {
self.queued_requests.push_back(request);
}
}
},
Poll::Ready(None) => {
unreachable!("channel can't close")
}
@@ -288,15 +269,6 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
peer.state = req.peer_state();
}
self.prepare_inflight_block_request(peer_id, req)
}
/// Tracks an inflight request and converts it into a peer request.
fn prepare_inflight_block_request(
&mut self,
peer_id: PeerId,
req: DownloadRequest<N>,
) -> BlockRequest {
match req {
DownloadRequest::GetBlockHeaders { request, response, .. } => {
let inflight = Request { request: request.clone(), response };
@@ -327,23 +299,12 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
}
}
/// Returns a queued followup request the peer can serve.
///
/// This is an immediate scheduling shortcut after a successful response. It skips queued
/// requests whose hard requirements do not match this peer, leaving them for the regular peer
/// selection path.
/// Returns a new followup request for the peer.
///
/// Caution: this expects that the peer is _not_ closed.
fn followup_request(&mut self, peer_id: PeerId) -> Option<BlockResponseOutcome> {
let peer = self.peers.get_mut(&peer_id)?;
let req_idx = self.queued_requests.iter().position(|req| {
// Find the first queued request this peer can serve.
peer.satisfies(&req.best_peer_requirements())
})?;
let req = self.queued_requests.remove(req_idx).expect("valid request index");
peer.state = req.peer_state();
let req = self.prepare_inflight_block_request(peer_id, req);
let req = self.queued_requests.pop_front()?;
let req = self.prepare_block_request(peer_id, req);
Some(BlockResponseOutcome::Request(peer_id, req))
}
@@ -515,16 +476,6 @@ impl Peer {
self.range_info.as_ref().map(|info| info.range())
}
/// Returns whether this peer can serve requests with the given hard requirements.
fn satisfies(&self, requirement: &BestPeerRequirements) -> bool {
match requirement {
BestPeerRequirements::EthVersion(ver) => self.capabilities.supports_eth_at_least(ver),
BestPeerRequirements::None |
BestPeerRequirements::FullBlock |
BestPeerRequirements::FullBlockRange(_) => true,
}
}
/// Returns true if this peer has a better range than the other peer for serving the requested
/// range.
///
@@ -651,7 +602,6 @@ pub(crate) enum DownloadRequest<N: NetworkPrimitives> {
request: Vec<B256>,
response: oneshot::Sender<PeerRequestResult<BlockAccessLists>>,
priority: Priority,
requirement: BalRequirement,
},
/// Download receipts for the given block hashes and send response through channel
GetReceipts {
@@ -689,21 +639,6 @@ impl<N: NetworkPrimitives> DownloadRequest<N> {
self.get_priority().is_normal()
}
/// Returns `true` if this is an optional BAL request.
const fn is_optional_bal(&self) -> bool {
matches!(self, Self::GetBlockAccessLists { requirement: BalRequirement::Optional, .. })
}
/// Sends an error response to the waiting caller.
fn send_err_response(self, err: RequestError) {
let _ = match self {
Self::GetBlockHeaders { response, .. } => response.send(Err(err)).ok(),
Self::GetBlockBodies { response, .. } => response.send(Err(err)).ok(),
Self::GetBlockAccessLists { response, .. } => response.send(Err(err)).ok(),
Self::GetReceipts { response, .. } => response.send(Err(err)).ok(),
};
}
/// Returns the best peer requirements for this request.
fn best_peer_requirements(&self) -> BestPeerRequirements {
match self {
@@ -1469,98 +1404,6 @@ mod tests {
assert!(matches!(outcome, Some(BlockResponseOutcome::Request(pid, _)) if pid == peer_id));
}
#[tokio::test]
async fn test_followup_skips_request_peer_cannot_serve() {
let (mut fetcher, peer_id) = fetcher_with_peer();
let peer_71 = B512::random();
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
fetcher.new_active_peer(
peer_71,
B256::random(),
100,
caps_71,
Arc::new(AtomicU64::new(10)),
None,
);
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
let (followup_tx, _followup_rx) = oneshot::channel();
fetcher.queued_requests.push_back(DownloadRequest::GetBlockAccessLists {
request: vec![B256::random()],
response: followup_tx,
priority: Priority::Normal,
requirement: BalRequirement::Optional,
});
let _rx = insert_inflight_receipts(&mut fetcher, peer_id);
let resp = ReceiptsResponse::new(vec![vec![]]);
assert!(fetcher.on_receipts_response(peer_id, Ok(resp)).is_none());
assert!(fetcher.peers[&peer_id].state.is_idle());
assert!(!fetcher.inflight_bals_requests.contains_key(&peer_id));
assert!(matches!(
fetcher.queued_requests.front(),
Some(DownloadRequest::GetBlockAccessLists {
requirement: BalRequirement::Optional,
..
})
));
}
#[tokio::test]
async fn test_followup_uses_first_satisfiable_request() {
let (mut fetcher, peer_id) = fetcher_with_peer();
let peer_71 = B512::random();
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
fetcher.new_active_peer(
peer_71,
B256::random(),
100,
caps_71,
Arc::new(AtomicU64::new(10)),
None,
);
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
let (bal_tx, _bal_rx) = oneshot::channel();
fetcher.queued_requests.push_back(DownloadRequest::GetBlockAccessLists {
request: vec![B256::random()],
response: bal_tx,
priority: Priority::Normal,
requirement: BalRequirement::Optional,
});
let (bodies_tx, _bodies_rx) = oneshot::channel();
fetcher.queued_requests.push_back(DownloadRequest::GetBlockBodies {
request: vec![B256::random()],
response: bodies_tx,
priority: Priority::Normal,
range_hint: None,
});
let _rx = insert_inflight_receipts(&mut fetcher, peer_id);
let resp = ReceiptsResponse::new(vec![vec![]]);
let outcome = fetcher.on_receipts_response(peer_id, Ok(resp));
assert!(matches!(
outcome,
Some(BlockResponseOutcome::Request(pid, BlockRequest::GetBlockBodies(_))) if pid == peer_id
));
assert!(fetcher.inflight_bodies_requests.contains_key(&peer_id));
assert!(matches!(fetcher.peers[&peer_id].state, PeerState::GetBlockBodies));
assert_eq!(fetcher.queued_requests.len(), 1);
assert!(matches!(
fetcher.queued_requests.front(),
Some(DownloadRequest::GetBlockAccessLists {
requirement: BalRequirement::Optional,
..
})
));
}
#[tokio::test]
async fn test_prepare_block_request_creates_inflight_receipts() {
let (mut fetcher, peer_id) = fetcher_with_peer();
@@ -1698,7 +1541,6 @@ mod tests {
request: vec![],
response: tx,
priority: Priority::Normal,
requirement: BalRequirement::Mandatory,
});
let waker = noop_waker();
@@ -1741,138 +1583,4 @@ mod tests {
assert_eq!(peer_id, peer_71);
}
}
#[tokio::test]
async fn test_optional_bal_request_rejected_without_eth71_peer() {
use futures::task::noop_waker;
use std::task::{Context, Poll};
let manager = PeersManager::new(PeersConfig::default());
let mut fetcher =
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
let peer_old = B512::random();
let caps_old = Arc::new(Capabilities::new(vec![]));
fetcher.new_active_peer(
peer_old,
B256::random(),
100,
caps_old,
Arc::new(AtomicU64::new(10)),
None,
);
let (tx, rx) = oneshot::channel();
fetcher
.download_requests_tx
.send(DownloadRequest::GetBlockAccessLists {
request: vec![],
response: tx,
priority: Priority::Normal,
requirement: BalRequirement::Optional,
})
.unwrap();
let waker = noop_waker();
let mut cx = Context::from_waker(&waker);
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
assert!(fetcher.queued_requests.is_empty());
assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability);
}
#[tokio::test]
async fn test_optional_bal_request_waits_for_busy_eth71_peer() {
use futures::task::noop_waker;
use std::task::{Context, Poll};
let manager = PeersManager::new(PeersConfig::default());
let mut fetcher =
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
let peer_71 = B512::random();
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
fetcher.new_active_peer(
peer_71,
B256::random(),
100,
caps_71,
Arc::new(AtomicU64::new(10)),
None,
);
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
let (tx, _rx) = oneshot::channel();
fetcher
.download_requests_tx
.send(DownloadRequest::GetBlockAccessLists {
request: vec![],
response: tx,
priority: Priority::Normal,
requirement: BalRequirement::Optional,
})
.unwrap();
let waker = noop_waker();
let mut cx = Context::from_waker(&waker);
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
assert_eq!(fetcher.queued_requests.len(), 1);
}
#[tokio::test]
async fn test_queued_optional_bal_request_rejected_after_eth71_disconnect() {
use futures::task::noop_waker;
use std::task::{Context, Poll};
let manager = PeersManager::new(PeersConfig::default());
let mut fetcher =
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
let peer_old = B512::random();
let caps_old = Arc::new(Capabilities::new(vec![]));
fetcher.new_active_peer(
peer_old,
B256::random(),
100,
caps_old,
Arc::new(AtomicU64::new(10)),
None,
);
let peer_71 = B512::random();
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
fetcher.new_active_peer(
peer_71,
B256::random(),
100,
caps_71,
Arc::new(AtomicU64::new(10)),
None,
);
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
let (tx, rx) = oneshot::channel();
fetcher
.download_requests_tx
.send(DownloadRequest::GetBlockAccessLists {
request: vec![],
response: tx,
priority: Priority::Normal,
requirement: BalRequirement::Optional,
})
.unwrap();
let waker = noop_waker();
let mut cx = Context::from_waker(&waker);
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
assert_eq!(fetcher.queued_requests.len(), 1);
fetcher.on_session_closed(&peer_71);
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
assert!(fetcher.queued_requests.is_empty());
assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability);
}
}

View File

@@ -318,7 +318,6 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
extra_protocols,
handshake,
eth_max_message_size,
network_mode.is_stake(),
);
let state = NetworkState::new(

View File

@@ -577,9 +577,6 @@ pub struct AnnouncedTxTypesMetrics {
/// Histogram for tracking frequency of EIP-7702 transaction type
pub(crate) eip7702: Histogram,
/// Histogram for tracking frequency of unknown/other transaction types
pub(crate) other: Histogram,
}
/// Counts the number of transactions by their type in a block or collection.
@@ -602,9 +599,6 @@ pub struct TxTypesCounter {
/// Count of transactions conforming to EIP-7702 (Restricted Storage Windows).
pub(crate) eip7702: usize,
/// Count of unknown/other transaction types not matching any known EIP.
pub(crate) other: usize,
}
impl TxTypesCounter {
@@ -627,10 +621,6 @@ impl TxTypesCounter {
}
}
}
pub(crate) const fn increase_other(&mut self) {
self.other += 1;
}
}
impl AnnouncedTxTypesMetrics {
@@ -642,6 +632,5 @@ impl AnnouncedTxTypesMetrics {
self.eip1559.record(tx_types_counter.eip1559 as f64);
self.eip4844.record(tx_types_counter.eip4844 as f64);
self.eip7702.record(tx_types_counter.eip7702 as f64);
self.other.record(tx_types_counter.other as f64);
}
}

View File

@@ -79,9 +79,6 @@ const TIMEOUT_SCALING: u32 = 3;
/// before reading any more messages from the remote peer, throttling the peer.
const MAX_QUEUED_OUTGOING_RESPONSES: usize = 4;
/// Minimum capacity to retain for buffered incoming requests from the remote peer.
const MIN_RECEIVED_REQUESTS_CAPACITY: usize = 1;
/// Soft limit for the total number of buffered outgoing broadcast items (e.g. transaction hashes).
///
/// Many small broadcast messages carrying a single tx hash each are equivalent in cost to one
@@ -207,8 +204,8 @@ impl<N: NetworkPrimitives> ActiveSession<N> {
/// Shrinks the capacity of the internal buffers.
pub fn shrink_to_fit(&mut self) {
self.received_requests_from_remote.shrink_to(MIN_RECEIVED_REQUESTS_CAPACITY);
self.queued_outgoing.shrink_to(MAX_QUEUED_OUTGOING_RESPONSES);
self.received_requests_from_remote.shrink_to_fit();
self.queued_outgoing.shrink_to_fit();
}
/// Returns how many responses we've currently queued up.
@@ -1093,8 +1090,8 @@ impl<N: NetworkPrimitives> QueuedOutgoingMessages<N> {
self.count.increment(1);
}
pub(crate) fn shrink_to(&mut self, min_capacity: usize) {
self.messages.shrink_to(min_capacity);
pub(crate) fn shrink_to_fit(&mut self) {
self.messages.shrink_to_fit();
}
}

View File

@@ -93,15 +93,6 @@ impl<N: NetworkPrimitives> EthRlpxConnection<N> {
Self::Satellite(conn) => conn.primary_mut().start_send_raw(msg),
}
}
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
/// RLP decoding to avoid memory amplification from deserializing blocks that will be discarded.
pub fn set_reject_block_announcements(&mut self, reject: bool) {
match self {
Self::EthOnly(conn) => conn.set_reject_block_announcements(reject),
Self::Satellite(conn) => conn.primary_mut().set_reject_block_announcements(reject),
}
}
}
impl<N: NetworkPrimitives> From<EthPeerConnection<N>> for EthRlpxConnection<N> {

View File

@@ -123,9 +123,6 @@ pub struct SessionManager<N: NetworkPrimitives> {
/// Shared local range information that gets propagated to active sessions.
/// This represents the range of blocks that this node can serve to other peers.
local_range_info: BlockRangeInfo,
/// When true, block announcement messages (`NewBlock`, `NewBlockHashes`) are rejected before
/// RLP decoding on new sessions to avoid memory amplification.
reject_block_announcements: bool,
}
// === impl SessionManager ===
@@ -143,7 +140,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
extra_protocols: RlpxSubProtocols,
handshake: Arc<dyn EthRlpxHandshake>,
eth_max_message_size: usize,
reject_block_announcements: bool,
) -> Self {
let (pending_sessions_tx, pending_sessions_rx) = mpsc::channel(config.session_event_buffer);
let (active_session_tx, active_session_rx) = mpsc::channel(config.session_event_buffer);
@@ -180,7 +176,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
handshake,
eth_max_message_size,
local_range_info,
reject_block_announcements,
}
}
@@ -501,7 +496,7 @@ impl<N: NetworkPrimitives> SessionManager<N> {
local_addr,
peer_id,
capabilities,
mut conn,
conn,
status,
direction,
client_id,
@@ -568,10 +563,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
BlockRangeInfo::new(update.earliest, update.latest, update.latest_hash)
});
if self.reject_block_announcements {
conn.set_reject_block_announcements(true);
}
let session = ActiveSession {
next_id: 0,
remote_peer_id: peer_id,

View File

@@ -27,8 +27,7 @@ use reth_network_api::{
};
use reth_network_peers::PeerId;
use reth_storage_api::{
noop::NoopProvider, BalProvider, BlockReader, BlockReaderIdExt, HeaderProvider,
StateProviderFactory,
noop::NoopProvider, BlockReader, BlockReaderIdExt, HeaderProvider, StateProviderFactory,
};
use reth_tasks::Runtime;
use reth_tokio_util::EventStream;
@@ -248,7 +247,6 @@ where
Receipt = reth_ethereum_primitives::Receipt,
Header = alloy_consensus::Header,
> + HeaderProvider
+ BalProvider
+ Clone
+ Unpin
+ 'static,
@@ -321,7 +319,6 @@ where
Receipt = reth_ethereum_primitives::Receipt,
Header = alloy_consensus::Header,
> + HeaderProvider
+ BalProvider
+ Unpin
+ 'static,
Pool: TransactionPool<
@@ -465,10 +462,7 @@ where
}
/// Set a new request handler that's connected to the peer's network
pub fn install_request_handler(&mut self)
where
C: BalProvider,
{
pub fn install_request_handler(&mut self) {
let (tx, rx) = channel(ETH_REQUEST_CHANNEL_CAPACITY);
self.network.set_eth_request_handler(tx);
let peers = self.network.peers_handle();
@@ -579,7 +573,6 @@ where
Receipt = reth_ethereum_primitives::Receipt,
Header = alloy_consensus::Header,
> + HeaderProvider
+ BalProvider
+ Unpin
+ 'static,
Pool: TransactionPool<

Some files were not shown because too many files have changed in this diff Show More