mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
4 Commits
devnet4
...
mediocrego
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
e9cdd1fbf8 | ||
|
|
4ebc6a18c6 | ||
|
|
7fa5608b45 | ||
|
|
ac6106e6f3 |
116
.github/scripts/bench-reth-build.sh
vendored
116
.github/scripts/bench-reth-build.sh
vendored
@@ -1,21 +1,24 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds reth binaries for benchmarking from local source only.
|
||||
# Builds (or fetches from cache) reth binaries for benchmarking.
|
||||
#
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit>
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
|
||||
#
|
||||
# baseline — build the baseline binary at <commit> (merge-base)
|
||||
# baseline — build/fetch the baseline binary at <commit> (merge-base)
|
||||
# source-dir must be checked out at <commit>
|
||||
# feature — build the candidate binary + reth-bench at <commit>
|
||||
# feature — build/fetch the candidate binary + reth-bench at <commit>
|
||||
# source-dir must be checked out at <commit>
|
||||
# optional branch-sha is the PR head commit for cache key
|
||||
#
|
||||
# Outputs:
|
||||
# baseline: <source-dir>/target/profiling/reth (or reth-bb if BENCH_BIG_BLOCKS=true)
|
||||
# feature: <source-dir>/target/profiling/reth (or reth-bb), reth-bench installed to cargo bin
|
||||
#
|
||||
# Required: mc (MinIO client) with a configured alias
|
||||
# Optional env: BENCH_BIG_BLOCKS (true/false) — build reth-bb instead of reth
|
||||
set -euxo pipefail
|
||||
|
||||
MC="mc"
|
||||
MODE="$1"
|
||||
SOURCE_DIR="$2"
|
||||
COMMIT="$3"
|
||||
@@ -39,38 +42,103 @@ if [ "${BENCH_TRACY:-off}" != "off" ]; then
|
||||
EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
|
||||
fi
|
||||
|
||||
# Build the requested node binary with the benchmark profile.
|
||||
build_node_binary() {
|
||||
local features_arg=""
|
||||
local workspace_arg=""
|
||||
# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
|
||||
if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
|
||||
BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
|
||||
else
|
||||
BUILD_SUFFIX=""
|
||||
fi
|
||||
|
||||
cd "$SOURCE_DIR"
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
features_arg="--features ${EXTRA_FEATURES}"
|
||||
workspace_arg="--workspace"
|
||||
# Verify a cached reth binary was built from the expected commit.
|
||||
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
|
||||
verify_binary() {
|
||||
local binary="$1" expected_commit="$2"
|
||||
local version binary_sha
|
||||
version=$("$binary" --version 2>/dev/null) || return 1
|
||||
binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
|
||||
if [ -z "$binary_sha" ]; then
|
||||
echo "Warning: could not extract commit SHA from version output"
|
||||
return 1
|
||||
fi
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg
|
||||
if [ "$binary_sha" = "$expected_commit" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
|
||||
return 1
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
baseline|main)
|
||||
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
|
||||
build_node_binary
|
||||
BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null; then
|
||||
echo "Cache hit for baseline (${COMMIT}), downloading ${NODE_BIN}..."
|
||||
if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
|
||||
verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached baseline binary is stale or download failed, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
FEATURES_ARG=""
|
||||
WORKSPACE_ARG=""
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
FEATURES_ARG="--features ${EXTRA_FEATURES}"
|
||||
WORKSPACE_ARG="--workspace"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling $NODE_PKG $WORKSPACE_ARG $FEATURES_ARG
|
||||
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
|
||||
fi
|
||||
;;
|
||||
|
||||
feature|branch)
|
||||
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
|
||||
rustup show active-toolchain || rustup default stable
|
||||
build_node_binary
|
||||
make -C "$SOURCE_DIR" install-reth-bench
|
||||
BRANCH_SHA="${4:-$COMMIT}"
|
||||
BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null && $MC stat --no-list "${BUCKET}/reth-bench" &>/dev/null; then
|
||||
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
|
||||
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench && \
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" /home/ubuntu/.cargo/bin/reth-bench && \
|
||||
verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached feature binary is stale or download failed, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
rustup show active-toolchain || rustup default stable
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# Can't use `make profiling` when adding features; build explicitly
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling --workspace $NODE_PKG --features "${EXTRA_FEATURES}"
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling $NODE_PKG
|
||||
fi
|
||||
make install-reth-bench
|
||||
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
|
||||
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
|
||||
fi
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit>"
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
37
.github/scripts/bench-reth-local.sh
vendored
37
.github/scripts/bench-reth-local.sh
vendored
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# local-reth-bench.sh — Run the reth Engine API benchmark locally.
|
||||
#
|
||||
# Replicates the CI bench.yml workflow (build, local snapshot validation, system tuning,
|
||||
# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
|
||||
# interleaved B-F-F-B execution, summary, charts) without any GitHub
|
||||
# Actions glue (no PR comments, no artifact upload, no Slack).
|
||||
#
|
||||
@@ -21,17 +21,15 @@
|
||||
# Requires: the reth repo at RETH_REPO (default: ~/reth)
|
||||
#
|
||||
# Dependencies (install before first run):
|
||||
# schelk, cpupower, taskset, stdbuf, python3, curl,
|
||||
# make, uv, jq, Rust toolchain (cargo/rustup)
|
||||
# Optional:
|
||||
# mc for Tracy profile upload
|
||||
# mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
|
||||
# make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
|
||||
#
|
||||
# The script delegates to the existing bench-reth-*.sh scripts in the reth
|
||||
# repo for the actual build, snapshot, and run steps.
|
||||
set -euxo pipefail
|
||||
|
||||
# ── PATH ──────────────────────────────────────────────────────────────
|
||||
# Ensure cargo and user-local bins (uv) are visible
|
||||
# Ensure cargo and user-local bins (mc, uv) are visible
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────
|
||||
@@ -108,7 +106,7 @@ fi
|
||||
|
||||
# ── Check dependencies ───────────────────────────────────────────────
|
||||
missing=()
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq cargo; do
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -240,14 +238,19 @@ echo " Baseline src : $BASELINE_SRC"
|
||||
echo " Feature src : $FEATURE_SRC"
|
||||
echo
|
||||
|
||||
# ── Step 3: Validate local snapshot ──────────────────────────────────
|
||||
echo "▸ Validating local snapshot..."
|
||||
# ── Step 3: Check / download snapshot ────────────────────────────────
|
||||
echo "▸ Checking snapshot..."
|
||||
cd "$RETH_REPO"
|
||||
"${SCRIPTS_DIR}/bench-reth-snapshot.sh"
|
||||
echo " Snapshot is ready."
|
||||
SNAPSHOT_NEEDED=false
|
||||
if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
|
||||
SNAPSHOT_NEEDED=true
|
||||
echo " Snapshot needs update."
|
||||
else
|
||||
echo " Snapshot is up-to-date."
|
||||
fi
|
||||
echo
|
||||
|
||||
# ── Step 4: Build binaries in parallel ───────────────────────────────
|
||||
# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
|
||||
echo "▸ Building binaries (parallel)..."
|
||||
cd "$RETH_REPO"
|
||||
|
||||
@@ -259,11 +262,19 @@ PID_BASELINE=$!
|
||||
"${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
|
||||
PID_FEATURE=$!
|
||||
|
||||
PID_SNAPSHOT=
|
||||
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
|
||||
echo " Also downloading snapshot in parallel..."
|
||||
"${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
|
||||
PID_SNAPSHOT=$!
|
||||
fi
|
||||
|
||||
wait $PID_BASELINE || FAIL=1
|
||||
wait $PID_FEATURE || FAIL=1
|
||||
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
|
||||
|
||||
if [ $FAIL -ne 0 ]; then
|
||||
echo "Error: one or more build tasks failed"
|
||||
echo "Error: one or more parallel tasks failed (builds / snapshot)"
|
||||
exit 1
|
||||
fi
|
||||
echo " Binaries built successfully."
|
||||
|
||||
10
.github/scripts/bench-reth-run.sh
vendored
10
.github/scripts/bench-reth-run.sh
vendored
@@ -88,16 +88,10 @@ trap cleanup EXIT
|
||||
# Stop any leftover reth process in the scope, then recover schelk state.
|
||||
sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
|
||||
sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
|
||||
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
|
||||
sudo schelk recover -y --kill || true
|
||||
|
||||
# Mount
|
||||
sudo schelk mount -y || true
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Failed to mount benchmark datadir at ${DATADIR}"
|
||||
ls -la "$SCHELK_MOUNT" || true
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
sudo schelk mount -y
|
||||
sync
|
||||
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
|
||||
echo "=== Cache state after drop ==="
|
||||
|
||||
137
.github/scripts/bench-reth-snapshot.sh
vendored
137
.github/scripts/bench-reth-snapshot.sh
vendored
@@ -1,56 +1,129 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Validates that the benchmark snapshot has already been populated into the
|
||||
# local schelk volume.
|
||||
# Downloads the latest snapshot into the schelk volume using
|
||||
# `reth download` with progress reporting to the GitHub PR comment.
|
||||
#
|
||||
# Skips the download if the manifest content hasn't changed since
|
||||
# the last successful download (checked via SHA-256 of the manifest).
|
||||
#
|
||||
# Usage: bench-reth-snapshot.sh [--check]
|
||||
# --check Exit 0 if the local snapshot is ready, 10 if it is missing.
|
||||
# --check Only check if a download is needed; exits 0 if up-to-date, 10 if not.
|
||||
#
|
||||
# Required env:
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# Optional env:
|
||||
# BENCH_BIG_BLOCKS – true when validating the big-blocks snapshot datadir
|
||||
# BENCH_SNAPSHOT_NAME – expected snapshot label for log/error output
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# BENCH_RETH_BINARY – path to the reth binary
|
||||
# GITHUB_TOKEN – token for GitHub API calls (only for download)
|
||||
# BENCH_COMMENT_ID – PR comment ID to update (optional)
|
||||
# BENCH_REPO – owner/repo (e.g. paradigmxyz/reth)
|
||||
# BENCH_JOB_URL – link to the Actions job
|
||||
# BENCH_ACTOR – user who triggered the benchmark
|
||||
# BENCH_CONFIG – config summary line
|
||||
set -euxo pipefail
|
||||
|
||||
: "${SCHELK_MOUNT:?SCHELK_MOUNT must be set}"
|
||||
|
||||
MC="mc"
|
||||
BUCKET="minio/reth-snapshots"
|
||||
# Allow overriding the snapshot name (e.g. for big-blocks mode where the
|
||||
# big-blocks manifest specifies which base snapshot to use).
|
||||
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable}"
|
||||
MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
|
||||
DATADIR_NAME="datadir"
|
||||
HASH_MODE_SUFFIX=""
|
||||
if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
DATADIR_NAME="datadir-big-blocks"
|
||||
HASH_MODE_SUFFIX="-big-blocks"
|
||||
fi
|
||||
DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
|
||||
HASH_FILE="$HOME/.reth-bench-snapshot-hash${HASH_MODE_SUFFIX}"
|
||||
|
||||
describe_snapshot() {
|
||||
if [ -n "${BENCH_SNAPSHOT_NAME:-}" ]; then
|
||||
printf '%s' "${BENCH_SNAPSHOT_NAME}"
|
||||
elif [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
printf '%s' 'big-block weekly snapshot'
|
||||
else
|
||||
printf '%s' 'benchmark snapshot'
|
||||
fi
|
||||
# Fetch manifest and compute content hash for reliable freshness check
|
||||
MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
|
||||
exit 2
|
||||
}
|
||||
REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
|
||||
|
||||
snapshot_ready() {
|
||||
[ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ]
|
||||
}
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
|
||||
EXPECTED_SNAPSHOT="$(describe_snapshot)"
|
||||
|
||||
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
|
||||
sudo schelk mount -y || true
|
||||
|
||||
if snapshot_ready; then
|
||||
echo "Found local ${EXPECTED_SNAPSHOT} at ${DATADIR}"
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "::error::Missing local ${EXPECTED_SNAPSHOT} at ${DATADIR}. Benchmarks no longer download snapshots; pre-populate the local schelk data first."
|
||||
ls -la "$SCHELK_MOUNT" || true
|
||||
ls -la "$DATADIR" || true
|
||||
|
||||
echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 10
|
||||
fi
|
||||
|
||||
exit 1
|
||||
RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
|
||||
if [ ! -x "$RETH" ]; then
|
||||
echo "::error::reth binary not found or not executable at $RETH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve the MinIO HTTP endpoint from the mc alias so reth can
|
||||
# fetch archives over HTTP (the manifest's embedded base_url points
|
||||
# to the cluster-internal address which is unreachable from runners).
|
||||
MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
|
||||
if [ -z "$MINIO_ENDPOINT" ]; then
|
||||
echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
|
||||
exit 1
|
||||
fi
|
||||
BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/${SNAPSHOT_NAME}"
|
||||
|
||||
# Rewrite manifest's base_url with the runner-reachable endpoint
|
||||
MANIFEST_TMP=$(mktemp --suffix=.json)
|
||||
trap 'rm -f -- "$MANIFEST_TMP"' EXIT
|
||||
echo "$MANIFEST_CONTENT" \
|
||||
| jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
|
||||
|
||||
# Prepare mount. If a previous run left the volume mounted, recover first.
|
||||
sudo schelk recover -y --kill || true
|
||||
sudo schelk mount -y
|
||||
sudo rm -rf "$DATADIR"
|
||||
sudo mkdir -p "$DATADIR"
|
||||
# reth download runs as current user (not root), needs write access
|
||||
sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
|
||||
|
||||
update_comment() {
|
||||
local status="$1"
|
||||
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
|
||||
local body
|
||||
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
|
||||
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
|
||||
curl -sf -X PATCH \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
|
||||
-d "$(jq -nc --arg body "$body" '{body: $body}')" \
|
||||
> /dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
update_comment "Downloading snapshot…"
|
||||
|
||||
# Download using reth download (manifest-path with rewritten base_url)
|
||||
"$RETH" download \
|
||||
--manifest-path "$MANIFEST_TMP" \
|
||||
-y \
|
||||
--minimal \
|
||||
--datadir "$DATADIR"
|
||||
|
||||
update_comment "Downloading snapshot… done"
|
||||
echo "Snapshot download complete"
|
||||
|
||||
# Sanity check: verify expected directories exist
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Promote the new snapshot to become the schelk baseline (virgin volume).
|
||||
# This copies changed blocks from scratch → virgin so that future
|
||||
# `schelk recover` calls restore to this new state.
|
||||
sync
|
||||
sudo schelk promote -y
|
||||
|
||||
# Save manifest hash
|
||||
echo "$REMOTE_HASH" > "$HASH_FILE"
|
||||
echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
|
||||
33
.github/scripts/bench-reth-summary.py
vendored
33
.github/scripts/bench-reth-summary.py
vendored
@@ -111,14 +111,6 @@ def compute_stats(combined: list[dict]) -> dict:
|
||||
wall_clock_s = sum(total_latencies_ms) / 1_000
|
||||
mean_total_lat_ms = sum(total_latencies_ms) / n
|
||||
|
||||
# Persistence wait mean (for main table)
|
||||
persist_values_ms = []
|
||||
for r in combined:
|
||||
v = r.get("persistence_wait_us")
|
||||
if v is not None:
|
||||
persist_values_ms.append(v / 1_000)
|
||||
mean_persist_ms = sum(persist_values_ms) / len(persist_values_ms) if persist_values_ms else 0.0
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_ms": mean_lat,
|
||||
@@ -129,7 +121,6 @@ def compute_stats(combined: list[dict]) -> dict:
|
||||
"mean_mgas_s": mean_mgas_s,
|
||||
"wall_clock_s": wall_clock_s,
|
||||
"mean_total_lat_ms": mean_total_lat_ms,
|
||||
"mean_persist_ms": mean_persist_ms,
|
||||
}
|
||||
|
||||
|
||||
@@ -154,7 +145,7 @@ def compute_wait_stats(combined: list[dict], field: str) -> dict:
|
||||
|
||||
def _paired_data(
|
||||
baseline: list[dict], feature: list[dict]
|
||||
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float], list[float]]:
|
||||
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float]]:
|
||||
"""Match blocks and return paired latencies and per-block diffs.
|
||||
|
||||
Returns:
|
||||
@@ -162,7 +153,6 @@ def _paired_data(
|
||||
lat_diffs_ms: list of feature − baseline latency diffs in ms
|
||||
mgas_diffs: list of feature − baseline Mgas/s diffs
|
||||
total_lat_diffs_ms: list of feature − baseline total latency diffs in ms
|
||||
persist_diffs_ms: list of feature − baseline persistence wait diffs in ms
|
||||
"""
|
||||
baseline_by_block = {r["block_number"]: r for r in baseline}
|
||||
feature_by_block = {r["block_number"]: r for r in feature}
|
||||
@@ -172,7 +162,6 @@ def _paired_data(
|
||||
lat_diffs_ms = []
|
||||
mgas_diffs = []
|
||||
total_lat_diffs_ms = []
|
||||
persist_diffs_ms = []
|
||||
for bn in common_blocks:
|
||||
b = baseline_by_block[bn]
|
||||
f = feature_by_block[bn]
|
||||
@@ -190,10 +179,7 @@ def _paired_data(
|
||||
total_lat_diffs_ms.append(
|
||||
f["total_latency_us"] / 1_000 - b["total_latency_us"] / 1_000
|
||||
)
|
||||
b_persist = (b.get("persistence_wait_us") or 0) / 1_000
|
||||
f_persist = (f.get("persistence_wait_us") or 0) / 1_000
|
||||
persist_diffs_ms.append(f_persist - b_persist)
|
||||
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms, persist_diffs_ms
|
||||
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms
|
||||
|
||||
|
||||
def compute_paired_stats(
|
||||
@@ -209,15 +195,13 @@ def compute_paired_stats(
|
||||
all_lat_diffs = []
|
||||
all_mgas_diffs = []
|
||||
all_total_lat_diffs = []
|
||||
all_persist_diffs = []
|
||||
blocks_per_pair = []
|
||||
for baseline, feature in zip(baseline_runs, feature_runs):
|
||||
pairs, lat_diffs, mgas_diffs, total_lat_diffs, persist_diffs = _paired_data(baseline, feature)
|
||||
pairs, lat_diffs, mgas_diffs, total_lat_diffs = _paired_data(baseline, feature)
|
||||
all_pairs.extend(pairs)
|
||||
all_lat_diffs.extend(lat_diffs)
|
||||
all_mgas_diffs.extend(mgas_diffs)
|
||||
all_total_lat_diffs.extend(total_lat_diffs)
|
||||
all_persist_diffs.extend(persist_diffs)
|
||||
blocks_per_pair.append(len(pairs))
|
||||
|
||||
if not all_lat_diffs:
|
||||
@@ -261,11 +245,6 @@ def compute_paired_stats(
|
||||
total_se = std_total_diff / math.sqrt(len(all_total_lat_diffs)) if all_total_lat_diffs else 0.0
|
||||
wall_clock_ci_ms = T_CRITICAL * total_se
|
||||
|
||||
mean_persist_diff = sum(all_persist_diffs) / len(all_persist_diffs) if all_persist_diffs else 0.0
|
||||
std_persist_diff = stddev(all_persist_diffs, mean_persist_diff) if len(all_persist_diffs) > 1 else 0.0
|
||||
persist_se = std_persist_diff / math.sqrt(len(all_persist_diffs)) if all_persist_diffs else 0.0
|
||||
persist_ci_ms = T_CRITICAL * persist_se
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_diff_ms": mean_diff,
|
||||
@@ -279,7 +258,6 @@ def compute_paired_stats(
|
||||
"mean_mgas_diff": mean_mgas_diff,
|
||||
"mgas_ci": mgas_ci,
|
||||
"wall_clock_ci_ms": wall_clock_ci_ms,
|
||||
"persist_ci_ms": persist_ci_ms,
|
||||
"blocks": max(blocks_per_pair),
|
||||
}
|
||||
|
||||
@@ -358,7 +336,6 @@ def compute_changes(
|
||||
("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
|
||||
("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
|
||||
("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True),
|
||||
("persist_wait", "mean_persist_ms", "persist_ci_ms", "mean_persist_ms", True),
|
||||
]
|
||||
changes = {}
|
||||
for name, stat_key, ci_key, base_key, lower_is_better in metrics:
|
||||
@@ -400,8 +377,6 @@ def generate_comparison_table(
|
||||
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
|
||||
p99_pct = pct(run1["p99_ms"], run2["p99_ms"])
|
||||
|
||||
persist_pct = pct(run1["mean_persist_ms"], run2["mean_persist_ms"])
|
||||
|
||||
# Bootstrap CIs as % of baseline percentile
|
||||
p50_ci_pct = paired["p50_ci_ms"] / run1["p50_ms"] * 100.0 if run1["p50_ms"] > 0 else 0.0
|
||||
p90_ci_pct = paired["p90_ci_ms"] / run1["p90_ms"] * 100.0 if run1["p90_ms"] > 0 else 0.0
|
||||
@@ -411,7 +386,6 @@ def generate_comparison_table(
|
||||
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
|
||||
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
|
||||
wall_ci_pct = paired["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0
|
||||
persist_ci_pct = paired["persist_ci_ms"] / run1["mean_persist_ms"] * 100.0 if run1["mean_persist_ms"] > 0 else 0.0
|
||||
|
||||
base_url = f"https://github.com/{repo}/commit"
|
||||
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
|
||||
@@ -427,7 +401,6 @@ def generate_comparison_table(
|
||||
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
|
||||
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
|
||||
f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, lower_is_better=True)} |",
|
||||
f"| Persist Wait | {fmt_ms(run1['mean_persist_ms'])} | {fmt_ms(run2['mean_persist_ms'])} | {change_str(persist_pct, persist_ci_pct, lower_is_better=True)} |",
|
||||
"",
|
||||
]
|
||||
meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}"]
|
||||
|
||||
1
.github/scripts/bench-utils.js
vendored
1
.github/scripts/bench-utils.js
vendored
@@ -83,7 +83,6 @@ function metricRows(summary) {
|
||||
{ label: 'P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) },
|
||||
{ label: 'Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
|
||||
{ label: 'Wall Clock', baseline: fmtS(b.wall_clock_s), feature: fmtS(f.wall_clock_s), change: fmtChange(c.wall_clock) },
|
||||
{ label: 'Persist Wait', baseline: fmtMs(b.mean_persist_ms || 0), feature: fmtMs(f.mean_persist_ms || 0), change: fmtChange(c.persist_wait) },
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
244
.github/scripts/fetch-grafana-dashboard.py
vendored
244
.github/scripts/fetch-grafana-dashboard.py
vendored
@@ -1,244 +0,0 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch a Grafana dashboard and convert it to the portable import format.
|
||||
|
||||
Fetches the dashboard via API, replaces internal datasource/variable references
|
||||
with template variables, and adds __inputs/__requires/__elements so the JSON is
|
||||
importable on any Grafana instance.
|
||||
|
||||
Usage:
|
||||
export FETCH_GRAFANA_DASHBOARD_URL=https://<NAMESPACE>.grafana.net
|
||||
export FETCH_GRAFANA_DASHBOARD_TOKEN=glsa_...
|
||||
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py <dashboard-uid> > output.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
PANEL_TYPE_NAMES = {
|
||||
"bargauge": "Bar gauge",
|
||||
"gauge": "Gauge",
|
||||
"heatmap": "Heatmap",
|
||||
"piechart": "Pie chart",
|
||||
"stat": "Stat",
|
||||
"table": "Table",
|
||||
"timeseries": "Time series",
|
||||
"barchart": "Bar chart",
|
||||
"text": "Text",
|
||||
"dashlist": "Dashboard list",
|
||||
"logs": "Logs",
|
||||
"nodeGraph": "Node Graph",
|
||||
"histogram": "Histogram",
|
||||
"candlestick": "Candlestick",
|
||||
"state-timeline": "State timeline",
|
||||
"status-history": "Status history",
|
||||
"geomap": "Geomap",
|
||||
"canvas": "Canvas",
|
||||
"news": "News",
|
||||
"xychart": "XY Chart",
|
||||
"trend": "Trend",
|
||||
"datagrid": "Datagrid",
|
||||
"flamegraph": "Flame Graph",
|
||||
"traces": "Traces",
|
||||
}
|
||||
|
||||
|
||||
def fetch_json(base_url: str, token: str, path: str) -> dict:
|
||||
url = f"{base_url}{path}"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def fetch_dashboard(base_url: str, token: str, uid: str) -> dict:
|
||||
return fetch_json(base_url, token, f"/api/dashboards/uid/{uid}")
|
||||
|
||||
|
||||
def fetch_grafana_version(base_url: str) -> str:
|
||||
req = urllib.request.Request(f"{base_url}/api/health")
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
data = json.loads(resp.read())
|
||||
# version string like "13.0.0-23940615780.patch2" -> take just the semver part
|
||||
version = data.get("version", "")
|
||||
# strip build metadata after the first hyphen if it looks like a pre-release
|
||||
parts = version.split("-")
|
||||
return parts[0] if parts else version
|
||||
|
||||
|
||||
def collect_panel_types(panels: list) -> set[str]:
|
||||
types = set()
|
||||
for panel in panels:
|
||||
ptype = panel.get("type", "")
|
||||
if ptype and ptype != "row":
|
||||
types.add(ptype)
|
||||
# nested panels inside collapsed rows
|
||||
for sub in panel.get("panels", []):
|
||||
sub_type = sub.get("type", "")
|
||||
if sub_type and sub_type != "row":
|
||||
types.add(sub_type)
|
||||
return types
|
||||
|
||||
|
||||
def has_expression_datasource(dashboard: dict) -> bool:
|
||||
return "__expr__" in json.dumps(dashboard)
|
||||
|
||||
|
||||
def make_exportable(dashboard: dict, grafana_version: str = "") -> dict:
|
||||
dash = json.loads(json.dumps(dashboard)) # deep copy
|
||||
|
||||
# --- Strip internal fields ---
|
||||
dash.pop("id", None)
|
||||
|
||||
# --- Rewrite links: point to the public repo instead of internal ---
|
||||
dash["links"] = [
|
||||
{
|
||||
"asDropdown": False,
|
||||
"icon": "external link",
|
||||
"includeVars": False,
|
||||
"keepTime": False,
|
||||
"tags": [],
|
||||
"targetBlank": True,
|
||||
"title": "Source (GitHub)",
|
||||
"tooltip": "View source file in repository",
|
||||
"type": "link",
|
||||
"url": "https://github.com/paradigmxyz/reth/tree/main/etc/grafana/dashboards",
|
||||
}
|
||||
]
|
||||
|
||||
# --- Datasource: victoriametrics -> prometheus ---
|
||||
dash_str = json.dumps(dash)
|
||||
dash_str = dash_str.replace("victoriametrics-metrics-datasource", "prometheus")
|
||||
dash = json.loads(dash_str)
|
||||
|
||||
# --- Templating: instance_label constant -> ${VAR_INSTANCE_LABEL} ---
|
||||
# Also strip default-value fields the API returns that are not needed for import
|
||||
STRIP_VAR_DEFAULTS = {"allowCustomValue", "regexApplyTo"}
|
||||
for var in dash.get("templating", {}).get("list", []):
|
||||
if var.get("name") == "instance_label" and var.get("type") == "constant":
|
||||
var["query"] = "${VAR_INSTANCE_LABEL}"
|
||||
var["current"] = {
|
||||
"value": "${VAR_INSTANCE_LABEL}",
|
||||
"text": "${VAR_INSTANCE_LABEL}",
|
||||
"selected": False,
|
||||
}
|
||||
var["options"] = [
|
||||
{
|
||||
"value": "${VAR_INSTANCE_LABEL}",
|
||||
"text": "${VAR_INSTANCE_LABEL}",
|
||||
"selected": False,
|
||||
}
|
||||
]
|
||||
# Clear current values for query/datasource vars (not meaningful for import)
|
||||
elif var.get("type") in ("query", "datasource"):
|
||||
var["current"] = {}
|
||||
# Remove noisy default fields
|
||||
for field in STRIP_VAR_DEFAULTS:
|
||||
var.pop(field, None)
|
||||
# Strip falsy defaults on query/datasource vars (API returns them, export omits them)
|
||||
if var.get("type") in ("query", "datasource"):
|
||||
for field in ("hide", "multi", "skipUrlSync"):
|
||||
if not var.get(field):
|
||||
var.pop(field, None)
|
||||
|
||||
# --- Build __inputs ---
|
||||
inputs = [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus",
|
||||
},
|
||||
]
|
||||
|
||||
if has_expression_datasource(dash):
|
||||
inputs.append(
|
||||
{
|
||||
"name": "DS_EXPRESSION",
|
||||
"label": "Expression",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "__expr__",
|
||||
}
|
||||
)
|
||||
|
||||
inputs.append(
|
||||
{
|
||||
"name": "VAR_INSTANCE_LABEL",
|
||||
"type": "constant",
|
||||
"label": "Instance Label",
|
||||
"value": "job",
|
||||
"description": "",
|
||||
}
|
||||
)
|
||||
|
||||
# --- Build __requires ---
|
||||
requires = []
|
||||
|
||||
if has_expression_datasource(dash):
|
||||
requires.append({"type": "datasource", "id": "__expr__", "version": "1.0.0"})
|
||||
|
||||
panel_types = collect_panel_types(dash.get("panels", []))
|
||||
for pt in sorted(panel_types):
|
||||
requires.append(
|
||||
{
|
||||
"type": "panel",
|
||||
"id": pt,
|
||||
"name": PANEL_TYPE_NAMES.get(pt, pt),
|
||||
"version": "",
|
||||
}
|
||||
)
|
||||
|
||||
requires.append(
|
||||
{"type": "grafana", "id": "grafana", "name": "Grafana", "version": grafana_version}
|
||||
)
|
||||
requires.append(
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "1.0.0",
|
||||
}
|
||||
)
|
||||
|
||||
# --- Assemble output (with __inputs/__requires/__elements first) ---
|
||||
output = {
|
||||
"__inputs": inputs,
|
||||
"__elements": {},
|
||||
"__requires": requires,
|
||||
}
|
||||
output.update(dash)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <dashboard-uid>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
uid = sys.argv[1]
|
||||
base_url = os.environ.get("FETCH_GRAFANA_DASHBOARD_URL", "").rstrip("/")
|
||||
token = os.environ.get("FETCH_GRAFANA_DASHBOARD_TOKEN", "")
|
||||
|
||||
if not base_url or not token:
|
||||
print(
|
||||
"Error: FETCH_GRAFANA_DASHBOARD_URL and FETCH_GRAFANA_DASHBOARD_TOKEN env vars required",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
resp = fetch_dashboard(base_url, token, uid)
|
||||
dashboard = resp["dashboard"]
|
||||
|
||||
grafana_version = fetch_grafana_version(base_url)
|
||||
exported = make_exportable(dashboard, grafana_version)
|
||||
print(json.dumps(exported, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
81
.github/workflows/bench-scheduled.yml
vendored
81
.github/workflows/bench-scheduled.yml
vendored
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
- name: Alert on long-running hourly
|
||||
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -154,7 +154,7 @@ jobs:
|
||||
|
||||
- name: Alert on stale nightly
|
||||
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -278,7 +278,7 @@ jobs:
|
||||
|
||||
- name: Resolve job URL
|
||||
id: job-url
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
@@ -307,6 +307,12 @@ jobs:
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
@@ -334,7 +340,7 @@ jobs:
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -360,30 +366,35 @@ jobs:
|
||||
echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Validate local snapshot
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
@@ -407,6 +418,15 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -660,7 +680,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
@@ -739,7 +759,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -894,7 +914,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -905,8 +925,8 @@ jobs:
|
||||
if (!token || !channel) return;
|
||||
|
||||
const steps_status = [
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
|
||||
@@ -928,7 +948,7 @@ jobs:
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n<@U0AAA8F0JEM> investigate this` },
|
||||
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n@ai investigate this` },
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
@@ -955,11 +975,6 @@ jobs:
|
||||
}),
|
||||
});
|
||||
|
||||
- name: Clean build outputs
|
||||
if: always()
|
||||
run: |
|
||||
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
|
||||
217
.github/workflows/bench.yml
vendored
217
.github/workflows/bench.yml
vendored
@@ -14,7 +14,7 @@ on:
|
||||
blocks:
|
||||
description: "Number of blocks to benchmark"
|
||||
required: false
|
||||
default: "500"
|
||||
default: "200"
|
||||
type: string
|
||||
big_blocks:
|
||||
description: "Use big blocks mode (pre-generated merged payloads with reth-bb)"
|
||||
@@ -34,7 +34,7 @@ on:
|
||||
warmup:
|
||||
description: "Number of warmup blocks"
|
||||
required: false
|
||||
default: "200"
|
||||
default: "100"
|
||||
type: string
|
||||
baseline:
|
||||
description: "Baseline git ref (default: merge-base)"
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
steps:
|
||||
- name: Check org membership
|
||||
if: github.event_name == 'issue_comment'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Parse arguments
|
||||
id: args
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -164,9 +164,9 @@ jobs:
|
||||
|
||||
if (context.eventName === 'workflow_dispatch') {
|
||||
actor = '${{ github.actor }}';
|
||||
blocks = '${{ github.event.inputs.blocks }}' || '500';
|
||||
warmup = '${{ github.event.inputs.warmup }}' || '200';
|
||||
if (warmup !== '200') explicitWarmup = true;
|
||||
blocks = '${{ github.event.inputs.blocks }}' || '200';
|
||||
warmup = '${{ github.event.inputs.warmup }}' || '100';
|
||||
if (warmup !== '100') explicitWarmup = true;
|
||||
baseline = '${{ github.event.inputs.baseline }}';
|
||||
feature = '${{ github.event.inputs.feature }}';
|
||||
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
|
||||
@@ -205,7 +205,7 @@ jobs:
|
||||
const enumArgs = new Map([['bal', validBalModes], ['slack', validSlackModes]]);
|
||||
const durationArgs = new Set(['wait-time']);
|
||||
const stringArgs = new Set(['baseline-args', 'feature-args']);
|
||||
const defaults = { blocks: '500', warmup: '200', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const defaults = { blocks: '200', warmup: '100', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const unknown = [];
|
||||
const invalid = [];
|
||||
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
|
||||
@@ -359,7 +359,7 @@ jobs:
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -445,7 +445,7 @@ jobs:
|
||||
|
||||
- name: Poll queue position
|
||||
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -560,7 +560,7 @@ jobs:
|
||||
|
||||
- name: Resolve checkout ref
|
||||
id: checkout-ref
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
if (!process.env.BENCH_PR) {
|
||||
@@ -586,7 +586,7 @@ jobs:
|
||||
|
||||
- name: Resolve job URL and update status
|
||||
if: env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -650,6 +650,12 @@ jobs:
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
@@ -684,7 +690,7 @@ jobs:
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -696,7 +702,7 @@ jobs:
|
||||
# Build binaries
|
||||
- name: Resolve PR head branch
|
||||
id: pr-info
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
if (process.env.BENCH_PR) {
|
||||
@@ -714,7 +720,7 @@ jobs:
|
||||
|
||||
- name: Resolve refs
|
||||
id: refs
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const { execSync } = require('child_process');
|
||||
@@ -760,68 +766,79 @@ jobs:
|
||||
core.setOutput('feature-ref', featureRef);
|
||||
core.setOutput('feature-name', featureName);
|
||||
|
||||
- name: Validate local big blocks
|
||||
- name: Check big-blocks freshness
|
||||
if: env.BENCH_BIG_BLOCKS == 'true'
|
||||
id: big-blocks-check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
|
||||
PAYLOAD_DIR="$BIG_BLOCKS_DIR/payloads"
|
||||
MANIFEST="$BIG_BLOCKS_DIR/manifest.json"
|
||||
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
|
||||
|
||||
if [ ! -f "$MANIFEST" ]; then
|
||||
echo "::error::Missing local big-blocks manifest at $MANIFEST"
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
|
||||
HASH_FILE="$HOME/.reth-bench-big-blocks-hash"
|
||||
echo "Fetching big-blocks manifest from $MANIFEST..."
|
||||
BB_MANIFEST=$($MC cat "$MANIFEST" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch big-blocks manifest from $MANIFEST"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
BASE_SNAPSHOT=$(jq -r '.base_snapshot // empty' "$MANIFEST")
|
||||
}
|
||||
BASE_SNAPSHOT=$(echo "$BB_MANIFEST" | jq -r '.base_snapshot // empty')
|
||||
if [ -z "$BASE_SNAPSHOT" ]; then
|
||||
echo "::error::Big-blocks manifest missing base_snapshot field"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if [ ! -d "$PAYLOAD_DIR" ]; then
|
||||
echo "::error::Missing local big-block payload directory at $PAYLOAD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
PAYLOAD_COUNT=$(find "$PAYLOAD_DIR" -name '*.json' | wc -l)
|
||||
if [ "$PAYLOAD_COUNT" -eq 0 ]; then
|
||||
echo "::error::No payload files found in $PAYLOAD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
|
||||
echo "Payload files: $PAYLOAD_COUNT"
|
||||
echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Validate local snapshot
|
||||
REMOTE_HASH=$(echo "$BB_MANIFEST" | sha256sum | awk '{print $1}')
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Big blocks up-to-date (hash: ${REMOTE_HASH:0:16}…)"
|
||||
echo "needed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Big blocks need update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
echo "needed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "remote-hash=${REMOTE_HASH}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
|
||||
- name: Update status (snapshot needed)
|
||||
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const s = require('./.github/scripts/bench-update-status.js');
|
||||
await s({github, context, status: 'Building binaries (snapshot update pending)...'});
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
|
||||
FEATURE_REF="${{ steps.refs.outputs.feature-ref }}"
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
@@ -845,6 +862,15 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -884,11 +910,8 @@ jobs:
|
||||
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
||||
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
||||
done || true
|
||||
# Replace any stale PM QoS holders left behind by earlier benchmark jobs.
|
||||
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
|
||||
# Prevent deep C-states (avoids wake-up latency jitter)
|
||||
sudo bash -c 'exec 3<>/dev/cpu_dma_latency; printf "\0\0\0\0" >&3; exec -a bench-cpu-dma-latency sleep infinity' &
|
||||
echo "BENCH_CPU_DMA_LATENCY_PID=$!" >> "$GITHUB_ENV"
|
||||
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
||||
# Move all IRQs to core 0 (housekeeping core)
|
||||
for irq in /proc/irq/*/smp_affinity_list; do
|
||||
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
||||
@@ -915,6 +938,45 @@ jobs:
|
||||
rm -rf "$BENCH_WORK_DIR"
|
||||
mkdir -p "$BENCH_WORK_DIR"
|
||||
|
||||
- name: Download big blocks
|
||||
if: env.BENCH_BIG_BLOCKS == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
|
||||
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
|
||||
if [ "${{ steps.big-blocks-check.outputs.needed }}" = "false" ]; then
|
||||
echo "Big blocks cached at $BIG_BLOCKS_DIR, skipping download"
|
||||
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
|
||||
exit 0
|
||||
fi
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
|
||||
rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
|
||||
|
||||
# Download and parse manifest
|
||||
echo "Downloading manifest from $MANIFEST..."
|
||||
$MC cat "$MANIFEST" > "$BIG_BLOCKS_DIR/manifest.json"
|
||||
UPLOAD_PATH=$(jq -r '.upload_path' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
COUNT=$(jq -r '.count' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
TARGET_GAS=$(jq -r '.target_gas' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
echo "Manifest: count=$COUNT, target_gas=$TARGET_GAS, archive=$UPLOAD_PATH"
|
||||
|
||||
# Download and extract archive
|
||||
ARCHIVE="minio/$UPLOAD_PATH"
|
||||
echo "Downloading big blocks from $ARCHIVE..."
|
||||
$MC cat "$ARCHIVE" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
|
||||
echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
|
||||
|
||||
# Verify expected directory structure
|
||||
if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
|
||||
echo "::error::Big blocks archive missing expected payloads/ directory"
|
||||
ls -laR "$BIG_BLOCKS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
|
||||
# Save manifest hash for freshness check on next run
|
||||
echo "${{ steps.big-blocks-check.outputs.remote-hash }}" > "$HOME/.reth-bench-big-blocks-hash"
|
||||
|
||||
- name: Start metrics proxy
|
||||
run: |
|
||||
BENCH_ID="ci-${{ github.run_id }}"
|
||||
@@ -937,7 +999,7 @@ jobs:
|
||||
|
||||
- name: Update status (running benchmarks)
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1199,7 +1261,7 @@ jobs:
|
||||
|
||||
- name: Compare & comment
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1279,7 +1341,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const jobSummary = require('./.github/scripts/bench-job-summary.js');
|
||||
@@ -1293,7 +1355,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -1304,16 +1366,14 @@ jobs:
|
||||
|
||||
- name: Update status (failed)
|
||||
if: failure() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const steps_status = [
|
||||
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
@@ -1340,18 +1400,16 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const steps_status = [
|
||||
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
@@ -1364,7 +1422,7 @@ jobs:
|
||||
|
||||
- name: Update status (cancelled)
|
||||
if: cancelled() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1375,11 +1433,6 @@ jobs:
|
||||
body: `cc @${process.env.BENCH_ACTOR}\n\n⚠️ Benchmark cancelled. [View logs](${jobUrl})`,
|
||||
});
|
||||
|
||||
- name: Clean build outputs
|
||||
if: always()
|
||||
run: |
|
||||
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
@@ -1392,9 +1445,5 @@ jobs:
|
||||
done
|
||||
# Restore amd-pstate to active (EPP) mode with powersave governor
|
||||
echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
|
||||
if [ -n "${BENCH_CPU_DMA_LATENCY_PID:-}" ]; then
|
||||
sudo kill "$BENCH_CPU_DMA_LATENCY_PID" 2>/dev/null || true
|
||||
fi
|
||||
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
|
||||
sudo cpupower frequency-set -g powersave 2>/dev/null || true
|
||||
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|
||||
|
||||
2
.github/workflows/book.yml
vendored
2
.github/workflows/book.yml
vendored
@@ -50,7 +50,7 @@ jobs:
|
||||
uses: actions/configure-pages@v6
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v5
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
with:
|
||||
path: "./docs/vocs/docs/dist"
|
||||
|
||||
|
||||
62
.github/workflows/fetch-grafana-dashboard.yml
vendored
62
.github/workflows/fetch-grafana-dashboard.yml
vendored
@@ -1,62 +0,0 @@
|
||||
name: Fetch Grafana Dashboard
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dashboard_uid:
|
||||
description: "Grafana dashboard UID to export"
|
||||
required: true
|
||||
default: "2k8BXz24x"
|
||||
target_path:
|
||||
description: "Target file path in the repo (e.g. etc/grafana/dashboards/overview.json)"
|
||||
required: true
|
||||
default: "etc/grafana/dashboards/overview.json"
|
||||
|
||||
jobs:
|
||||
fetch:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-python@v6
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Fetch dashboard from Grafana
|
||||
env:
|
||||
FETCH_GRAFANA_DASHBOARD_URL: ${{ secrets.FETCH_GRAFANA_DASHBOARD_URL }}
|
||||
FETCH_GRAFANA_DASHBOARD_TOKEN: ${{ secrets.FETCH_GRAFANA_DASHBOARD_TOKEN }}
|
||||
run: |
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py "${{ inputs.dashboard_uid }}" \
|
||||
> "${{ inputs.target_path }}"
|
||||
|
||||
- name: Check for changes
|
||||
id: diff
|
||||
run: |
|
||||
if git diff --quiet "${{ inputs.target_path }}"; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No changes detected."
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create pull request
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
TARGET="${{ inputs.target_path }}"
|
||||
FILENAME="$(basename "$TARGET")"
|
||||
BRANCH="chore/sync-grafana-${FILENAME%.*}-$(date +%Y%m%d-%H%M%S)"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$BRANCH"
|
||||
git add "$TARGET"
|
||||
git commit -m "chore: update Grafana dashboard ${FILENAME}"
|
||||
git push origin "$BRANCH"
|
||||
gh pr create \
|
||||
--title "chore: update Grafana dashboard ${FILENAME}" \
|
||||
--body "Automated export from Grafana (dashboard UID: \`${{ inputs.dashboard_uid }}\`, target: \`${TARGET}\`)."
|
||||
25
.github/workflows/grafana.yml
vendored
25
.github/workflows/grafana.yml
vendored
@@ -11,22 +11,11 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Validate dashboard format
|
||||
- name: Check for ${DS_PROMETHEUS} in overview.json
|
||||
run: |
|
||||
python3 -c "
|
||||
import json, sys
|
||||
with open('etc/grafana/dashboards/overview.json') as f:
|
||||
d = json.load(f)
|
||||
errors = []
|
||||
if '__inputs' not in d:
|
||||
errors.append('missing __inputs')
|
||||
if '__requires' not in d:
|
||||
errors.append('missing __requires')
|
||||
if d.get('id') is not None:
|
||||
errors.append('contains internal id field — use export-dashboard.py')
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f'Error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print('✓ overview.json is a valid exported dashboard')
|
||||
"
|
||||
if grep -Fn '${DS_PROMETHEUS}' etc/grafana/dashboards/overview.json; then
|
||||
echo "Error: overview.json contains '\${DS_PROMETHEUS}' placeholder"
|
||||
echo "Please replace it with '\${datasource}'"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ overview.json does not contain '\${DS_PROMETHEUS}' placeholder"
|
||||
|
||||
2
.github/workflows/label-pr.yml
vendored
2
.github/workflows/label-pr.yml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Label PRs
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const label_pr = require('./.github/scripts/label_pr.js')
|
||||
|
||||
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@@ -195,7 +195,7 @@ jobs:
|
||||
fi
|
||||
|
||||
body=$(cat <<- "ENDBODY"
|
||||

|
||||

|
||||
|
||||
## Testing Checklist (DELETE ME)
|
||||
|
||||
|
||||
995
Cargo.lock
generated
995
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
109
Cargo.toml
109
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "2.1.0"
|
||||
version = "2.0.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.93"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -129,7 +129,6 @@ members = [
|
||||
"examples/custom-node-components/",
|
||||
"examples/custom-payload-builder/",
|
||||
"examples/custom-rlpx-subprotocol",
|
||||
"examples/custom-auth-http-middleware",
|
||||
"examples/custom-rpc-middleware",
|
||||
"examples/db-access",
|
||||
"examples/exex-subscription",
|
||||
@@ -326,8 +325,8 @@ reth-cli = { path = "crates/cli/cli" }
|
||||
reth-cli-commands = { path = "crates/cli/commands" }
|
||||
reth-cli-runner = { path = "crates/cli/runner" }
|
||||
reth-cli-util = { path = "crates/cli/util" }
|
||||
reth-codecs = { version = "0.3.1", default-features = false }
|
||||
reth-codecs-derive = "0.3.1"
|
||||
reth-codecs = { version = "0.3.0", default-features = false }
|
||||
reth-codecs-derive = "0.3.0"
|
||||
reth-config = { path = "crates/config", default-features = false }
|
||||
reth-consensus = { path = "crates/consensus/consensus", default-features = false }
|
||||
reth-consensus-common = { path = "crates/consensus/common", default-features = false }
|
||||
@@ -395,7 +394,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
|
||||
reth-payload-primitives = { path = "crates/payload/primitives" }
|
||||
reth-payload-validator = { path = "crates/payload/validator" }
|
||||
reth-payload-util = { path = "crates/payload/util" }
|
||||
reth-primitives-traits = { version = "0.3.1", default-features = false }
|
||||
reth-primitives-traits = { version = "0.3.0", default-features = false }
|
||||
reth-provider = { path = "crates/storage/provider" }
|
||||
reth-prune = { path = "crates/prune/prune" }
|
||||
reth-prune-types = { path = "crates/prune/types", default-features = false }
|
||||
@@ -411,7 +410,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal
|
||||
reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
|
||||
reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
|
||||
reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
|
||||
reth-rpc-traits = { version = "0.3.1", default-features = false }
|
||||
reth-rpc-traits = { version = "0.3.0", default-features = false }
|
||||
reth-stages = { path = "crates/stages/stages" }
|
||||
reth-stages-api = { path = "crates/stages/api" }
|
||||
reth-stages-types = { path = "crates/stages/types", default-features = false }
|
||||
@@ -430,17 +429,17 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
|
||||
reth-trie-db = { path = "crates/trie/db" }
|
||||
reth-trie-parallel = { path = "crates/trie/parallel" }
|
||||
reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.1", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.0", default-features = false }
|
||||
|
||||
# revm
|
||||
revm = { version = "=37.0.0", default-features = false }
|
||||
revm-bytecode = { version = "=10.0.0", default-features = false }
|
||||
revm-database = { version = "=13.0.0", default-features = false }
|
||||
revm-state = { version = "=11.0.0", default-features = false }
|
||||
revm-primitives = { version = "=23.0.0", default-features = false }
|
||||
revm-interpreter = { version = "=35.0.0", default-features = false }
|
||||
revm-database-interface = { version = "=11.0.0", default-features = false }
|
||||
revm-inspectors = "=0.39.0"
|
||||
revm = { version = "37.0.0", default-features = false }
|
||||
revm-bytecode = { version = "10.0.0", default-features = false }
|
||||
revm-database = { version = "13.0.0", default-features = false }
|
||||
revm-state = { version = "11.0.0", default-features = false }
|
||||
revm-primitives = { version = "23.0.0", default-features = false }
|
||||
revm-interpreter = { version = "35.0.0", default-features = false }
|
||||
revm-database-interface = { version = "11.0.0", default-features = false }
|
||||
revm-inspectors = "0.38.0"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.6"
|
||||
@@ -449,40 +448,40 @@ alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.33", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.4", default-features = false }
|
||||
alloy-evm = { version = "0.33.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.32.0", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.7"
|
||||
|
||||
alloy-consensus = { version = "2.0.1", default-features = false }
|
||||
alloy-contract = { version = "2.0.1", default-features = false }
|
||||
alloy-eips = { version = "2.0.1", default-features = false }
|
||||
alloy-genesis = { version = "2.0.1", default-features = false }
|
||||
alloy-json-rpc = { version = "2.0.1", default-features = false }
|
||||
alloy-network = { version = "2.0.1", default-features = false }
|
||||
alloy-network-primitives = { version = "2.0.1", default-features = false }
|
||||
alloy-provider = { version = "2.0.1", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-client = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types = { version = "2.0.1", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "2.0.1", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "2.0.1", default-features = false }
|
||||
alloy-serde = { version = "2.0.1", default-features = false }
|
||||
alloy-signer = { version = "2.0.1", default-features = false }
|
||||
alloy-signer-local = { version = "2.0.1", default-features = false }
|
||||
alloy-transport = { version = "2.0.1" }
|
||||
alloy-transport-http = { version = "2.0.1", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "2.0.1", default-features = false }
|
||||
alloy-transport-ws = { version = "2.0.1", default-features = false }
|
||||
alloy-consensus = { version = "2.0.0", default-features = false }
|
||||
alloy-contract = { version = "2.0.0", default-features = false }
|
||||
alloy-eips = { version = "2.0.0", default-features = false }
|
||||
alloy-genesis = { version = "2.0.0", default-features = false }
|
||||
alloy-json-rpc = { version = "2.0.0", default-features = false }
|
||||
alloy-network = { version = "2.0.0", default-features = false }
|
||||
alloy-network-primitives = { version = "2.0.0", default-features = false }
|
||||
alloy-provider = { version = "2.0.0", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-client = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types = { version = "2.0.0", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "2.0.0", default-features = false }
|
||||
alloy-serde = { version = "2.0.0", default-features = false }
|
||||
alloy-signer = { version = "2.0.0", default-features = false }
|
||||
alloy-signer-local = { version = "2.0.0", default-features = false }
|
||||
alloy-transport = { version = "2.0.0" }
|
||||
alloy-transport-http = { version = "2.0.0", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "2.0.0", default-features = false }
|
||||
alloy-transport-ws = { version = "2.0.0", default-features = false }
|
||||
|
||||
# misc
|
||||
either = { version = "1.15.0", default-features = false }
|
||||
@@ -507,7 +506,6 @@ eyre = "0.6"
|
||||
fdlimit = "0.3.0"
|
||||
fixed-map = { version = "0.9", default-features = false }
|
||||
humantime = "2.1"
|
||||
imbl = "7"
|
||||
humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
@@ -700,24 +698,3 @@ vergen-git2 = "9.1.0"
|
||||
|
||||
# networking
|
||||
ipnet = "2.11"
|
||||
|
||||
[patch.crates-io]
|
||||
revm = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-context = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-database = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-state = { git = "https://github.com/bluealloy/revm", rev = "fe2549d85fb9e201e7b629f8b47bcca46d49aa1d" }
|
||||
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "a2c7a41977b468d016a339f560acb76e002766f3" }
|
||||
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "da7633f6bc9554f5a6e60773ef21b8e9d6e0cca6" }
|
||||
reth-codecs = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
|
||||
reth-codecs-derive = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
|
||||
reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
|
||||
reth-rpc-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
|
||||
reth-zstd-compressors = { git = "https://github.com/paradigmxyz/reth-core", rev = "c763480b9fa51957fbdb69b7caead5dfc4e3752c" }
|
||||
|
||||
@@ -69,7 +69,6 @@ default = [
|
||||
"jemalloc",
|
||||
"reth-cli-util/jemalloc",
|
||||
"asm-keccak",
|
||||
"keccak-cache-global",
|
||||
"min-debug-logs",
|
||||
]
|
||||
|
||||
@@ -90,12 +89,6 @@ asm-keccak = [
|
||||
"revm-primitives/asm-keccak",
|
||||
]
|
||||
|
||||
keccak-cache-global = [
|
||||
"reth-node-core/keccak-cache-global",
|
||||
"reth-node-ethereum/keccak-cache-global",
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
]
|
||||
|
||||
min-debug-logs = [
|
||||
"tracing/release_max_level_debug",
|
||||
"reth-ethereum-cli/min-debug-logs",
|
||||
|
||||
@@ -238,7 +238,6 @@ where
|
||||
withdrawals: prev_segment.ctx.withdrawals.clone(),
|
||||
extra_data: prev_segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: prev_segment.ctx.tx_count_hint,
|
||||
slot_number: prev_segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
// Clone the next segment's data before we consume inner.
|
||||
@@ -253,7 +252,6 @@ where
|
||||
withdrawals: new_segment.ctx.withdrawals.clone(),
|
||||
extra_data: new_segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: new_segment.ctx.tx_count_hint,
|
||||
slot_number: new_segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
plan.next_segment += 1;
|
||||
@@ -366,7 +364,6 @@ where
|
||||
withdrawals: seg0.ctx.withdrawals.clone(),
|
||||
extra_data: seg0.ctx.extra_data.clone(),
|
||||
tx_count_hint: seg0.ctx.tx_count_hint,
|
||||
slot_number: seg0.ctx.slot_number,
|
||||
};
|
||||
|
||||
let inner = self.inner_mut();
|
||||
@@ -425,7 +422,6 @@ where
|
||||
withdrawals: last_seg.ctx.withdrawals.clone(),
|
||||
extra_data: last_seg.ctx.extra_data.clone(),
|
||||
tx_count_hint: last_seg.ctx.tx_count_hint,
|
||||
slot_number: last_seg.ctx.slot_number,
|
||||
};
|
||||
self.inner_mut().ctx = last_ctx;
|
||||
}
|
||||
|
||||
@@ -266,7 +266,6 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: ctx.withdrawals.map(|w| std::borrow::Cow::Owned(w.into_owned())),
|
||||
extra_data: ctx.extra_data,
|
||||
slot_number: ctx.slot_number,
|
||||
};
|
||||
BigBlockSegment { start_tx, evm_env, ctx }
|
||||
})
|
||||
|
||||
@@ -176,7 +176,6 @@ impl BbAddOns {
|
||||
BasicEngineApiBuilder::default(),
|
||||
BasicEngineValidatorBuilder::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,8 +21,6 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) auth_provider: RootProvider<AnyNetwork>,
|
||||
/// The block provider is used for block queries.
|
||||
pub(crate) block_provider: RootProvider<AnyNetwork>,
|
||||
/// The local regular RPC provider is used for non-authenticated node RPCs like `testing_*`.
|
||||
pub(crate) local_rpc_provider: RootProvider<AnyNetwork>,
|
||||
/// The benchmark mode, which defines whether the benchmark should run for a closed or open
|
||||
/// range of blocks.
|
||||
pub(crate) benchmark_mode: BenchMode,
|
||||
@@ -85,11 +83,6 @@ impl BenchContext {
|
||||
let client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let auth_provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
let local_rpc_url = Url::parse(&bench_args.local_rpc_url)?;
|
||||
info!(target: "reth-bench", "Connecting to local regular RPC at {} for testing namespace calls", local_rpc_url);
|
||||
let local_rpc_provider =
|
||||
RootProvider::<AnyNetwork>::new(ClientBuilder::default().http(local_rpc_url));
|
||||
|
||||
// Computes the block range for the benchmark.
|
||||
//
|
||||
// - If `--advance` is provided, fetches the latest block from the engine and sets:
|
||||
@@ -166,7 +159,6 @@ impl BenchContext {
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
local_rpc_provider,
|
||||
benchmark_mode,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
|
||||
@@ -10,7 +10,7 @@ use alloy_eips::{
|
||||
eip1559::BaseFeeParams,
|
||||
eip7840::BlobParams,
|
||||
eip7928::{AccountChanges, BlockAccessList, SlotChanges},
|
||||
Typed2718,
|
||||
BlockNumberOrTag, Typed2718,
|
||||
};
|
||||
use alloy_primitives::{Bloom, Bytes, B256};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
@@ -32,8 +32,6 @@ use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, future::Future};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
|
||||
/// A single transaction with its gas used and raw encoded bytes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RawTransaction {
|
||||
@@ -671,6 +669,20 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_block_access_list(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
) -> eyre::Result<BlockAccessList> {
|
||||
provider
|
||||
.client()
|
||||
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.and_then(|block_access_list: Option<BlockAccessList>| {
|
||||
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn merge_block_access_list(
|
||||
merged: &mut BlockAccessList,
|
||||
incoming: BlockAccessList,
|
||||
|
||||
@@ -1,7 +1,5 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use alloy_eips::{eip7928::BlockAccessList, BlockNumberOrTag};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use eyre::Result;
|
||||
use std::{
|
||||
io::{BufReader, Read},
|
||||
@@ -71,21 +69,6 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches the block access list for a given block number using the provided provider.
|
||||
pub(crate) async fn fetch_block_access_list(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
) -> eyre::Result<BlockAccessList> {
|
||||
provider
|
||||
.client()
|
||||
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.and_then(|block_access_list: Option<BlockAccessList>| {
|
||||
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::{fetch_block_access_list, parse_duration},
|
||||
helpers::parse_duration,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
@@ -14,24 +14,14 @@ use crate::{
|
||||
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
|
||||
},
|
||||
};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::Encodable2718;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{
|
||||
ext::DebugApi,
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
Provider, RootProvider,
|
||||
};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionData, ExecutionPayloadEnvelopeV5, ForkchoiceState, PayloadAttributes,
|
||||
};
|
||||
use alloy_provider::{ext::DebugApi, Provider};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use clap::Parser;
|
||||
use eyre::{bail, ensure, Context, OptionExt};
|
||||
use eyre::{Context, OptionExt};
|
||||
use futures::{stream, StreamExt, TryStreamExt};
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use reth_rpc_api::{RethNewPayloadInput, TestingBuildBlockRequestV1};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -42,22 +32,6 @@ pub struct Command {
|
||||
#[arg(long, value_name = "RPC_URL", verbatim_doc_comment)]
|
||||
rpc_url: String,
|
||||
|
||||
/// Build a separate fork with `testing_buildBlockV1` and alternate forkchoice updates between
|
||||
/// the canonical chain and that fork on every block while the fork grows up to the configured
|
||||
/// depth.
|
||||
///
|
||||
/// This requires enabling the hidden `testing` RPC module on the target node,
|
||||
/// for example with `reth node --http --http.api eth,testing`.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "DEPTH",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "8",
|
||||
value_parser = parse_reorg_depth,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
reorg: Option<usize>,
|
||||
|
||||
/// How long to wait after a forkchoice update before sending the next payload.
|
||||
///
|
||||
/// Accepts a duration string (e.g. `100ms`, `2s`) or a bare integer treated as
|
||||
@@ -101,87 +75,22 @@ pub struct Command {
|
||||
)]
|
||||
rpc_block_buffer_size: usize,
|
||||
|
||||
/// Weather to enable bal by default or not.
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
enable_bal: bool,
|
||||
|
||||
#[command(flatten)]
|
||||
benchmark: BenchmarkArgs,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PreparedBuiltBlock {
|
||||
block_hash: B256,
|
||||
params: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct QueuedForkBlock {
|
||||
block_number: u64,
|
||||
prepared: PreparedBuiltBlock,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ReorgState {
|
||||
depth: usize,
|
||||
fork_length: usize,
|
||||
branch_point_hash: Option<B256>,
|
||||
fork_parent_hash: Option<B256>,
|
||||
}
|
||||
|
||||
impl ReorgState {
|
||||
const fn new(depth: usize) -> Self {
|
||||
Self { depth, fork_length: 0, branch_point_hash: None, fork_parent_hash: None }
|
||||
}
|
||||
|
||||
const fn push_fork_head(&mut self, canonical_parent_hash: B256, fork_head_hash: B256) {
|
||||
if self.fork_length == 0 {
|
||||
self.branch_point_hash = Some(canonical_parent_hash);
|
||||
}
|
||||
self.fork_length += 1;
|
||||
self.fork_parent_hash = Some(fork_head_hash);
|
||||
}
|
||||
|
||||
fn forkchoice_state(&self, fork_head_hash: B256) -> eyre::Result<ForkchoiceState> {
|
||||
let branch_point_hash = self.branch_point_hash.ok_or_eyre("missing reorg branch point")?;
|
||||
|
||||
Ok(ForkchoiceState {
|
||||
head_block_hash: fork_head_hash,
|
||||
safe_block_hash: branch_point_hash,
|
||||
finalized_block_hash: branch_point_hash,
|
||||
})
|
||||
}
|
||||
|
||||
const fn reset(&mut self) {
|
||||
self.fork_length = 0;
|
||||
self.branch_point_hash = None;
|
||||
self.fork_parent_hash = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark new-payload-fcu` command
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.reorg.is_some() && self.benchmark.rlp_blocks {
|
||||
bail!("--reorg cannot be combined with --rlp-blocks")
|
||||
}
|
||||
if self.reorg.is_some() && self.enable_bal {
|
||||
bail!("--reorg cannot be combined with --enable-bal")
|
||||
}
|
||||
|
||||
// Log mode configuration
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!(target: "reth-bench", "Using wait-time mode with {}ms minimum interval between blocks", duration.as_millis());
|
||||
}
|
||||
if let Some(depth) = self.reorg {
|
||||
info!(target: "reth-bench", depth, "Using testing_buildBlockV1 reorg mode");
|
||||
}
|
||||
|
||||
let BenchContext {
|
||||
benchmark_mode,
|
||||
block_provider,
|
||||
auth_provider,
|
||||
local_rpc_provider,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
@@ -269,8 +178,7 @@ impl Command {
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
let mut reorg_state = self.reorg.map(ReorgState::new);
|
||||
let mut queued_fork_block = None;
|
||||
|
||||
while let Some((block, head, safe, finalized, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = blocks.try_next().await?;
|
||||
@@ -280,39 +188,27 @@ impl Command {
|
||||
let gas_used = block.header.gas_used;
|
||||
let gas_limit = block.header.gas_limit;
|
||||
let block_number = block.header.number;
|
||||
let canonical_parent_hash = block.header.parent_hash;
|
||||
let transaction_count = block.transactions.len() as u64;
|
||||
let deferred_branch_start_block = reorg_state
|
||||
.as_ref()
|
||||
.filter(|state| state.fork_length == 0 && queued_fork_block.is_none())
|
||||
.map(|_| block.clone());
|
||||
let canonical_forkchoice_state = ForkchoiceState {
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: head,
|
||||
safe_block_hash: safe,
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let bal = if rlp.is_none() &&
|
||||
(block.header.block_access_list_hash.is_some() || self.enable_bal)
|
||||
{
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
@@ -333,12 +229,17 @@ impl Command {
|
||||
};
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, canonical_forkchoice_state)
|
||||
.await?;
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency =
|
||||
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
|
||||
let total_latency = if server_timings.is_some() {
|
||||
// When using server-side latency for newPayload, derive total from the
|
||||
// independently measured components to avoid mixing server-side and
|
||||
// client-side (network-inclusive) timings.
|
||||
np_latency + fcu_latency
|
||||
} else {
|
||||
start.elapsed()
|
||||
};
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
@@ -348,88 +249,6 @@ impl Command {
|
||||
total_latency,
|
||||
};
|
||||
|
||||
if let Some(reorg_state) = reorg_state.as_mut() {
|
||||
if queued_fork_block.is_none() && reorg_state.fork_length == 0 {
|
||||
// A branch start uses a canonical parent, so it can be built lazily here
|
||||
// instead of being queued ahead of time.
|
||||
let block = deferred_branch_start_block
|
||||
.as_ref()
|
||||
.ok_or_eyre("missing deferred fork block for reorg branch start")?;
|
||||
queued_fork_block = Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
&local_rpc_provider,
|
||||
block,
|
||||
canonical_parent_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
});
|
||||
}
|
||||
|
||||
let queued = queued_fork_block
|
||||
.take()
|
||||
.ok_or_eyre("missing queued fork block for reorg replay")?;
|
||||
ensure!(
|
||||
queued.block_number == block_number,
|
||||
"queued fork block {} does not match source block {}",
|
||||
queued.block_number,
|
||||
block_number
|
||||
);
|
||||
let prepared = queued.prepared;
|
||||
|
||||
call_new_payload_with_reth(&auth_provider, None, prepared.params).await?;
|
||||
|
||||
reorg_state.push_fork_head(canonical_parent_hash, prepared.block_hash);
|
||||
let forkchoice_state = reorg_state.forkchoice_state(prepared.block_hash)?;
|
||||
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
branch_point = %forkchoice_state.safe_block_hash,
|
||||
fork_head = %prepared.block_hash,
|
||||
fork_depth = reorg_state.fork_length,
|
||||
max_reorg_depth = reorg_state.depth,
|
||||
"Switching forkchoice to reorg branch"
|
||||
);
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, None, forkchoice_state).await?;
|
||||
let _fork_fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let next_fork_block_number = block_number + 1;
|
||||
if reorg_state.fork_length < reorg_state.depth {
|
||||
queued_fork_block = queue_fork_block(
|
||||
&block_provider,
|
||||
&local_rpc_provider,
|
||||
&benchmark_mode,
|
||||
next_fork_block_number,
|
||||
Some(prepared.block_hash),
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
reorg_depth = reorg_state.depth,
|
||||
"Resetting reorg branch after reaching max depth"
|
||||
);
|
||||
|
||||
// `testing_buildBlockV1` resolves the parent from canonical state, so switch
|
||||
// back to the source chain before reseeding the next queued fork block.
|
||||
call_forkchoice_updated_with_reth(
|
||||
&auth_provider,
|
||||
version,
|
||||
canonical_forkchoice_state,
|
||||
)
|
||||
.await?;
|
||||
|
||||
reorg_state.reset();
|
||||
queued_fork_block = None;
|
||||
}
|
||||
}
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
@@ -486,155 +305,3 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_built_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<PreparedBuiltBlock> {
|
||||
const MAX_BUILD_ATTEMPTS: usize = 10;
|
||||
const BUILD_RETRY_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
let request = build_block_request(block, parent_block_hash)?;
|
||||
let built_payload: ExecutionPayloadEnvelopeV5 = {
|
||||
let mut attempts_remaining = MAX_BUILD_ATTEMPTS;
|
||||
|
||||
loop {
|
||||
match block_provider.client().request("testing_buildBlockV1", [request.clone()]).await {
|
||||
Ok(payload) => break payload,
|
||||
Err(err) if attempts_remaining > 1 && is_retryable_build_block_error(&err) => {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
block_number = block.header.number,
|
||||
%parent_block_hash,
|
||||
attempts_remaining,
|
||||
error = %err,
|
||||
"Retrying testing_buildBlockV1 after transient fork build failure"
|
||||
);
|
||||
attempts_remaining -= 1;
|
||||
tokio::time::sleep(BUILD_RETRY_INTERVAL).await;
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(err).wrap_err_with(|| {
|
||||
format!(
|
||||
"Failed to build block {} via testing_buildBlockV1",
|
||||
block.header.number
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let payload = &built_payload.execution_payload.payload_inner.payload_inner;
|
||||
let block_hash = payload.block_hash;
|
||||
let (payload, sidecar) = built_payload
|
||||
.into_payload_and_sidecar(block.header.parent_beacon_block_root.unwrap_or_default());
|
||||
// Fork payloads are built immediately before the next `testing_buildBlockV1` call. Leaving
|
||||
// reth's default persistence wait enabled here gives the regular RPC side a consistent base
|
||||
// state for the next synthetic fork block build.
|
||||
let params = serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(ExecutionData { payload, sidecar }),
|
||||
None::<bool>,
|
||||
no_wait_for_caches.then_some(false),
|
||||
))?;
|
||||
|
||||
Ok(PreparedBuiltBlock { block_hash, params })
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn queue_fork_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
local_rpc_provider: &RootProvider<AnyNetwork>,
|
||||
benchmark_mode: &crate::bench_mode::BenchMode,
|
||||
block_number: u64,
|
||||
parent_block_hash: Option<B256>,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<Option<QueuedForkBlock>> {
|
||||
if !benchmark_mode.contains(block_number) {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
let future_block = block_provider
|
||||
.get_block_by_number(alloy_eips::BlockNumberOrTag::Number(block_number))
|
||||
.full()
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch block by number {block_number}"))?
|
||||
.ok_or_eyre("Block not found")?;
|
||||
let parent_block_hash = parent_block_hash.unwrap_or(future_block.header.parent_hash);
|
||||
|
||||
Ok(Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
local_rpc_provider,
|
||||
&future_block,
|
||||
parent_block_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
}))
|
||||
}
|
||||
|
||||
fn is_retryable_build_block_error(err: &alloy_transport::TransportError) -> bool {
|
||||
let message = err.to_string();
|
||||
message.contains("block not found: hash") ||
|
||||
message.contains("block hash not found for block number")
|
||||
}
|
||||
|
||||
fn build_block_request(
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
) -> eyre::Result<TestingBuildBlockRequestV1> {
|
||||
let mut transactions = block
|
||||
.clone()
|
||||
.try_into_transactions()
|
||||
.map_err(|_| eyre::eyre!("Block transactions must be fetched in full for --reorg"))?
|
||||
.into_iter()
|
||||
.map(|tx| {
|
||||
let tx: TxEnvelope =
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type in RPC block"))?;
|
||||
if tx.is_eip4844() {
|
||||
return Ok(None)
|
||||
}
|
||||
Ok(Some(tx.encoded_2718().into()))
|
||||
})
|
||||
.filter_map(|tx| tx.transpose())
|
||||
.collect::<eyre::Result<Vec<_>>>()?;
|
||||
|
||||
// `testing_buildBlockV1` only takes raw transaction bytes, so we exclude blob transactions
|
||||
// from the synthetic fork blocks rather than trying to reconstruct their sidecars.
|
||||
// Keep only 90% of the remaining transactions so the alternate branch produces a materially
|
||||
// different post-state instead of only differing by header data.
|
||||
let keep = transactions.len().saturating_mul(9) / 10;
|
||||
transactions.truncate(keep);
|
||||
|
||||
let rpc_block = block.clone().into_inner();
|
||||
|
||||
Ok(TestingBuildBlockRequestV1 {
|
||||
parent_block_hash,
|
||||
payload_attributes: PayloadAttributes {
|
||||
timestamp: block.header.timestamp,
|
||||
prev_randao: block.header.mix_hash.unwrap_or_default(),
|
||||
suggested_fee_recipient: block.header.beneficiary,
|
||||
withdrawals: rpc_block.withdrawals.map(|withdrawals| withdrawals.into_inner()),
|
||||
parent_beacon_block_root: block.header.parent_beacon_block_root,
|
||||
slot_number: block.header.slot_number,
|
||||
},
|
||||
transactions,
|
||||
extra_data: Some(block.header.extra_data.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_reorg_depth(value: &str) -> Result<usize, String> {
|
||||
let depth = value
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("invalid reorg depth {value:?}, expected a positive integer"))?;
|
||||
|
||||
if depth == 0 {
|
||||
return Err("reorg depth must be greater than 0".to_string())
|
||||
}
|
||||
|
||||
Ok(depth)
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::fetch_block_access_list,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
|
||||
@@ -54,7 +53,6 @@ impl Command {
|
||||
rlp_blocks,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
@@ -71,9 +69,7 @@ impl Command {
|
||||
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
|
||||
|
||||
let block_provider_clone = block_provider.clone();
|
||||
tokio::task::spawn(async move {
|
||||
let block_provider = block_provider_clone;
|
||||
while benchmark_mode.contains(next_block) {
|
||||
let block_res = block_provider
|
||||
.get_block_by_number(next_block.into())
|
||||
@@ -127,19 +123,12 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
@@ -18,7 +18,7 @@ use alloy_primitives::B256;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV6,
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV4,
|
||||
ExecutionPayloadSidecar, ExecutionPayloadV4, ForkchoiceState, JwtSecret, PraguePayloadFields,
|
||||
};
|
||||
use clap::Parser;
|
||||
@@ -315,7 +315,7 @@ impl Command {
|
||||
let requests =
|
||||
execution_data.sidecar.requests().cloned().unwrap_or_default().to_vec();
|
||||
(
|
||||
Some(EngineApiMessageVersion::V6),
|
||||
Some(EngineApiMessageVersion::V4),
|
||||
serde_json::to_value((
|
||||
execution_data.payload.clone(),
|
||||
Vec::<B256>::new(),
|
||||
@@ -423,7 +423,7 @@ impl Command {
|
||||
/// Load and parse all payload files from the directory.
|
||||
///
|
||||
/// Tries to load each file as a [`BigBlockPayload`] first (which includes `env_switches`),
|
||||
/// falling back to [`ExecutionPayloadEnvelopeV6`] for backwards compatibility.
|
||||
/// falling back to [`ExecutionPayloadEnvelopeV4`] for backwards compatibility.
|
||||
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
|
||||
let mut payloads = Vec::new();
|
||||
|
||||
@@ -450,11 +450,12 @@ impl Command {
|
||||
let name_str = name.to_string_lossy();
|
||||
let index = if let Some(rest) = name_str.strip_prefix("payload_block_") {
|
||||
rest.strip_suffix(".json")?.parse::<u64>().ok()?
|
||||
} else {
|
||||
let rest = name_str.strip_prefix("big_block_")?;
|
||||
} else if let Some(rest) = name_str.strip_prefix("big_block_") {
|
||||
// "big_block_FROM_to_TO.json" — use FROM as the index
|
||||
let rest = rest.strip_suffix(".json")?;
|
||||
rest.split("_to_").next()?.parse::<u64>().ok()?
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
Some((index, e.path()))
|
||||
})
|
||||
@@ -480,10 +481,10 @@ impl Command {
|
||||
{
|
||||
(big_block.execution_data, big_block.big_block_data, big_block.block_access_list)
|
||||
} else {
|
||||
let envelope: ExecutionPayloadEnvelopeV6 = serde_json::from_str(&content)
|
||||
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
let execution_data = ExecutionData {
|
||||
payload: envelope.execution_payload.clone().into(),
|
||||
payload: envelope.envelope_inner.execution_payload.clone().into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
|
||||
@@ -1,8 +1,6 @@
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4,
|
||||
};
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
|
||||
|
||||
/// Configuration for invalidating payload fields
|
||||
#[derive(Debug, Default)]
|
||||
@@ -23,7 +21,6 @@ pub(super) struct InvalidationConfig {
|
||||
pub(super) block_hash: Option<B256>,
|
||||
pub(super) blob_gas_used: Option<u64>,
|
||||
pub(super) excess_blob_gas: Option<u64>,
|
||||
pub(super) slot_number: Option<u64>,
|
||||
|
||||
// Auto-invalidation flags
|
||||
pub(super) invalidate_parent_hash: bool,
|
||||
@@ -38,8 +35,6 @@ pub(super) struct InvalidationConfig {
|
||||
pub(super) invalidate_withdrawals: bool,
|
||||
pub(super) invalidate_blob_gas_used: bool,
|
||||
pub(super) invalidate_excess_blob_gas: bool,
|
||||
pub(super) invalidate_block_access_list: bool,
|
||||
pub(super) invalidate_slot_number: bool,
|
||||
}
|
||||
|
||||
impl InvalidationConfig {
|
||||
@@ -221,30 +216,4 @@ impl InvalidationConfig {
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V4 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v4(&self, payload: &mut ExecutionPayloadV4) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v3(&mut payload.payload_inner);
|
||||
|
||||
// Explicit override for slot_number
|
||||
if let Some(slot_number) = self.slot_number {
|
||||
payload.slot_number = slot_number;
|
||||
changes.push(format!("slot_number = {slot_number}"));
|
||||
}
|
||||
|
||||
// Handle block access list invalidation (V4+)
|
||||
if self.invalidate_block_access_list {
|
||||
let fake_bal = Bytes::from_static(&[0x01, 0x02, 0x03]);
|
||||
payload.block_access_list = fake_bal.clone();
|
||||
changes.push(format!("block_access_list = {fake_bal} (auto-invalidated)"));
|
||||
}
|
||||
|
||||
// Handle slot number invalidation (V4+)
|
||||
if self.invalidate_slot_number {
|
||||
payload.slot_number = u64::MAX;
|
||||
changes.push("slot_number = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,12 @@
|
||||
//! Command for sending invalid payloads to test Engine API rejection.
|
||||
|
||||
mod invalidation;
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use invalidation::InvalidationConfig;
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use alloy_provider::{
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
RootProvider,
|
||||
};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
@@ -111,9 +105,6 @@ pub struct Command {
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
requests_hash: Option<B256>,
|
||||
|
||||
/// Override the slot number with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
slot_number: Option<u64>,
|
||||
// ==================== Auto-Invalidation Flags ====================
|
||||
/// Invalidate the parent hash by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
@@ -167,14 +158,6 @@ pub struct Command {
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_requests_hash: bool,
|
||||
|
||||
/// Invalidate the block access list by setting it to a random value (EIP-7928).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_access_list: bool,
|
||||
|
||||
/// Invalidate the slot number by setting it to an random value.(EIP-7843).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_slot_number: bool,
|
||||
|
||||
// ==================== Meta Flags ====================
|
||||
/// Skip block hash recalculation after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
@@ -216,7 +199,6 @@ impl Command {
|
||||
block_hash: self.block_hash,
|
||||
blob_gas_used: self.blob_gas_used,
|
||||
excess_blob_gas: self.excess_blob_gas,
|
||||
slot_number: self.slot_number,
|
||||
invalidate_parent_hash: self.invalidate_parent_hash,
|
||||
invalidate_state_root: self.invalidate_state_root,
|
||||
invalidate_receipts_root: self.invalidate_receipts_root,
|
||||
@@ -229,8 +211,6 @@ impl Command {
|
||||
invalidate_withdrawals: self.invalidate_withdrawals,
|
||||
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
|
||||
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
|
||||
invalidate_block_access_list: self.invalidate_block_access_list,
|
||||
invalidate_slot_number: self.invalidate_slot_number,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -254,21 +234,15 @@ impl Command {
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let use_v5 = block.header.block_access_list_hash.is_some();
|
||||
let requests_hash = self.requests_hash.or(block.header.requests_hash);
|
||||
|
||||
let mut execution_payload = if use_v5 {
|
||||
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
|
||||
} else {
|
||||
ExecutionPayload::from_block_slow(&block).0
|
||||
};
|
||||
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
|
||||
let changes = match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
|
||||
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
|
||||
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
|
||||
ExecutionPayload::V4(p) => config.apply_to_payload_v4(p),
|
||||
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
|
||||
};
|
||||
|
||||
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
|
||||
@@ -338,13 +312,7 @@ impl Command {
|
||||
match self.mode {
|
||||
Mode::Execute => {
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v5 {
|
||||
"engine_newPayloadV5"
|
||||
} else if use_v4 {
|
||||
"engine_newPayloadV4"
|
||||
} else {
|
||||
"engine_newPayloadV3"
|
||||
};
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
@@ -385,17 +353,4 @@ impl Command {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
|
||||
let rpc_url = self
|
||||
.rpc_url
|
||||
.as_deref()
|
||||
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
|
||||
let client = ClientBuilder::default()
|
||||
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
let bal = fetch_block_access_list(&provider, block_number).await?;
|
||||
Ok(alloy_rlp::encode(bal).into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,11 +1,6 @@
|
||||
use super::helpers::{fetch_block_access_list, load_jwt_secret, read_input};
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_primitives::Bytes;
|
||||
use alloy_provider::{
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
RootProvider,
|
||||
};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
@@ -74,9 +69,6 @@ impl Command {
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let use_v5 = block.header.block_access_list_hash.is_some();
|
||||
|
||||
// Extract parent beacon block root
|
||||
let parent_beacon_block_root = block.header.parent_beacon_block_root;
|
||||
|
||||
@@ -84,14 +76,10 @@ impl Command {
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
|
||||
// V5 payloads must carry the full RLP-encoded block access list, not just the hash stored
|
||||
// in the header.
|
||||
let execution_payload = if use_v5 {
|
||||
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
|
||||
} else {
|
||||
ExecutionPayload::from_block_slow(&block).0
|
||||
};
|
||||
// Convert to execution payload
|
||||
let execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
|
||||
// Create JSON request data
|
||||
let json_request = if use_v4 {
|
||||
@@ -114,13 +102,7 @@ impl Command {
|
||||
Mode::Execute => {
|
||||
// Create cast command
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v5 {
|
||||
"engine_newPayloadV5"
|
||||
} else if use_v4 {
|
||||
"engine_newPayloadV4"
|
||||
} else {
|
||||
"engine_newPayloadV3"
|
||||
};
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
@@ -164,17 +146,4 @@ impl Command {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
|
||||
let rpc_url = self
|
||||
.rpc_url
|
||||
.as_deref()
|
||||
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
|
||||
let client = ClientBuilder::default()
|
||||
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
let bal = fetch_block_access_list(&provider, block_number).await?;
|
||||
Ok(alloy_rlp::encode(bal).into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,6 @@
|
||||
//! before sending additional calls.
|
||||
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::eip7928::BlockAccessList;
|
||||
use alloy_primitives::Bytes;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
|
||||
use alloy_rpc_types_engine::{
|
||||
@@ -44,14 +43,6 @@ pub trait EngineApiValidWaitExt<N>: Send + Sync {
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated>;
|
||||
|
||||
/// Calls `engine_forkChoiceUpdatedV4` with the given [`ForkchoiceState`] and optional
|
||||
/// [`PayloadAttributes`], and waits until the response is VALID.
|
||||
async fn fork_choice_updated_v4_wait(
|
||||
&self,
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -171,40 +162,6 @@ where
|
||||
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
async fn fork_choice_updated_v4_wait(
|
||||
&self,
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
method = "engine_forkchoiceUpdatedV3",
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Sending forkchoiceUpdated"
|
||||
);
|
||||
|
||||
let mut status =
|
||||
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(
|
||||
target: "reth-bench",
|
||||
?status,
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Invalid forkchoiceUpdatedV4 message",
|
||||
);
|
||||
panic!("Invalid forkchoiceUpdatedV4: {status:?}");
|
||||
}
|
||||
status =
|
||||
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
|
||||
}
|
||||
|
||||
Ok(status)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
|
||||
@@ -219,7 +176,6 @@ pub(crate) fn block_to_new_payload(
|
||||
reth_new_payload: bool,
|
||||
wait_for_persistence: WaitForPersistence,
|
||||
no_wait_for_caches: bool,
|
||||
bal: Option<BlockAccessList>,
|
||||
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
|
||||
let block_number = block.header.number;
|
||||
let wait_for_persistence = wait_for_persistence.rpc_value(block_number);
|
||||
@@ -242,11 +198,7 @@ pub(crate) fn block_to_new_payload(
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
let block_access_list = alloy_rlp::encode(bal.unwrap_or_default());
|
||||
|
||||
let (payload, sidecar) =
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, block_access_list.into());
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
let (version, params, execution_data) = payload_to_new_payload(payload, sidecar, None)?;
|
||||
|
||||
if reth_new_payload {
|
||||
@@ -275,22 +227,6 @@ pub(crate) fn payload_to_new_payload(
|
||||
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
|
||||
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V4(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
|
||||
let version = target_version.unwrap_or(EngineApiMessageVersion::V6);
|
||||
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
|
||||
(
|
||||
version,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
@@ -330,6 +266,22 @@ pub(crate) fn payload_to_new_payload(
|
||||
ExecutionPayload::V1(payload) => {
|
||||
(EngineApiMessageVersion::V1, serde_json::to_value((payload,))?)
|
||||
}
|
||||
ExecutionPayload::V4(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
|
||||
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
|
||||
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
|
||||
(
|
||||
version,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
Ok((version, params, execution_data))
|
||||
@@ -434,10 +386,10 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
|
||||
match message_version {
|
||||
EngineApiMessageVersion::V3 |
|
||||
EngineApiMessageVersion::V4 |
|
||||
EngineApiMessageVersion::V5 |
|
||||
EngineApiMessageVersion::V6 => {
|
||||
provider.fork_choice_updated_v4_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
|
||||
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
EngineApiMessageVersion::V2 => {
|
||||
|
||||
@@ -150,22 +150,16 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
// commands can proceed.
|
||||
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
|
||||
reth_fs_util::create_dir_all(&rocksdb_path)?;
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level);
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
} else {
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write());
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
.build()?
|
||||
} else {
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?
|
||||
};
|
||||
|
||||
let provider_factory =
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{hex, Address, BlockHash, B256};
|
||||
use alloy_primitives::{hex, BlockHash};
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
static_file::{
|
||||
@@ -10,20 +10,16 @@ use reth_db::{
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbDupCursorRO},
|
||||
database::Database,
|
||||
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
|
||||
table::{Compress, Decompress, DupSort, Table},
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
RawKey, RawTable, TableViewer,
|
||||
RawKey, RawTable, Receipts, TableViewer, Transactions,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, ChangeSetReader, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageChangeSetReader;
|
||||
use tracing::error;
|
||||
@@ -77,55 +73,6 @@ enum Subcommand {
|
||||
#[arg(long)]
|
||||
raw: bool,
|
||||
},
|
||||
/// Gets the content of a RocksDB table for the given key
|
||||
///
|
||||
/// For history tables (accounts-history, storages-history), you can pass a plain address
|
||||
/// instead of a full JSON ShardedKey. Use --block to query a specific block number
|
||||
/// (seeks to the shard containing that block), or --all-shards to list all shards for
|
||||
/// the address.
|
||||
///
|
||||
/// Examples:
|
||||
/// reth db get rocksdb accounts-history 0xdBBE3D8c2d2b22A2611c5A94A9a12C2fCD49Eb29
|
||||
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --block 1000000
|
||||
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --all-shards
|
||||
/// reth db get rocksdb storages-history 0xdBBE...Eb29 --storage-key 0x0000...0003
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: RocksDbTable,
|
||||
|
||||
/// The key to get content for. For history tables, this can be a plain address.
|
||||
#[arg(value_parser = maybe_json_value_parser)]
|
||||
key: String,
|
||||
|
||||
/// Target block number for history tables. Seeks to the shard containing this block.
|
||||
/// Defaults to the latest shard if not specified.
|
||||
#[arg(long)]
|
||||
block: Option<u64>,
|
||||
|
||||
/// Storage key for storages-history table lookups.
|
||||
#[arg(long)]
|
||||
storage_key: Option<String>,
|
||||
|
||||
/// List all shards for the given key (history tables only).
|
||||
#[arg(long)]
|
||||
all_shards: bool,
|
||||
|
||||
/// Output bytes instead of human-readable decoded value
|
||||
#[arg(long)]
|
||||
raw: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// RocksDB tables that can be queried.
|
||||
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -135,9 +82,6 @@ impl Command {
|
||||
Subcommand::Mdbx { table, key, subkey, end_key, end_subkey, raw } => {
|
||||
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
|
||||
}
|
||||
Subcommand::Rocksdb { table, key, block, storage_key, all_shards, raw } => {
|
||||
get_rocksdb(tool, table, &key, block, storage_key.as_deref(), all_shards, raw)?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, key, subkey, raw } => {
|
||||
if let StaticFileSegment::StorageChangeSets = segment {
|
||||
let storage_key =
|
||||
@@ -264,12 +208,15 @@ impl Command {
|
||||
);
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
let transaction = TxTy::<N>::decompress(content[0].as_slice())?;
|
||||
let transaction = <<Transactions as Table>::Value>::decompress(
|
||||
content[0].as_slice(),
|
||||
)?;
|
||||
println!("{}", serde_json::to_string_pretty(&transaction)?);
|
||||
}
|
||||
StaticFileSegment::Receipts => {
|
||||
let receipt =
|
||||
ReceiptTy::<N>::decompress(content[0].as_slice())?;
|
||||
let receipt = <<Receipts as Table>::Value>::decompress(
|
||||
content[0].as_slice(),
|
||||
)?;
|
||||
println!("{}", serde_json::to_string_pretty(&receipt)?);
|
||||
}
|
||||
StaticFileSegment::TransactionSenders => {
|
||||
@@ -299,208 +246,6 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a value from a RocksDB table by key.
|
||||
fn get_rocksdb<N: ProviderNodeTypes>(
|
||||
tool: &DbTool<N>,
|
||||
table: RocksDbTable,
|
||||
key: &str,
|
||||
block: Option<u64>,
|
||||
storage_key: Option<&str>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
if block.is_some() || all_shards || storage_key.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block, --all-shards, and --storage-key are only supported for history tables"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, key, raw)
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
if storage_key.is_some() {
|
||||
return Err(eyre::eyre!("--storage-key is only supported for storages-history"));
|
||||
}
|
||||
get_rocksdb_account_history(&rocksdb, key, block, all_shards, raw)
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
get_rocksdb_storage_history(&rocksdb, key, storage_key, block, all_shards, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to parse a key string as a plain address, falling back to JSON `ShardedKey` parsing.
|
||||
fn parse_address(key: &str) -> eyre::Result<Address> {
|
||||
// Strip surrounding quotes that `maybe_json_value_parser` may have added
|
||||
let stripped = key.trim_matches('"');
|
||||
stripped.parse::<Address>().map_err(|e| eyre::eyre!("failed to parse address: {e}"))
|
||||
}
|
||||
|
||||
/// Gets account history from RocksDB with ergonomic key parsing.
|
||||
///
|
||||
/// Accepts a plain address and uses seek to find the relevant shard.
|
||||
fn get_rocksdb_account_history(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key: &str,
|
||||
block: Option<u64>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
// Try parsing as a plain address first, fall back to full JSON ShardedKey
|
||||
match parse_address(key) {
|
||||
Ok(address) => {
|
||||
let block_number = block.unwrap_or(u64::MAX);
|
||||
let seek_key = ShardedKey::new(address, block_number);
|
||||
|
||||
if all_shards {
|
||||
// Iterate all shards: seek from (address, 0) until address changes
|
||||
let start = ShardedKey::new(address, 0);
|
||||
let iter = rocksdb.iter_from::<tables::AccountsHistory>(start)?;
|
||||
for result in iter {
|
||||
let (k, v) = result?;
|
||||
if k.key != address {
|
||||
break;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"highest_block_number": k.highest_block_number,
|
||||
"value": v,
|
||||
}))?
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Seek to the first shard with highest_block_number >= target
|
||||
let mut iter = rocksdb.iter_from::<tables::AccountsHistory>(seek_key)?;
|
||||
match iter.next() {
|
||||
Some(Ok((k, v))) if k.key == address => {
|
||||
if raw {
|
||||
let raw_val = rocksdb.get_raw::<tables::AccountsHistory>(k)?;
|
||||
if let Some(bytes) = raw_val {
|
||||
println!("{}", hex::encode_prefixed(&bytes));
|
||||
}
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&v)?);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
// Fall back to full JSON key parsing (e.g.
|
||||
// `{"key":"0x...","highest_block_number":...}`)
|
||||
if all_shards || block.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block and --all-shards require a plain address, not a JSON key"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::AccountsHistory>(rocksdb, key, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets storage history from RocksDB with ergonomic key parsing.
|
||||
///
|
||||
/// Accepts a plain address + optional `--storage-key` and uses seek.
|
||||
fn get_rocksdb_storage_history(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key: &str,
|
||||
storage_key: Option<&str>,
|
||||
block: Option<u64>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
match parse_address(key) {
|
||||
Ok(address) => {
|
||||
let storage_key = storage_key
|
||||
.map(|s| s.trim_matches('"').parse::<B256>())
|
||||
.transpose()
|
||||
.map_err(|e| eyre::eyre!("failed to parse storage key: {e}"))?
|
||||
.unwrap_or_default();
|
||||
let block_number = block.unwrap_or(u64::MAX);
|
||||
let seek_key = StorageShardedKey::new(address, storage_key, block_number);
|
||||
|
||||
if all_shards {
|
||||
let start = StorageShardedKey::new(address, storage_key, 0);
|
||||
let iter = rocksdb.iter_from::<tables::StoragesHistory>(start)?;
|
||||
for result in iter {
|
||||
let (k, v) = result?;
|
||||
if k.address != address || k.sharded_key.key != storage_key {
|
||||
break;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"highest_block_number": k.sharded_key.highest_block_number,
|
||||
"value": v,
|
||||
}))?
|
||||
);
|
||||
}
|
||||
} else {
|
||||
let mut iter = rocksdb.iter_from::<tables::StoragesHistory>(seek_key)?;
|
||||
match iter.next() {
|
||||
Some(Ok((k, v)))
|
||||
if k.address == address && k.sharded_key.key == storage_key =>
|
||||
{
|
||||
if raw {
|
||||
let raw_val = rocksdb.get_raw::<tables::StoragesHistory>(k)?;
|
||||
if let Some(bytes) = raw_val {
|
||||
println!("{}", hex::encode_prefixed(&bytes));
|
||||
}
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&v)?);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
if all_shards || block.is_some() || storage_key.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block, --all-shards, and --storage-key require a plain address, not a JSON key"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::StoragesHistory>(rocksdb, key, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a value from a specific RocksDB table by exact key and prints it.
|
||||
fn get_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key_str: &str,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
let key = table_key::<T>(key_str)?;
|
||||
|
||||
if raw {
|
||||
let content = rocksdb.get_raw::<T>(key)?;
|
||||
match content {
|
||||
Some(bytes) => println!("{}", hex::encode_prefixed(&bytes)),
|
||||
None => error!(target: "reth::cli", "No content for the given table key."),
|
||||
}
|
||||
} else {
|
||||
let content = rocksdb.get::<T>(key)?;
|
||||
match content {
|
||||
Some(value) => println!("{}", serde_json::to_string_pretty(&value)?),
|
||||
None => error!(target: "reth::cli", "No content for the given table key."),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get an instance of key for given table
|
||||
pub(crate) fn table_key<T: Table>(key: &str) -> Result<T::Key, eyre::Error> {
|
||||
serde_json::from_str(key).map_err(|e| eyre::eyre!(e))
|
||||
|
||||
@@ -1,361 +0,0 @@
|
||||
//! `reth db migrate-v2` command for migrating v1 storage layout to v2.
|
||||
//!
|
||||
//! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to
|
||||
//! static files, clears recomputable tables (senders, indices, trie, plain
|
||||
//! state), compacts MDBX, then runs the pipeline to rebuild them.
|
||||
|
||||
use crate::common::CliNodeTypes;
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
mdbx::{self, ffi},
|
||||
models::StorageBeforeTx,
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
database::Database,
|
||||
table::Table,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider,
|
||||
MetadataWriter, ProviderFactory, PruneCheckpointReader, StageCheckpointWriter,
|
||||
StaticFileProviderFactory, StaticFileWriter, StorageSettings,
|
||||
};
|
||||
use reth_prune_types::PruneSegment;
|
||||
use reth_stages_types::{StageCheckpoint, StageId};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StageCheckpointReader;
|
||||
use tracing::info;
|
||||
|
||||
/// `reth db migrate-v2` command
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command;
|
||||
|
||||
impl Command {
|
||||
/// Execute the full v1 → v2 migration:
|
||||
///
|
||||
/// 1. Migrate changesets + receipts to static files
|
||||
/// 2. Flip `StorageSettings` to v2
|
||||
/// 3. Clear recomputable MDBX tables + reset stage checkpoints
|
||||
/// 4. Compact MDBX
|
||||
pub async fn execute<N: CliNodeTypes>(
|
||||
self,
|
||||
provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N::Primitives: reth_primitives_traits::NodePrimitives<
|
||||
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
|
||||
>,
|
||||
{
|
||||
// === Phase 0: Preflight ===
|
||||
info!(target: "reth::cli", "Starting v1 → v2 storage migration");
|
||||
|
||||
let provider = provider_factory.provider()?;
|
||||
let current_settings = provider.storage_settings()?;
|
||||
|
||||
if current_settings.is_some_and(|s| s.is_v2()) {
|
||||
info!(target: "reth::cli", "Storage is already v2, nothing to do");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let tip =
|
||||
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number).unwrap_or(0);
|
||||
|
||||
info!(target: "reth::cli", tip, "Chain tip block number");
|
||||
|
||||
let sf_provider = provider_factory.static_file_provider();
|
||||
|
||||
for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets]
|
||||
{
|
||||
if sf_provider.get_highest_static_file_block(segment).is_some() {
|
||||
eyre::bail!(
|
||||
"Static file segment {segment:?} already contains data. \
|
||||
Cannot migrate — target must be empty."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
drop(provider);
|
||||
|
||||
// === Phase 1: Migrate changesets → static files ===
|
||||
Self::migrate_account_changesets(&provider_factory, tip)?;
|
||||
Self::migrate_storage_changesets(&provider_factory, tip)?;
|
||||
|
||||
// === Phase 2: Migrate receipts → static files ===
|
||||
Self::migrate_receipts::<NodeTypesWithDBAdapter<N, DatabaseEnv>>(&provider_factory, tip)?;
|
||||
|
||||
// === Phase 3: Flip metadata to v2 ===
|
||||
info!(target: "reth::cli", "Writing StorageSettings v2 metadata");
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
provider_rw.write_storage_settings(StorageSettings::v2())?;
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
info!(target: "reth::cli", "Storage settings updated to v2");
|
||||
|
||||
// === Phase 4: Clear recomputable tables ===
|
||||
Self::clear_recomputable_tables(&provider_factory)?;
|
||||
|
||||
// === Phase 5: Compact MDBX (before pipeline, so it runs on a smaller DB) ===
|
||||
let db_path = provider_factory.db_ref().path();
|
||||
Self::compact_mdbx(provider_factory.db_ref())?;
|
||||
|
||||
// Drop to release DB handle for swap
|
||||
drop(provider_factory);
|
||||
|
||||
let compact_path = db_path.with_file_name("db_compact");
|
||||
Self::swap_compacted_db(&db_path, &compact_path)?;
|
||||
|
||||
// === Phase 6: Reopen DB and run pipeline ===
|
||||
// The caller will reopen the environment and run the pipeline.
|
||||
// We return here — the pipeline step is handled in mod.rs after
|
||||
// reopening the database with the compacted copy.
|
||||
info!(target: "reth::cli", "Migration complete. You should now restart the node and let it run the pipeline to rebuild the remaining data.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_account_changesets<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Migrating AccountChangeSets → static files");
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let sf_provider = factory.static_file_provider();
|
||||
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::AccountChangeSets>()?;
|
||||
|
||||
let first_block = provider
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(first_block))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Some(Ok((block_number, _))) = walker.peek() {
|
||||
if *block_number != block {
|
||||
break;
|
||||
}
|
||||
let (_, entry) = walker.next().expect("peeked")?;
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
count += entries.len() as u64;
|
||||
writer.append_account_changeset(entries, block)?;
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
|
||||
info!(target: "reth::cli", count, "AccountChangeSets migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_storage_changesets<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Migrating StorageChangeSets → static files");
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let sf_provider = factory.static_file_provider();
|
||||
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::StorageChangeSets>()?;
|
||||
|
||||
let first_block = provider
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(Default::default()))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Some(Ok((key, _))) = walker.peek() {
|
||||
if key.block_number() != block {
|
||||
break;
|
||||
}
|
||||
let (key, entry) = walker.next().expect("peeked")?;
|
||||
entries.push(StorageBeforeTx {
|
||||
address: key.address(),
|
||||
key: entry.key,
|
||||
value: entry.value,
|
||||
});
|
||||
}
|
||||
|
||||
count += entries.len() as u64;
|
||||
writer.append_storage_changeset(entries, block)?;
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
|
||||
info!(target: "reth::cli", count, "StorageChangeSets migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_receipts<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N::Primitives: reth_primitives_traits::NodePrimitives<
|
||||
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
|
||||
>,
|
||||
{
|
||||
let provider = factory.provider()?;
|
||||
if !provider.prune_modes_ref().receipts_log_filter.is_empty() {
|
||||
info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX");
|
||||
return Ok(());
|
||||
}
|
||||
drop(provider);
|
||||
|
||||
let sf_provider = factory.static_file_provider();
|
||||
let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts);
|
||||
|
||||
if existing.is_some_and(|b| b >= tip) {
|
||||
info!(target: "reth::cli", "Receipts already in static files, skipping");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Migrating Receipts → static files");
|
||||
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let prune_start = provider
|
||||
.get_prune_checkpoint(PruneSegment::Receipts)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
|
||||
|
||||
let block_range = first_block..=tip;
|
||||
|
||||
let segment = reth_static_file::segments::Receipts;
|
||||
reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?;
|
||||
|
||||
sf_provider.commit()?;
|
||||
|
||||
info!(target: "reth::cli", "Receipts migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clears tables that can be recomputed by the pipeline and resets their
|
||||
/// stage checkpoints.
|
||||
fn clear_recomputable_tables<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Clearing recomputable MDBX tables");
|
||||
let db = factory.db_ref();
|
||||
|
||||
macro_rules! clear_table {
|
||||
($table:ty) => {{
|
||||
let tx = db.tx_mut()?;
|
||||
tx.clear::<$table>()?;
|
||||
tx.commit()?;
|
||||
info!(target: "reth::cli", table = <$table as Table>::NAME, "Cleared");
|
||||
}};
|
||||
}
|
||||
|
||||
// Migrated changeset tables (now in static files)
|
||||
clear_table!(tables::AccountChangeSets);
|
||||
clear_table!(tables::StorageChangeSets);
|
||||
|
||||
// Senders — rebuilt by SenderRecovery
|
||||
clear_table!(tables::TransactionSenders);
|
||||
|
||||
// Indices — rebuilt by TransactionLookup / IndexAccountHistory / IndexStorageHistory
|
||||
clear_table!(tables::TransactionHashNumbers);
|
||||
clear_table!(tables::AccountsHistory);
|
||||
clear_table!(tables::StoragesHistory);
|
||||
|
||||
// Plain state — superseded by hashed state in v2
|
||||
clear_table!(tables::PlainAccountState);
|
||||
clear_table!(tables::PlainStorageState);
|
||||
|
||||
// Trie — rebuilt by MerkleExecute
|
||||
clear_table!(tables::AccountsTrie);
|
||||
clear_table!(tables::StoragesTrie);
|
||||
|
||||
// Reset stage checkpoints so the pipeline rebuilds everything
|
||||
info!(target: "reth::cli", "Resetting stage checkpoints");
|
||||
let provider_rw = factory.database_provider_rw()?;
|
||||
for stage in [
|
||||
StageId::SenderRecovery,
|
||||
StageId::TransactionLookup,
|
||||
StageId::IndexAccountHistory,
|
||||
StageId::IndexStorageHistory,
|
||||
StageId::MerkleExecute,
|
||||
StageId::MerkleUnwind,
|
||||
] {
|
||||
provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(0))?;
|
||||
info!(target: "reth::cli", %stage, "Checkpoint reset to 0");
|
||||
}
|
||||
provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
info!(target: "reth::cli", "Recomputable tables cleared");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a compacted copy of the MDBX database.
|
||||
fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
|
||||
let db_path = db.path();
|
||||
let compact_path = db_path.with_file_name("db_compact");
|
||||
|
||||
reth_fs_util::create_dir_all(&compact_path)?;
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?compact_path, "Compacting MDBX database");
|
||||
|
||||
let compact_dest = compact_path.join("mdbx.dat");
|
||||
let dest_cstr = std::ffi::CString::new(
|
||||
compact_dest.to_str().ok_or_else(|| eyre::eyre!("compact path must be valid UTF-8"))?,
|
||||
)?;
|
||||
|
||||
let flags = ffi::MDBX_CP_COMPACT | ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
|
||||
|
||||
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
|
||||
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
|
||||
});
|
||||
|
||||
if rc != 0 {
|
||||
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
|
||||
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
|
||||
});
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "MDBX compaction complete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Swaps the original MDBX database with a compacted copy.
|
||||
fn swap_compacted_db(
|
||||
db_path: &std::path::Path,
|
||||
compact_path: &std::path::Path,
|
||||
) -> eyre::Result<()> {
|
||||
let backup_path = db_path.with_file_name("db_pre_compact");
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database");
|
||||
|
||||
std::fs::rename(db_path, &backup_path)?;
|
||||
|
||||
if let Err(e) = std::fs::rename(compact_path, db_path) {
|
||||
let _ = std::fs::rename(&backup_path, db_path);
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
std::fs::remove_dir_all(&backup_path)?;
|
||||
|
||||
info!(target: "reth::cli", "Database compaction swap complete");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -16,7 +16,6 @@ mod copy;
|
||||
mod diff;
|
||||
mod get;
|
||||
mod list;
|
||||
mod migrate_v2;
|
||||
mod prune_checkpoints;
|
||||
mod repair_trie;
|
||||
mod settings;
|
||||
@@ -78,9 +77,6 @@ pub enum Subcommands {
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
/// Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
|
||||
#[command(name = "migrate-v2")]
|
||||
MigrateV2(migrate_v2::Command),
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
|
||||
@@ -235,13 +231,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::MigrateV2(command) => {
|
||||
let Environment { provider_factory, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, ctx.task_executor.clone())?;
|
||||
|
||||
// Migrate changesets+receipts, clear tables, compact MDBX
|
||||
command.execute::<N>(provider_factory).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
@@ -1,368 +0,0 @@
|
||||
use super::{
|
||||
extract::{extract_archive_raw, streaming_download_and_extract, CompressionFormat},
|
||||
fetch::ArchiveFetcher,
|
||||
manifest::SnapshotArchive,
|
||||
planning::{PlannedArchive, PlannedDownloads},
|
||||
progress::{
|
||||
spawn_progress_display, ArchiveDownloadProgress, ArchiveExtractionProgress,
|
||||
ArchiveVerificationProgress, DownloadRequestLimiter, SharedProgress,
|
||||
},
|
||||
session::{ArchiveProcessContext, DownloadSession},
|
||||
verify::OutputVerifier,
|
||||
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
|
||||
};
|
||||
use eyre::Result;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::task;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const DOWNLOAD_CACHE_DIR: &str = ".download-cache";
|
||||
|
||||
/// Runs all planned modular archive downloads for one command invocation.
|
||||
pub(crate) async fn run_modular_downloads(
|
||||
planned_downloads: PlannedDownloads,
|
||||
target_dir: &Path,
|
||||
download_concurrency: usize,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let download_cache_dir = target_dir.join(DOWNLOAD_CACHE_DIR);
|
||||
fs::create_dir_all(&download_cache_dir)?;
|
||||
|
||||
let shared = SharedProgress::new(
|
||||
planned_downloads.total_download_size,
|
||||
planned_downloads.total_output_size,
|
||||
planned_downloads.total_archives() as u64,
|
||||
cancel_token.clone(),
|
||||
);
|
||||
let session = DownloadSession::new(
|
||||
Some(Arc::clone(&shared)),
|
||||
Some(DownloadRequestLimiter::new(download_concurrency)),
|
||||
cancel_token,
|
||||
);
|
||||
let ctx =
|
||||
ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session);
|
||||
|
||||
ModularDownloadJob::new(ctx, download_concurrency).run(planned_downloads).await
|
||||
}
|
||||
|
||||
/// Schedules modular archive work for one run of `reth download`.
|
||||
struct ModularDownloadJob {
|
||||
/// Shared paths and session state for each archive in this job.
|
||||
ctx: ArchiveProcessContext,
|
||||
/// Maximum number of archives processed at once.
|
||||
archive_concurrency: usize,
|
||||
}
|
||||
|
||||
impl ModularDownloadJob {
|
||||
/// Creates the modular download job for one command run.
|
||||
const fn new(ctx: ArchiveProcessContext, archive_concurrency: usize) -> Self {
|
||||
Self { ctx, archive_concurrency }
|
||||
}
|
||||
|
||||
/// Runs all planned archives and waits for the shared progress task to finish.
|
||||
async fn run(self, planned_downloads: PlannedDownloads) -> Result<()> {
|
||||
let shared = Arc::clone(
|
||||
self.ctx.session().progress().expect("modular downloads always use shared progress"),
|
||||
);
|
||||
let progress_handle = spawn_progress_display(Arc::clone(&shared));
|
||||
let ctx = self.ctx.clone();
|
||||
let results: Vec<Result<()>> = stream::iter(planned_downloads.archives)
|
||||
.map(move |archive| {
|
||||
let ctx = ctx.clone();
|
||||
async move { Self::process_archive(ctx, archive).await }
|
||||
})
|
||||
.buffer_unordered(self.archive_concurrency)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
shared.done.store(true, Ordering::Relaxed);
|
||||
let _ = progress_handle.await;
|
||||
|
||||
for result in results {
|
||||
result?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs one archive on the blocking pool so fetch and extraction stay off the async executor.
|
||||
async fn process_archive(ctx: ArchiveProcessContext, archive: PlannedArchive) -> Result<()> {
|
||||
task::spawn_blocking(move || ArchiveProcessor::new(archive, ctx).run()).await??;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit retry states for one modular archive.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ArchiveAttemptState {
|
||||
/// Start or restart one full archive attempt.
|
||||
RunAttempt,
|
||||
/// Check whether the extracted outputs verify.
|
||||
VerifyOutputs,
|
||||
/// Wait and decide whether another full attempt should run.
|
||||
RetryAttempt,
|
||||
/// Finish successfully.
|
||||
Complete,
|
||||
/// Stop with an error after retries are exhausted.
|
||||
Fail,
|
||||
}
|
||||
|
||||
/// Processes one modular archive from reuse check through extraction and verification.
|
||||
struct ArchiveProcessor {
|
||||
/// The concrete archive and component being processed.
|
||||
archive: PlannedArchive,
|
||||
/// Shared paths and session state for this archive attempt.
|
||||
ctx: ArchiveProcessContext,
|
||||
}
|
||||
|
||||
impl ArchiveProcessor {
|
||||
/// Creates a processor for one archive and the shared download context.
|
||||
fn new(archive: PlannedArchive, ctx: ArchiveProcessContext) -> Self {
|
||||
Self { archive, ctx }
|
||||
}
|
||||
|
||||
/// Runs the archive retry state machine until outputs are verified or retries are exhausted.
|
||||
fn run(self) -> Result<()> {
|
||||
let archive = self.archive();
|
||||
if self.try_reuse_outputs()? {
|
||||
info!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, "Skipping already verified plain files");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mode = ArchiveMode::new(&self.ctx)?;
|
||||
let format = CompressionFormat::from_url(&archive.file_name)?;
|
||||
let mut attempt = 1;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
let mut state = ArchiveAttemptState::RunAttempt;
|
||||
|
||||
loop {
|
||||
match state {
|
||||
ArchiveAttemptState::RunAttempt => {
|
||||
self.cleanup_outputs();
|
||||
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
file = %archive.file_name,
|
||||
component = %self.archive.component,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
"Retrying archive from scratch"
|
||||
);
|
||||
}
|
||||
|
||||
match self.run_attempt(mode, format) {
|
||||
Ok(()) => state = ArchiveAttemptState::VerifyOutputs,
|
||||
Err(error) if mode.retries_fetch_errors() => {
|
||||
warn!(target: "reth::cli",
|
||||
file = %archive.file_name,
|
||||
component = %self.archive.component,
|
||||
attempt,
|
||||
err = %format_args!("{error:#}"),
|
||||
"Archive attempt failed, retrying from scratch"
|
||||
);
|
||||
last_error = Some(error);
|
||||
state = ArchiveAttemptState::RetryAttempt;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::VerifyOutputs => {
|
||||
if self.verify_outputs_with_progress()? {
|
||||
state = ArchiveAttemptState::Complete;
|
||||
} else {
|
||||
warn!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, attempt, "Archive extracted, but output verification failed, retrying");
|
||||
state = ArchiveAttemptState::RetryAttempt;
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::RetryAttempt => {
|
||||
if attempt >= MAX_DOWNLOAD_RETRIES {
|
||||
state = ArchiveAttemptState::Fail;
|
||||
} else {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
attempt += 1;
|
||||
state = ArchiveAttemptState::RunAttempt;
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::Complete => return Ok(()),
|
||||
ArchiveAttemptState::Fail => {
|
||||
if let Some(error) = last_error {
|
||||
return Err(error.wrap_err(format!(
|
||||
"Failed after {} attempts for {}",
|
||||
MAX_DOWNLOAD_RETRIES, archive.file_name
|
||||
)));
|
||||
}
|
||||
|
||||
eyre::bail!(
|
||||
"Failed integrity validation after {} attempts for {}",
|
||||
MAX_DOWNLOAD_RETRIES,
|
||||
archive.file_name
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the concrete archive being fetched or verified.
|
||||
fn archive(&self) -> &SnapshotArchive {
|
||||
&self.archive.archive
|
||||
}
|
||||
|
||||
/// Returns the verifier for this archive's output files.
|
||||
fn output_verifier(&self) -> OutputVerifier<'_> {
|
||||
OutputVerifier::new(self.ctx.target_dir())
|
||||
}
|
||||
|
||||
/// Returns `true` if this archive can be reused from existing verified outputs.
|
||||
/// Returns `false` if a fresh archive attempt is still needed.
|
||||
fn try_reuse_outputs(&self) -> Result<bool> {
|
||||
if self.verify_outputs()? {
|
||||
self.mark_complete();
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Removes any partial outputs before a fresh archive attempt.
|
||||
fn cleanup_outputs(&self) {
|
||||
self.output_verifier().cleanup(&self.archive().output_files);
|
||||
}
|
||||
|
||||
/// Returns `true` if all declared plain outputs verify.
|
||||
/// Returns `false` if any output is missing or does not match.
|
||||
fn verify_outputs(&self) -> Result<bool> {
|
||||
self.output_verifier().verify(&self.archive().output_files)
|
||||
}
|
||||
|
||||
/// Records archive completion in shared progress once outputs verify.
|
||||
fn mark_complete(&self) {
|
||||
self.ctx.session().record_reused_archive(self.archive().size, self.archive().output_size());
|
||||
}
|
||||
|
||||
/// Executes one archive attempt according to the selected cache-vs-stream mode.
|
||||
fn run_attempt(&self, mode: ArchiveMode, format: CompressionFormat) -> Result<()> {
|
||||
mode.execute(self, format)
|
||||
}
|
||||
|
||||
/// Downloads the archive into the cache, then extracts from the cached file.
|
||||
fn run_cached_attempt(&self, format: CompressionFormat) -> Result<()> {
|
||||
let cache_dir =
|
||||
self.ctx.cache_dir().ok_or_else(|| eyre::eyre!("Missing download cache directory"))?;
|
||||
let fetcher =
|
||||
ArchiveFetcher::new(self.archive().url.clone(), cache_dir, self.ctx.session().clone());
|
||||
|
||||
if self.archive.ty == super::manifest::SnapshotComponentType::State {
|
||||
debug!(target: "reth::cli", url = %self.archive().url, "Downloading state snapshot archive");
|
||||
}
|
||||
|
||||
let download_result = {
|
||||
let mut download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
|
||||
let result = fetcher.download(Some(&mut download_progress));
|
||||
if let Ok(ref downloaded) = result &&
|
||||
download_progress.has_tracked_bytes()
|
||||
{
|
||||
download_progress.complete(downloaded.size);
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
let downloaded = match download_result {
|
||||
Ok(downloaded) => downloaded,
|
||||
Err(error) => {
|
||||
fetcher.cleanup_downloaded_files();
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
info!(target: "reth::cli",
|
||||
file = %self.archive().file_name,
|
||||
component = %self.archive.component,
|
||||
size = %super::progress::DownloadProgress::format_size(downloaded.size),
|
||||
"Archive download complete"
|
||||
);
|
||||
|
||||
let extract_result = self.extract_cached_archive(&downloaded.path, format);
|
||||
fetcher.cleanup_downloaded_files();
|
||||
extract_result
|
||||
}
|
||||
|
||||
/// Streams the archive directly into extraction without keeping a cached copy.
|
||||
fn run_streaming_attempt(&self, format: CompressionFormat) -> Result<()> {
|
||||
let _download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
|
||||
streaming_download_and_extract(
|
||||
&self.archive().url,
|
||||
format,
|
||||
self.ctx.target_dir(),
|
||||
self.ctx.session(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Extracts a cached archive file while updating shared extraction activity.
|
||||
fn extract_cached_archive(&self, archive_path: &Path, format: CompressionFormat) -> Result<()> {
|
||||
let mut extraction_progress = ArchiveExtractionProgress::new(self.ctx.session().progress());
|
||||
let file = fs::open(archive_path)?;
|
||||
let result = extract_archive_raw(
|
||||
file,
|
||||
format,
|
||||
self.ctx.target_dir(),
|
||||
Some(&mut extraction_progress),
|
||||
);
|
||||
extraction_progress.finish();
|
||||
result
|
||||
}
|
||||
|
||||
/// Returns `true` if all declared plain outputs verify while updating shared verification
|
||||
/// progress.
|
||||
fn verify_outputs_with_progress(&self) -> Result<bool> {
|
||||
let mut verification_progress =
|
||||
ArchiveVerificationProgress::new(self.ctx.session().progress());
|
||||
let verified = self
|
||||
.output_verifier()
|
||||
.verify_with_progress(&self.archive().output_files, Some(&mut verification_progress))?;
|
||||
if verified {
|
||||
verification_progress.complete(self.archive().output_size());
|
||||
}
|
||||
Ok(verified)
|
||||
}
|
||||
}
|
||||
|
||||
/// Chooses whether an archive attempt uses the cache or streams directly.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ArchiveMode {
|
||||
/// Download the archive to the cache, then extract it.
|
||||
Cached,
|
||||
/// Stream the archive directly into extraction.
|
||||
Streaming,
|
||||
}
|
||||
|
||||
impl ArchiveMode {
|
||||
/// Picks the archive mode from the process context.
|
||||
fn new(ctx: &ArchiveProcessContext) -> Result<Self> {
|
||||
if ctx.cache_dir().is_some() {
|
||||
ctx.session().require_request_limiter()?;
|
||||
return Ok(Self::Cached)
|
||||
}
|
||||
|
||||
Ok(Self::Streaming)
|
||||
}
|
||||
|
||||
/// Returns `true` when fetch failures should retry the whole archive attempt.
|
||||
const fn retries_fetch_errors(&self) -> bool {
|
||||
matches!(self, Self::Cached)
|
||||
}
|
||||
|
||||
/// Runs the selected archive mode for a single attempt.
|
||||
fn execute(&self, processor: &ArchiveProcessor, format: CompressionFormat) -> Result<()> {
|
||||
match self {
|
||||
Self::Cached => processor.run_cached_attempt(format),
|
||||
Self::Streaming => processor.run_streaming_attempt(format),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -248,7 +248,6 @@ fn selection_to_prune_mode(
|
||||
ComponentSelection::Distance(d) => {
|
||||
Some(PruneMode::Distance(min_distance.map_or(d, |min| d.max(min))))
|
||||
}
|
||||
ComponentSelection::Since(block) => Some(PruneMode::Before(block)),
|
||||
ComponentSelection::None => Some(min_distance.map_or(PruneMode::Full, PruneMode::Distance)),
|
||||
}
|
||||
}
|
||||
@@ -270,7 +269,6 @@ pub(crate) fn describe_prune_config(config: &Config) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Formats one prune mode for the generated config summary.
|
||||
fn format_mode(mode: &PruneMode) -> String {
|
||||
match mode {
|
||||
PruneMode::Full => "\"full\"".to_string(),
|
||||
@@ -455,36 +453,6 @@ mod tests {
|
||||
assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Distance(10_064)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selections_since_maps_to_before_prune_mode() {
|
||||
let mut selections = BTreeMap::new();
|
||||
selections.insert(SnapshotComponentType::State, ComponentSelection::All);
|
||||
selections.insert(SnapshotComponentType::Headers, ComponentSelection::All);
|
||||
selections
|
||||
.insert(SnapshotComponentType::Transactions, ComponentSelection::Since(15_537_394));
|
||||
selections.insert(SnapshotComponentType::Receipts, ComponentSelection::Since(15_537_394));
|
||||
selections.insert(
|
||||
SnapshotComponentType::AccountChangesets,
|
||||
ComponentSelection::Since(15_537_394),
|
||||
);
|
||||
selections.insert(
|
||||
SnapshotComponentType::StorageChangesets,
|
||||
ComponentSelection::Since(15_537_394),
|
||||
);
|
||||
|
||||
let config = config_for_selections(
|
||||
&selections,
|
||||
&empty_manifest(),
|
||||
None,
|
||||
None::<&reth_chainspec::ChainSpec>,
|
||||
);
|
||||
|
||||
assert_eq!(config.prune.segments.bodies_history, Some(PruneMode::Before(15_537_394)));
|
||||
assert_eq!(config.prune.segments.receipts, Some(PruneMode::Before(15_537_394)));
|
||||
assert_eq!(config.prune.segments.account_history, Some(PruneMode::Before(15_537_394)));
|
||||
assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Before(15_537_394)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_preset_matches_default_full_prune_config() {
|
||||
let mut selections = BTreeMap::new();
|
||||
|
||||
@@ -1,490 +0,0 @@
|
||||
use super::{
|
||||
fetch::{ArchiveFetcher, DownloadedArchive},
|
||||
progress::{
|
||||
ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress,
|
||||
DownloadRequestLimiter, ProgressReader, SharedProgress, SharedProgressReader,
|
||||
},
|
||||
session::DownloadSession,
|
||||
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
|
||||
};
|
||||
use eyre::{Result, WrapErr};
|
||||
use lz4::Decoder;
|
||||
use reqwest::blocking::Client as BlockingClient;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
io::Read,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tar::Archive;
|
||||
use tokio::task;
|
||||
use tracing::{info, warn};
|
||||
use url::Url;
|
||||
use zstd::stream::read::Decoder as ZstdDecoder;
|
||||
|
||||
const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
|
||||
const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
|
||||
const STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE: u64 = 64 * 1024 * 1024;
|
||||
const EXTRACTION_PROGRESS_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Supported compression formats for snapshots
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) enum CompressionFormat {
|
||||
/// LZ4-compressed tar archive.
|
||||
Lz4,
|
||||
/// Zstandard-compressed tar archive.
|
||||
Zstd,
|
||||
}
|
||||
|
||||
impl CompressionFormat {
|
||||
/// Detect compression format from file extension
|
||||
pub(crate) fn from_url(url: &str) -> Result<Self> {
|
||||
let path =
|
||||
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
|
||||
|
||||
if path.ends_with(EXTENSION_TAR_LZ4) {
|
||||
Ok(Self::Lz4)
|
||||
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
Ok(Self::Zstd)
|
||||
} else {
|
||||
Err(eyre::eyre!(
|
||||
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
|
||||
path
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a compressed tar archive to the target directory with progress tracking.
|
||||
fn extract_archive<R: Read>(
|
||||
reader: R,
|
||||
total_size: u64,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let progress_reader = ProgressReader::new(reader, total_size, cancel_token);
|
||||
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
let decoder = Decoder::new(progress_reader)?;
|
||||
Archive::new(decoder).unpack(target_dir)?;
|
||||
}
|
||||
CompressionFormat::Zstd => {
|
||||
let decoder = ZstdDecoder::new(progress_reader)?;
|
||||
Archive::new(decoder).unpack(target_dir)?;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extracts a compressed tar archive without progress tracking.
|
||||
pub(crate) fn extract_archive_raw<R: Read>(
|
||||
reader: R,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
unpack_archive(Archive::new(Decoder::new(reader)?), target_dir, progress)?;
|
||||
}
|
||||
CompressionFormat::Zstd => {
|
||||
unpack_archive(Archive::new(ZstdDecoder::new(reader)?), target_dir, progress)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unpack_archive<R: Read>(
|
||||
mut archive: Archive<R>,
|
||||
target_dir: &Path,
|
||||
mut progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
let entries = archive.entries().wrap_err_with(|| {
|
||||
format!("failed to read archive entries for `{}`", target_dir.display())
|
||||
})?;
|
||||
|
||||
for entry in entries {
|
||||
let mut entry = entry.wrap_err_with(|| {
|
||||
format!("failed to read archive entry for `{}`", target_dir.display())
|
||||
})?;
|
||||
extract_entry_with_progress(&mut entry, target_dir, progress.as_deref_mut())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_entry_with_progress<R: Read>(
|
||||
entry: &mut tar::Entry<'_, R>,
|
||||
target_dir: &Path,
|
||||
progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
let size = entry.header().entry_size().unwrap_or(0);
|
||||
let entry_type = entry.header().entry_type();
|
||||
|
||||
if !entry_type.is_file() || size == 0 {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
if let Some(progress) = progress {
|
||||
progress.record_extracted(size);
|
||||
}
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let Some(entry_path) = entry_destination_path(entry, target_dir)? else {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop));
|
||||
let unpack_result = entry
|
||||
.unpack_in(target_dir)
|
||||
.wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display()));
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
|
||||
let monitor_result = monitor.join();
|
||||
unpack_result?;
|
||||
|
||||
monitor_result.map_err(|_| eyre::eyre!("extraction progress monitor panicked"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn entry_destination_path<R: Read>(
|
||||
entry: &tar::Entry<'_, R>,
|
||||
target_dir: &Path,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
let mut file_dst = target_dir.to_path_buf();
|
||||
let path = entry.path().wrap_err("invalid path in archive entry")?;
|
||||
|
||||
for part in path.components() {
|
||||
match part {
|
||||
Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
|
||||
Component::ParentDir => return Ok(None),
|
||||
Component::Normal(part) => file_dst.push(part),
|
||||
}
|
||||
}
|
||||
|
||||
if file_dst == target_dir {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
Ok(Some(file_dst))
|
||||
}
|
||||
|
||||
fn spawn_extraction_progress_monitor(
|
||||
entry_path: PathBuf,
|
||||
progress: ArchiveExtractionProgressHandle,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> thread::JoinHandle<()> {
|
||||
thread::spawn(move || {
|
||||
let mut extracted = 0_u64;
|
||||
|
||||
loop {
|
||||
record_extracted_file_bytes(&entry_path, &progress, &mut extracted);
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
thread::sleep(EXTRACTION_PROGRESS_POLL_INTERVAL);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn record_extracted_file_bytes(
|
||||
entry_path: &Path,
|
||||
progress: &ArchiveExtractionProgressHandle,
|
||||
extracted: &mut u64,
|
||||
) {
|
||||
let Ok(meta) = fs::metadata(entry_path) else { return };
|
||||
let len = meta.len();
|
||||
if len > *extracted {
|
||||
progress.record_extracted(len - *extracted);
|
||||
*extracted = len;
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a snapshot from a local file.
|
||||
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let total_size = file.metadata()?.len();
|
||||
info!(target: "reth::cli",
|
||||
file = %path.display(),
|
||||
size = %DownloadProgress::format_size(total_size),
|
||||
"Extracting local archive"
|
||||
);
|
||||
let start = Instant::now();
|
||||
extract_archive(file, total_size, format, target_dir, CancellationToken::new())?;
|
||||
info!(target: "reth::cli",
|
||||
file = %path.display(),
|
||||
elapsed = %DownloadProgress::format_duration(start.elapsed()),
|
||||
"Local extraction complete"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Streams a remote archive directly into the extractor without writing to disk.
|
||||
///
|
||||
/// On failure, retries from scratch up to [`MAX_DOWNLOAD_RETRIES`] times.
|
||||
pub(crate) fn streaming_download_and_extract(
|
||||
url: &str,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
session: &DownloadSession,
|
||||
) -> Result<()> {
|
||||
let shared = session.progress();
|
||||
let quiet = session.progress().is_some();
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
"Retrying streaming download from scratch"
|
||||
);
|
||||
}
|
||||
|
||||
let client = BlockingClient::builder().connect_timeout(Duration::from_secs(30)).build()?;
|
||||
let _request_permit = session
|
||||
.request_limiter()
|
||||
.map(|limiter| limiter.acquire(session.progress(), session.cancel_token()))
|
||||
.transpose()?;
|
||||
|
||||
let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
|
||||
Ok(r) => r,
|
||||
Err(error) => {
|
||||
let err = eyre::Error::from(error);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
warn!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
err = %err,
|
||||
"Streaming request failed, retrying"
|
||||
);
|
||||
}
|
||||
last_error = Some(err);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !quiet && let Some(size) = response.content_length() {
|
||||
info!(target: "reth::cli",
|
||||
url = %url,
|
||||
size = %DownloadProgress::format_size(size),
|
||||
"Streaming archive"
|
||||
);
|
||||
}
|
||||
|
||||
let result = if let Some(progress) = shared {
|
||||
let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) };
|
||||
extract_archive_raw(reader, format, target_dir, None)
|
||||
} else {
|
||||
let total_size = response.content_length().unwrap_or(0);
|
||||
extract_archive(
|
||||
response,
|
||||
total_size,
|
||||
format,
|
||||
target_dir,
|
||||
session.cancel_token().clone(),
|
||||
)
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(error) => {
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
warn!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
err = %error,
|
||||
"Streaming extraction failed, retrying"
|
||||
);
|
||||
}
|
||||
last_error = Some(error);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or_else(|| {
|
||||
eyre::eyre!("Streaming download failed after {MAX_DOWNLOAD_RETRIES} attempts")
|
||||
}))
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
|
||||
fn download_and_extract(
|
||||
url: &str,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
session: DownloadSession,
|
||||
) -> Result<()> {
|
||||
let quiet = session.progress().is_some();
|
||||
let fetcher = ArchiveFetcher::new(url.to_string(), target_dir, session.clone());
|
||||
let DownloadedArchive { path: downloaded_path, size: total_size } = fetcher.download(None)?;
|
||||
|
||||
let file_name =
|
||||
downloaded_path.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
|
||||
|
||||
if !quiet {
|
||||
info!(target: "reth::cli",
|
||||
file = %file_name,
|
||||
size = %DownloadProgress::format_size(total_size),
|
||||
"Extracting archive"
|
||||
);
|
||||
}
|
||||
let file = fs::open(&downloaded_path)?;
|
||||
|
||||
if quiet {
|
||||
extract_archive_raw(file, format, target_dir, None)?;
|
||||
} else {
|
||||
extract_archive(file, total_size, format, target_dir, session.cancel_token().clone())?;
|
||||
info!(target: "reth::cli",
|
||||
file = %file_name,
|
||||
"Extraction complete"
|
||||
);
|
||||
}
|
||||
|
||||
fetcher.cleanup_downloaded_files();
|
||||
session.record_archive_output_complete(total_size);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
///
|
||||
/// Supports `file://` URLs for local files and HTTP(S) URLs for remote downloads.
|
||||
/// When `resumable` is true, downloads to a `.part` file first with HTTP Range resume
|
||||
/// support. Otherwise streams directly into the extractor.
|
||||
fn blocking_download_and_extract(
|
||||
url: &str,
|
||||
target_dir: &Path,
|
||||
shared: Option<Arc<SharedProgress>>,
|
||||
resumable: bool,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
|
||||
if let Ok(parsed_url) = Url::parse(url) &&
|
||||
parsed_url.scheme() == "file"
|
||||
{
|
||||
let session = DownloadSession::new(shared, request_limiter, cancel_token);
|
||||
let file_path = parsed_url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
|
||||
let result = extract_from_file(&file_path, format, target_dir);
|
||||
if result.is_ok() {
|
||||
session.record_archive_output_complete(file_path.metadata()?.len());
|
||||
}
|
||||
result
|
||||
} else if let Some(request_limiter) = request_limiter {
|
||||
download_and_extract(
|
||||
url,
|
||||
format,
|
||||
target_dir,
|
||||
DownloadSession::new(shared, Some(request_limiter), cancel_token),
|
||||
)
|
||||
} else if resumable {
|
||||
let session =
|
||||
DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token);
|
||||
download_and_extract(url, format, target_dir, session)
|
||||
} else {
|
||||
let session = DownloadSession::new(shared, None, cancel_token);
|
||||
let result = streaming_download_and_extract(url, format, target_dir, &session);
|
||||
if result.is_ok() {
|
||||
session.record_archive_output_complete(0);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot archive asynchronously.
|
||||
///
|
||||
/// When `shared` is provided, download progress is reported to the shared
|
||||
/// counter for aggregated display. Otherwise uses a local progress bar.
|
||||
/// When `resumable` is true, uses two-phase download with `.part` files.
|
||||
pub(crate) async fn stream_and_extract(
|
||||
url: &str,
|
||||
target_dir: &Path,
|
||||
shared: Option<Arc<SharedProgress>>,
|
||||
resumable: bool,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let target_dir = target_dir.to_path_buf();
|
||||
let url = url.to_string();
|
||||
task::spawn_blocking(move || {
|
||||
blocking_download_and_extract(
|
||||
&url,
|
||||
&target_dir,
|
||||
shared,
|
||||
resumable,
|
||||
request_limiter,
|
||||
cancel_token,
|
||||
)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_detection() {
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
|
||||
}
|
||||
}
|
||||
File diff suppressed because it is too large
Load Diff
@@ -10,10 +10,6 @@ use std::{
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
fn is_zero(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
/// A snapshot manifest describes available components for a snapshot at a given block height.
|
||||
///
|
||||
/// Each component is either a single archive (state) or a set of chunked archives (static file
|
||||
@@ -66,12 +62,6 @@ pub struct SingleArchive {
|
||||
pub file: String,
|
||||
/// Compressed archive size in bytes.
|
||||
pub size: u64,
|
||||
/// Total extracted plain-output size in bytes.
|
||||
///
|
||||
/// Older manifests may omit this, in which case downloaders should derive it from
|
||||
/// `output_files`.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
pub decompressed_size: u64,
|
||||
/// Optional BLAKE3 checksum of the compressed archive.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub blake3: Option<String>,
|
||||
@@ -93,12 +83,6 @@ pub struct ChunkedArchive {
|
||||
/// Computed during manifest generation. Older manifests may omit this.
|
||||
#[serde(default)]
|
||||
pub chunk_sizes: Vec<u64>,
|
||||
/// Extracted plain-output size of each chunk in bytes, ordered from first to last.
|
||||
///
|
||||
/// Older manifests may omit this, in which case downloaders should derive it from
|
||||
/// `chunk_output_files`.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub chunk_decompressed_sizes: Vec<u64>,
|
||||
/// Expected extracted plain files per chunk, ordered from first to last.
|
||||
///
|
||||
/// This is the authoritative integrity source for the modular download path.
|
||||
@@ -117,9 +101,9 @@ pub struct OutputFileChecksum {
|
||||
pub blake3: String,
|
||||
}
|
||||
|
||||
/// A concrete snapshot archive with its download and verification metadata.
|
||||
/// A single archive with concrete URL and optional integrity metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct SnapshotArchive {
|
||||
pub struct ArchiveDescriptor {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub size: u64,
|
||||
@@ -127,13 +111,6 @@ pub struct SnapshotArchive {
|
||||
pub output_files: Vec<OutputFileChecksum>,
|
||||
}
|
||||
|
||||
impl SnapshotArchive {
|
||||
/// Returns the total extracted plain-output size for this archive.
|
||||
pub fn output_size(&self) -> u64 {
|
||||
self.output_files.iter().map(|file| file.size).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// How much of a component to download.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ComponentSelection {
|
||||
@@ -142,9 +119,6 @@ pub enum ComponentSelection {
|
||||
/// Download only the most recent chunks covering at least `distance` blocks.
|
||||
/// Maps to `PruneMode::Distance(distance)` in the generated config.
|
||||
Distance(u64),
|
||||
/// Download chunks starting at the specified block number.
|
||||
/// Maps to `PruneMode::Before(block)` in the generated config.
|
||||
Since(u64),
|
||||
/// Don't download this component at all.
|
||||
/// Maps to `PruneMode::Full` for tx-based segments, or a minimal distance for others.
|
||||
None,
|
||||
@@ -155,7 +129,6 @@ impl std::fmt::Display for ComponentSelection {
|
||||
match self {
|
||||
Self::All => write!(f, "All"),
|
||||
Self::Distance(d) => write!(f, "Last {d} blocks"),
|
||||
Self::Since(block) => write!(f, "Since block {block}"),
|
||||
Self::None => write!(f, "None"),
|
||||
}
|
||||
}
|
||||
@@ -338,19 +311,19 @@ impl SnapshotManifest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns concrete snapshot archives for a component, optionally limited to distance.
|
||||
pub fn snapshot_archives_for_distance(
|
||||
/// Returns concrete archive descriptors for a component, optionally limited to distance.
|
||||
pub fn archive_descriptors_for_distance(
|
||||
&self,
|
||||
ty: SnapshotComponentType,
|
||||
distance: Option<u64>,
|
||||
) -> Vec<SnapshotArchive> {
|
||||
) -> Vec<ArchiveDescriptor> {
|
||||
let Some(component) = self.component(ty) else {
|
||||
return vec![];
|
||||
};
|
||||
|
||||
match component {
|
||||
ComponentManifest::Single(single) => {
|
||||
vec![SnapshotArchive {
|
||||
vec![ArchiveDescriptor {
|
||||
url: format!("{}/{}", self.base_url_or_empty(), single.file),
|
||||
file_name: single.file.clone(),
|
||||
size: single.size,
|
||||
@@ -380,7 +353,7 @@ impl SnapshotManifest {
|
||||
let output_files =
|
||||
chunked.chunk_output_files.get(i as usize).cloned().unwrap_or_default();
|
||||
|
||||
SnapshotArchive {
|
||||
ArchiveDescriptor {
|
||||
url: format!("{}/{}", self.base_url_or_empty(), file_name),
|
||||
file_name,
|
||||
size,
|
||||
@@ -421,36 +394,6 @@ impl SnapshotManifest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the exact extracted plain-output size for a component given a distance selection.
|
||||
pub fn output_size_for_distance(
|
||||
&self,
|
||||
ty: SnapshotComponentType,
|
||||
distance: Option<u64>,
|
||||
) -> u64 {
|
||||
let Some(component) = self.component(ty) else {
|
||||
return 0;
|
||||
};
|
||||
|
||||
match component {
|
||||
ComponentManifest::Single(single) => single.output_size(),
|
||||
ComponentManifest::Chunked(chunked) => {
|
||||
let num_chunks = chunked.num_chunks();
|
||||
let start_chunk = match distance {
|
||||
Some(dist) => {
|
||||
let needed = dist.min(chunked.total_blocks);
|
||||
let needed_chunks = needed.div_ceil(chunked.blocks_per_file);
|
||||
num_chunks.saturating_sub(needed_chunks)
|
||||
}
|
||||
None => 0,
|
||||
};
|
||||
|
||||
(start_chunk..num_chunks)
|
||||
.map(|index| chunked.chunk_output_size(index as usize))
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of chunks that would be downloaded for a given distance.
|
||||
pub fn chunks_for_distance(&self, ty: SnapshotComponentType, distance: Option<u64>) -> u64 {
|
||||
let Some(ComponentManifest::Chunked(chunked)) = self.component(ty) else {
|
||||
@@ -474,14 +417,6 @@ impl ComponentManifest {
|
||||
Self::Chunked(c) => c.chunk_sizes.iter().sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total extracted plain-output size for this component.
|
||||
pub fn total_output_size(&self) -> u64 {
|
||||
match self {
|
||||
Self::Single(single) => single.output_size(),
|
||||
Self::Chunked(chunked) => chunked.total_output_size(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkedArchive {
|
||||
@@ -489,39 +424,6 @@ impl ChunkedArchive {
|
||||
pub fn num_chunks(&self) -> u64 {
|
||||
self.total_blocks.div_ceil(self.blocks_per_file)
|
||||
}
|
||||
|
||||
/// Returns the extracted plain-output size for one chunk.
|
||||
pub fn chunk_output_size(&self, index: usize) -> u64 {
|
||||
self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| {
|
||||
self.chunk_output_files
|
||||
.get(index)
|
||||
.map(|files| files.iter().map(|file| file.size).sum())
|
||||
.unwrap_or(0)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the total extracted plain-output size across all chunks.
|
||||
pub fn total_output_size(&self) -> u64 {
|
||||
if !self.chunk_decompressed_sizes.is_empty() {
|
||||
self.chunk_decompressed_sizes.iter().sum()
|
||||
} else {
|
||||
self.chunk_output_files
|
||||
.iter()
|
||||
.map(|files| files.iter().map(|file| file.size).sum::<u64>())
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SingleArchive {
|
||||
/// Returns the total extracted plain-output size for this archive.
|
||||
pub fn output_size(&self) -> u64 {
|
||||
if self.decompressed_size != 0 {
|
||||
self.decompressed_size
|
||||
} else {
|
||||
self.output_files.iter().map(|file| file.size).sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a snapshot manifest from a URL.
|
||||
@@ -610,10 +512,6 @@ pub fn generate_manifest(
|
||||
blocks_per_file,
|
||||
total_blocks: block,
|
||||
chunk_sizes,
|
||||
chunk_decompressed_sizes: chunk_output_files
|
||||
.iter()
|
||||
.map(|files| files.iter().map(|file| file.size).sum())
|
||||
.collect(),
|
||||
chunk_output_files,
|
||||
}),
|
||||
);
|
||||
@@ -630,7 +528,6 @@ pub fn generate_manifest(
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: state_size,
|
||||
decompressed_size: state_output_files.iter().map(|file| file.size).sum(),
|
||||
blake3: None,
|
||||
output_files: state_output_files,
|
||||
}),
|
||||
@@ -645,7 +542,6 @@ pub fn generate_manifest(
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: rocksdb_size,
|
||||
decompressed_size: rocksdb_output_files.iter().map(|file| file.size).sum(),
|
||||
blake3: None,
|
||||
output_files: rocksdb_output_files,
|
||||
}),
|
||||
@@ -914,7 +810,6 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 0,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
}),
|
||||
@@ -925,7 +820,6 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_500_000,
|
||||
chunk_sizes: vec![80_000, 100_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![], vec![], vec![]],
|
||||
}),
|
||||
);
|
||||
@@ -935,7 +829,6 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_500_000,
|
||||
chunk_sizes: vec![40_000, 50_000, 60_000],
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![], vec![], vec![]],
|
||||
}),
|
||||
);
|
||||
@@ -986,7 +879,6 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: 777,
|
||||
decompressed_size: 0,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
}),
|
||||
@@ -1044,7 +936,6 @@ mod tests {
|
||||
fn component_selection_display() {
|
||||
assert_eq!(ComponentSelection::All.to_string(), "All");
|
||||
assert_eq!(ComponentSelection::Distance(10_064).to_string(), "Last 10064 blocks");
|
||||
assert_eq!(ComponentSelection::Since(15_537_394).to_string(), "Since block 15537394");
|
||||
assert_eq!(ComponentSelection::None.to_string(), "None");
|
||||
}
|
||||
|
||||
@@ -1059,7 +950,6 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 24_396_822,
|
||||
chunk_sizes: vec![100; 49], // 49 chunks
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![]; 49],
|
||||
}),
|
||||
);
|
||||
@@ -1102,68 +992,6 @@ mod tests {
|
||||
assert_eq!(m.size_for_distance(SnapshotComponentType::Receipts, None), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_size_for_distance_uses_manifest_or_output_files() {
|
||||
let m = test_manifest();
|
||||
assert_eq!(m.output_size_for_distance(SnapshotComponentType::Transactions, None), 0);
|
||||
|
||||
let mut components = BTreeMap::new();
|
||||
components.insert(
|
||||
"state".to_string(),
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 1_000,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
size: 1_000,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
components.insert(
|
||||
"transactions".to_string(),
|
||||
ComponentManifest::Chunked(ChunkedArchive {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![80_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![111, 222],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
|
||||
size: 111,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_500000_999999.bin".to_string(),
|
||||
size: 222,
|
||||
blake3: "h1".to_string(),
|
||||
}],
|
||||
],
|
||||
}),
|
||||
);
|
||||
let manifest = SnapshotManifest {
|
||||
block: 1_000_000,
|
||||
chain_id: 1,
|
||||
storage_version: 2,
|
||||
timestamp: 0,
|
||||
base_url: Some("https://example.com".to_string()),
|
||||
reth_version: None,
|
||||
components,
|
||||
};
|
||||
|
||||
assert_eq!(manifest.output_size_for_distance(SnapshotComponentType::State, None), 1_000);
|
||||
assert_eq!(
|
||||
manifest.output_size_for_distance(SnapshotComponentType::Transactions, None),
|
||||
333
|
||||
);
|
||||
assert_eq!(
|
||||
manifest.output_size_for_distance(SnapshotComponentType::Transactions, Some(500_000)),
|
||||
222
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_descriptors_include_checksum_metadata() {
|
||||
let mut components = BTreeMap::new();
|
||||
@@ -1172,7 +1000,6 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 1_000,
|
||||
blake3: Some("abc123".to_string()),
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
@@ -1187,7 +1014,6 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![80_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![111, 222],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
|
||||
@@ -1213,13 +1039,13 @@ mod tests {
|
||||
components,
|
||||
};
|
||||
|
||||
let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None);
|
||||
let state = m.archive_descriptors_for_distance(SnapshotComponentType::State, None);
|
||||
assert_eq!(state.len(), 1);
|
||||
assert_eq!(state[0].file_name, "state.tar.zst");
|
||||
assert_eq!(state[0].blake3.as_deref(), Some("abc123"));
|
||||
assert_eq!(state[0].output_files.len(), 1);
|
||||
|
||||
let tx = m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, None);
|
||||
let tx = m.archive_descriptors_for_distance(SnapshotComponentType::Transactions, None);
|
||||
assert_eq!(tx.len(), 2);
|
||||
assert_eq!(tx[0].blake3, None);
|
||||
assert_eq!(tx[1].blake3, None);
|
||||
@@ -1242,7 +1068,6 @@ mod tests {
|
||||
panic!("state should be a single archive")
|
||||
};
|
||||
assert_eq!(state.file, "state.tar.zst");
|
||||
assert!(state.decompressed_size > 0);
|
||||
assert!(!state.output_files.is_empty());
|
||||
assert_eq!(state.output_files[0].path, "db/mdbx.dat");
|
||||
assert!(output.path().join("state.tar.zst").exists());
|
||||
@@ -1267,7 +1092,6 @@ mod tests {
|
||||
panic!("rocksdb indices should be a single archive")
|
||||
};
|
||||
assert_eq!(rocksdb.file, "rocksdb_indices.tar.zst");
|
||||
assert!(rocksdb.decompressed_size > 0);
|
||||
assert!(!rocksdb.output_files.is_empty());
|
||||
assert_eq!(rocksdb.output_files[0].path, "rocksdb/CURRENT");
|
||||
assert!(output.path().join("rocksdb_indices.tar.zst").exists());
|
||||
|
||||
@@ -45,7 +45,6 @@ pub struct SnapshotManifestCommand {
|
||||
}
|
||||
|
||||
impl SnapshotManifestCommand {
|
||||
/// Packages snapshot archives and writes the manifest file.
|
||||
pub fn execute(self) -> Result<()> {
|
||||
let block = match self.block {
|
||||
Some(block) => block,
|
||||
@@ -89,7 +88,6 @@ impl SnapshotManifestCommand {
|
||||
}
|
||||
}
|
||||
|
||||
/// Infers the snapshot block from the source datadir.
|
||||
fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
if let Ok(block) = infer_snapshot_block_from_db(source_datadir) {
|
||||
return Ok(block);
|
||||
@@ -104,7 +102,6 @@ fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
Ok(block)
|
||||
}
|
||||
|
||||
/// Reads the snapshot block from the source database Finish stage checkpoint.
|
||||
fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let candidates = [source_datadir.join("db"), source_datadir.to_path_buf()];
|
||||
|
||||
@@ -129,7 +126,6 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64>
|
||||
)
|
||||
}
|
||||
|
||||
/// Infers the snapshot block from the highest header static-file range.
|
||||
fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let max_end = header_ranges(source_datadir)?
|
||||
.into_iter()
|
||||
@@ -139,7 +135,6 @@ fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result
|
||||
Ok(max_end)
|
||||
}
|
||||
|
||||
/// Infers the static-file block span from header file ranges.
|
||||
fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let mut inferred = None;
|
||||
for (start, end) in header_ranges(source_datadir)? {
|
||||
@@ -166,7 +161,6 @@ fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Collects header static-file ranges from the source datadir.
|
||||
fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
|
||||
let static_files_dir = source_datadir.join("static_files");
|
||||
let static_files_dir =
|
||||
@@ -189,7 +183,6 @@ fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
/// Parses the block range from a header static-file name.
|
||||
fn parse_headers_range(file_name: &str) -> Option<(u64, u64)> {
|
||||
let remainder = file_name.strip_prefix("static_file_headers_")?;
|
||||
let (start, end_with_suffix) = remainder.split_once('_')?;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,322 +0,0 @@
|
||||
use super::{manifest::*, verify::OutputVerifier};
|
||||
use eyre::Result;
|
||||
use std::{collections::BTreeMap, path::Path};
|
||||
use tracing::info;
|
||||
|
||||
/// One archive selected from the manifest, along with its component name.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PlannedArchive {
|
||||
/// Snapshot component type this archive belongs to.
|
||||
pub(crate) ty: SnapshotComponentType,
|
||||
/// User-facing component name used in logs.
|
||||
pub(crate) component: String,
|
||||
/// Concrete snapshot archive metadata resolved from the manifest.
|
||||
pub(crate) archive: SnapshotArchive,
|
||||
}
|
||||
|
||||
/// The archive list for a modular snapshot download.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct PlannedDownloads {
|
||||
/// Concrete archives that still need reuse checks or processing.
|
||||
pub(crate) archives: Vec<PlannedArchive>,
|
||||
/// Total compressed download size of all planned archives.
|
||||
pub(crate) total_download_size: u64,
|
||||
/// Total extracted plain-output size of all planned archives.
|
||||
pub(crate) total_output_size: u64,
|
||||
}
|
||||
|
||||
impl PlannedDownloads {
|
||||
/// Returns the number of concrete archives queued for this snapshot selection.
|
||||
pub(crate) const fn total_archives(&self) -> usize {
|
||||
self.archives.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the sort priority used to schedule archives.
|
||||
pub(crate) const fn archive_priority_rank(ty: SnapshotComponentType) -> u8 {
|
||||
match ty {
|
||||
SnapshotComponentType::State => 0,
|
||||
SnapshotComponentType::RocksdbIndices => 1,
|
||||
_ => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Startup summary showing how much of the selected work can be reused.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub(crate) struct DownloadStartupSummary {
|
||||
/// Archives whose declared outputs already verify on disk.
|
||||
pub(crate) reusable: usize,
|
||||
/// Archives that still need to be downloaded or retried.
|
||||
pub(crate) needs_download: usize,
|
||||
}
|
||||
|
||||
/// Checks selected archives against existing output files before work begins.
|
||||
pub(crate) fn summarize_download_startup(
|
||||
all_downloads: &[PlannedArchive],
|
||||
target_dir: &Path,
|
||||
) -> Result<DownloadStartupSummary> {
|
||||
let mut summary = DownloadStartupSummary::default();
|
||||
let verifier = OutputVerifier::new(target_dir);
|
||||
|
||||
for planned in all_downloads {
|
||||
if verifier.verify(&planned.archive.output_files)? {
|
||||
summary.reusable += 1;
|
||||
} else {
|
||||
summary.needs_download += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Converts a selection into the manifest distance form used for archive lookup.
|
||||
fn selection_archive_distance(
|
||||
selection: &ComponentSelection,
|
||||
snapshot_block: u64,
|
||||
) -> Option<Option<u64>> {
|
||||
match selection {
|
||||
ComponentSelection::All => Some(None),
|
||||
ComponentSelection::Distance(distance) => Some(Some(*distance)),
|
||||
ComponentSelection::Since(block) => Some(Some(snapshot_block.saturating_sub(*block) + 1)),
|
||||
ComponentSelection::None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sorts planned archives into a stable processing order.
|
||||
fn sort_planned_archives(all_downloads: &mut [PlannedArchive]) {
|
||||
all_downloads.sort_by(|a, b| {
|
||||
archive_priority_rank(a.ty)
|
||||
.cmp(&archive_priority_rank(b.ty))
|
||||
.then_with(|| a.component.cmp(&b.component))
|
||||
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
|
||||
});
|
||||
}
|
||||
|
||||
/// Expands component selections into the archives that need to be processed.
|
||||
pub(crate) fn collect_planned_archives(
|
||||
manifest: &SnapshotManifest,
|
||||
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
|
||||
) -> Result<PlannedDownloads> {
|
||||
let mut archives = Vec::new();
|
||||
let mut total_download_size = 0;
|
||||
let mut total_output_size = 0;
|
||||
|
||||
for (ty, selection) in selections {
|
||||
let Some(distance) = selection_archive_distance(selection, manifest.block) else {
|
||||
continue;
|
||||
};
|
||||
total_download_size += manifest.size_for_distance(*ty, distance);
|
||||
total_output_size += manifest.output_size_for_distance(*ty, distance);
|
||||
|
||||
let snapshot_archives = manifest.snapshot_archives_for_distance(*ty, distance);
|
||||
let component = ty.display_name().to_string();
|
||||
if !snapshot_archives.is_empty() {
|
||||
info!(target: "reth::cli",
|
||||
component = %component,
|
||||
archives = snapshot_archives.len(),
|
||||
selection = %selection,
|
||||
"Queued component for download"
|
||||
);
|
||||
}
|
||||
|
||||
for archive in snapshot_archives {
|
||||
if archive.output_files.is_empty() {
|
||||
eyre::bail!(
|
||||
"Invalid modular manifest: {} is missing plain output checksum metadata",
|
||||
archive.file_name
|
||||
);
|
||||
}
|
||||
|
||||
archives.push(PlannedArchive { ty: *ty, component: component.clone(), archive });
|
||||
}
|
||||
}
|
||||
|
||||
sort_planned_archives(&mut archives);
|
||||
Ok(PlannedDownloads { archives, total_download_size, total_output_size })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn summarize_download_startup_counts_reusable_and_needs_download() {
|
||||
let dir = tempdir().unwrap();
|
||||
let target_dir = dir.path();
|
||||
let ok_file = target_dir.join("ok.bin");
|
||||
std::fs::write(&ok_file, vec![1_u8; 4]).unwrap();
|
||||
let ok_hash = blake3::hash(&[1_u8; 4]).to_hex().to_string();
|
||||
|
||||
let planned = vec![
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::State,
|
||||
component: "State".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/ok.tar.zst".to_string(),
|
||||
file_name: "ok.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "ok.bin".to_string(),
|
||||
size: 4,
|
||||
blake3: ok_hash,
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Headers,
|
||||
component: "Headers".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/missing.tar.zst".to_string(),
|
||||
file_name: "missing.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "missing.bin".to_string(),
|
||||
size: 1,
|
||||
blake3: "deadbeef".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Transactions,
|
||||
component: "Transactions".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/bad-size.tar.zst".to_string(),
|
||||
file_name: "bad-size.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let summary = summarize_download_startup(&planned, target_dir).unwrap();
|
||||
assert_eq!(summary.reusable, 1);
|
||||
assert_eq!(summary.needs_download, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_priority_prefers_state_then_rocksdb() {
|
||||
let mut planned = [
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Transactions,
|
||||
component: "Transactions".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u3".to_string(),
|
||||
file_name: "t.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "a".to_string(),
|
||||
size: 1,
|
||||
blake3: "x".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::RocksdbIndices,
|
||||
component: "RocksDB Indices".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u2".to_string(),
|
||||
file_name: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "b".to_string(),
|
||||
size: 1,
|
||||
blake3: "y".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::State,
|
||||
component: "State (mdbx)".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u1".to_string(),
|
||||
file_name: "state.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "c".to_string(),
|
||||
size: 1,
|
||||
blake3: "z".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
planned.sort_by(|a, b| {
|
||||
archive_priority_rank(a.ty)
|
||||
.cmp(&archive_priority_rank(b.ty))
|
||||
.then_with(|| a.component.cmp(&b.component))
|
||||
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
|
||||
});
|
||||
|
||||
assert_eq!(planned[0].ty, SnapshotComponentType::State);
|
||||
assert_eq!(planned[1].ty, SnapshotComponentType::RocksdbIndices);
|
||||
assert_eq!(planned[2].ty, SnapshotComponentType::Transactions);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_planned_archives_tracks_download_and_output_totals() {
|
||||
let mut components = BTreeMap::new();
|
||||
components.insert(
|
||||
"state".to_string(),
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 10,
|
||||
decompressed_size: 100,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
size: 100,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
components.insert(
|
||||
"transactions".to_string(),
|
||||
ComponentManifest::Chunked(ChunkedArchive {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![20, 30],
|
||||
chunk_decompressed_sizes: vec![200, 300],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/tx-0".to_string(),
|
||||
size: 200,
|
||||
blake3: "h1".to_string(),
|
||||
}],
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/tx-1".to_string(),
|
||||
size: 300,
|
||||
blake3: "h2".to_string(),
|
||||
}],
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
let manifest = SnapshotManifest {
|
||||
block: 1_000_000,
|
||||
chain_id: 1,
|
||||
storage_version: 2,
|
||||
timestamp: 0,
|
||||
base_url: Some("https://example.com".to_string()),
|
||||
reth_version: None,
|
||||
components,
|
||||
};
|
||||
|
||||
let selections = BTreeMap::from([
|
||||
(SnapshotComponentType::State, ComponentSelection::All),
|
||||
(SnapshotComponentType::Transactions, ComponentSelection::Distance(500_000)),
|
||||
]);
|
||||
|
||||
let planned = collect_planned_archives(&manifest, &selections).unwrap();
|
||||
|
||||
assert_eq!(planned.total_download_size, 40);
|
||||
assert_eq!(planned.total_output_size, 400);
|
||||
assert_eq!(planned.archives.len(), 2);
|
||||
}
|
||||
}
|
||||
@@ -1,844 +0,0 @@
|
||||
use eyre::Result;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use std::{
|
||||
io::{self, Read, Write},
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc, Condvar, Mutex,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
|
||||
|
||||
/// Tracks download progress and throttles display updates to every 100ms.
|
||||
pub(crate) struct DownloadProgress {
|
||||
/// Bytes copied so far for this single download.
|
||||
pub(crate) downloaded: u64,
|
||||
/// Total bytes expected for this single download.
|
||||
total_size: u64,
|
||||
/// Time when the progress line was last printed.
|
||||
last_displayed: Instant,
|
||||
/// Time when this progress tracker started.
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
impl DownloadProgress {
|
||||
/// Creates new progress tracker with given total size
|
||||
pub(crate) fn new(total_size: u64) -> Self {
|
||||
let now = Instant::now();
|
||||
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
|
||||
}
|
||||
|
||||
/// Converts bytes to human readable format (B, KB, MB, GB)
|
||||
pub(crate) fn format_size(size: u64) -> String {
|
||||
let mut size = size as f64;
|
||||
let mut unit_index = 0;
|
||||
|
||||
while size >= 1024.0 && unit_index < BYTE_UNITS.len() - 1 {
|
||||
size /= 1024.0;
|
||||
unit_index += 1;
|
||||
}
|
||||
|
||||
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
|
||||
}
|
||||
|
||||
/// Format duration as human readable string
|
||||
pub(crate) fn format_duration(duration: Duration) -> String {
|
||||
let secs = duration.as_secs();
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m {}s", secs / 60, secs % 60)
|
||||
} else {
|
||||
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates progress bar (for single-archive legacy downloads)
|
||||
pub(crate) fn update(&mut self, chunk_size: u64) -> Result<()> {
|
||||
self.downloaded += chunk_size;
|
||||
|
||||
if self.last_displayed.elapsed() >= Duration::from_millis(100) {
|
||||
let formatted_downloaded = Self::format_size(self.downloaded);
|
||||
let formatted_total = Self::format_size(self.total_size);
|
||||
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
|
||||
|
||||
let elapsed = self.started_at.elapsed();
|
||||
let eta = if self.downloaded > 0 {
|
||||
let remaining = self.total_size.saturating_sub(self.downloaded);
|
||||
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
|
||||
if speed > 0.0 {
|
||||
Duration::from_secs_f64(remaining as f64 / speed)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
}
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let eta_str = Self::format_duration(eta);
|
||||
|
||||
print!(
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
|
||||
);
|
||||
io::stdout().flush()?;
|
||||
self.last_displayed = Instant::now();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct PhaseStart {
|
||||
started_at: Instant,
|
||||
baseline_bytes: u64,
|
||||
}
|
||||
|
||||
/// Shared progress counters for parallel downloads.
|
||||
pub(crate) struct SharedProgress {
|
||||
/// Raw HTTP bytes fetched during this session, including retries.
|
||||
pub(crate) session_fetched_bytes: AtomicU64,
|
||||
/// Compressed bytes from archives that have fully downloaded.
|
||||
pub(crate) completed_download_bytes: AtomicU64,
|
||||
/// Compressed bytes written for currently active archive download attempts.
|
||||
pub(crate) active_download_bytes: AtomicU64,
|
||||
/// Total compressed bytes expected across all planned archives.
|
||||
pub(crate) total_download_bytes: u64,
|
||||
/// Plain-output bytes from archives that have fully verified.
|
||||
pub(crate) completed_output_bytes: AtomicU64,
|
||||
/// Plain-output bytes unpacked by currently active extractions.
|
||||
pub(crate) active_extracted_output_bytes: AtomicU64,
|
||||
/// Plain-output bytes hashed by currently active verifications.
|
||||
pub(crate) active_verified_output_bytes: AtomicU64,
|
||||
/// Total plain-output bytes expected across all planned archives.
|
||||
pub(crate) total_output_bytes: u64,
|
||||
/// Total number of planned archives.
|
||||
pub(crate) total_archives: u64,
|
||||
/// Time when the modular download job started.
|
||||
pub(crate) started_at: Instant,
|
||||
/// Time and baseline when the current extraction phase started.
|
||||
extraction_phase: Mutex<Option<PhaseStart>>,
|
||||
/// Time and baseline when the current verification phase started.
|
||||
verification_phase: Mutex<Option<PhaseStart>>,
|
||||
/// Number of archives that have fully finished.
|
||||
pub(crate) archives_done: AtomicU64,
|
||||
/// Number of archives currently in the fetch phase.
|
||||
pub(crate) active_downloads: AtomicU64,
|
||||
/// Number of in-flight HTTP requests.
|
||||
pub(crate) active_download_requests: AtomicU64,
|
||||
/// Number of archives currently extracting.
|
||||
pub(crate) active_extractions: AtomicU64,
|
||||
/// Number of archives currently verifying extracted outputs.
|
||||
pub(crate) active_verifications: AtomicU64,
|
||||
/// Signals the background progress task to exit.
|
||||
pub(crate) done: AtomicBool,
|
||||
/// Cancellation token shared by the whole command.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl SharedProgress {
|
||||
/// Creates the shared progress state for a modular download job.
|
||||
pub(crate) fn new(
|
||||
total_download_bytes: u64,
|
||||
total_output_bytes: u64,
|
||||
total_archives: u64,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
session_fetched_bytes: AtomicU64::new(0),
|
||||
completed_download_bytes: AtomicU64::new(0),
|
||||
active_download_bytes: AtomicU64::new(0),
|
||||
total_download_bytes,
|
||||
completed_output_bytes: AtomicU64::new(0),
|
||||
active_extracted_output_bytes: AtomicU64::new(0),
|
||||
active_verified_output_bytes: AtomicU64::new(0),
|
||||
total_output_bytes,
|
||||
total_archives,
|
||||
started_at: Instant::now(),
|
||||
extraction_phase: Mutex::new(None),
|
||||
verification_phase: Mutex::new(None),
|
||||
archives_done: AtomicU64::new(0),
|
||||
active_downloads: AtomicU64::new(0),
|
||||
active_download_requests: AtomicU64::new(0),
|
||||
active_extractions: AtomicU64::new(0),
|
||||
active_verifications: AtomicU64::new(0),
|
||||
done: AtomicBool::new(false),
|
||||
cancel_token,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether the whole command has been cancelled.
|
||||
pub(crate) fn is_cancelled(&self) -> bool {
|
||||
self.cancel_token.is_cancelled()
|
||||
}
|
||||
|
||||
/// Adds raw session traffic bytes without affecting logical progress.
|
||||
pub(crate) fn record_session_fetched_bytes(&self, bytes: u64) {
|
||||
self.session_fetched_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn add_active_download_bytes(&self, bytes: u64) {
|
||||
self.active_download_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn sub_active_download_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_download_bytes, bytes);
|
||||
}
|
||||
|
||||
fn add_active_extracted_output_bytes(&self, bytes: u64) {
|
||||
self.active_extracted_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn sub_active_extracted_output_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_extracted_output_bytes, bytes);
|
||||
}
|
||||
|
||||
fn add_active_verified_output_bytes(&self, bytes: u64) {
|
||||
self.active_verified_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn sub_active_verified_output_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_verified_output_bytes, bytes);
|
||||
}
|
||||
|
||||
/// Records an archive whose outputs were already present locally.
|
||||
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
|
||||
self.completed_download_bytes.fetch_add(download_bytes, Ordering::Relaxed);
|
||||
self.completed_output_bytes.fetch_add(output_bytes, Ordering::Relaxed);
|
||||
self.archives_done.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records an archive whose compressed download completed successfully.
|
||||
pub(crate) fn record_archive_download_complete(&self, bytes: u64) {
|
||||
self.completed_download_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records an archive whose extracted outputs have fully verified.
|
||||
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
|
||||
self.completed_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.archives_done.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Returns logical compressed download progress.
|
||||
pub(crate) fn logical_downloaded_bytes(&self) -> u64 {
|
||||
(self.completed_download_bytes.load(Ordering::Relaxed) +
|
||||
self.active_download_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_download_bytes)
|
||||
}
|
||||
|
||||
/// Returns verified plain-output bytes.
|
||||
pub(crate) fn verified_output_bytes(&self) -> u64 {
|
||||
self.completed_output_bytes.load(Ordering::Relaxed).min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Returns plain-output bytes currently represented by extraction progress.
|
||||
pub(crate) fn extracting_output_bytes(&self) -> u64 {
|
||||
(self.completed_output_bytes.load(Ordering::Relaxed) +
|
||||
self.active_extracted_output_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Returns plain-output bytes currently represented by verification progress.
|
||||
pub(crate) fn verifying_output_bytes(&self) -> u64 {
|
||||
(self.completed_output_bytes.load(Ordering::Relaxed) +
|
||||
self.active_verified_output_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
fn restart_phase(slot: &Mutex<Option<PhaseStart>>, baseline_bytes: u64) {
|
||||
*slot.lock().unwrap() = Some(PhaseStart { started_at: Instant::now(), baseline_bytes });
|
||||
}
|
||||
|
||||
fn phase_eta(
|
||||
slot: &Mutex<Option<PhaseStart>>,
|
||||
current_bytes: u64,
|
||||
total_bytes: u64,
|
||||
) -> Option<Duration> {
|
||||
let phase = *slot.lock().unwrap();
|
||||
let phase = phase?;
|
||||
let done = current_bytes.saturating_sub(phase.baseline_bytes);
|
||||
let total = total_bytes.saturating_sub(phase.baseline_bytes);
|
||||
eta_from_progress(phase.started_at.elapsed(), done, total)
|
||||
}
|
||||
|
||||
fn extraction_eta(&self, current_bytes: u64) -> Option<Duration> {
|
||||
Self::phase_eta(&self.extraction_phase, current_bytes, self.total_output_bytes)
|
||||
}
|
||||
|
||||
fn verification_eta(&self, current_bytes: u64) -> Option<Duration> {
|
||||
Self::phase_eta(&self.verification_phase, current_bytes, self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Marks one archive as actively downloading.
|
||||
pub(crate) fn download_started(&self) {
|
||||
self.active_downloads.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Marks one archive download as finished.
|
||||
pub(crate) fn download_finished(&self) {
|
||||
sub_bytes(&self.active_downloads, 1);
|
||||
}
|
||||
|
||||
/// Marks one HTTP request as in flight.
|
||||
pub(crate) fn request_started(&self) {
|
||||
self.active_download_requests.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Marks one HTTP request as finished.
|
||||
pub(crate) fn request_finished(&self) {
|
||||
sub_bytes(&self.active_download_requests, 1);
|
||||
}
|
||||
|
||||
/// Marks one archive as actively extracting.
|
||||
pub(crate) fn extraction_started(&self) {
|
||||
if self.active_extractions.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
Self::restart_phase(
|
||||
&self.extraction_phase,
|
||||
self.completed_output_bytes.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one archive extraction as finished.
|
||||
pub(crate) fn extraction_finished(&self) {
|
||||
sub_bytes(&self.active_extractions, 1);
|
||||
}
|
||||
|
||||
/// Marks one archive as actively verifying outputs.
|
||||
pub(crate) fn verification_started(&self) {
|
||||
if self.active_verifications.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
Self::restart_phase(
|
||||
&self.verification_phase,
|
||||
self.completed_output_bytes.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one archive verification as finished.
|
||||
pub(crate) fn verification_finished(&self) {
|
||||
sub_bytes(&self.active_verifications, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn sub_bytes(counter: &AtomicU64, bytes: u64) {
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_sub(bytes))
|
||||
});
|
||||
}
|
||||
|
||||
fn eta_from_progress(elapsed: Duration, done: u64, total: u64) -> Option<Duration> {
|
||||
if done == 0 || done >= total {
|
||||
return None;
|
||||
}
|
||||
|
||||
let secs = elapsed.as_secs_f64();
|
||||
if secs <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let speed = done as f64 / secs;
|
||||
if speed <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(Duration::from_secs_f64((total - done) as f64 / speed))
|
||||
}
|
||||
|
||||
fn format_percent(done: u64, total: u64) -> String {
|
||||
if total == 0 {
|
||||
return "100.0%".to_string();
|
||||
}
|
||||
|
||||
format!("{:.1}%", (done as f64 / total as f64) * 100.0)
|
||||
}
|
||||
|
||||
fn format_eta(eta: Option<Duration>) -> String {
|
||||
eta.map(DownloadProgress::format_duration).unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
|
||||
/// Global request limit for the blocking downloader.
|
||||
///
|
||||
/// This uses `Mutex + Condvar` because the segmented path runs blocking reqwest
|
||||
/// clients on OS threads.
|
||||
pub(crate) struct DownloadRequestLimiter {
|
||||
/// Maximum number of in-flight HTTP requests.
|
||||
limit: usize,
|
||||
/// Current number of acquired request slots.
|
||||
active: Mutex<usize>,
|
||||
/// Wakes blocked threads when a slot is released.
|
||||
notify: Condvar,
|
||||
}
|
||||
|
||||
impl DownloadRequestLimiter {
|
||||
/// Creates the shared request limiter.
|
||||
pub(crate) fn new(limit: usize) -> Arc<Self> {
|
||||
Arc::new(Self { limit: limit.max(1), active: Mutex::new(0), notify: Condvar::new() })
|
||||
}
|
||||
|
||||
/// Returns the configured request limit.
|
||||
pub(crate) fn max_concurrency(&self) -> usize {
|
||||
self.limit
|
||||
}
|
||||
|
||||
pub(crate) fn acquire<'a>(
|
||||
&'a self,
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
cancel_token: &CancellationToken,
|
||||
) -> Result<DownloadRequestPermit<'a>> {
|
||||
let mut active = self.active.lock().unwrap();
|
||||
loop {
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(eyre::eyre!("Download cancelled"));
|
||||
}
|
||||
|
||||
if *active < self.limit {
|
||||
*active += 1;
|
||||
if let Some(progress) = progress {
|
||||
progress.request_started();
|
||||
}
|
||||
return Ok(DownloadRequestPermit { limiter: self, progress });
|
||||
}
|
||||
|
||||
// Wake periodically so cancellation can interrupt waiters even if
|
||||
// no request finishes.
|
||||
let (next_active, _) =
|
||||
self.notify.wait_timeout(active, Duration::from_millis(100)).unwrap();
|
||||
active = next_active;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII permit for one in-flight HTTP request.
|
||||
///
|
||||
/// Dropping the permit releases a slot in the shared request limit and updates
|
||||
/// the live progress counters.
|
||||
pub(crate) struct DownloadRequestPermit<'a> {
|
||||
/// Limiter that owns the request slot.
|
||||
limiter: &'a DownloadRequestLimiter,
|
||||
/// Shared progress counters updated when the permit drops.
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
}
|
||||
|
||||
impl Drop for DownloadRequestPermit<'_> {
|
||||
/// Releases the request slot and updates shared progress counters.
|
||||
fn drop(&mut self) {
|
||||
let mut active = self.limiter.active.lock().unwrap();
|
||||
*active = active.saturating_sub(1);
|
||||
drop(active);
|
||||
self.limiter.notify.notify_one();
|
||||
|
||||
if let Some(progress) = self.progress {
|
||||
progress.request_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive download attempt.
|
||||
pub(crate) struct ArchiveDownloadProgress<'a> {
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
downloaded: u64,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveDownloadProgress<'a> {
|
||||
/// Starts tracking one archive download attempt.
|
||||
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.download_started();
|
||||
}
|
||||
Self { progress, downloaded: 0, completed: false }
|
||||
}
|
||||
|
||||
/// Adds logical compressed bytes written by this attempt.
|
||||
pub(crate) fn record_downloaded(&mut self, bytes: u64) {
|
||||
self.downloaded += bytes;
|
||||
if let Some(progress) = self.progress {
|
||||
progress.add_active_download_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this tracker has recorded any logical bytes itself.
|
||||
pub(crate) fn has_tracked_bytes(&self) -> bool {
|
||||
self.downloaded > 0
|
||||
}
|
||||
|
||||
/// Moves this archive from active download bytes into completed download bytes.
|
||||
pub(crate) fn complete(&mut self, total_bytes: u64) {
|
||||
if self.completed {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_download_bytes(self.downloaded);
|
||||
progress.record_archive_download_complete(total_bytes);
|
||||
}
|
||||
self.downloaded = 0;
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveDownloadProgress<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_download_bytes(self.downloaded);
|
||||
progress.download_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive extraction attempt.
|
||||
pub(crate) struct ArchiveExtractionProgress {
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
extracted: Arc<AtomicU64>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
/// Cloneable handle for reporting extracted bytes from background monitoring.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ArchiveExtractionProgressHandle {
|
||||
progress: Arc<SharedProgress>,
|
||||
extracted: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl ArchiveExtractionProgress {
|
||||
/// Starts tracking one archive extraction attempt.
|
||||
pub(crate) fn new(progress: Option<&Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.extraction_started();
|
||||
}
|
||||
Self {
|
||||
progress: progress.cloned(),
|
||||
extracted: Arc::new(AtomicU64::new(0)),
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a cloneable handle that can report extraction progress from another thread.
|
||||
pub(crate) fn handle(&self) -> Option<ArchiveExtractionProgressHandle> {
|
||||
Some(ArchiveExtractionProgressHandle {
|
||||
progress: Arc::clone(self.progress.as_ref()?),
|
||||
extracted: Arc::clone(&self.extracted),
|
||||
})
|
||||
}
|
||||
|
||||
/// Adds plain-output bytes extracted by this attempt.
|
||||
pub(crate) fn record_extracted(&mut self, bytes: u64) {
|
||||
if let Some(handle) = self.handle() {
|
||||
handle.record_extracted(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ends extraction tracking before verification begins.
|
||||
pub(crate) fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
|
||||
}
|
||||
self.finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveExtractionProgress {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
|
||||
progress.extraction_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ArchiveExtractionProgressHandle {
|
||||
/// Adds plain-output bytes extracted by this attempt.
|
||||
pub(crate) fn record_extracted(&self, bytes: u64) {
|
||||
self.extracted.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.progress.add_active_extracted_output_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive verification attempt.
|
||||
pub(crate) struct ArchiveVerificationProgress<'a> {
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
verified: u64,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveVerificationProgress<'a> {
|
||||
/// Starts tracking one archive verification attempt.
|
||||
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.verification_started();
|
||||
}
|
||||
Self { progress, verified: 0, completed: false }
|
||||
}
|
||||
|
||||
/// Adds plain-output bytes hashed by this verification attempt.
|
||||
pub(crate) fn record_verified(&mut self, bytes: u64) {
|
||||
self.verified += bytes;
|
||||
if let Some(progress) = self.progress {
|
||||
progress.add_active_verified_output_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Moves this archive from active verification bytes into completed output bytes.
|
||||
pub(crate) fn complete(&mut self, total_bytes: u64) {
|
||||
if self.completed {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_verified_output_bytes(self.verified);
|
||||
progress.record_archive_output_complete(total_bytes);
|
||||
}
|
||||
self.verified = 0;
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveVerificationProgress<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_verified_output_bytes(self.verified);
|
||||
progress.verification_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Adapter to track progress while reading (used for extraction in legacy path)
|
||||
pub(crate) struct ProgressReader<R> {
|
||||
/// Wrapped reader that provides archive bytes.
|
||||
reader: R,
|
||||
/// Per-download progress tracker for legacy paths.
|
||||
progress: DownloadProgress,
|
||||
/// Cancellation token checked between reads.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl<R: Read> ProgressReader<R> {
|
||||
/// Wraps a reader with per-download progress tracking.
|
||||
pub(crate) fn new(reader: R, total_size: u64, cancel_token: CancellationToken) -> Self {
|
||||
Self { reader, progress: DownloadProgress::new(total_size), cancel_token }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for ProgressReader<R> {
|
||||
/// Reads bytes, checks cancellation, and updates the local progress bar.
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let bytes = self.reader.read(buf)?;
|
||||
if bytes > 0 &&
|
||||
let Err(error) = self.progress.update(bytes as u64)
|
||||
{
|
||||
return Err(io::Error::other(error));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that bumps a shared atomic counter while writing data.
|
||||
/// Used for parallel downloads where a single display task shows aggregated progress.
|
||||
pub(crate) struct SharedProgressWriter<'a, W> {
|
||||
/// Wrapped writer receiving downloaded bytes.
|
||||
pub(crate) inner: W,
|
||||
/// Shared counters updated as bytes are written.
|
||||
pub(crate) progress: Arc<SharedProgress>,
|
||||
/// Optional callback for logical bytes written by the current archive attempt.
|
||||
pub(crate) on_written: Option<&'a mut dyn FnMut(u64)>,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for SharedProgressWriter<'_, W> {
|
||||
/// Writes bytes and records them in shared progress.
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
if self.progress.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let n = self.inner.write(buf)?;
|
||||
self.progress.record_session_fetched_bytes(n as u64);
|
||||
if let Some(on_written) = self.on_written.as_deref_mut() {
|
||||
on_written(n as u64);
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Flushes the wrapped writer.
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that bumps a shared atomic counter while reading data.
|
||||
/// Used for streaming downloads where a single display task shows aggregated progress.
|
||||
pub(crate) struct SharedProgressReader<R> {
|
||||
/// Wrapped reader producing streamed bytes.
|
||||
pub(crate) inner: R,
|
||||
/// Shared counters updated as bytes are read.
|
||||
pub(crate) progress: Arc<SharedProgress>,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for SharedProgressReader<R> {
|
||||
/// Reads bytes and records them in shared progress.
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
if self.progress.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let n = self.inner.read(buf)?;
|
||||
self.progress.record_session_fetched_bytes(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns a background task that prints aggregated download progress.
|
||||
/// Returns a handle; drop it (or call `.abort()`) to stop.
|
||||
pub(crate) fn spawn_progress_display(progress: Arc<SharedProgress>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(3));
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
if progress.done.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
|
||||
let download_total = progress.total_download_bytes;
|
||||
let output_total = progress.total_output_bytes;
|
||||
if download_total == 0 && output_total == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let done = progress.archives_done.load(Ordering::Relaxed);
|
||||
let all = progress.total_archives;
|
||||
let active_downloads = progress.active_downloads.load(Ordering::Relaxed);
|
||||
let active_requests = progress.active_download_requests.load(Ordering::Relaxed);
|
||||
let active_extractions = progress.active_extractions.load(Ordering::Relaxed);
|
||||
let active_verifications = progress.active_verifications.load(Ordering::Relaxed);
|
||||
let downloaded = progress.logical_downloaded_bytes();
|
||||
let extracted = progress.extracting_output_bytes();
|
||||
let verified = progress.verifying_output_bytes();
|
||||
let elapsed = DownloadProgress::format_duration(progress.started_at.elapsed());
|
||||
let download_total_display = DownloadProgress::format_size(download_total);
|
||||
let output_total_display = DownloadProgress::format_size(output_total);
|
||||
let downloaded_display = DownloadProgress::format_size(downloaded);
|
||||
let extracted_display = DownloadProgress::format_size(extracted);
|
||||
let active_download_phase = active_downloads > 0 || active_requests > 0;
|
||||
|
||||
if active_download_phase {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(downloaded, download_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(eta_from_progress(progress.started_at.elapsed(), downloaded, download_total)),
|
||||
bytes = format_args!("{downloaded_display}/{download_total_display}"),
|
||||
"Downloading snapshot archives"
|
||||
);
|
||||
} else if active_extractions > 0 {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(extracted, output_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(progress.extraction_eta(extracted)),
|
||||
bytes = format_args!("{extracted_display}/{output_total_display}"),
|
||||
"Extracting snapshot archives"
|
||||
);
|
||||
} else if active_verifications > 0 {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(verified, output_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(progress.verification_eta(verified)),
|
||||
bytes = format_args!("{}/{output_total_display}", DownloadProgress::format_size(verified)),
|
||||
"Verifying snapshot archives"
|
||||
);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let completed = progress.verified_output_bytes();
|
||||
let completed_display = DownloadProgress::format_size(completed);
|
||||
let output_total = DownloadProgress::format_size(progress.total_output_bytes);
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{}/{}", progress.total_archives, progress.total_archives),
|
||||
progress = "100.0%",
|
||||
elapsed = %DownloadProgress::format_duration(progress.started_at.elapsed()),
|
||||
eta = "0s",
|
||||
bytes = format_args!("{completed_display}/{output_total}"),
|
||||
"Snapshot archive processing complete"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[test]
|
||||
fn shared_progress_separates_session_fetch_from_logical_progress() {
|
||||
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
|
||||
|
||||
progress.record_session_fetched_bytes(10);
|
||||
progress.record_session_fetched_bytes(10);
|
||||
progress.record_archive_download_complete(10);
|
||||
progress.record_archive_output_complete(20);
|
||||
|
||||
assert_eq!(progress.session_fetched_bytes.load(Ordering::Relaxed), 20);
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 10);
|
||||
assert_eq!(progress.verified_output_bytes(), 20);
|
||||
assert_eq!(progress.archives_done.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_download_progress_rolls_back_unfinished_attempts() {
|
||||
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
|
||||
|
||||
{
|
||||
let mut download = ArchiveDownloadProgress::new(Some(&progress));
|
||||
download.record_downloaded(4);
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 4);
|
||||
}
|
||||
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 0);
|
||||
assert_eq!(progress.active_downloads.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_phase_baseline_restarts_after_idle() {
|
||||
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
|
||||
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.completed_output_bytes.store(25, Ordering::Relaxed);
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.extraction_finished();
|
||||
progress.extraction_finished();
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_phase_baseline_restarts_after_idle() {
|
||||
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
|
||||
|
||||
progress.verification_started();
|
||||
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.completed_output_bytes.store(40, Ordering::Relaxed);
|
||||
progress.verification_started();
|
||||
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.verification_finished();
|
||||
progress.verification_finished();
|
||||
progress.verification_started();
|
||||
assert_eq!(
|
||||
progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes,
|
||||
40
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -1,100 +0,0 @@
|
||||
use super::progress::{DownloadRequestLimiter, SharedProgress};
|
||||
use eyre::Result;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
/// Shared state for one run of `reth download`.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DownloadSession {
|
||||
/// Shared progress counters for this command, when enabled.
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
/// Shared limit for concurrent HTTP requests, when enabled.
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
/// Cancellation token shared by the whole command.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl DownloadSession {
|
||||
/// Stores the shared progress, request limiter, and cancellation token.
|
||||
pub(crate) fn new(
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Self {
|
||||
Self { progress, request_limiter, cancel_token }
|
||||
}
|
||||
|
||||
/// Returns the shared progress tracker, if this flow uses one.
|
||||
pub(crate) fn progress(&self) -> Option<&Arc<SharedProgress>> {
|
||||
self.progress.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the shared HTTP request limiter, if this flow uses one.
|
||||
pub(crate) fn request_limiter(&self) -> Option<&Arc<DownloadRequestLimiter>> {
|
||||
self.request_limiter.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the request limiter or errors if the caller needs one.
|
||||
pub(crate) fn require_request_limiter(&self) -> Result<&Arc<DownloadRequestLimiter>> {
|
||||
self.request_limiter().ok_or_else(|| eyre::eyre!("Missing download request limiter"))
|
||||
}
|
||||
|
||||
/// Returns the cancellation token for this command.
|
||||
pub(crate) fn cancel_token(&self) -> &CancellationToken {
|
||||
&self.cancel_token
|
||||
}
|
||||
|
||||
/// Records one archive whose outputs were already reusable on disk.
|
||||
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
|
||||
if let Some(progress) = self.progress() {
|
||||
progress.record_reused_archive(download_bytes, output_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Records one archive whose extracted outputs fully verified.
|
||||
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
|
||||
if let Some(progress) = self.progress() {
|
||||
progress.record_archive_output_complete(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Paths used while processing one archive, plus the shared download session.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ArchiveProcessContext {
|
||||
/// Directory where extracted output files are written.
|
||||
target_dir: PathBuf,
|
||||
/// Directory used for cached archive downloads, when enabled.
|
||||
cache_dir: Option<PathBuf>,
|
||||
/// Shared command-scoped download state.
|
||||
session: DownloadSession,
|
||||
}
|
||||
|
||||
impl ArchiveProcessContext {
|
||||
/// Creates the context used while processing modular archives.
|
||||
pub(crate) fn new(
|
||||
target_dir: PathBuf,
|
||||
cache_dir: Option<PathBuf>,
|
||||
session: DownloadSession,
|
||||
) -> Self {
|
||||
Self { target_dir, cache_dir, session }
|
||||
}
|
||||
|
||||
/// Returns the directory where extracted outputs should be written.
|
||||
pub(crate) fn target_dir(&self) -> &Path {
|
||||
&self.target_dir
|
||||
}
|
||||
|
||||
/// Returns the cache directory for two-phase downloads, if enabled.
|
||||
pub(crate) fn cache_dir(&self) -> Option<&Path> {
|
||||
self.cache_dir.as_deref()
|
||||
}
|
||||
|
||||
/// Returns the shared download session.
|
||||
pub(crate) fn session(&self) -> &DownloadSession {
|
||||
&self.session
|
||||
}
|
||||
}
|
||||
@@ -1,230 +0,0 @@
|
||||
use super::{manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults};
|
||||
use eyre::{Result, WrapErr};
|
||||
use reqwest::Client;
|
||||
use reth_fs_util as fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
|
||||
/// An entry from the snapshot discovery API listing.
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct SnapshotApiEntry {
|
||||
#[serde(deserialize_with = "deserialize_string_or_u64")]
|
||||
chain_id: u64,
|
||||
#[serde(deserialize_with = "deserialize_string_or_u64")]
|
||||
block: u64,
|
||||
#[serde(default)]
|
||||
date: Option<String>,
|
||||
#[serde(default)]
|
||||
profile: Option<String>,
|
||||
metadata_url: String,
|
||||
#[serde(default)]
|
||||
size: u64,
|
||||
}
|
||||
|
||||
impl SnapshotApiEntry {
|
||||
/// Returns whether this discovery entry points to a modular manifest.
|
||||
fn is_modular(&self) -> bool {
|
||||
self.metadata_url.ends_with("manifest.json")
|
||||
}
|
||||
}
|
||||
|
||||
/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
|
||||
///
|
||||
/// Queries the configured snapshot API and returns the manifest URL for the most
|
||||
/// recent modular snapshot matching the requested chain.
|
||||
pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
|
||||
let defaults = DownloadDefaults::get_global();
|
||||
let api_url = &*defaults.snapshot_api_url;
|
||||
|
||||
info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
|
||||
|
||||
let entries = fetch_snapshot_api_entries(chain_id).await?;
|
||||
let entry =
|
||||
entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"No modular snapshot manifest found for chain \
|
||||
{chain_id} at {api_url}\n\n\
|
||||
You can provide a manifest URL directly with --manifest-url, or\n\
|
||||
use a direct snapshot URL with -u from:\n\
|
||||
\t- {}\n\n\
|
||||
Use --list to see all available snapshots.",
|
||||
api_url.trim_end_matches("/api/snapshots"),
|
||||
)
|
||||
})?;
|
||||
|
||||
info!(target: "reth::cli",
|
||||
block = entry.block,
|
||||
url = %entry.metadata_url,
|
||||
"Found latest snapshot manifest"
|
||||
);
|
||||
|
||||
Ok(entry.metadata_url.clone())
|
||||
}
|
||||
|
||||
/// Deserializes a JSON value that may be either a number or a string-encoded number.
|
||||
fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
use serde::Deserialize;
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
match &value {
|
||||
serde_json::Value::Number(n) => {
|
||||
n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
|
||||
}
|
||||
serde_json::Value::String(s) => {
|
||||
s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
|
||||
}
|
||||
_ => Err(serde::de::Error::custom("expected number or string")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
|
||||
pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
|
||||
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
|
||||
|
||||
let entries: Vec<SnapshotApiEntry> = Client::new()
|
||||
.get(api_url)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
Ok(entries.into_iter().filter(|entry| entry.chain_id == chain_id).collect())
|
||||
}
|
||||
|
||||
/// Prints a formatted table of available modular snapshots.
|
||||
pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
|
||||
let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
|
||||
|
||||
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
|
||||
println!(
|
||||
"Available snapshots for chain {chain_id} ({}):\n",
|
||||
api_url.trim_end_matches("/api/snapshots"),
|
||||
);
|
||||
println!("{:<12} {:>10} {:<10} {:>10} MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
|
||||
println!("{}", "-".repeat(100));
|
||||
|
||||
for entry in &modular {
|
||||
let date = entry.date.as_deref().unwrap_or("-");
|
||||
let profile = entry.profile.as_deref().unwrap_or("-");
|
||||
let size = if entry.size > 0 {
|
||||
DownloadProgress::format_size(entry.size)
|
||||
} else {
|
||||
"-".to_string()
|
||||
};
|
||||
|
||||
println!(
|
||||
"{date:<12} {:>10} {profile:<10} {size:>10} {}",
|
||||
entry.block, entry.metadata_url
|
||||
);
|
||||
}
|
||||
|
||||
if modular.is_empty() {
|
||||
println!(" (no modular snapshots found)");
|
||||
}
|
||||
|
||||
println!(
|
||||
"\nTo download a specific snapshot, copy its manifest URL and run:\n \
|
||||
reth download --manifest-url <URL>"
|
||||
);
|
||||
}
|
||||
|
||||
/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path.
|
||||
pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
|
||||
if let Ok(parsed) = Url::parse(source) {
|
||||
return match parsed.scheme() {
|
||||
"http" | "https" => {
|
||||
let response = Client::new()
|
||||
.get(source)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.wrap_err_with(|| {
|
||||
let sources = DownloadDefaults::get_global()
|
||||
.available_snapshots
|
||||
.iter()
|
||||
.map(|snapshot| format!("\t- {snapshot}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
format!(
|
||||
"Failed to fetch snapshot manifest from {source}\n\n\
|
||||
The manifest endpoint may not be available for this snapshot source.\n\
|
||||
You can use a direct snapshot URL instead:\n\n\
|
||||
\treth download -u <snapshot-url>\n\n\
|
||||
Available snapshot sources:\n{sources}"
|
||||
)
|
||||
})?;
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
"file" => {
|
||||
let path = parsed
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
|
||||
let content = fs::read_to_string(path)?;
|
||||
Ok(serde_json::from_str(&content)?)
|
||||
}
|
||||
_ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
|
||||
};
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(source)?;
|
||||
Ok(serde_json::from_str(&content)?)
|
||||
}
|
||||
|
||||
/// Resolves the base URL used to join relative archive paths in a manifest.
|
||||
pub(crate) fn resolve_manifest_base_url(
|
||||
manifest: &SnapshotManifest,
|
||||
source: &str,
|
||||
) -> Result<String> {
|
||||
if let Some(base_url) = manifest.base_url.as_deref() &&
|
||||
!base_url.is_empty()
|
||||
{
|
||||
return Ok(base_url.trim_end_matches('/').to_string());
|
||||
}
|
||||
|
||||
if let Ok(mut url) = Url::parse(source) {
|
||||
if url.scheme() == "file" {
|
||||
let mut path = url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
|
||||
path.pop();
|
||||
let mut base = Url::from_directory_path(path)
|
||||
.map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
|
||||
.to_string();
|
||||
if base.ends_with('/') {
|
||||
base.pop();
|
||||
}
|
||||
return Ok(base);
|
||||
}
|
||||
|
||||
{
|
||||
let mut segments = url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
|
||||
segments.pop_if_empty();
|
||||
segments.pop();
|
||||
}
|
||||
return Ok(url.as_str().trim_end_matches('/').to_string());
|
||||
}
|
||||
|
||||
let path = Path::new(source);
|
||||
let manifest_dir = if path.is_absolute() {
|
||||
path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
|
||||
} else {
|
||||
let joined = std::env::current_dir()?.join(path);
|
||||
joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
|
||||
};
|
||||
|
||||
let mut base = Url::from_directory_path(&manifest_dir)
|
||||
.map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
|
||||
.to_string();
|
||||
if base.ends_with('/') {
|
||||
base.pop();
|
||||
}
|
||||
Ok(base)
|
||||
}
|
||||
@@ -262,7 +262,6 @@ impl SelectorApp {
|
||||
ComponentSelection::None => return 0,
|
||||
ComponentSelection::All => None,
|
||||
ComponentSelection::Distance(d) => Some(d),
|
||||
ComponentSelection::Since(block) => Some(self.manifest.block - block + 1),
|
||||
};
|
||||
self.groups[group_idx]
|
||||
.types
|
||||
@@ -345,7 +344,6 @@ fn format_selection(sel: &ComponentSelection) -> String {
|
||||
match sel {
|
||||
ComponentSelection::All => "All".to_string(),
|
||||
ComponentSelection::Distance(d) => format!("Last {d} blocks"),
|
||||
ComponentSelection::Since(block) => format!("Since block {block}"),
|
||||
ComponentSelection::None => "None".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,84 +0,0 @@
|
||||
use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress};
|
||||
use blake3::Hasher;
|
||||
use eyre::Result;
|
||||
use reth_fs_util as fs;
|
||||
use std::{io::Read, path::Path};
|
||||
|
||||
/// Verifies and cleans up extracted output files in one target directory.
|
||||
pub(crate) struct OutputVerifier<'a> {
|
||||
/// Directory containing the output files declared by the manifest.
|
||||
target_dir: &'a Path,
|
||||
}
|
||||
|
||||
impl<'a> OutputVerifier<'a> {
|
||||
/// Creates a verifier for one extraction target directory.
|
||||
pub(crate) const fn new(target_dir: &'a Path) -> Self {
|
||||
Self { target_dir }
|
||||
}
|
||||
|
||||
/// Returns `true` only when every declared output file exists and matches size and BLAKE3.
|
||||
/// Returns `false` if any file is missing, mismatched, or no outputs were declared.
|
||||
pub(crate) fn verify(&self, output_files: &[OutputFileChecksum]) -> Result<bool> {
|
||||
self.verify_with_progress(output_files, None)
|
||||
}
|
||||
|
||||
/// Returns `true` only when every declared output file exists and matches size and BLAKE3,
|
||||
/// updating the optional verification progress as file bytes are hashed.
|
||||
pub(crate) fn verify_with_progress(
|
||||
&self,
|
||||
output_files: &[OutputFileChecksum],
|
||||
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
|
||||
) -> Result<bool> {
|
||||
if output_files.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
for expected in output_files {
|
||||
let output_path = self.target_dir.join(&expected.path);
|
||||
let meta = match fs::metadata(&output_path) {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
if meta.len() != expected.size {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let actual = Self::file_blake3_hex(&output_path, progress.as_deref_mut())?;
|
||||
if !actual.eq_ignore_ascii_case(&expected.blake3) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Removes any declared output files so a fresh archive attempt can restart cleanly.
|
||||
pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) {
|
||||
for output in output_files {
|
||||
let _ = fs::remove_file(self.target_dir.join(&output.path));
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the hex-encoded BLAKE3 checksum for one plain output file.
|
||||
fn file_blake3_hex(
|
||||
path: &Path,
|
||||
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
|
||||
) -> Result<String> {
|
||||
let mut file = fs::open(path)?;
|
||||
let mut hasher = Hasher::new();
|
||||
let mut buf = [0_u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let n = file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
if let Some(progress) = progress.as_deref_mut() {
|
||||
progress.record_verified(n as u64);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
}
|
||||
@@ -78,10 +78,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
|
||||
self.env.init::<N>(AccessRights::RW, runtime)?;
|
||||
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
|
||||
if self.without_evm {
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
|
||||
// ensure header, total difficulty and header hash are provided
|
||||
let header = self.header.ok_or_else(|| eyre::eyre!("Header file must be provided"))?;
|
||||
let header = without_evm::read_header_from_file::<
|
||||
@@ -107,22 +106,23 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
|
||||
// SAFETY: it's safe to commit static files, since in the event of a crash, they
|
||||
// will be unwound according to database checkpoints.
|
||||
//
|
||||
// Necessary to commit, so the header is accessible to init_from_state_dump
|
||||
// Necessary to commit, so the header is accessible to provider_rw and
|
||||
// init_state_dump
|
||||
static_file_provider.commit()?;
|
||||
} else if last_block_number > 0 && last_block_number < header.number() {
|
||||
return Err(eyre::eyre!(
|
||||
"Data directory should be empty when calling init-state with --without-evm."
|
||||
));
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Initiating state dump");
|
||||
|
||||
let reader = BufReader::new(reth_fs_util::open(self.state)?);
|
||||
|
||||
let hash = init_from_state_dump(reader, &provider_factory, config.stages.etl)?;
|
||||
let hash = init_from_state_dump(reader, &provider_rw, config.stages.etl)?;
|
||||
|
||||
provider_rw.commit()?;
|
||||
|
||||
info!(target: "reth::cli", hash = ?hash, "Genesis block written");
|
||||
Ok(())
|
||||
|
||||
@@ -50,13 +50,8 @@ where
|
||||
info!(target: "reth::cli", new_tip = ?header.num_hash(), "Setting up dummy EVM chain before importing state.");
|
||||
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
// Write EVM dummy data up to `header - 1` block. Skip when the supplied
|
||||
// header is at block 0: `header.number() - 1` would underflow in u64 to
|
||||
// `u64::MAX`, sending `append_dummy_chain` into a 1..=u64::MAX loop that
|
||||
// exhausts memory before failing.
|
||||
if header.number() > 0 {
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
}
|
||||
// Write EVM dummy data up to `header - 1` block
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
|
||||
info!(target: "reth::cli", "Appending first valid block.");
|
||||
|
||||
@@ -196,13 +191,7 @@ mod tests {
|
||||
use alloy_primitives::{address, b256};
|
||||
use reth_db_common::init::init_genesis;
|
||||
use reth_provider::{test_utils::create_test_provider_factory, DatabaseProviderFactory};
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
use std::io::Write;
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
@@ -275,45 +264,4 @@ mod tests {
|
||||
|
||||
assert_eq!(actual_next_height, expected_next_height);
|
||||
}
|
||||
|
||||
/// Regression: a header at block 0 used to send `append_dummy_chain` into
|
||||
/// a `1..=u64::MAX` loop because `header.number() - 1` underflowed in
|
||||
/// u64. The guard `if header.number() > 0` skips the dummy-chain step
|
||||
/// when there is no pre-genesis range to backfill, so `header_factory`
|
||||
/// is never invoked.
|
||||
#[test]
|
||||
fn test_setup_without_evm_skips_dummy_chain_for_genesis_header() {
|
||||
let header = Header { number: 0, ..Default::default() };
|
||||
let header_hash = header.hash_slow();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
init_genesis(&provider_factory).unwrap();
|
||||
let provider_rw = provider_factory.database_provider_rw().unwrap();
|
||||
|
||||
let factory_calls = Arc::new(AtomicU64::new(0));
|
||||
let factory_calls_inner = Arc::clone(&factory_calls);
|
||||
|
||||
// The Result of `setup_without_evm` itself is not asserted: with
|
||||
// `number == 0` plus a genesis already written by `init_genesis`,
|
||||
// the subsequent `append_first_block` may legitimately fail. The
|
||||
// bug under test is the OOM in the dummy-chain loop, observable
|
||||
// through the factory-call counter below.
|
||||
let _ = setup_without_evm(
|
||||
&provider_rw,
|
||||
SealedHeader::new(header, header_hash),
|
||||
move |number| {
|
||||
// Bound calls so a regression cannot exhaust the test
|
||||
// runner's memory; the only correct value here is 0.
|
||||
let n = factory_calls_inner.fetch_add(1, Ordering::Relaxed);
|
||||
assert!(n < 8, "header_factory must not be invoked for a genesis-block header");
|
||||
Header { number, ..Default::default() }
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
factory_calls.load(Ordering::Relaxed),
|
||||
0,
|
||||
"append_dummy_chain must be skipped when header.number() == 0"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
|
||||
)
|
||||
}
|
||||
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
|
||||
let default_secret_key_path = data_dir.p2p_secret();
|
||||
let p2p_secret_key = self.network.secret_key(default_secret_key_path)?;
|
||||
|
||||
@@ -5,7 +5,6 @@ use crate::common::{
|
||||
EnvironmentArgs,
|
||||
};
|
||||
use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt};
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use clap::Parser;
|
||||
use eyre::WrapErr;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
|
||||
@@ -13,19 +12,14 @@ use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_consensus::FullConsensus;
|
||||
use reth_evm::{execute::Executor, ConfigureEvm};
|
||||
use reth_primitives_traits::{format_gas_throughput, Account, BlockBody, GotExpected};
|
||||
use reth_primitives_traits::{format_gas_throughput, BlockBody, GotExpected};
|
||||
use reth_provider::{
|
||||
BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider,
|
||||
StaticFileProviderFactory, TransactionVariant,
|
||||
};
|
||||
use reth_revm::{
|
||||
database::StateProviderDatabase,
|
||||
db::{states::reverts::AccountInfoRevert, BundleState},
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_stages::stages::calculate_gas_used_from_headers;
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
@@ -74,18 +68,13 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>> Command<C> {
|
||||
/// Execute `re-execute` command
|
||||
pub async fn execute<N>(
|
||||
mut self,
|
||||
self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
{
|
||||
// Default to 4GB RocksDB block cache for re-execute unless explicitly set.
|
||||
if self.env.db.rocksdb_block_cache_size.is_none() {
|
||||
self.env.db.rocksdb_block_cache_size = Some(4 << 30);
|
||||
}
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO, runtime)?;
|
||||
|
||||
let components = components(provider_factory.chain_spec());
|
||||
@@ -119,6 +108,15 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
min_block..=max_block,
|
||||
)?;
|
||||
|
||||
let db_at = {
|
||||
let provider_factory = provider_factory.clone();
|
||||
move |block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider_factory.history_by_block_number(block_number).unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let skip_invalid_blocks = self.skip_invalid_blocks;
|
||||
let blocks_per_chunk = self.blocks_per_chunk;
|
||||
let (stats_tx, mut stats_rx) = mpsc::unbounded_channel();
|
||||
@@ -134,23 +132,13 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let provider_factory = provider_factory.clone();
|
||||
let evm_config = components.evm_config().clone();
|
||||
let consensus = components.consensus().clone();
|
||||
let db_at = db_at.clone();
|
||||
let stats_tx = stats_tx.clone();
|
||||
let info_tx = info_tx.clone();
|
||||
let cancellation = cancellation.clone();
|
||||
let next_block = Arc::clone(&next_block);
|
||||
tasks.spawn_blocking(move || {
|
||||
let executor_lifetime = Duration::from_secs(600);
|
||||
let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety();
|
||||
|
||||
let db_at = {
|
||||
|block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider
|
||||
.history_by_block_number(block_number)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
let executor_lifetime = Duration::from_secs(120);
|
||||
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
@@ -257,31 +245,14 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let _ = stats_tx.send(block.gas_used());
|
||||
|
||||
// Reset DB once in a while to avoid OOM or read tx timeouts
|
||||
if executor.size_hint() > 5_000_000 ||
|
||||
if executor.size_hint() > 1_000_000 ||
|
||||
executor_created.elapsed() > executor_lifetime
|
||||
{
|
||||
let last_block = block.number();
|
||||
let old_executor = std::mem::replace(
|
||||
&mut executor,
|
||||
evm_config.batch_executor(db_at(last_block)),
|
||||
);
|
||||
let bundle = old_executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
last_block,
|
||||
)?;
|
||||
executor =
|
||||
evm_config.batch_executor(db_at(block.number()));
|
||||
executor_created = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Full verification at chunk end for remaining unverified blocks
|
||||
let bundle = executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
chunk_end - 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
eyre::Ok(())
|
||||
@@ -362,98 +333,3 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies reverts against database changesets.
|
||||
///
|
||||
/// For each block, reverts must match changeset entries exactly. No extra slots/accounts
|
||||
/// in reverts for non-destroyed accounts. Destroyed accounts may have extra changeset slots
|
||||
/// (from DB storage wipe) absent from reverts.
|
||||
fn verify_bundle_against_changesets<P>(
|
||||
provider: &P,
|
||||
bundle: &BundleState,
|
||||
last_block: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
P: ChangeSetReader + StorageChangeSetReader,
|
||||
{
|
||||
// Verify reverts against changesets per block
|
||||
for (i, block_reverts) in bundle.reverts.iter().rev().enumerate() {
|
||||
let block_number = last_block - i as u64;
|
||||
|
||||
let mut cs_accounts: HashMap<Address, Option<Account>> = provider
|
||||
.account_block_changeset(block_number)?
|
||||
.into_iter()
|
||||
.map(|cs| (cs.address, cs.info))
|
||||
.collect();
|
||||
|
||||
let mut cs_storage: HashMap<Address, HashMap<B256, U256>> = HashMap::new();
|
||||
for (bna, entry) in provider.storage_changeset(block_number)? {
|
||||
cs_storage.entry(bna.address()).or_default().insert(entry.key, entry.value);
|
||||
}
|
||||
|
||||
for (addr, revert) in block_reverts {
|
||||
// Verify account info
|
||||
match &revert.account {
|
||||
AccountInfoRevert::DoNothing => {
|
||||
eyre::ensure!(
|
||||
!cs_accounts.contains_key(addr),
|
||||
"Block {block_number}: account {addr} in changeset but revert is DoNothing",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::DeleteIt => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is DeleteIt but not in changeset")
|
||||
})?;
|
||||
eyre::ensure!(
|
||||
cs_info.is_none(),
|
||||
"Block {block_number}: account {addr} revert is DeleteIt but changeset has {cs_info:?}",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::RevertTo(info) => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is RevertTo but not in changeset")
|
||||
})?;
|
||||
let revert_acct = Some(Account::from(info));
|
||||
eyre::ensure!(
|
||||
revert_acct == cs_info,
|
||||
"Block {block_number}: account {addr} info mismatch: revert={revert_acct:?} cs={cs_info:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify storage slots — remove matched changeset entries as we go
|
||||
let mut cs_slots = cs_storage.get_mut(addr);
|
||||
for (slot_key, revert_slot) in &revert.storage {
|
||||
let b256_key = B256::from(*slot_key);
|
||||
match cs_slots.as_mut().and_then(|s| s.remove(&b256_key)) {
|
||||
Some(cs_value) => eyre::ensure!(
|
||||
revert_slot.to_previous_value() == cs_value,
|
||||
"Block {block_number}: {addr} slot {b256_key} mismatch: \
|
||||
revert={} cs={cs_value}",
|
||||
revert_slot.to_previous_value(),
|
||||
),
|
||||
None => eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} slot {b256_key} in reverts but not in changeset",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_storage slots for this address must be from a destroyed account
|
||||
if let Some(remaining) = cs_slots.filter(|s| !s.is_empty()) {
|
||||
eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} has {} unmatched storage slots in changeset",
|
||||
remaining.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_accounts entries had no corresponding revert
|
||||
if let Some(addr) = cs_accounts.keys().next() {
|
||||
eyre::bail!("Block {block_number}: account {addr} in changeset but not in reverts");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use reth_db_api::{
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_node_api::{HeaderTy, TxTy};
|
||||
use reth_node_api::HeaderTy;
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
|
||||
@@ -88,7 +88,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
|
||||
)
|
||||
})??;
|
||||
output_db.update(|tx| {
|
||||
tx.import_table_with_range::<tables::BlockOmmers<HeaderTy<N>>, _>(
|
||||
tx.import_table_with_range::<tables::BlockOmmers, _>(
|
||||
&db_tool.provider_factory.db_ref().tx()?,
|
||||
Some(from),
|
||||
to,
|
||||
@@ -110,7 +110,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
|
||||
})??;
|
||||
|
||||
output_db.update(|tx| {
|
||||
tx.import_table_with_range::<tables::Transactions<TxTy<N>>, _>(
|
||||
tx.import_table_with_range::<tables::Transactions, _>(
|
||||
&db_tool.provider_factory.db_ref().tx()?,
|
||||
Some(from_tx),
|
||||
to_tx,
|
||||
|
||||
@@ -29,10 +29,7 @@ use execution::dump_execution_stage;
|
||||
mod merkle;
|
||||
use merkle::dump_merkle_stage;
|
||||
|
||||
/// `reth dump-stage` command.
|
||||
///
|
||||
/// Note: mutates the source datadir (unwinds hashing/merkle/execution before copying tables).
|
||||
/// Stop the node and back up the datadir first.
|
||||
/// `reth dump-stage` command
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
@@ -103,9 +100,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
Comp: CliNodeComponents<N>,
|
||||
F: FnOnce(Arc<C::ChainSpec>) -> Comp,
|
||||
{
|
||||
// `unwind_and_copy` opens a RW provider on the source datadir, so open RW here.
|
||||
let Environment { provider_factory, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, runtime.clone())?;
|
||||
self.env.init::<N>(AccessRights::RO, runtime.clone())?;
|
||||
let tool = DbTool::new(provider_factory)?;
|
||||
let components = components(tool.chain());
|
||||
let evm_config = components.evm_config().clone();
|
||||
|
||||
@@ -210,7 +210,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let consensus = Arc::new(components.consensus().clone());
|
||||
|
||||
let mut config = config;
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
config.peers.trusted_nodes.extend(self.network.trusted_peers.clone());
|
||||
|
||||
let network_secret_path = self
|
||||
|
||||
@@ -30,16 +30,10 @@
|
||||
|
||||
extern crate alloc;
|
||||
|
||||
use alloc::{
|
||||
boxed::Box,
|
||||
fmt::Debug,
|
||||
string::{String, ToString},
|
||||
sync::Arc,
|
||||
vec::Vec,
|
||||
};
|
||||
use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
|
||||
use core::{error::Error, fmt::Display};
|
||||
use core::error::Error;
|
||||
|
||||
/// Pre-computed receipt root and logs bloom.
|
||||
///
|
||||
@@ -110,18 +104,6 @@ pub trait Consensus<B: Block>: HeaderValidator<B::Header> {
|
||||
/// Note: validating blocks does not include other validations of the Consensus
|
||||
fn validate_block_pre_execution(&self, block: &SealedBlock<B>) -> Result<(), ConsensusError>;
|
||||
|
||||
/// Returns `true` if the given consensus error is transient and may resolve on its own.
|
||||
///
|
||||
/// On fast chains, clock skew between nodes can cause a valid block's timestamp to
|
||||
/// appear briefly in the future. Caching such blocks as permanently invalid would
|
||||
/// prevent them from being re-validated once the local clock catches up.
|
||||
///
|
||||
/// Transient errors will not cause the block hash to be cached as permanently invalid,
|
||||
/// allowing the block to be re-validated later.
|
||||
fn is_transient_error(&self, _error: &ConsensusError) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Validate a block disregarding world state using an optional pre-computed transaction root.
|
||||
///
|
||||
/// If `transaction_root` is provided, the implementation should use the pre-computed
|
||||
@@ -474,49 +456,19 @@ pub enum ConsensusError {
|
||||
/// EIP-7825: Transaction gas limit exceeds maximum allowed
|
||||
#[error(transparent)]
|
||||
TransactionGasLimitTooHigh(Box<TxGasLimitTooHighErr>),
|
||||
/// Any additional consensus error, for example L2-specific errors.
|
||||
/// Other, likely an injected L2 error.
|
||||
#[error("{0}")]
|
||||
Other(String),
|
||||
/// Other unspecified error.
|
||||
#[error(transparent)]
|
||||
Other(#[from] Arc<dyn Error + Send + Sync>),
|
||||
Custom(#[from] Arc<dyn Error + Send + Sync>),
|
||||
}
|
||||
|
||||
impl ConsensusError {
|
||||
/// Returns a new [`ConsensusError::Other`] instance with the given error.
|
||||
pub fn other<E>(error: E) -> Self
|
||||
where
|
||||
E: Error + Send + Sync + 'static,
|
||||
{
|
||||
Self::Other(Arc::new(error))
|
||||
}
|
||||
|
||||
/// Returns a new [`ConsensusError::Other`] instance with the given message.
|
||||
pub fn msg(msg: impl Display) -> Self {
|
||||
Self::other(MessageError(msg.to_string()))
|
||||
}
|
||||
|
||||
/// Returns `true` if the error is a state root error.
|
||||
pub const fn is_state_root_error(&self) -> bool {
|
||||
matches!(self, Self::BodyStateRootDiff(_))
|
||||
}
|
||||
|
||||
/// Returns the arbitrary error if it is [`ConsensusError::Other`].
|
||||
pub fn as_other(&self) -> Option<&(dyn Error + Send + Sync + 'static)> {
|
||||
match self {
|
||||
Self::Other(err) => Some(err.as_ref()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to the [`ConsensusError::Other`] value if it is of that type.
|
||||
/// Returns `None` otherwise.
|
||||
pub fn downcast_other_ref<T: Error + 'static>(&self) -> Option<&T> {
|
||||
let other = self.as_other()?;
|
||||
other.downcast_ref()
|
||||
}
|
||||
|
||||
/// Returns `true` if this type is a [`ConsensusError::Other`] of that error type.
|
||||
pub fn is_other<T: Error + 'static>(&self) -> bool {
|
||||
self.as_other().map(|err| err.is::<T>()).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<InvalidTransactionError> for ConsensusError {
|
||||
@@ -548,10 +500,6 @@ pub struct TxGasLimitTooHighErr {
|
||||
pub max_allowed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct MessageError(String);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -561,31 +509,24 @@ mod tests {
|
||||
struct CustomL2Error;
|
||||
|
||||
#[test]
|
||||
fn test_other_error_conversion() {
|
||||
let consensus_err = ConsensusError::other(CustomL2Error);
|
||||
assert!(matches!(consensus_err, ConsensusError::Other(_)));
|
||||
fn test_custom_error_conversion() {
|
||||
// Test conversion from custom error to ConsensusError
|
||||
let custom_err = CustomL2Error;
|
||||
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
|
||||
let consensus_err: ConsensusError = arc_err.into();
|
||||
|
||||
// Verify it's the Custom variant
|
||||
assert!(matches!(consensus_err, ConsensusError::Custom(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_other_error_display() {
|
||||
let consensus_err = ConsensusError::other(CustomL2Error);
|
||||
fn test_custom_error_display() {
|
||||
let custom_err = CustomL2Error;
|
||||
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
|
||||
let consensus_err: ConsensusError = arc_err.into();
|
||||
|
||||
// Verify the error message is preserved through transparent attribute
|
||||
let error_message = format!("{}", consensus_err);
|
||||
assert_eq!(error_message, "Custom L2 consensus error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_other_error_downcast() {
|
||||
let consensus_err = ConsensusError::other(CustomL2Error);
|
||||
|
||||
assert!(consensus_err.is_other::<CustomL2Error>());
|
||||
assert!(consensus_err.downcast_other_ref::<CustomL2Error>().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_other_msg() {
|
||||
let consensus_err = ConsensusError::msg("consensus message");
|
||||
|
||||
assert_eq!(consensus_err.to_string(), "consensus message");
|
||||
assert!(consensus_err.downcast_other_ref::<MessageError>().is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -19,7 +19,6 @@ use reth_trie::{
|
||||
MultiProofTargets, StorageMultiProof, StorageProof, TrieInput,
|
||||
};
|
||||
use std::{
|
||||
fmt,
|
||||
sync::{
|
||||
atomic::{AtomicU64, AtomicUsize, Ordering},
|
||||
Arc,
|
||||
@@ -148,29 +147,6 @@ pub enum CachedStatus<T> {
|
||||
Cached(T),
|
||||
}
|
||||
|
||||
/// The source that is using the execution cache.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum CachedStateMetricsSource {
|
||||
/// Engine (validation).
|
||||
Engine,
|
||||
/// Payload builder.
|
||||
Builder,
|
||||
/// Tests.
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
Test,
|
||||
}
|
||||
|
||||
impl fmt::Display for CachedStateMetricsSource {
|
||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||
match self {
|
||||
Self::Engine => f.write_str("engine"),
|
||||
Self::Builder => f.write_str("builder"),
|
||||
#[cfg(any(test, feature = "test-utils"))]
|
||||
Self::Test => f.write_str("test"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Metrics for the cached state provider, showing hits / misses for each cache
|
||||
#[derive(Metrics, Clone)]
|
||||
#[metrics(scope = "sync.caching")]
|
||||
@@ -246,10 +222,9 @@ impl CachedStateMetrics {
|
||||
self.account_cache_collisions.set(0);
|
||||
}
|
||||
|
||||
/// Returns a new zeroed-out instance of [`CachedStateMetrics`] with a `source` label
|
||||
/// to distinguish between different callers (e.g., engine vs builder).
|
||||
pub fn zeroed(source: CachedStateMetricsSource) -> Self {
|
||||
let zeroed = Self::new_with_labels(&[("source", source.to_string())]);
|
||||
/// Returns a new zeroed-out instance of [`CachedStateMetrics`].
|
||||
pub fn zeroed() -> Self {
|
||||
let zeroed = Self::default();
|
||||
zeroed.reset();
|
||||
zeroed
|
||||
}
|
||||
@@ -944,15 +919,27 @@ pub struct SavedCache {
|
||||
/// The caches used for the provider.
|
||||
caches: ExecutionCache,
|
||||
|
||||
/// Metrics for the cached state provider (includes size/capacity/collisions from fixed-cache)
|
||||
metrics: CachedStateMetrics,
|
||||
|
||||
/// A guard to track in-flight usage of this cache.
|
||||
/// The cache is considered available if the strong count is 1.
|
||||
usage_guard: Arc<()>,
|
||||
|
||||
/// Whether to skip cache metrics recording (can be expensive with large cached state).
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl SavedCache {
|
||||
/// Creates a new instance with the internals
|
||||
pub fn new(hash: B256, caches: ExecutionCache) -> Self {
|
||||
Self { hash, caches, usage_guard: Arc::new(()) }
|
||||
pub fn new(hash: B256, caches: ExecutionCache, metrics: CachedStateMetrics) -> Self {
|
||||
Self { hash, caches, metrics, usage_guard: Arc::new(()), disable_cache_metrics: false }
|
||||
}
|
||||
|
||||
/// Sets whether to disable cache metrics recording.
|
||||
pub const fn with_disable_cache_metrics(mut self, disable: bool) -> Self {
|
||||
self.disable_cache_metrics = disable;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the hash for this cache
|
||||
@@ -960,6 +947,11 @@ impl SavedCache {
|
||||
self.hash
|
||||
}
|
||||
|
||||
/// Splits the cache into its caches, metrics, and `disable_cache_metrics` flag, consuming it.
|
||||
pub fn split(self) -> (ExecutionCache, CachedStateMetrics, bool) {
|
||||
(self.caches, self.metrics, self.disable_cache_metrics)
|
||||
}
|
||||
|
||||
/// Returns true if the cache is available for use (no other tasks are currently using it).
|
||||
pub fn is_available(&self) -> bool {
|
||||
Arc::strong_count(&self.usage_guard) == 1
|
||||
@@ -975,11 +967,20 @@ impl SavedCache {
|
||||
&self.caches
|
||||
}
|
||||
|
||||
/// Returns the metrics associated with this cache.
|
||||
pub const fn metrics(&self) -> &CachedStateMetrics {
|
||||
&self.metrics
|
||||
}
|
||||
|
||||
/// Updates the cache metrics (size/capacity/collisions) from the stats handlers.
|
||||
pub fn update_metrics(&self, metrics: Option<&CachedStateMetrics>) {
|
||||
if let Some(metrics) = metrics {
|
||||
self.caches.update_metrics(metrics);
|
||||
///
|
||||
/// Note: This can be expensive with large cached state. Use
|
||||
/// `with_disable_cache_metrics(true)` to skip.
|
||||
pub fn update_metrics(&self) {
|
||||
if self.disable_cache_metrics {
|
||||
return
|
||||
}
|
||||
self.caches.update_metrics(&self.metrics);
|
||||
}
|
||||
|
||||
/// Clears all caches, resetting them to empty state,
|
||||
@@ -1016,11 +1017,8 @@ mod tests {
|
||||
provider.extend_accounts(vec![(address, account)]);
|
||||
|
||||
let caches = ExecutionCache::new(1000);
|
||||
let state_provider = CachedStateProvider::new(
|
||||
provider,
|
||||
caches,
|
||||
CachedStateMetrics::zeroed(CachedStateMetricsSource::Test),
|
||||
);
|
||||
let state_provider =
|
||||
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
|
||||
|
||||
let res = state_provider.storage(address, storage_key);
|
||||
assert!(res.is_ok());
|
||||
@@ -1039,11 +1037,8 @@ mod tests {
|
||||
provider.extend_accounts(vec![(address, account)]);
|
||||
|
||||
let caches = ExecutionCache::new(1000);
|
||||
let state_provider = CachedStateProvider::new(
|
||||
provider,
|
||||
caches,
|
||||
CachedStateMetrics::zeroed(CachedStateMetricsSource::Test),
|
||||
);
|
||||
let state_provider =
|
||||
CachedStateProvider::new(provider, caches, CachedStateMetrics::zeroed());
|
||||
|
||||
let res = state_provider.storage(address, storage_key);
|
||||
assert!(res.is_ok());
|
||||
@@ -1080,7 +1075,7 @@ mod tests {
|
||||
#[test]
|
||||
fn test_saved_cache_is_available() {
|
||||
let execution_cache = ExecutionCache::new(1000);
|
||||
let cache = SavedCache::new(B256::ZERO, execution_cache);
|
||||
let cache = SavedCache::new(B256::ZERO, execution_cache, CachedStateMetrics::zeroed());
|
||||
|
||||
assert!(cache.is_available(), "Cache should be available initially");
|
||||
|
||||
@@ -1092,7 +1087,8 @@ mod tests {
|
||||
#[test]
|
||||
fn test_saved_cache_multiple_references() {
|
||||
let execution_cache = ExecutionCache::new(1000);
|
||||
let cache = SavedCache::new(B256::from([2u8; 32]), execution_cache);
|
||||
let cache =
|
||||
SavedCache::new(B256::from([2u8; 32]), execution_cache, CachedStateMetrics::zeroed());
|
||||
|
||||
let guard1 = cache.clone_guard_for_test();
|
||||
let guard2 = cache.clone_guard_for_test();
|
||||
|
||||
@@ -165,7 +165,11 @@ mod tests {
|
||||
let hash = B256::from([1u8; 32]);
|
||||
|
||||
cache.update_with_guard(|slot| {
|
||||
*slot = Some(SavedCache::new(hash, ExecutionCache::new(1_000)))
|
||||
*slot = Some(SavedCache::new(
|
||||
hash,
|
||||
ExecutionCache::new(1_000),
|
||||
CachedStateMetrics::zeroed(),
|
||||
))
|
||||
});
|
||||
|
||||
let first = cache.get_cache_for(hash);
|
||||
@@ -181,7 +185,11 @@ mod tests {
|
||||
let hash = B256::from([2u8; 32]);
|
||||
|
||||
cache.update_with_guard(|slot| {
|
||||
*slot = Some(SavedCache::new(hash, ExecutionCache::new(1_000)))
|
||||
*slot = Some(SavedCache::new(
|
||||
hash,
|
||||
ExecutionCache::new(1_000),
|
||||
CachedStateMetrics::zeroed(),
|
||||
))
|
||||
});
|
||||
|
||||
let checked_out = cache.get_cache_for(hash);
|
||||
@@ -199,7 +207,11 @@ mod tests {
|
||||
let hash_b = B256::from([0xBB; 32]);
|
||||
|
||||
cache.update_with_guard(|slot| {
|
||||
*slot = Some(SavedCache::new(hash_a, ExecutionCache::new(1_000)))
|
||||
*slot = Some(SavedCache::new(
|
||||
hash_a,
|
||||
ExecutionCache::new(1_000),
|
||||
CachedStateMetrics::zeroed(),
|
||||
))
|
||||
});
|
||||
|
||||
let checked_out = cache.get_cache_for(hash_b);
|
||||
|
||||
@@ -141,11 +141,6 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
|
||||
last_header: SealedHeaderFor<<T::BuiltPayload as BuiltPayload>::Primitives>,
|
||||
/// Stores latest mined blocks.
|
||||
last_block_hashes: VecDeque<B256>,
|
||||
/// Optional sleep duration between initiating payload building and resolving.
|
||||
///
|
||||
/// When set, the miner sleeps after `fork_choice_updated` before calling
|
||||
/// `resolve_kind`, giving the payload job time for multiple rebuild attempts.
|
||||
payload_wait_time: Option<Duration>,
|
||||
}
|
||||
|
||||
impl<T, B, Pool> LocalMiner<T, B, Pool>
|
||||
@@ -175,16 +170,9 @@ where
|
||||
payload_builder,
|
||||
last_block_hashes: VecDeque::from([last_header.hash()]),
|
||||
last_header,
|
||||
payload_wait_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the payload wait time, if any.
|
||||
pub const fn with_payload_wait_time_opt(mut self, wait_time: Option<Duration>) -> Self {
|
||||
self.payload_wait_time = wait_time;
|
||||
self
|
||||
}
|
||||
|
||||
/// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads.
|
||||
pub async fn run(mut self) {
|
||||
let mut fcu_interval = tokio::time::interval(Duration::from_secs(1));
|
||||
@@ -250,10 +238,6 @@ where
|
||||
|
||||
let payload_id = res.payload_id.ok_or_eyre("No payload id")?;
|
||||
|
||||
if let Some(wait_time) = self.payload_wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
}
|
||||
|
||||
let Some(Ok(payload)) =
|
||||
self.payload_builder.resolve_kind(payload_id, PayloadKind::WaitForPending).await
|
||||
else {
|
||||
|
||||
@@ -57,7 +57,7 @@ where
|
||||
.chain_spec
|
||||
.is_cancun_active_at_timestamp(timestamp)
|
||||
.then(B256::random),
|
||||
slot_number: self.chain_spec.is_amsterdam_active_at_timestamp(timestamp).then_some(0),
|
||||
slot_number: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -4,20 +4,17 @@ use alloy_eips::merge::EPOCH_SLOTS;
|
||||
use core::time::Duration;
|
||||
|
||||
/// Triggers persistence when the number of canonical blocks in memory exceeds this threshold.
|
||||
pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 2;
|
||||
pub const DEFAULT_PERSISTENCE_THRESHOLD: u64 = 12;
|
||||
|
||||
/// Maximum canonical-minus-persisted gap before engine API processing is stalled.
|
||||
pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 16;
|
||||
pub const DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD: u64 = 44;
|
||||
|
||||
/// How close to the canonical head we persist blocks.
|
||||
pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0;
|
||||
pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 10;
|
||||
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 5;
|
||||
|
||||
/// Default number of cache hits before an invalid header entry is evicted and reprocessed.
|
||||
pub const DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
|
||||
|
||||
/// Gas threshold below which the small block chunk size is used.
|
||||
pub const SMALL_BLOCK_GAS_THRESHOLD: u64 = 20_000_000;
|
||||
|
||||
@@ -105,11 +102,6 @@ pub struct TreeConfig {
|
||||
block_buffer_limit: u32,
|
||||
/// Number of invalid headers to keep in cache.
|
||||
max_invalid_header_cache_length: u32,
|
||||
/// Number of cache hits before an invalid header entry is evicted and reprocessed.
|
||||
///
|
||||
/// Setting this to `0` effectively disables the cache because entries are evicted on the
|
||||
/// first lookup.
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
/// Maximum number of blocks to execute sequentially in a batch.
|
||||
///
|
||||
/// This is used as a cutoff to prevent long-running sequential block execution when we receive
|
||||
@@ -178,23 +170,6 @@ pub struct TreeConfig {
|
||||
share_execution_cache_with_payload_builder: bool,
|
||||
/// Whether to share sparse trie with the payload builder.
|
||||
share_sparse_trie_with_payload_builder: bool,
|
||||
/// Whether to suppress persistence cycles while building a payload.
|
||||
///
|
||||
/// When enabled, persistence is deferred from the moment an FCU with payload attributes
|
||||
/// arrives until the next FCU without attributes. This avoids persistence I/O competing
|
||||
/// with block building on latency-sensitive chains.
|
||||
suppress_persistence_during_build: bool,
|
||||
/// Whether to disable BAL (Block Access List, EIP-7928) based parallel execution.
|
||||
/// When disabled, falls back to transaction-based prewarming even when a BAL is available.
|
||||
disable_bal_parallel_execution: bool,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
/// When disabled, the BAL hashed post state is not sent to the multiproof task for
|
||||
/// early parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
/// Whether to disable BAL (Block Access List) batched IO during prewarming.
|
||||
/// When disabled, falls back to individual per-slot storage reads instead of
|
||||
/// batched cursor reads via `storage_range`.
|
||||
disable_bal_batch_io: bool,
|
||||
/// Maximum random jitter applied before each proof computation (trie-debug only).
|
||||
/// When set, each proof worker sleeps for a random duration up to this value
|
||||
/// before starting a proof calculation.
|
||||
@@ -214,7 +189,6 @@ impl Default for TreeConfig {
|
||||
persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
|
||||
block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT,
|
||||
max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH,
|
||||
invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
|
||||
max_execute_block_batch_size: DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE,
|
||||
legacy_state_root: false,
|
||||
always_compare_trie_updates: false,
|
||||
@@ -238,10 +212,6 @@ impl Default for TreeConfig {
|
||||
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
|
||||
share_execution_cache_with_payload_builder: false,
|
||||
share_sparse_trie_with_payload_builder: false,
|
||||
suppress_persistence_during_build: false,
|
||||
disable_bal_parallel_execution: false,
|
||||
disable_bal_parallel_state_root: false,
|
||||
disable_bal_batch_io: false,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
proof_jitter: None,
|
||||
}
|
||||
@@ -257,7 +227,6 @@ impl TreeConfig {
|
||||
persistence_backpressure_threshold: u64,
|
||||
block_buffer_limit: u32,
|
||||
max_invalid_header_cache_length: u32,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
max_execute_block_batch_size: usize,
|
||||
legacy_state_root: bool,
|
||||
always_compare_trie_updates: bool,
|
||||
@@ -291,7 +260,6 @@ impl TreeConfig {
|
||||
persistence_backpressure_threshold,
|
||||
block_buffer_limit,
|
||||
max_invalid_header_cache_length,
|
||||
invalid_header_hit_eviction_threshold,
|
||||
max_execute_block_batch_size,
|
||||
legacy_state_root,
|
||||
always_compare_trie_updates,
|
||||
@@ -315,10 +283,6 @@ impl TreeConfig {
|
||||
state_root_task_timeout,
|
||||
share_execution_cache_with_payload_builder,
|
||||
share_sparse_trie_with_payload_builder,
|
||||
suppress_persistence_during_build: false,
|
||||
disable_bal_parallel_execution: false,
|
||||
disable_bal_parallel_state_root: false,
|
||||
disable_bal_batch_io: false,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
proof_jitter: None,
|
||||
}
|
||||
@@ -349,14 +313,6 @@ impl TreeConfig {
|
||||
self.max_invalid_header_cache_length
|
||||
}
|
||||
|
||||
/// Return the invalid header cache hit eviction threshold.
|
||||
///
|
||||
/// Setting this to `0` effectively disables the cache because entries are evicted on the
|
||||
/// first lookup.
|
||||
pub const fn invalid_header_hit_eviction_threshold(&self) -> u8 {
|
||||
self.invalid_header_hit_eviction_threshold
|
||||
}
|
||||
|
||||
/// Return the maximum execute block batch size.
|
||||
pub const fn max_execute_block_batch_size(&self) -> usize {
|
||||
self.max_execute_block_batch_size
|
||||
@@ -487,15 +443,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for the invalid header cache hit eviction threshold.
|
||||
pub const fn with_invalid_header_hit_eviction_threshold(
|
||||
mut self,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
) -> Self {
|
||||
self.invalid_header_hit_eviction_threshold = invalid_header_hit_eviction_threshold;
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for maximum execute block batch size.
|
||||
pub const fn with_max_execute_block_batch_size(
|
||||
mut self,
|
||||
@@ -699,56 +646,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether persistence is suppressed during payload building.
|
||||
pub const fn suppress_persistence_during_build(&self) -> bool {
|
||||
self.suppress_persistence_during_build
|
||||
}
|
||||
|
||||
/// Setter for whether to suppress persistence during payload building.
|
||||
pub const fn with_suppress_persistence_during_build(mut self, value: bool) -> Self {
|
||||
self.suppress_persistence_during_build = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL-based parallel execution is disabled.
|
||||
pub const fn disable_bal_parallel_execution(&self) -> bool {
|
||||
self.disable_bal_parallel_execution
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL-based parallel execution.
|
||||
pub const fn without_bal_parallel_execution(
|
||||
mut self,
|
||||
disable_bal_parallel_execution: bool,
|
||||
) -> Self {
|
||||
self.disable_bal_parallel_execution = disable_bal_parallel_execution;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL-driven parallel state root computation is disabled.
|
||||
pub const fn disable_bal_parallel_state_root(&self) -> bool {
|
||||
self.disable_bal_parallel_state_root
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL-driven parallel state root computation.
|
||||
pub const fn without_bal_parallel_state_root(
|
||||
mut self,
|
||||
disable_bal_parallel_state_root: bool,
|
||||
) -> Self {
|
||||
self.disable_bal_parallel_state_root = disable_bal_parallel_state_root;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL batched IO is disabled.
|
||||
pub const fn disable_bal_batch_io(&self) -> bool {
|
||||
self.disable_bal_batch_io
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL batched IO.
|
||||
pub const fn without_bal_batch_io(mut self, disable_bal_batch_io: bool) -> Self {
|
||||
self.disable_bal_batch_io = disable_bal_batch_io;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the proof jitter duration, if configured (trie-debug only).
|
||||
#[cfg(feature = "trie-debug")]
|
||||
pub const fn proof_jitter(&self) -> Option<Duration> {
|
||||
|
||||
@@ -102,8 +102,8 @@ where
|
||||
self.sync_metrics_tx.send(MetricEvent::SyncHeight { height: new_tip_num });
|
||||
let _ = sender.send(PersistenceResult { last_block, commit_duration: None });
|
||||
}
|
||||
PersistenceAction::SaveBlocks(blocks, sender) => {
|
||||
let result = self.on_save_blocks(blocks)?;
|
||||
PersistenceAction::SaveBlocks(batch, sender) => {
|
||||
let result = self.on_save_blocks(batch)?;
|
||||
let result_number = result.last_block.map(|b| b.number);
|
||||
|
||||
let _ = sender.send(result);
|
||||
@@ -144,14 +144,18 @@ where
|
||||
Ok(new_tip_hash.map(|hash| BlockNumHash { hash, number: new_tip_num }))
|
||||
}
|
||||
|
||||
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = blocks.len()))]
|
||||
#[instrument(level = "debug", target = "engine::persistence", skip_all, fields(block_count = batch.blocks.len()))]
|
||||
fn on_save_blocks(
|
||||
&mut self,
|
||||
blocks: Vec<ExecutedBlock<N::Primitives>>,
|
||||
batch: SaveBlocksBatch<N::Primitives>,
|
||||
) -> Result<PersistenceResult, PersistenceError> {
|
||||
let first_block = blocks.first().map(|b| b.recovered_block.num_hash());
|
||||
let last_block = blocks.last().map(|b| b.recovered_block.num_hash());
|
||||
let block_count = blocks.len();
|
||||
let SaveBlocksBatch { blocks, persist_count } = batch;
|
||||
let (blocks_to_persist, buffer_blocks) = blocks.split_at(persist_count);
|
||||
let blocks_to_persist = blocks_to_persist.to_vec();
|
||||
|
||||
let first_block = blocks_to_persist.first().map(|b| b.recovered_block.num_hash());
|
||||
let last_block = blocks_to_persist.last().map(|b| b.recovered_block.num_hash());
|
||||
let block_count = blocks_to_persist.len();
|
||||
|
||||
let pending_finalized = self.pending_finalized_block.take();
|
||||
let pending_safe = self.pending_safe_block.take();
|
||||
@@ -162,7 +166,7 @@ where
|
||||
|
||||
if let Some(last) = last_block {
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
provider_rw.save_blocks(blocks, SaveBlocksMode::Full)?;
|
||||
provider_rw.save_blocks(blocks_to_persist, SaveBlocksMode::Full, buffer_blocks)?;
|
||||
|
||||
if let Some(finalized) = pending_finalized {
|
||||
provider_rw.save_finalized_block_number(finalized.min(last.number))?;
|
||||
@@ -216,6 +220,28 @@ pub enum PersistenceError {
|
||||
ProviderError(#[from] ProviderError),
|
||||
}
|
||||
|
||||
/// A batch of blocks passed to the persistence service.
|
||||
///
|
||||
/// Contains all canonical blocks collected during a persistence cycle. Only the first
|
||||
/// `persist_count` blocks are written to disk; the remaining blocks are included for
|
||||
/// downstream consumers but are not persisted yet.
|
||||
#[derive(Debug, Clone, PartialEq)]
|
||||
pub struct SaveBlocksBatch<N: NodePrimitives = EthPrimitives> {
|
||||
/// All blocks in the batch, ordered oldest to newest.
|
||||
pub blocks: Vec<ExecutedBlock<N>>,
|
||||
/// Number of leading blocks to actually persist. The blocks at indices
|
||||
/// `[persist_count..]` are passed through but not written to storage.
|
||||
pub persist_count: usize,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> SaveBlocksBatch<N> {
|
||||
/// Creates a new batch where all blocks should be persisted.
|
||||
pub fn persist_all(blocks: Vec<ExecutedBlock<N>>) -> Self {
|
||||
let persist_count = blocks.len();
|
||||
Self { blocks, persist_count }
|
||||
}
|
||||
}
|
||||
|
||||
/// A signal to the persistence service that part of the tree state can be persisted.
|
||||
#[derive(Debug)]
|
||||
pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
|
||||
@@ -224,7 +250,7 @@ pub enum PersistenceAction<N: NodePrimitives = EthPrimitives> {
|
||||
///
|
||||
/// First, header, transaction, and receipt-related data should be written to static files.
|
||||
/// Then the execution history-related data will be written to the database.
|
||||
SaveBlocks(Vec<ExecutedBlock<N>>, CrossbeamSender<PersistenceResult>),
|
||||
SaveBlocks(SaveBlocksBatch<N>, CrossbeamSender<PersistenceResult>),
|
||||
|
||||
/// Removes block data above the given block number from the database.
|
||||
///
|
||||
@@ -308,10 +334,10 @@ impl<T: NodePrimitives> PersistenceHandle<T> {
|
||||
/// If there are no blocks to persist, then `None` is sent in the sender.
|
||||
pub fn save_blocks(
|
||||
&self,
|
||||
blocks: Vec<ExecutedBlock<T>>,
|
||||
batch: SaveBlocksBatch<T>,
|
||||
tx: CrossbeamSender<PersistenceResult>,
|
||||
) -> Result<(), SendError<PersistenceAction<T>>> {
|
||||
self.send_action(PersistenceAction::SaveBlocks(blocks, tx))
|
||||
self.send_action(PersistenceAction::SaveBlocks(batch, tx))
|
||||
}
|
||||
|
||||
/// Queues the finalized block number to be persisted on disk.
|
||||
@@ -407,7 +433,7 @@ mod tests {
|
||||
let blocks = vec![];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
|
||||
|
||||
let result = rx.recv().unwrap();
|
||||
assert!(result.last_block.is_none());
|
||||
@@ -426,7 +452,7 @@ mod tests {
|
||||
let blocks = vec![executed];
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
|
||||
|
||||
let result = rx.recv_timeout(std::time::Duration::from_secs(10)).expect("test timed out");
|
||||
|
||||
@@ -443,7 +469,7 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
|
||||
let result = rx.recv().unwrap();
|
||||
assert_eq!(last_hash, result.last_block.unwrap().hash);
|
||||
}
|
||||
@@ -460,7 +486,7 @@ mod tests {
|
||||
let last_hash = blocks.last().unwrap().recovered_block().hash();
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
|
||||
handle.save_blocks(blocks, tx).unwrap();
|
||||
handle.save_blocks(SaveBlocksBatch::persist_all(blocks), tx).unwrap();
|
||||
|
||||
let result = rx.recv().unwrap();
|
||||
assert_eq!(last_hash, result.last_block.unwrap().hash);
|
||||
@@ -555,7 +581,7 @@ mod tests {
|
||||
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw().unwrap();
|
||||
provider_rw.save_blocks(blocks_a, SaveBlocksMode::Full).unwrap();
|
||||
provider_rw.save_blocks(blocks_a, SaveBlocksMode::Full, &[]).unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
}
|
||||
|
||||
@@ -612,7 +638,7 @@ mod tests {
|
||||
provider_rw.commit().unwrap();
|
||||
|
||||
let provider_rw = pf.database_provider_rw().unwrap();
|
||||
provider_rw.save_blocks(vec![block_b2], SaveBlocksMode::Full).unwrap();
|
||||
provider_rw.save_blocks(vec![block_b2], SaveBlocksMode::Full, &[]).unwrap();
|
||||
provider_rw.commit().unwrap();
|
||||
});
|
||||
|
||||
|
||||
@@ -8,28 +8,25 @@ use schnellru::{ByLength, LruMap};
|
||||
use std::fmt::Debug;
|
||||
use tracing::warn;
|
||||
|
||||
/// The max hit counter for invalid headers in the cache before it is forcefully evicted.
|
||||
///
|
||||
/// In other words, if a header is referenced more than this number of times, it will be evicted to
|
||||
/// allow for reprocessing.
|
||||
const INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
|
||||
|
||||
/// Keeps track of invalid headers.
|
||||
#[derive(Debug)]
|
||||
pub struct InvalidHeaderCache {
|
||||
/// This maps a header hash to a reference to its invalid ancestor.
|
||||
headers: LruMap<B256, HeaderEntry>,
|
||||
/// Number of cache hits before an invalid header entry is evicted and reprocessed.
|
||||
hit_eviction_threshold: u8,
|
||||
/// Metrics for the cache.
|
||||
metrics: InvalidHeaderCacheMetrics,
|
||||
}
|
||||
|
||||
impl InvalidHeaderCache {
|
||||
/// Invalid header cache constructor.
|
||||
///
|
||||
/// Setting `hit_eviction_threshold` to `0` effectively disables the cache because entries are
|
||||
/// evicted on the first lookup.
|
||||
pub fn new(max_length: u32, hit_eviction_threshold: u8) -> Self {
|
||||
Self {
|
||||
headers: LruMap::new(ByLength::new(max_length)),
|
||||
hit_eviction_threshold,
|
||||
metrics: Default::default(),
|
||||
}
|
||||
pub fn new(max_length: u32) -> Self {
|
||||
Self { headers: LruMap::new(ByLength::new(max_length)), metrics: Default::default() }
|
||||
}
|
||||
|
||||
fn insert_entry(&mut self, hash: B256, header: BlockWithParent) {
|
||||
@@ -44,7 +41,7 @@ impl InvalidHeaderCache {
|
||||
{
|
||||
let entry = self.headers.get(hash)?;
|
||||
entry.hit_count += 1;
|
||||
if entry.hit_count < self.hit_eviction_threshold {
|
||||
if entry.hit_count < INVALID_HEADER_HIT_EVICTION_THRESHOLD {
|
||||
return Some(entry.header)
|
||||
}
|
||||
}
|
||||
@@ -113,28 +110,17 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_hit_eviction() {
|
||||
let hit_eviction_threshold = 3;
|
||||
let mut cache = InvalidHeaderCache::new(10, hit_eviction_threshold);
|
||||
let mut cache = InvalidHeaderCache::new(10);
|
||||
let header = Header::default();
|
||||
let header = SealedHeader::seal_slow(header);
|
||||
cache.insert(header.block_with_parent());
|
||||
assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, 0);
|
||||
|
||||
for hit in 1..hit_eviction_threshold {
|
||||
for hit in 1..INVALID_HEADER_HIT_EVICTION_THRESHOLD {
|
||||
assert!(cache.get(&header.hash()).is_some());
|
||||
assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, hit);
|
||||
}
|
||||
|
||||
assert!(cache.get(&header.hash()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_zero_hit_eviction_threshold_effectively_disables_cache() {
|
||||
let mut cache = InvalidHeaderCache::new(10, 0);
|
||||
let header = SealedHeader::seal_slow(Header::default());
|
||||
cache.insert(header.block_with_parent());
|
||||
|
||||
assert!(cache.get(&header.hash()).is_none());
|
||||
assert_eq!(cache.headers.len(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@ use crate::{
|
||||
backfill::{BackfillAction, BackfillSyncState},
|
||||
chain::FromOrchestrator,
|
||||
engine::{DownloadRequest, EngineApiEvent, EngineApiKind, EngineApiRequest, FromEngine},
|
||||
persistence::PersistenceHandle,
|
||||
persistence::{PersistenceHandle, SaveBlocksBatch},
|
||||
tree::{error::InsertPayloadError, payload_validator::TreeCtx},
|
||||
};
|
||||
use alloy_consensus::BlockHeader;
|
||||
@@ -11,7 +11,7 @@ use alloy_primitives::B256;
|
||||
use alloy_rpc_types_engine::{
|
||||
ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError,
|
||||
};
|
||||
use error::{InsertBlockError, InsertBlockFatalError, InsertBlockValidationError};
|
||||
use error::{InsertBlockError, InsertBlockFatalError};
|
||||
use reth_chain_state::{
|
||||
CanonicalInMemoryState, ComputedTrieData, ExecutedBlock, ExecutionTimingStats,
|
||||
MemoryOverlayStateProvider, NewCanonicalChain,
|
||||
@@ -71,8 +71,7 @@ pub use payload_validator::{BasicEngineValidator, EngineValidator};
|
||||
pub use persistence_state::PersistenceState;
|
||||
pub use reth_engine_primitives::TreeConfig;
|
||||
pub use reth_execution_cache::{
|
||||
CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider, ExecutionCache,
|
||||
PayloadExecutionCache, SavedCache,
|
||||
CachedStateMetrics, CachedStateProvider, ExecutionCache, PayloadExecutionCache, SavedCache,
|
||||
};
|
||||
|
||||
pub mod state;
|
||||
@@ -151,15 +150,11 @@ impl<N: NodePrimitives> EngineApiTreeState<N> {
|
||||
fn new(
|
||||
block_buffer_limit: u32,
|
||||
max_invalid_header_cache_length: u32,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
canonical_block: BlockNumHash,
|
||||
engine_kind: EngineApiKind,
|
||||
) -> Self {
|
||||
Self {
|
||||
invalid_headers: InvalidHeaderCache::new(
|
||||
max_invalid_header_cache_length,
|
||||
invalid_header_hit_eviction_threshold,
|
||||
),
|
||||
invalid_headers: InvalidHeaderCache::new(max_invalid_header_cache_length),
|
||||
buffer: BlockBuffer::new(block_buffer_limit),
|
||||
tree_state: TreeState::new(canonical_block, engine_kind),
|
||||
forkchoice_state_tracker: ForkchoiceStateTracker::default(),
|
||||
@@ -309,9 +304,6 @@ where
|
||||
/// Stored here (not in `ExecutedBlock`) to avoid leaking observability concerns into the block
|
||||
/// type. Entries are removed when blocks are persisted or invalidated.
|
||||
execution_timing_stats: HashMap<B256, Box<ExecutionTimingStats>>,
|
||||
/// Set when an FCU with payload attributes is received, cleared on the next FCU without.
|
||||
/// Suppresses persistence cycles during payload building.
|
||||
building_payload: bool,
|
||||
/// Task runtime for spawning blocking work on named, reusable threads.
|
||||
runtime: reth_tasks::Runtime,
|
||||
}
|
||||
@@ -403,7 +395,6 @@ where
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
execution_timing_stats: HashMap::new(),
|
||||
building_payload: false,
|
||||
runtime,
|
||||
}
|
||||
}
|
||||
@@ -440,7 +431,6 @@ where
|
||||
let state = EngineApiTreeState::new(
|
||||
config.block_buffer_limit(),
|
||||
config.max_invalid_header_cache_length(),
|
||||
config.invalid_header_hit_eviction_threshold(),
|
||||
header.num_hash(),
|
||||
kind,
|
||||
);
|
||||
@@ -1121,8 +1111,6 @@ where
|
||||
) -> ProviderResult<TreeOutcome<OnForkChoiceUpdated>> {
|
||||
trace!(target: "engine::tree", ?attrs, "invoked forkchoice update");
|
||||
|
||||
self.building_payload = attrs.is_some() && self.config.suppress_persistence_during_build();
|
||||
|
||||
// Record metrics
|
||||
self.record_forkchoice_metrics();
|
||||
|
||||
@@ -1361,22 +1349,25 @@ where
|
||||
|
||||
/// Helper method to save blocks and set the persistence state. This ensures we keep track of
|
||||
/// the current persistence action while we're saving blocks.
|
||||
fn persist_blocks(&mut self, blocks_to_persist: Vec<ExecutedBlock<N>>) {
|
||||
if blocks_to_persist.is_empty() {
|
||||
fn persist_blocks(&mut self, batch: SaveBlocksBatch<N>) {
|
||||
if batch.blocks.is_empty() {
|
||||
debug!(target: "engine::tree", "Returned empty set of blocks to persist");
|
||||
return
|
||||
}
|
||||
|
||||
// NOTE: checked non-empty above
|
||||
let highest_num_hash = blocks_to_persist
|
||||
// The highest block to be persisted determines the persistence state tracking.
|
||||
// Use persist_count to find the highest block that will actually be written to disk.
|
||||
let highest_num_hash = batch
|
||||
.blocks
|
||||
.iter()
|
||||
.take(batch.persist_count)
|
||||
.max_by_key(|block| block.recovered_block().number())
|
||||
.map(|b| b.recovered_block().num_hash())
|
||||
.expect("Checked non-empty persisting blocks");
|
||||
|
||||
debug!(target: "engine::tree", count=blocks_to_persist.len(), blocks = ?blocks_to_persist.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(), "Persisting blocks");
|
||||
debug!(target: "engine::tree", count=batch.blocks.len(), persist_count=batch.persist_count, blocks = ?batch.blocks.iter().map(|block| block.recovered_block().num_hash()).collect::<Vec<_>>(), "Persisting blocks");
|
||||
let (tx, rx) = crossbeam_channel::bounded(1);
|
||||
let _ = self.persistence.save_blocks(blocks_to_persist, tx);
|
||||
let _ = self.persistence.save_blocks(batch, tx);
|
||||
|
||||
self.persistence_state.start_save(highest_num_hash, rx);
|
||||
}
|
||||
@@ -1390,9 +1381,8 @@ where
|
||||
if let Some(new_tip_num) = self.find_disk_reorg()? {
|
||||
self.remove_blocks(new_tip_num)
|
||||
} else if self.should_persist() {
|
||||
let blocks_to_persist =
|
||||
self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?;
|
||||
self.persist_blocks(blocks_to_persist);
|
||||
let batch = self.get_canonical_blocks_to_persist(PersistTarget::Threshold)?;
|
||||
self.persist_blocks(batch);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1423,15 +1413,15 @@ where
|
||||
self.on_persistence_complete(result, start_time)?;
|
||||
}
|
||||
|
||||
let blocks_to_persist = self.get_canonical_blocks_to_persist(PersistTarget::Head)?;
|
||||
let batch = self.get_canonical_blocks_to_persist(PersistTarget::Head)?;
|
||||
|
||||
if blocks_to_persist.is_empty() {
|
||||
if batch.blocks.is_empty() {
|
||||
debug!(target: "engine::tree", "persistence complete, signaling termination");
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
debug!(target: "engine::tree", count = blocks_to_persist.len(), "persisting remaining blocks before shutdown");
|
||||
self.persist_blocks(blocks_to_persist);
|
||||
debug!(target: "engine::tree", count = batch.blocks.len(), "persisting remaining blocks before shutdown");
|
||||
self.persist_blocks(batch);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -2016,13 +2006,9 @@ where
|
||||
}
|
||||
|
||||
/// Returns true if the canonical chain length minus the last persisted
|
||||
/// block is greater than or equal to the persistence threshold,
|
||||
/// backfill is not running, and no payload is currently being built.
|
||||
/// block is greater than or equal to the persistence threshold and
|
||||
/// backfill is not running.
|
||||
pub const fn should_persist(&self) -> bool {
|
||||
if self.building_payload {
|
||||
return false
|
||||
}
|
||||
|
||||
if !self.backfill_sync_state.is_idle() {
|
||||
// can't persist if backfill is running
|
||||
return false
|
||||
@@ -2034,16 +2020,20 @@ where
|
||||
}
|
||||
|
||||
/// Returns a batch of consecutive canonical blocks to persist in the range
|
||||
/// `(last_persisted_number .. target]`. The expected order is oldest -> newest.
|
||||
/// `(last_persisted_number .. canonical_head]`. The expected order is oldest -> newest.
|
||||
///
|
||||
/// All blocks above `last_persisted_number` are included in the batch, but only
|
||||
/// those up to the persistence target (determined by [`PersistTarget`]) are marked
|
||||
/// for actual persistence via [`SaveBlocksBatch::persist_count`].
|
||||
fn get_canonical_blocks_to_persist(
|
||||
&self,
|
||||
target: PersistTarget,
|
||||
) -> Result<Vec<ExecutedBlock<N>>, AdvancePersistenceError> {
|
||||
) -> Result<SaveBlocksBatch<N>, AdvancePersistenceError> {
|
||||
// We will calculate the state root using the database, so we need to be sure there are no
|
||||
// changes
|
||||
debug_assert!(!self.persistence_state.in_progress());
|
||||
|
||||
let mut blocks_to_persist = Vec::new();
|
||||
let mut all_blocks = Vec::new();
|
||||
let mut current_hash = self.state.tree_state.canonical_block_hash();
|
||||
let last_persisted_number = self.persistence_state.last_persisted_block.number;
|
||||
let canonical_head_number = self.state.tree_state.canonical_block_number();
|
||||
@@ -2068,17 +2058,18 @@ where
|
||||
break;
|
||||
}
|
||||
|
||||
if block.recovered_block().number() <= target_number {
|
||||
blocks_to_persist.push(block.clone());
|
||||
}
|
||||
|
||||
all_blocks.push(block.clone());
|
||||
current_hash = block.recovered_block().parent_hash();
|
||||
}
|
||||
|
||||
// Reverse the order so that the oldest block comes first
|
||||
blocks_to_persist.reverse();
|
||||
all_blocks.reverse();
|
||||
|
||||
Ok(blocks_to_persist)
|
||||
// Only blocks up to target_number are persisted; the rest are buffer blocks
|
||||
let persist_count =
|
||||
all_blocks.iter().filter(|b| b.recovered_block().number() <= target_number).count();
|
||||
|
||||
Ok(SaveBlocksBatch { blocks: all_blocks, persist_count })
|
||||
}
|
||||
|
||||
/// This clears the blocks from the in-memory tree state that have been persisted to the
|
||||
@@ -3024,22 +3015,8 @@ where
|
||||
);
|
||||
let latest_valid_hash = self.latest_valid_hash_for_invalid_payload(block.parent_hash())?;
|
||||
|
||||
// keep track of the invalid header unless the consensus impl considers it transient
|
||||
let is_transient = match &validation_err {
|
||||
InsertBlockValidationError::Consensus(err) => self.consensus.is_transient_error(err),
|
||||
_ => false,
|
||||
};
|
||||
if is_transient {
|
||||
warn!(
|
||||
target: "engine::tree",
|
||||
invalid_hash=%block.hash(),
|
||||
invalid_number=block.number(),
|
||||
%validation_err,
|
||||
"Skipping invalid header cache insert for transient validation error",
|
||||
);
|
||||
} else {
|
||||
self.state.invalid_headers.insert(block.block_with_parent());
|
||||
}
|
||||
// keep track of the invalid header
|
||||
self.state.invalid_headers.insert(block.block_with_parent());
|
||||
self.emit_event(EngineApiEvent::BeaconConsensus(ConsensusEngineEvent::InvalidBlock(
|
||||
Box::new(block),
|
||||
)));
|
||||
|
||||
@@ -4,10 +4,10 @@ use super::precompile_cache::PrecompileCacheMap;
|
||||
use crate::tree::{
|
||||
payload_processor::prewarm::{PrewarmCacheTask, PrewarmContext, PrewarmMode, PrewarmTaskEvent},
|
||||
sparse_trie::SparseTrieCacheTask,
|
||||
CacheWaitDurations, CachedStateMetrics, CachedStateMetricsSource, ExecutionCache,
|
||||
PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig, WaitForCaches,
|
||||
CacheWaitDurations, CachedStateMetrics, ExecutionCache, PayloadExecutionCache, SavedCache,
|
||||
StateProviderBuilder, TreeConfig, WaitForCaches,
|
||||
};
|
||||
use alloy_eip7928::bal::DecodedBal;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal};
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
@@ -96,8 +96,6 @@ where
|
||||
executor: Runtime,
|
||||
/// The most recent cache used for execution.
|
||||
execution_cache: PayloadExecutionCache,
|
||||
/// Metrics for the execution cache.
|
||||
cache_metrics: Option<CachedStateMetrics>,
|
||||
/// Metrics for trie operations
|
||||
trie_metrics: MultiProofTaskMetrics,
|
||||
/// Cross-block cache size in bytes.
|
||||
@@ -122,12 +120,8 @@ where
|
||||
sparse_trie_max_hot_accounts: usize,
|
||||
/// Whether sparse trie cache pruning is fully disabled.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Whether to disable BAL-based parallel execution (falls back to tx-based prewarming).
|
||||
disable_bal_parallel_execution: bool,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
/// Whether BAL batched IO is disabled.
|
||||
disable_bal_batch_io: bool,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
@@ -161,11 +155,7 @@ where
|
||||
sparse_trie_max_hot_slots: config.sparse_trie_max_hot_slots(),
|
||||
sparse_trie_max_hot_accounts: config.sparse_trie_max_hot_accounts(),
|
||||
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
|
||||
cache_metrics: (!config.disable_cache_metrics())
|
||||
.then(|| CachedStateMetrics::zeroed(CachedStateMetricsSource::Engine)),
|
||||
disable_bal_parallel_execution: config.disable_bal_parallel_execution(),
|
||||
disable_bal_parallel_state_root: config.disable_bal_parallel_state_root(),
|
||||
disable_bal_batch_io: config.disable_bal_batch_io(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -250,6 +240,7 @@ where
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
multiproof_provider_factory: F,
|
||||
config: &TreeConfig,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -272,12 +263,13 @@ where
|
||||
halve_workers,
|
||||
config,
|
||||
);
|
||||
let install_state_hook = env.decoded_bal.is_none();
|
||||
let install_state_hook = bal.is_none();
|
||||
let prewarm_handle = self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
provider_builder,
|
||||
Some(state_root_handle.updates_tx().clone()),
|
||||
bal,
|
||||
);
|
||||
|
||||
PayloadHandle {
|
||||
@@ -298,13 +290,14 @@ where
|
||||
env: ExecutionEnv<Evm>,
|
||||
transactions: I,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx) =
|
||||
self.spawn_tx_iterator(transactions, env.transaction_count);
|
||||
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None);
|
||||
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal);
|
||||
PayloadHandle {
|
||||
state_root_handle: None,
|
||||
install_state_hook: false,
|
||||
@@ -462,7 +455,7 @@ where
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor",
|
||||
skip_all,
|
||||
fields(bal=%env.decoded_bal.is_some())
|
||||
fields(bal=%bal.is_some())
|
||||
)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
@@ -470,6 +463,7 @@ where
|
||||
transactions: mpsc::Receiver<(usize, impl ExecutableTxFor<Evm> + Clone + Send + 'static)>,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_sparse_trie_task: Option<CrossbeamSender<StateRootMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -480,7 +474,7 @@ where
|
||||
let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash));
|
||||
|
||||
let executed_tx_index = Arc::new(AtomicUsize::new(0));
|
||||
let maybe_decoded_bal = env.decoded_bal.clone();
|
||||
|
||||
// configure prewarming
|
||||
let prewarm_ctx = PrewarmContext {
|
||||
env,
|
||||
@@ -488,13 +482,10 @@ where
|
||||
saved_cache: saved_cache.clone(),
|
||||
provider: provider_builder,
|
||||
metrics: PrewarmMetrics::default(),
|
||||
cache_metrics: self.cache_metrics.clone(),
|
||||
terminate_execution: Arc::new(AtomicBool::new(false)),
|
||||
executed_tx_index: Arc::clone(&executed_tx_index),
|
||||
precompile_cache_disabled: self.precompile_cache_disabled,
|
||||
precompile_cache_map: self.precompile_cache_map.clone(),
|
||||
disable_bal_parallel_state_root: self.disable_bal_parallel_state_root,
|
||||
disable_bal_batch_io: self.disable_bal_batch_io,
|
||||
};
|
||||
|
||||
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
|
||||
@@ -503,16 +494,14 @@ where
|
||||
prewarm_ctx,
|
||||
to_sparse_trie_task,
|
||||
);
|
||||
|
||||
{
|
||||
let to_prewarm_task = to_prewarm_task.clone();
|
||||
let disable_bal_parallel_execution = self.disable_bal_parallel_execution;
|
||||
self.executor.spawn_blocking_named("prewarm", move || {
|
||||
let mode = if skip_prewarm {
|
||||
PrewarmMode::Skipped
|
||||
} else if let Some(decoded_bal) =
|
||||
maybe_decoded_bal.filter(|_| !disable_bal_parallel_execution)
|
||||
{
|
||||
PrewarmMode::BlockAccessList(decoded_bal)
|
||||
} else if let Some(bal) = bal {
|
||||
PrewarmMode::BlockAccessList(bal)
|
||||
} else {
|
||||
PrewarmMode::Transactions(transactions)
|
||||
};
|
||||
@@ -520,12 +509,7 @@ where
|
||||
});
|
||||
}
|
||||
|
||||
CacheTaskHandle {
|
||||
saved_cache,
|
||||
to_prewarm_task: Some(to_prewarm_task),
|
||||
executed_tx_index,
|
||||
cache_metrics: self.cache_metrics.clone(),
|
||||
}
|
||||
CacheTaskHandle { saved_cache, to_prewarm_task: Some(to_prewarm_task), executed_tx_index }
|
||||
}
|
||||
|
||||
/// Returns the cache for the given parent hash.
|
||||
@@ -541,10 +525,10 @@ where
|
||||
debug!("creating new execution cache on cache miss");
|
||||
let start = Instant::now();
|
||||
let cache = ExecutionCache::new(self.cross_block_cache_size);
|
||||
if let Some(metrics) = &self.cache_metrics {
|
||||
metrics.record_cache_creation(start.elapsed());
|
||||
}
|
||||
SavedCache::new(parent_hash, cache)
|
||||
let metrics = CachedStateMetrics::zeroed();
|
||||
metrics.record_cache_creation(start.elapsed());
|
||||
SavedCache::new(parent_hash, cache, metrics)
|
||||
.with_disable_cache_metrics(self.disable_cache_metrics)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -606,7 +590,6 @@ where
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
parent_state_root,
|
||||
chunk_size,
|
||||
);
|
||||
|
||||
@@ -692,7 +675,7 @@ where
|
||||
block_with_parent: BlockWithParent,
|
||||
bundle_state: &BundleState,
|
||||
) {
|
||||
let cache_metrics = self.cache_metrics.clone();
|
||||
let disable_cache_metrics = self.disable_cache_metrics;
|
||||
self.execution_cache.update_with_guard(|cached| {
|
||||
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
|
||||
debug!(
|
||||
@@ -704,19 +687,25 @@ where
|
||||
}
|
||||
|
||||
// Take existing cache (if any) or create fresh caches
|
||||
let caches = match cached.take() {
|
||||
Some(existing) => existing.cache().clone(),
|
||||
None => ExecutionCache::new(self.cross_block_cache_size),
|
||||
let (caches, cache_metrics, _) = match cached.take() {
|
||||
Some(existing) => existing.split(),
|
||||
None => (
|
||||
ExecutionCache::new(self.cross_block_cache_size),
|
||||
CachedStateMetrics::zeroed(),
|
||||
false,
|
||||
),
|
||||
};
|
||||
|
||||
// Insert the block's bundle state into cache
|
||||
let new_cache = SavedCache::new(block_with_parent.block.hash, caches);
|
||||
let new_cache =
|
||||
SavedCache::new(block_with_parent.block.hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
if new_cache.cache().insert_state(bundle_state).is_err() {
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
return
|
||||
}
|
||||
new_cache.update_metrics(cache_metrics.as_ref());
|
||||
new_cache.update_metrics();
|
||||
|
||||
// Replace with the updated cache
|
||||
*cached = Some(new_cache);
|
||||
@@ -810,9 +799,9 @@ impl<Tx, Err, R: Send + Sync + 'static> PayloadHandle<Tx, Err, R> {
|
||||
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.cache().clone())
|
||||
}
|
||||
|
||||
/// Returns engine cache metrics if a cache exists for prewarming.
|
||||
/// Returns a clone of the cache metrics used by prewarming
|
||||
pub fn cache_metrics(&self) -> Option<CachedStateMetrics> {
|
||||
self.prewarm_handle.cache_metrics.clone()
|
||||
self.prewarm_handle.saved_cache.as_ref().map(|cache| cache.metrics().clone())
|
||||
}
|
||||
|
||||
/// Returns a reference to the shared executed transaction index counter.
|
||||
@@ -863,8 +852,6 @@ pub struct CacheTaskHandle<R> {
|
||||
/// Shared counter tracking the next transaction index to be executed by the main execution
|
||||
/// loop. Prewarm workers skip transactions below this index.
|
||||
executed_tx_index: Arc<AtomicUsize>,
|
||||
/// Metrics for the execution cache.
|
||||
cache_metrics: Option<CachedStateMetrics>,
|
||||
}
|
||||
|
||||
impl<R: Send + Sync + 'static> CacheTaskHandle<R> {
|
||||
@@ -933,9 +920,6 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
/// Withdrawals included in the block.
|
||||
/// Used to generate prefetch targets for withdrawal addresses.
|
||||
pub withdrawals: Option<Vec<Withdrawal>>,
|
||||
/// Optional decoded BAL for the block.
|
||||
/// Used to validate and optimize execution.
|
||||
pub decoded_bal: Option<Arc<DecodedBal>>,
|
||||
}
|
||||
|
||||
impl<Evm: ConfigureEvm> ExecutionEnv<Evm>
|
||||
@@ -953,7 +937,6 @@ where
|
||||
transaction_count: 0,
|
||||
gas_used: 0,
|
||||
withdrawals: None,
|
||||
decoded_bal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -963,7 +946,8 @@ mod tests {
|
||||
use crate::tree::{
|
||||
payload_processor::{evm_state_to_hashed_post_state, ExecutionEnv, PayloadProcessor},
|
||||
precompile_cache::PrecompileCacheMap,
|
||||
ExecutionCache, PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig,
|
||||
CachedStateMetrics, ExecutionCache, PayloadExecutionCache, SavedCache,
|
||||
StateProviderBuilder, TreeConfig,
|
||||
};
|
||||
use alloy_eips::eip1898::{BlockNumHash, BlockWithParent};
|
||||
use alloy_evm::block::StateChangeSource;
|
||||
@@ -975,7 +959,7 @@ mod tests {
|
||||
use reth_evm_ethereum::EthEvmConfig;
|
||||
use reth_primitives_traits::{Account, Recovered, StorageEntry};
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory},
|
||||
providers::{BlockchainProvider, OverlayStateProviderFactory},
|
||||
test_utils::create_test_provider_factory_with_chain_spec,
|
||||
ChainSpecProvider, HashingWriter,
|
||||
};
|
||||
@@ -989,7 +973,7 @@ mod tests {
|
||||
|
||||
fn make_saved_cache(hash: B256) -> SavedCache {
|
||||
let execution_cache = ExecutionCache::new(1_000);
|
||||
SavedCache::new(hash, execution_cache)
|
||||
SavedCache::new(hash, execution_cache, CachedStateMetrics::zeroed())
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -1159,16 +1143,19 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
let mut account = revm_state::Account::default();
|
||||
account.info = AccountInfo {
|
||||
balance: U256::from(rng.random::<u64>()),
|
||||
nonce: rng.random::<u64>(),
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: Some(Default::default()),
|
||||
account_id: None,
|
||||
let account = revm_state::Account {
|
||||
info: AccountInfo {
|
||||
balance: U256::from(rng.random::<u64>()),
|
||||
nonce: rng.random::<u64>(),
|
||||
code_hash: KECCAK_EMPTY,
|
||||
code: Some(Default::default()),
|
||||
account_id: None,
|
||||
},
|
||||
original_info: Box::new(AccountInfo::default()),
|
||||
storage,
|
||||
status: AccountStatus::Touched,
|
||||
transaction_id: 0,
|
||||
};
|
||||
account.storage = storage;
|
||||
account.status = AccountStatus::Touched;
|
||||
|
||||
state_update.insert(address, account);
|
||||
}
|
||||
@@ -1247,11 +1234,9 @@ mod tests {
|
||||
std::convert::identity,
|
||||
),
|
||||
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
|
||||
OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
OverlayBuilder::new(ChangesetCache::new()),
|
||||
),
|
||||
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
|
||||
&TreeConfig::default(),
|
||||
None, // No BAL for test
|
||||
);
|
||||
|
||||
let mut state_hook = handle.state_hook().expect("state hook is None");
|
||||
|
||||
@@ -14,11 +14,10 @@
|
||||
use crate::tree::{
|
||||
payload_processor::multiproof::StateRootMessage,
|
||||
precompile_cache::{CachedPrecompile, PrecompileCacheMap},
|
||||
CachedStateMetrics, CachedStateProvider, ExecutionEnv, PayloadExecutionCache, SavedCache,
|
||||
StateProviderBuilder,
|
||||
CachedStateProvider, ExecutionEnv, PayloadExecutionCache, SavedCache, StateProviderBuilder,
|
||||
};
|
||||
use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_eip7928::bal::DecodedBal;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{keccak256, StorageKey, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
@@ -48,7 +47,7 @@ pub enum PrewarmMode<Tx> {
|
||||
/// Prewarm by executing transactions from a stream, each paired with its block index.
|
||||
Transactions(Receiver<(usize, Tx)>),
|
||||
/// Prewarm by prefetching slots from a Block Access List.
|
||||
BlockAccessList(Arc<DecodedBal>),
|
||||
BlockAccessList(Arc<BlockAccessList>),
|
||||
/// Transaction prewarming is skipped (e.g. small blocks where the overhead exceeds the
|
||||
/// benefit). No workers are spawned.
|
||||
Skipped,
|
||||
@@ -277,20 +276,19 @@ where
|
||||
) {
|
||||
let start = Instant::now();
|
||||
|
||||
let Self {
|
||||
execution_cache,
|
||||
ctx: PrewarmContext { env, metrics, cache_metrics, saved_cache, .. },
|
||||
..
|
||||
} = self;
|
||||
let Self { execution_cache, ctx: PrewarmContext { env, metrics, saved_cache, .. }, .. } =
|
||||
self;
|
||||
let hash = env.hash;
|
||||
|
||||
if let Some(saved_cache) = saved_cache {
|
||||
debug!(target: "engine::caching", parent_hash=?hash, "Updating execution cache");
|
||||
// Perform all cache operations atomically under the lock
|
||||
execution_cache.update_with_guard(|cached| {
|
||||
// consumes the `SavedCache` held by the prewarming task, which releases its usage
|
||||
// guard
|
||||
let caches = saved_cache.cache().clone();
|
||||
let new_cache = SavedCache::new(hash, caches);
|
||||
let (caches, cache_metrics, disable_cache_metrics) = saved_cache.split();
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
@@ -301,7 +299,7 @@ where
|
||||
return;
|
||||
}
|
||||
|
||||
new_cache.update_metrics(cache_metrics.as_ref());
|
||||
new_cache.update_metrics();
|
||||
|
||||
if valid_block_rx.recv().is_ok() {
|
||||
// Replace the shared cache with the new one; the previous cache (if any) is
|
||||
@@ -331,10 +329,9 @@ where
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn run_bal_prewarm(
|
||||
&self,
|
||||
decoded_bal: Arc<DecodedBal>,
|
||||
bal: Arc<BlockAccessList>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
) {
|
||||
let bal = decoded_bal.as_bal();
|
||||
if bal.is_empty() {
|
||||
if let Some(to_sparse_trie_task) = self.to_sparse_trie_task.as_ref() {
|
||||
let _ = to_sparse_trie_task.send(StateRootMessage::FinishedStateUpdates);
|
||||
@@ -356,8 +353,8 @@ where
|
||||
let parent_span = Span::current();
|
||||
let prefetch_parent_span = parent_span.clone();
|
||||
let stream_parent_span = parent_span;
|
||||
let prefetch_bal = Arc::clone(&decoded_bal);
|
||||
let stream_bal = Arc::clone(&decoded_bal);
|
||||
let prefetch_bal = Arc::clone(&bal);
|
||||
let stream_bal = Arc::clone(&bal);
|
||||
let (prefetch_tx, prefetch_rx) = oneshot::channel();
|
||||
let (stream_tx, stream_rx) = oneshot::channel();
|
||||
|
||||
@@ -368,12 +365,12 @@ where
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
parent: &prefetch_parent_span,
|
||||
"bal_prefetch_storage",
|
||||
bal_accounts = prefetch_bal.as_bal().len(),
|
||||
bal_accounts = prefetch_bal.len(),
|
||||
);
|
||||
let provider_parent_span = branch_span.clone();
|
||||
let _span = branch_span.entered();
|
||||
|
||||
prefetch_bal.as_bal().par_iter().for_each_init(
|
||||
prefetch_bal.par_iter().for_each_init(
|
||||
|| {
|
||||
(
|
||||
prefetch_ctx.clone(),
|
||||
@@ -401,12 +398,12 @@ where
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
parent: &stream_parent_span,
|
||||
"bal_hashed_state_stream",
|
||||
bal_accounts = stream_bal.as_bal().len(),
|
||||
bal_accounts = stream_bal.len(),
|
||||
);
|
||||
let provider_parent_span = branch_span.clone();
|
||||
let _span = branch_span.entered();
|
||||
|
||||
stream_bal.as_bal().par_iter().for_each_init(
|
||||
stream_bal.par_iter().for_each_init(
|
||||
|| (ctx.clone(), None::<Box<dyn AccountReader>>, provider_parent_span.clone()),
|
||||
|(ctx, provider, parent_span), account_changes| {
|
||||
ctx.send_bal_hashed_state(
|
||||
@@ -524,9 +521,6 @@ where
|
||||
pub provider: StateProviderBuilder<N, P>,
|
||||
/// The metrics for the prewarm task.
|
||||
pub metrics: PrewarmMetrics,
|
||||
/// Metrics for the execution cache.
|
||||
/// Metrics for the execution cache. `None` disables metrics recording.
|
||||
pub cache_metrics: Option<CachedStateMetrics>,
|
||||
/// An atomic bool that tells prewarm tasks to not start any more execution.
|
||||
pub terminate_execution: Arc<AtomicBool>,
|
||||
/// Shared counter tracking the next transaction index to be executed by the main execution
|
||||
@@ -537,10 +531,6 @@ where
|
||||
pub precompile_cache_disabled: bool,
|
||||
/// The precompile cache map.
|
||||
pub precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
pub disable_bal_parallel_state_root: bool,
|
||||
/// Whether BAL batched IO is disabled.
|
||||
pub disable_bal_batch_io: bool,
|
||||
}
|
||||
|
||||
/// Per-thread EVM state initialised by [`PrewarmContext::evm_for_ctx`] and stored in
|
||||
@@ -572,11 +562,9 @@ where
|
||||
// Use the caches to create a new provider with caching
|
||||
if let Some(saved_cache) = &self.saved_cache {
|
||||
let caches = saved_cache.cache().clone();
|
||||
state_provider = Box::new(CachedStateProvider::new_prewarm(
|
||||
state_provider,
|
||||
caches,
|
||||
self.cache_metrics.clone().unwrap_or_default(),
|
||||
));
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
state_provider =
|
||||
Box::new(CachedStateProvider::new_prewarm(state_provider, caches, cache_metrics));
|
||||
}
|
||||
|
||||
let state_provider = StateProviderDatabase::new(state_provider);
|
||||
@@ -636,9 +624,6 @@ where
|
||||
account_changes: &alloy_eip7928::AccountChanges,
|
||||
to_sparse_trie_task: &CrossbeamSender<StateRootMessage>,
|
||||
) {
|
||||
if self.disable_bal_parallel_state_root {
|
||||
return;
|
||||
}
|
||||
let address = account_changes.address;
|
||||
let mut hashed_address = None;
|
||||
|
||||
@@ -680,11 +665,8 @@ where
|
||||
};
|
||||
let boxed: Box<dyn AccountReader> = if let Some(saved) = &self.saved_cache {
|
||||
let caches = saved.cache().clone();
|
||||
Box::new(CachedStateProvider::new_prewarm(
|
||||
inner,
|
||||
caches,
|
||||
self.cache_metrics.clone().unwrap_or_default(),
|
||||
))
|
||||
let cache_metrics = saved.metrics().clone();
|
||||
Box::new(CachedStateProvider::new_prewarm(inner, caches, cache_metrics))
|
||||
} else {
|
||||
Box::new(inner)
|
||||
};
|
||||
@@ -779,11 +761,8 @@ where
|
||||
let saved_cache =
|
||||
self.saved_cache.as_ref().expect("BAL prewarm should only run with cache");
|
||||
let caches = saved_cache.cache().clone();
|
||||
slot.insert(CachedStateProvider::new_prewarm(
|
||||
built,
|
||||
caches,
|
||||
self.cache_metrics.clone().unwrap_or_default(),
|
||||
))
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
slot.insert(CachedStateProvider::new_prewarm(built, caches, cache_metrics))
|
||||
}
|
||||
};
|
||||
|
||||
|
||||
@@ -27,9 +27,8 @@ use reth_trie_parallel::{
|
||||
root::ParallelStateRootError,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
errors::{SparseStateTrieErrorKind, SparseTrieErrorKind, SparseTrieResult},
|
||||
ConfigurableSparseTrie, DeferredDrops, LeafUpdate, RevealableSparseTrie, SparseStateTrie,
|
||||
SparseTrie,
|
||||
errors::SparseTrieResult, ConfigurableSparseTrie, DeferredDrops, LeafUpdate,
|
||||
RevealableSparseTrie, SparseStateTrie, SparseTrie,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use tracing::{debug, debug_span, error, instrument, trace_span};
|
||||
@@ -47,8 +46,6 @@ pub(super) struct SparseTrieCacheTask<A = ConfigurableSparseTrie, S = Configurab
|
||||
updates: CrossbeamReceiver<SparseTrieTaskMessage>,
|
||||
/// `SparseStateTrie` used for computing the state root.
|
||||
trie: SparseStateTrie<A, S>,
|
||||
/// The parent block's state root.
|
||||
parent_state_root: B256,
|
||||
/// Handle to the proof worker pools (storage and account).
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
|
||||
@@ -123,7 +120,6 @@ where
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
parent_state_root: B256,
|
||||
chunk_size: usize,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
@@ -142,7 +138,6 @@ where
|
||||
updates: hashed_state_rx,
|
||||
proof_worker_handle,
|
||||
trie,
|
||||
parent_state_root,
|
||||
chunk_size,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
account_updates: Default::default(),
|
||||
@@ -364,25 +359,10 @@ where
|
||||
debug!(target: "engine::root", "All proofs processed, ending calculation");
|
||||
|
||||
let start = Instant::now();
|
||||
let (state_root, trie_updates) = match self.trie.root_with_updates() {
|
||||
Ok(result) => result,
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.kind(),
|
||||
SparseStateTrieErrorKind::Sparse(SparseTrieErrorKind::Blind)
|
||||
) =>
|
||||
{
|
||||
// A still-blind account trie means this block never changed state, so preserve
|
||||
// the cached parent root instead of fetching and revealing
|
||||
// the unchanged root node.
|
||||
(self.parent_state_root, TrieUpdates::default())
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(ParallelStateRootError::Other(format!(
|
||||
"could not calculate state root: {err:?}"
|
||||
)))
|
||||
}
|
||||
};
|
||||
let (state_root, trie_updates) =
|
||||
self.trie.root_with_updates(&self.proof_worker_handle).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
|
||||
})?;
|
||||
|
||||
#[cfg(feature = "trie-debug")]
|
||||
let debug_recorders = self.trie.take_debug_recorders();
|
||||
@@ -893,12 +873,6 @@ enum SparseTrieTaskMessage {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{keccak256, Address, B256, U256};
|
||||
use reth_provider::{
|
||||
providers::{OverlayBuilder, OverlayStateProviderFactory},
|
||||
test_utils::create_test_provider_factory,
|
||||
};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::proof_task::ProofTaskCtx;
|
||||
use reth_trie_sparse::ArenaParallelSparseTrie;
|
||||
|
||||
#[test]
|
||||
@@ -979,45 +953,4 @@ mod tests {
|
||||
assert_eq!(decoded.storage_root, storage_root);
|
||||
assert_eq!(account_rlp_buf, encoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() {
|
||||
let runtime = reth_tasks::Runtime::test();
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
OverlayBuilder::new(ChangesetCache::new()),
|
||||
);
|
||||
let proof_worker_handle =
|
||||
ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false);
|
||||
|
||||
let default_trie = RevealableSparseTrie::blind_from(ConfigurableSparseTrie::Arena(
|
||||
ArenaParallelSparseTrie::default(),
|
||||
));
|
||||
let trie = SparseStateTrie::default()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true);
|
||||
|
||||
let parent_state_root = B256::from([0x55; 32]);
|
||||
let (updates_tx, updates_rx) = crossbeam_channel::unbounded();
|
||||
let mut task = SparseTrieCacheTask::new_with_trie(
|
||||
&runtime,
|
||||
updates_rx,
|
||||
proof_worker_handle,
|
||||
MultiProofTaskMetrics::default(),
|
||||
trie,
|
||||
parent_state_root,
|
||||
1,
|
||||
);
|
||||
|
||||
updates_tx.send(StateRootMessage::FinishedStateUpdates).unwrap();
|
||||
drop(updates_tx);
|
||||
|
||||
let outcome = task.run().expect("state root computation should succeed");
|
||||
|
||||
assert_eq!(outcome.state_root, parent_state_root);
|
||||
assert!(outcome.trie_updates.is_empty());
|
||||
assert!(task.trie.state_trie_ref().is_none(), "blind trie should not be revealed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,10 +48,7 @@ use crate::tree::{
|
||||
PayloadHandle, StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches,
|
||||
};
|
||||
use alloy_consensus::transaction::{Either, TxHashRef};
|
||||
use alloy_eip7928::{
|
||||
bal::{Bal, DecodedBal},
|
||||
BlockAccessList,
|
||||
};
|
||||
use alloy_eip7928::{bal::Bal, BlockAccessList};
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
use alloy_primitives::{map::B256Set, B256};
|
||||
@@ -80,14 +77,13 @@ use reth_primitives_traits::{
|
||||
RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable,
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::{OverlayBuilder, OverlayStateProviderFactory},
|
||||
BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory,
|
||||
DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader,
|
||||
StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader,
|
||||
StorageChangeSetReader, StorageSettingsCache,
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State};
|
||||
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState};
|
||||
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
use revm_primitives::{Address, KECCAK_EMPTY};
|
||||
@@ -491,12 +487,6 @@ where
|
||||
.in_scope(|| self.evm_env_for(&input))
|
||||
.map_err(NewPayloadError::other)?;
|
||||
|
||||
// Extract the decoded BAL, if valid and available.
|
||||
let decoded_bal = ensure_ok!(input
|
||||
.try_decoded_access_list()
|
||||
.map_err(|err| { Box::<dyn std::error::Error + Send + Sync>::from(err) }))
|
||||
.map(Arc::new);
|
||||
|
||||
let env = ExecutionEnv {
|
||||
evm_env,
|
||||
hash: input.hash(),
|
||||
@@ -505,7 +495,6 @@ where
|
||||
transaction_count: input.transaction_count(),
|
||||
gas_used: input.gas_used(),
|
||||
withdrawals: input.withdrawals().map(|w| w.to_vec()),
|
||||
decoded_bal,
|
||||
};
|
||||
|
||||
// Plan the strategy used for state root computation.
|
||||
@@ -520,26 +509,33 @@ where
|
||||
// Get an iterator over the transactions in the payload
|
||||
let txs = self.tx_iterator_for(&input)?;
|
||||
|
||||
// Extract the BAL, if valid and available
|
||||
let block_access_list = ensure_ok!(input
|
||||
.block_access_list()
|
||||
.transpose()
|
||||
// Eventually gets converted to a `InsertBlockErrorKind::Other`
|
||||
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
|
||||
.map(Arc::new);
|
||||
|
||||
// Create lazy overlay from ancestors - this doesn't block, allowing execution to start
|
||||
// before the trie data is ready. The overlay will be computed on first access.
|
||||
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state());
|
||||
|
||||
// Create overlay factory for payload processor (StateRootTask path needs it for
|
||||
// multiproofs)
|
||||
let provider_factory = self.provider.clone();
|
||||
let overlay_builder = OverlayBuilder::new(self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(provider_factory.clone(), overlay_builder.clone());
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
|
||||
// Spawn the appropriate processor based on strategy
|
||||
let mut handle = ensure_ok!(self.spawn_payload_processor(
|
||||
env.clone(),
|
||||
txs,
|
||||
provider_builder.clone(),
|
||||
provider_builder,
|
||||
overlay_factory.clone(),
|
||||
strategy,
|
||||
block_access_list,
|
||||
));
|
||||
|
||||
// Create optional cache stats for detailed block logging
|
||||
@@ -668,7 +664,7 @@ where
|
||||
let task_result = ensure_ok_post_block!(
|
||||
self.await_state_root_with_timeout(
|
||||
&mut handle,
|
||||
provider_builder.clone(),
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
),
|
||||
block
|
||||
@@ -692,9 +688,7 @@ where
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
let _has_diff = self.compare_trie_updates_with_serial(
|
||||
provider_builder.clone(),
|
||||
provider_factory,
|
||||
overlay_builder,
|
||||
overlay_factory.clone(),
|
||||
&hashed_state,
|
||||
trie_updates.as_ref().clone(),
|
||||
);
|
||||
@@ -733,11 +727,7 @@ where
|
||||
}
|
||||
StateRootStrategy::Parallel => {
|
||||
debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm");
|
||||
match self.compute_state_root_parallel(
|
||||
provider_factory,
|
||||
overlay_builder,
|
||||
&hashed_state,
|
||||
) {
|
||||
match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) {
|
||||
Ok(result) => {
|
||||
let elapsed = root_time.elapsed();
|
||||
info!(
|
||||
@@ -773,9 +763,7 @@ where
|
||||
}
|
||||
|
||||
let (root, updates) = ensure_ok_post_block!(
|
||||
provider_builder
|
||||
.build()
|
||||
.and_then(|provider| Self::compute_state_root_serial(provider, &hashed_state)),
|
||||
Self::compute_state_root_serial(overlay_factory.clone(), &hashed_state),
|
||||
block
|
||||
);
|
||||
|
||||
@@ -1099,8 +1087,7 @@ where
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
fn compute_state_root_parallel(
|
||||
&self,
|
||||
provider_factory: P,
|
||||
overlay_builder: OverlayBuilder,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> Result<(B256, TrieUpdates), ParallelStateRootError> {
|
||||
let hashed_state = hashed_state.get();
|
||||
@@ -1108,24 +1095,34 @@ where
|
||||
// need to use the prefix sets which were generated from it to indicate to the
|
||||
// ParallelStateRoot which parts of the trie need to be recomputed.
|
||||
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()),
|
||||
);
|
||||
let overlay_factory =
|
||||
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
|
||||
ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone())
|
||||
.incremental_root_with_updates()
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in serial.
|
||||
///
|
||||
/// Uses the same provider construction path as main execution and computes the state root and
|
||||
/// trie updates for this block directly via
|
||||
/// [`reth_provider::StateRootProvider::state_root_with_updates`].
|
||||
/// Uses an overlay factory which provides the state of the parent block, along with the
|
||||
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
|
||||
/// trie updates for this block.
|
||||
fn compute_state_root_serial(
|
||||
state_provider: StateProviderBox,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
state_provider.state_root_with_updates(hashed_state.get().clone())
|
||||
let hashed_state = hashed_state.get();
|
||||
// The `hashed_state` argument will be taken into account as part of the overlay, but we
|
||||
// need to use the prefix sets which were generated from it to indicate to the
|
||||
// StateRoot which parts of the trie need to be recomputed.
|
||||
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
|
||||
let overlay_factory =
|
||||
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
|
||||
|
||||
let provider = overlay_factory.database_provider_ro()?;
|
||||
|
||||
Ok(StateRoot::new(&provider, &provider)
|
||||
.with_prefix_sets(prefix_sets)
|
||||
.root_with_updates()?)
|
||||
}
|
||||
|
||||
/// Awaits the state root from the background task, with an optional timeout fallback.
|
||||
@@ -1150,7 +1147,7 @@ where
|
||||
fn await_state_root_with_timeout<Tx, Err, R: Send + Sync + 'static>(
|
||||
&self,
|
||||
handle: &mut PayloadHandle<Tx, Err, R>,
|
||||
state_provider_builder: StateProviderBuilder<N, P>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<Result<StateRootComputeOutcome, ParallelStateRootError>> {
|
||||
let Some(timeout) = self.config.state_root_task_timeout() else {
|
||||
@@ -1175,11 +1172,10 @@ where
|
||||
let (seq_tx, seq_rx) =
|
||||
std::sync::mpsc::channel::<ProviderResult<(B256, TrieUpdates)>>();
|
||||
|
||||
let seq_overlay = overlay_factory;
|
||||
let seq_hashed_state = hashed_state.clone();
|
||||
self.payload_processor.executor().spawn_blocking_named("serial-root", move || {
|
||||
let result = state_provider_builder.build().and_then(|provider| {
|
||||
Self::compute_state_root_serial(provider, &seq_hashed_state)
|
||||
});
|
||||
let result = Self::compute_state_root_serial(seq_overlay, &seq_hashed_state);
|
||||
let _ = seq_tx.send(result);
|
||||
});
|
||||
|
||||
@@ -1243,18 +1239,13 @@ where
|
||||
/// updates.
|
||||
fn compare_trie_updates_with_serial(
|
||||
&self,
|
||||
state_provider_builder: StateProviderBuilder<N, P>,
|
||||
provider_factory: P,
|
||||
overlay_builder: OverlayBuilder,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) -> bool {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match state_provider_builder
|
||||
.build()
|
||||
.and_then(|provider| Self::compute_state_root_serial(provider, hashed_state))
|
||||
{
|
||||
match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
Ok((serial_root, serial_trie_updates)) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
@@ -1263,8 +1254,6 @@ where
|
||||
);
|
||||
|
||||
// Get a database provider to use as trie cursor factory
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(provider_factory, overlay_builder);
|
||||
match overlay_factory.database_provider_ro() {
|
||||
Ok(provider) => {
|
||||
match super::trie_updates::compare_trie_updates(
|
||||
@@ -1450,6 +1439,7 @@ where
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
strategy: StateRootStrategy,
|
||||
block_access_list: Option<Arc<BlockAccessList>>,
|
||||
) -> Result<
|
||||
PayloadHandle<
|
||||
impl ExecutableTxFor<Evm> + use<N, P, Evm, V, T>,
|
||||
@@ -1469,6 +1459,7 @@ where
|
||||
provider_builder,
|
||||
overlay_factory,
|
||||
&self.config,
|
||||
block_access_list,
|
||||
);
|
||||
|
||||
// record prewarming initialization duration
|
||||
@@ -1481,8 +1472,12 @@ where
|
||||
}
|
||||
StateRootStrategy::Parallel | StateRootStrategy::Synchronous => {
|
||||
let start = Instant::now();
|
||||
let handle =
|
||||
self.payload_processor.spawn_cache_exclusive(env, txs, provider_builder);
|
||||
let handle = self.payload_processor.spawn_cache_exclusive(
|
||||
env,
|
||||
txs,
|
||||
provider_builder,
|
||||
block_access_list,
|
||||
);
|
||||
|
||||
// Record prewarming initialization duration
|
||||
self.metrics
|
||||
@@ -2031,12 +2026,10 @@ where
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> Option<StateRootHandle> {
|
||||
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state);
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
self.provider.clone(),
|
||||
OverlayBuilder::new(self.changeset_cache.clone())
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay),
|
||||
);
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
|
||||
Some(self.payload_processor.spawn_state_root(
|
||||
overlay_factory,
|
||||
@@ -2117,17 +2110,6 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the decoded block access list, if present and successfully decoded.
|
||||
pub fn try_decoded_access_list(&self) -> Result<Option<DecodedBal>, alloy_rlp::Error> {
|
||||
match self {
|
||||
Self::Payload(payload) => payload
|
||||
.block_access_list()
|
||||
.map(|block_access_list| DecodedBal::from_rlp_bytes(block_access_list.clone()))
|
||||
.transpose(),
|
||||
Self::Block(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of transactions in the payload or block.
|
||||
pub fn transaction_count(&self) -> usize
|
||||
where
|
||||
@@ -2160,15 +2142,4 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
Self::Block(block) => block.gas_used(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the gas limit used by the block.
|
||||
pub fn gas_limit(&self) -> u64
|
||||
where
|
||||
T::ExecutionData: ExecutionPayload,
|
||||
{
|
||||
match self {
|
||||
Self::Payload(payload) => payload.gas_limit(),
|
||||
Self::Block(block) => block.gas_limit(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,6 @@ use reth_primitives_traits::dashmap::DashMap;
|
||||
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
|
||||
use revm_primitives::Address;
|
||||
use std::{hash::Hash, sync::Arc};
|
||||
use tracing::error;
|
||||
|
||||
/// Default max cache size for [`PrecompileCache`]
|
||||
const MAX_CACHE_SIZE: u32 = 10_000;
|
||||
@@ -88,14 +87,8 @@ impl<S> CacheEntry<S> {
|
||||
self.output.gas_used
|
||||
}
|
||||
|
||||
/// Converts the cache entry to a precompile result. Accepts state gas reservoir as input.
|
||||
///
|
||||
/// All cached precompiles are not expected to access/created state and thus reservoir is always
|
||||
/// kept as is.
|
||||
fn to_precompile_result(&self, reservoir: u64) -> PrecompileResult {
|
||||
let mut output = self.output.clone();
|
||||
output.reservoir = reservoir;
|
||||
Ok(output)
|
||||
fn to_precompile_result(&self) -> PrecompileResult {
|
||||
Ok(self.output.clone())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -182,34 +175,22 @@ where
|
||||
input.gas >= entry.gas_used()
|
||||
{
|
||||
self.increment_by_one_precompile_cache_hits();
|
||||
return entry.to_precompile_result(input.reservoir);
|
||||
return entry.to_precompile_result()
|
||||
}
|
||||
|
||||
let calldata = input.data;
|
||||
let reservoir = input.reservoir;
|
||||
let result = self.precompile.call(input);
|
||||
|
||||
match &result {
|
||||
// Only successful outputs are cacheable. Non-success statuses and errors must execute
|
||||
// again instead of poisoning the cache for subsequent calls.
|
||||
Ok(output) if output.is_success() => {
|
||||
// Sanity-check precompile output to ensure that it does not affect state gas in any
|
||||
// way.
|
||||
//
|
||||
// This does not fully protect us from caching stateful precompiles but might make
|
||||
// it obvious when the node is misconfigured.
|
||||
if output.reservoir != reservoir {
|
||||
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile decremented reservoir, skipping cache insertion");
|
||||
} else if output.state_gas_used != 0 {
|
||||
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile used state gas, skipping cache insertion");
|
||||
} else {
|
||||
let size = self.cache.insert(
|
||||
Bytes::copy_from_slice(calldata),
|
||||
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
|
||||
);
|
||||
self.set_precompile_cache_size_metric(size as f64);
|
||||
self.increment_by_one_precompile_cache_misses();
|
||||
}
|
||||
let size = self.cache.insert(
|
||||
Bytes::copy_from_slice(calldata),
|
||||
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
|
||||
);
|
||||
self.set_precompile_cache_size_metric(size as f64);
|
||||
self.increment_by_one_precompile_cache_misses();
|
||||
}
|
||||
_ => {
|
||||
self.increment_by_one_precompile_errors();
|
||||
@@ -265,7 +246,6 @@ mod tests {
|
||||
gas_used: 0,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: Bytes::default(),
|
||||
})
|
||||
})
|
||||
@@ -279,7 +259,6 @@ mod tests {
|
||||
gas_used: 50,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
|
||||
};
|
||||
|
||||
@@ -313,7 +292,6 @@ mod tests {
|
||||
gas_used: 5000,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
|
||||
})
|
||||
}
|
||||
@@ -330,7 +308,6 @@ mod tests {
|
||||
gas_used: 7000,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
use super::*;
|
||||
use crate::{
|
||||
persistence::PersistenceAction,
|
||||
persistence::{PersistenceAction, SaveBlocksBatch},
|
||||
tree::{
|
||||
payload_validator::{BasicEngineValidator, TreeCtx, ValidationOutcome},
|
||||
persistence_state::CurrentPersistenceAction,
|
||||
@@ -184,18 +184,11 @@ impl TestHarness {
|
||||
let payload_validator = MockEngineValidator;
|
||||
|
||||
let (from_tree_tx, from_tree_rx) = unbounded_channel();
|
||||
let tree_config =
|
||||
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true);
|
||||
|
||||
let header = chain_spec.genesis_header().clone();
|
||||
let header = SealedHeader::seal_slow(header);
|
||||
let engine_api_tree_state = EngineApiTreeState::new(
|
||||
10,
|
||||
10,
|
||||
tree_config.invalid_header_hit_eviction_threshold(),
|
||||
header.num_hash(),
|
||||
EngineApiKind::Ethereum,
|
||||
);
|
||||
let engine_api_tree_state =
|
||||
EngineApiTreeState::new(10, 10, header.num_hash(), EngineApiKind::Ethereum);
|
||||
let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None);
|
||||
|
||||
let (to_payload_service, _payload_command_rx) = unbounded_channel();
|
||||
@@ -224,7 +217,8 @@ impl TestHarness {
|
||||
persistence_handle,
|
||||
PersistenceState { last_persisted_block: BlockNumHash::default(), rx: None },
|
||||
payload_builder,
|
||||
tree_config,
|
||||
// always assume enough parallelism for tests
|
||||
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
|
||||
EngineApiKind::Ethereum,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
@@ -554,12 +548,14 @@ async fn test_tree_persist_blocks() {
|
||||
|
||||
let received_action =
|
||||
test_harness.action_rx.recv().expect("Failed to receive save blocks action");
|
||||
if let PersistenceAction::SaveBlocks(saved_blocks, _) = received_action {
|
||||
if let PersistenceAction::SaveBlocks(batch, _) = received_action {
|
||||
// all blocks are included in the batch
|
||||
assert_eq!(batch.blocks.len(), blocks.len());
|
||||
assert_eq!(batch.blocks, blocks);
|
||||
// only blocks.len() - tree_config.memory_block_buffer_target() will be
|
||||
// persisted
|
||||
let expected_persist_len = blocks.len() - tree_config.memory_block_buffer_target() as usize;
|
||||
assert_eq!(saved_blocks.len(), expected_persist_len);
|
||||
assert_eq!(saved_blocks, blocks[..expected_persist_len]);
|
||||
assert_eq!(batch.persist_count, expected_persist_len);
|
||||
} else {
|
||||
panic!("unexpected action received {received_action:?}");
|
||||
}
|
||||
@@ -824,10 +820,12 @@ async fn test_tree_state_on_new_head_reorg() {
|
||||
|
||||
// get rid of the prev action
|
||||
let received_action = test_harness.action_rx.recv().unwrap();
|
||||
let PersistenceAction::SaveBlocks(saved_blocks, sender) = received_action else {
|
||||
let PersistenceAction::SaveBlocks(batch, sender) = received_action else {
|
||||
panic!("received wrong action");
|
||||
};
|
||||
assert_eq!(saved_blocks, vec![blocks[0].clone(), blocks[1].clone()]);
|
||||
// all blocks above last_persisted are included, persist_count covers those up to the target
|
||||
assert_eq!(batch.persist_count, 2);
|
||||
assert_eq!(&batch.blocks[..batch.persist_count], &[blocks[0].clone(), blocks[1].clone()]);
|
||||
|
||||
// send the response so we can advance again
|
||||
sender
|
||||
@@ -985,8 +983,10 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
.try_into()
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
|
||||
for (i, item) in blocks_to_persist.iter().enumerate().take(expected_blocks_to_persist_length) {
|
||||
assert_eq!(blocks_to_persist.persist_count, expected_blocks_to_persist_length);
|
||||
for (i, item) in
|
||||
blocks_to_persist.blocks.iter().enumerate().take(expected_blocks_to_persist_length)
|
||||
{
|
||||
assert_eq!(item.recovered_block().number, last_persisted_block_number + i as u64 + 1);
|
||||
}
|
||||
|
||||
@@ -999,13 +999,16 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
|
||||
let blocks_to_persist =
|
||||
test_harness.tree.get_canonical_blocks_to_persist(PersistTarget::Threshold).unwrap();
|
||||
assert_eq!(blocks_to_persist.len(), expected_blocks_to_persist_length);
|
||||
assert_eq!(blocks_to_persist.persist_count, expected_blocks_to_persist_length);
|
||||
|
||||
// check that the fork block is not included in the blocks to persist
|
||||
assert!(!blocks_to_persist.iter().any(|b| b.recovered_block().hash() == fork_block_hash));
|
||||
assert!(!blocks_to_persist
|
||||
.blocks
|
||||
.iter()
|
||||
.any(|b| b.recovered_block().hash() == fork_block_hash));
|
||||
|
||||
// check that the original block 4 is still included
|
||||
assert!(blocks_to_persist.iter().any(|b| b.recovered_block().number == 4 &&
|
||||
assert!(blocks_to_persist.blocks.iter().any(|b| b.recovered_block().number == 4 &&
|
||||
b.recovered_block().hash() == blocks[4].recovered_block().hash()));
|
||||
|
||||
// check that if we advance persistence, the persistence action is the correct value
|
||||
@@ -1013,7 +1016,11 @@ async fn test_get_canonical_blocks_to_persist() {
|
||||
assert_eq!(
|
||||
test_harness.tree.persistence_state.current_action().cloned(),
|
||||
Some(CurrentPersistenceAction::SavingBlocks {
|
||||
highest: blocks_to_persist.last().unwrap().recovered_block().num_hash()
|
||||
highest: blocks_to_persist.blocks[..blocks_to_persist.persist_count]
|
||||
.last()
|
||||
.unwrap()
|
||||
.recovered_block()
|
||||
.num_hash()
|
||||
})
|
||||
);
|
||||
}
|
||||
@@ -2112,15 +2119,16 @@ mod forkchoice_updated_tests {
|
||||
break;
|
||||
}
|
||||
|
||||
if let Ok(PersistenceAction::SaveBlocks(saved_blocks, sender)) =
|
||||
if let Ok(PersistenceAction::SaveBlocks(batch, sender)) =
|
||||
action_rx.recv_timeout(std::time::Duration::from_millis(100))
|
||||
{
|
||||
if let Some(last) = saved_blocks.last() {
|
||||
let persisted = &batch.blocks[..batch.persist_count];
|
||||
if let Some(last) = persisted.last() {
|
||||
last_persisted_number = last.recovered_block().number;
|
||||
}
|
||||
sender
|
||||
.send(PersistenceResult {
|
||||
last_block: saved_blocks.last().map(|b| b.recovered_block().num_hash()),
|
||||
last_block: persisted.last().map(|b| b.recovered_block().num_hash()),
|
||||
commit_duration: Some(Duration::ZERO),
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
//! and injecting them into era1 files with `Era1Writer`.
|
||||
|
||||
use crate::calculate_td_by_number;
|
||||
use alloy_consensus::{BlockHeader, Sealable, TxReceipt};
|
||||
use alloy_primitives::{BlockNumber, U256};
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_primitives::{BlockNumber, B256, U256};
|
||||
use eyre::{eyre, Result};
|
||||
use reth_era::{
|
||||
common::file_ops::{EraFileId, StreamWriter},
|
||||
@@ -13,7 +13,7 @@ use reth_era::{
|
||||
types::{
|
||||
execution::{
|
||||
Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts,
|
||||
HeaderRecord, TotalDifficulty, MAX_BLOCKS_PER_ERA1,
|
||||
TotalDifficulty, MAX_BLOCKS_PER_ERA1,
|
||||
},
|
||||
group::{BlockIndex, Era1Id},
|
||||
},
|
||||
@@ -139,21 +139,17 @@ where
|
||||
|
||||
let headers = provider.headers_range(start_block..=end_block)?;
|
||||
|
||||
// Pre-compute accumulator from headers to determine filename
|
||||
let mut precompute_td = total_difficulty;
|
||||
let header_records: Vec<HeaderRecord> = headers
|
||||
.iter()
|
||||
.map(|h| {
|
||||
precompute_td += h.difficulty();
|
||||
HeaderRecord { block_hash: h.hash_slow(), total_difficulty: precompute_td }
|
||||
// Extract first 4 bytes of last block's state root as historical identifier
|
||||
let historical_root = headers
|
||||
.last()
|
||||
.map(|header| {
|
||||
let state_root = header.state_root();
|
||||
[state_root[0], state_root[1], state_root[2], state_root[3]]
|
||||
})
|
||||
.collect();
|
||||
let accumulator = Accumulator::from_header_records(&header_records)
|
||||
.map_err(|e| eyre!("Failed to compute accumulator: {e}"))?;
|
||||
let file_hash: [u8; 4] = accumulator.root[..4].try_into().unwrap();
|
||||
.unwrap_or([0u8; 4]);
|
||||
|
||||
let era1_id =
|
||||
Era1Id::new(&config.network, start_block, block_count as u32).with_hash(file_hash);
|
||||
let era1_id = Era1Id::new(&config.network, start_block, block_count as u32)
|
||||
.with_hash(historical_root);
|
||||
|
||||
let era1_id = if config.max_blocks_per_file == MAX_BLOCKS_PER_ERA1 as u64 {
|
||||
era1_id
|
||||
@@ -170,6 +166,7 @@ where
|
||||
let mut offsets = Vec::<i64>::with_capacity(block_count);
|
||||
let mut position = VERSION_ENTRY_SIZE as i64;
|
||||
let mut blocks_written = 0;
|
||||
let mut final_header_data = Vec::new();
|
||||
|
||||
for (i, header) in headers.into_iter().enumerate() {
|
||||
let expected_block_number = start_block + i as u64;
|
||||
@@ -181,6 +178,11 @@ where
|
||||
&mut total_difficulty,
|
||||
)?;
|
||||
|
||||
// Save last block's header data for accumulator
|
||||
if expected_block_number == end_block {
|
||||
final_header_data = compressed_header.data.clone();
|
||||
}
|
||||
|
||||
let difficulty = TotalDifficulty::new(total_difficulty);
|
||||
|
||||
let header_size = compressed_header.data.len() + ENTRY_HEADER_SIZE;
|
||||
@@ -216,12 +218,10 @@ where
|
||||
}
|
||||
}
|
||||
if blocks_written > 0 {
|
||||
// Convert absolute offsets to relative (measured from block-index entry start)
|
||||
let accumulator_entry_size = (ENTRY_HEADER_SIZE + 32) as i64;
|
||||
let block_index_position = position + accumulator_entry_size;
|
||||
let relative_offsets: Vec<i64> =
|
||||
offsets.iter().map(|&abs| abs - block_index_position).collect();
|
||||
let block_index = BlockIndex::new(start_block, relative_offsets);
|
||||
let accumulator_hash =
|
||||
B256::from_slice(&final_header_data[0..32.min(final_header_data.len())]);
|
||||
let accumulator = Accumulator::new(accumulator_hash);
|
||||
let block_index = BlockIndex::new(start_block, offsets);
|
||||
|
||||
writer.write_accumulator(&accumulator)?;
|
||||
writer.write_block_index(&block_index)?;
|
||||
@@ -310,9 +310,7 @@ where
|
||||
|
||||
let compressed_header = CompressedHeader::from_header(&header)?;
|
||||
let compressed_body = CompressedBody::from_body(&body)?;
|
||||
let receipts_with_bloom: Vec<_> =
|
||||
receipts.iter().map(|r| TxReceipt::with_bloom_ref(r)).collect();
|
||||
let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts_with_bloom)
|
||||
let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts)
|
||||
.map_err(|e| eyre!("Failed to compress receipts: {}", e))?;
|
||||
|
||||
Ok((compressed_header, compressed_body, compressed_receipts))
|
||||
|
||||
@@ -24,7 +24,6 @@ snap.workspace = true
|
||||
# ssz encoding and decoding
|
||||
ethereum_ssz.workspace = true
|
||||
ethereum_ssz_derive.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
eyre.workspace = true
|
||||
|
||||
@@ -76,7 +76,6 @@ use crate::{
|
||||
use alloy_consensus::{Block, BlockBody, Header};
|
||||
use alloy_primitives::{B256, U256};
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use sha2::{Digest, Sha256};
|
||||
use snap::{read::FrameDecoder, write::FrameEncoder};
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
@@ -494,73 +493,6 @@ impl Accumulator {
|
||||
|
||||
Ok(Self { root: B256::from(root) })
|
||||
}
|
||||
|
||||
/// Compute the accumulator from a list of header records.
|
||||
///
|
||||
/// Implements `hash_tree_root(List[HeaderRecord, 8192])` per the ERA1 spec:
|
||||
/// - Each leaf is `sha256(block_hash || total_difficulty_le_bytes32)`
|
||||
/// - Leaves are padded to `MAX_BLOCKS_PER_ERA1` (8192) with zero hashes
|
||||
/// - Binary Merkle tree is computed bottom-up
|
||||
/// - Final root is `sha256(merkle_root || le_bytes32(actual_count))`
|
||||
///
|
||||
/// Returns `Err` if `records` exceeds [`MAX_BLOCKS_PER_ERA1`].
|
||||
pub fn from_header_records(records: &[HeaderRecord]) -> Result<Self, E2sError> {
|
||||
let capacity = MAX_BLOCKS_PER_ERA1;
|
||||
|
||||
if records.len() > capacity {
|
||||
return Err(E2sError::Ssz(format!(
|
||||
"Too many header records: got {}, max {}",
|
||||
records.len(),
|
||||
capacity
|
||||
)));
|
||||
}
|
||||
|
||||
// Compute leaf hash for each header record
|
||||
let mut leaves = Vec::with_capacity(capacity);
|
||||
for record in records {
|
||||
let mut data = [0u8; 64];
|
||||
data[..32].copy_from_slice(record.block_hash.as_slice());
|
||||
data[32..].copy_from_slice(&record.total_difficulty.to_le_bytes::<32>());
|
||||
leaves.push(<[u8; 32]>::from(Sha256::digest(data)));
|
||||
}
|
||||
|
||||
// Pad to capacity with zero hashes
|
||||
leaves.resize(capacity, [0u8; 32]);
|
||||
|
||||
// Binary Merkle tree bottom-up (capacity is always a power of two)
|
||||
while leaves.len() > 1 {
|
||||
let mut next_level = Vec::with_capacity(leaves.len() / 2);
|
||||
for pair in leaves.chunks_exact(2) {
|
||||
let mut data = [0u8; 64];
|
||||
data[..32].copy_from_slice(&pair[0]);
|
||||
data[32..].copy_from_slice(&pair[1]);
|
||||
next_level.push(<[u8; 32]>::from(Sha256::digest(data)));
|
||||
}
|
||||
leaves = next_level;
|
||||
}
|
||||
|
||||
let merkle_root = leaves[0];
|
||||
|
||||
// mix_in_length: sha256(merkle_root || le_bytes32(actual_length))
|
||||
let mut mix = [0u8; 64];
|
||||
mix[..32].copy_from_slice(&merkle_root);
|
||||
let length = records.len() as u64;
|
||||
mix[32..40].copy_from_slice(&length.to_le_bytes());
|
||||
// remaining bytes stay zero (uint256 LE padding)
|
||||
|
||||
Ok(Self { root: B256::from(<[u8; 32]>::from(Sha256::digest(mix))) })
|
||||
}
|
||||
}
|
||||
|
||||
/// A header record used to compute the ERA1 accumulator.
|
||||
///
|
||||
/// Per the ERA1 spec: `header-record := { block-hash: Bytes32, total-difficulty: Uint256 }`
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HeaderRecord {
|
||||
/// The block hash (keccak256 of RLP-encoded header)
|
||||
pub block_hash: B256,
|
||||
/// The cumulative total difficulty at this block
|
||||
pub total_difficulty: U256,
|
||||
}
|
||||
|
||||
/// A block tuple in an Era1 file, containing all components for a single block
|
||||
@@ -759,44 +691,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accumulator_from_header_records_known_vectors() {
|
||||
// Known-answer vectors computed from the SSZ spec:
|
||||
// hash_tree_root(List[HeaderRecord, 8192])
|
||||
let expected_empty: B256 =
|
||||
"4a8c3a07c8d23adc5bac61157555c3c784d53d9bc110c1370809bd23cd93777d".parse().unwrap();
|
||||
let expected_single_zero: B256 =
|
||||
"81fd641249670887a731386e756a7a1538dc781b1b0bf016889045d350812817".parse().unwrap();
|
||||
let expected_single_nonzero: B256 =
|
||||
"ada35c48d81117f4fd588554cd4c4752356336e84cb41106dea1ceb4cfac8799".parse().unwrap();
|
||||
|
||||
// Empty list
|
||||
let acc_empty = Accumulator::from_header_records(&[]).unwrap();
|
||||
assert_eq!(acc_empty.root, expected_empty);
|
||||
|
||||
// Single record with zero values
|
||||
let records = vec![HeaderRecord { block_hash: B256::ZERO, total_difficulty: U256::ZERO }];
|
||||
let acc = Accumulator::from_header_records(&records).unwrap();
|
||||
assert_eq!(acc.root, expected_single_zero);
|
||||
|
||||
// Single record with non-zero values
|
||||
let records2 = vec![HeaderRecord {
|
||||
block_hash: B256::from([1u8; 32]),
|
||||
total_difficulty: U256::from(100u64),
|
||||
}];
|
||||
let acc2 = Accumulator::from_header_records(&records2).unwrap();
|
||||
assert_eq!(acc2.root, expected_single_nonzero);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accumulator_rejects_oversized_input() {
|
||||
let records = vec![
|
||||
HeaderRecord { block_hash: B256::ZERO, total_difficulty: U256::ZERO };
|
||||
MAX_BLOCKS_PER_ERA1 + 1
|
||||
];
|
||||
assert!(Accumulator::from_header_records(&records).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receipt_list_compression() {
|
||||
let receipts = create_test_receipts();
|
||||
|
||||
@@ -102,8 +102,8 @@ pub struct Era1Id {
|
||||
/// Number of blocks in the file
|
||||
pub block_count: u32,
|
||||
|
||||
/// Optional hash identifier for this file.
|
||||
/// First 4 bytes of the accumulator root hash.
|
||||
/// Optional hash identifier for this file
|
||||
/// First 4 bytes of the last historical root in the last state in the era file
|
||||
pub hash: Option<[u8; 4]>,
|
||||
|
||||
/// Whether to include era count in filename
|
||||
|
||||
@@ -491,6 +491,7 @@ mod tests {
|
||||
fn parse_env_filter_directives() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
|
||||
unsafe { std::env::set_var("RUST_LOG", "info,evm=debug") };
|
||||
let reth = Cli::try_parse_args_from([
|
||||
"reth",
|
||||
"init",
|
||||
|
||||
@@ -109,8 +109,13 @@ where
|
||||
result: &BlockExecutionResult<N::Receipt>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
) -> Result<(), ConsensusError> {
|
||||
let res =
|
||||
validate_block_post_execution(block, &self.chain_spec, result, receipt_root_bloom);
|
||||
let res = validate_block_post_execution(
|
||||
block,
|
||||
&self.chain_spec,
|
||||
&result.receipts,
|
||||
&result.requests,
|
||||
receipt_root_bloom,
|
||||
);
|
||||
|
||||
if self.skip_requests_hash_check &&
|
||||
let Err(ConsensusError::BodyRequestsHashDiff(_)) = &res
|
||||
|
||||
@@ -1,10 +1,9 @@
|
||||
use alloc::vec::Vec;
|
||||
use alloy_consensus::{proofs::calculate_receipt_root, BlockHeader, TxReceipt};
|
||||
use alloy_eips::Encodable2718;
|
||||
use alloy_eips::{eip7685::Requests, Encodable2718};
|
||||
use alloy_primitives::{Bloom, Bytes, B256};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_consensus::ConsensusError;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{
|
||||
receipt::gas_spent_by_transactions, Block, GotExpected, Receipt, RecoveredBlock,
|
||||
};
|
||||
@@ -19,7 +18,8 @@ use reth_primitives_traits::{
|
||||
pub fn validate_block_post_execution<B, R, ChainSpec>(
|
||||
block: &RecoveredBlock<B>,
|
||||
chain_spec: &ChainSpec,
|
||||
result: &BlockExecutionResult<R>,
|
||||
receipts: &[R],
|
||||
requests: &Requests,
|
||||
receipt_root_bloom: Option<(B256, Bloom)>,
|
||||
) -> Result<(), ConsensusError>
|
||||
where
|
||||
@@ -28,10 +28,12 @@ where
|
||||
ChainSpec: EthereumHardforks,
|
||||
{
|
||||
// Check if gas used matches the value set in header.
|
||||
if block.header().gas_used() != result.gas_used {
|
||||
let cumulative_gas_used =
|
||||
receipts.last().map(|receipt| receipt.cumulative_gas_used()).unwrap_or(0);
|
||||
if block.header().gas_used() != cumulative_gas_used {
|
||||
return Err(ConsensusError::BlockGasUsed {
|
||||
gas: GotExpected { got: result.gas_used, expected: block.header().gas_used() },
|
||||
gas_spent_by_tx: gas_spent_by_transactions(&result.receipts),
|
||||
gas: GotExpected { got: cumulative_gas_used, expected: block.header().gas_used() },
|
||||
gas_spent_by_tx: gas_spent_by_transactions(receipts),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -40,7 +42,7 @@ where
|
||||
// transaction This was replaced with is_success flag.
|
||||
// See more about EIP here: https://eips.ethereum.org/EIPS/eip-658
|
||||
if chain_spec.is_byzantium_active_at_block(block.header().number()) {
|
||||
let res = if let Some((receipts_root, logs_bloom)) = receipt_root_bloom {
|
||||
let result = if let Some((receipts_root, logs_bloom)) = receipt_root_bloom {
|
||||
compare_receipts_root_and_logs_bloom(
|
||||
receipts_root,
|
||||
logs_bloom,
|
||||
@@ -48,16 +50,11 @@ where
|
||||
block.header().logs_bloom(),
|
||||
)
|
||||
} else {
|
||||
verify_receipts(
|
||||
block.header().receipts_root(),
|
||||
block.header().logs_bloom(),
|
||||
&result.receipts,
|
||||
)
|
||||
verify_receipts(block.header().receipts_root(), block.header().logs_bloom(), receipts)
|
||||
};
|
||||
|
||||
if let Err(error) = res {
|
||||
let receipts = result
|
||||
.receipts
|
||||
if let Err(error) = result {
|
||||
let receipts = receipts
|
||||
.iter()
|
||||
.map(|r| Bytes::from(r.with_bloom_ref().encoded_2718()))
|
||||
.collect::<Vec<_>>();
|
||||
@@ -71,7 +68,7 @@ where
|
||||
let Some(header_requests_hash) = block.header().requests_hash() else {
|
||||
return Err(ConsensusError::RequestsHashMissing)
|
||||
};
|
||||
let requests_hash = result.requests.requests_hash();
|
||||
let requests_hash = requests.requests_hash();
|
||||
if requests_hash != header_requests_hash {
|
||||
return Err(ConsensusError::BodyRequestsHashDiff(
|
||||
GotExpected::new(requests_hash, header_requests_hash).into(),
|
||||
|
||||
@@ -195,7 +195,6 @@ where
|
||||
ommers: &block.body().ommers,
|
||||
withdrawals: block.body().withdrawals.as_ref().map(|w| Cow::Borrowed(w.as_slice())),
|
||||
extra_data: block.header().extra_data.clone(),
|
||||
slot_number: block.header().slot_number,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -211,7 +210,6 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: attributes.withdrawals.map(|w| Cow::Owned(w.into_inner())),
|
||||
extra_data: attributes.extra_data,
|
||||
slot_number: attributes.slot_number,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -276,7 +274,7 @@ where
|
||||
gas_limit: payload.payload.gas_limit(),
|
||||
basefee: payload.payload.saturated_base_fee_per_gas(),
|
||||
blob_excess_gas_and_price,
|
||||
slot_num: payload.payload.as_v4().map(|v4| v4.slot_number).unwrap_or_default(),
|
||||
slot_num: 0,
|
||||
};
|
||||
|
||||
Ok(EvmEnv { cfg_env, block_env })
|
||||
@@ -293,7 +291,6 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: payload.payload.withdrawals().map(|w| Cow::Borrowed(w.as_slice())),
|
||||
extra_data: payload.payload.as_v1().extra_data.clone(),
|
||||
slot_number: payload.payload.as_v4().map(|v4| v4.slot_number),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -37,6 +37,7 @@ reth-rpc-eth-types.workspace = true
|
||||
reth-engine-local.workspace = true
|
||||
reth-engine-primitives = { workspace = true, features = ["std"] }
|
||||
reth-payload-primitives.workspace = true
|
||||
|
||||
# ethereum
|
||||
alloy-eips.workspace = true
|
||||
alloy-network.workspace = true
|
||||
@@ -48,7 +49,7 @@ tokio.workspace = true
|
||||
|
||||
# revm with required ethereum features
|
||||
# Note: this must be kept to ensure all features are properly enabled/forwarded
|
||||
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit", "p256-aws-lc-rs"] }
|
||||
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] }
|
||||
|
||||
# misc
|
||||
eyre.workspace = true
|
||||
|
||||
@@ -26,10 +26,9 @@ use reth_node_builder::{
|
||||
},
|
||||
node::{FullNodeTypes, NodeTypes},
|
||||
rpc::{
|
||||
BasicEngineApiBuilder, BasicEngineValidatorBuilder, Either, EngineApiBuilder,
|
||||
EngineValidatorAddOn, EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity,
|
||||
PayloadValidatorBuilder, RethAuthHttpMiddleware, RethRpcAddOns, RethRpcMiddleware,
|
||||
RpcAddOns, RpcHandle, Stack,
|
||||
BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn,
|
||||
EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity, PayloadValidatorBuilder,
|
||||
RethRpcAddOns, RpcAddOns, RpcHandle,
|
||||
},
|
||||
BuilderContext, DebugNode, Node, NodeAdapter,
|
||||
};
|
||||
@@ -40,7 +39,7 @@ use reth_rpc::{
|
||||
TestingApi, ValidationApi,
|
||||
};
|
||||
use reth_rpc_api::servers::{BlockSubmissionValidationApiServer, TestingApiServer};
|
||||
use reth_rpc_builder::config::RethRpcServerConfig;
|
||||
use reth_rpc_builder::{config::RethRpcServerConfig, middleware::RethRpcMiddleware};
|
||||
use reth_rpc_eth_api::{
|
||||
helpers::{
|
||||
config::{EthConfigApiServer, EthConfigHandler},
|
||||
@@ -166,21 +165,17 @@ pub struct EthereumAddOns<
|
||||
EB = BasicEngineApiBuilder<PVB>,
|
||||
EVB = BasicEngineValidatorBuilder<PVB>,
|
||||
RpcMiddleware = Identity,
|
||||
AuthHttpMiddleware = Identity,
|
||||
> {
|
||||
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
|
||||
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>,
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
EthB: EthApiBuilder<N>,
|
||||
{
|
||||
/// Creates a new instance from the inner `RpcAddOns`.
|
||||
pub const fn new(
|
||||
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
|
||||
) -> Self {
|
||||
pub const fn new(inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
@@ -204,13 +199,11 @@ where
|
||||
BasicEngineApiBuilder::default(),
|
||||
BasicEngineValidatorBuilder::default(),
|
||||
Default::default(),
|
||||
Identity::new(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
EthB: EthApiBuilder<N>,
|
||||
@@ -219,7 +212,7 @@ where
|
||||
pub fn with_engine_api<T>(
|
||||
self,
|
||||
engine_api_builder: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware>
|
||||
where
|
||||
T: Send,
|
||||
{
|
||||
@@ -231,7 +224,7 @@ where
|
||||
pub fn with_payload_validator<V, T>(
|
||||
self,
|
||||
payload_validator_builder: T,
|
||||
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware, AuthHttpMiddleware> {
|
||||
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware> {
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.with_payload_validator(payload_validator_builder))
|
||||
}
|
||||
@@ -240,7 +233,7 @@ where
|
||||
pub fn with_rpc_middleware<T>(
|
||||
self,
|
||||
rpc_middleware: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T, AuthHttpMiddleware>
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T>
|
||||
where
|
||||
T: Send,
|
||||
{
|
||||
@@ -248,45 +241,6 @@ where
|
||||
EthereumAddOns::new(inner.with_rpc_middleware(rpc_middleware))
|
||||
}
|
||||
|
||||
/// Configures the HTTP transport middleware for the auth / Engine API server.
|
||||
pub fn with_auth_http_middleware<T>(
|
||||
self,
|
||||
auth_http_middleware: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, T>
|
||||
where
|
||||
T: Send,
|
||||
{
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.with_auth_http_middleware(auth_http_middleware))
|
||||
}
|
||||
|
||||
/// Stacks an additional HTTP transport middleware layer for the auth / Engine API server.
|
||||
pub fn layer_auth_http_middleware<T>(
|
||||
self,
|
||||
layer: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, Stack<AuthHttpMiddleware, T>> {
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.layer_auth_http_middleware(layer))
|
||||
}
|
||||
|
||||
/// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server.
|
||||
#[expect(clippy::type_complexity)]
|
||||
pub fn option_layer_auth_http_middleware<T>(
|
||||
self,
|
||||
layer: Option<T>,
|
||||
) -> EthereumAddOns<
|
||||
N,
|
||||
EthB,
|
||||
PVB,
|
||||
EB,
|
||||
EVB,
|
||||
RpcMiddleware,
|
||||
Stack<AuthHttpMiddleware, Either<T, Identity>>,
|
||||
> {
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.option_layer_auth_http_middleware(layer))
|
||||
}
|
||||
|
||||
/// Sets the tokio runtime for the RPC servers.
|
||||
///
|
||||
/// Caution: This runtime must not be created from within asynchronous context.
|
||||
@@ -296,8 +250,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> NodeAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> NodeAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
where
|
||||
N: FullNodeComponents<
|
||||
Types: NodeTypes<
|
||||
@@ -314,7 +268,6 @@ where
|
||||
EthApiError: FromEvmError<N::Evm>,
|
||||
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
|
||||
{
|
||||
type Handle = RpcHandle<N, EthB::EthApi>;
|
||||
|
||||
@@ -370,8 +323,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> RethRpcAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> RethRpcAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
where
|
||||
N: FullNodeComponents<
|
||||
Types: NodeTypes<
|
||||
@@ -388,7 +341,6 @@ where
|
||||
EthApiError: FromEvmError<N::Evm>,
|
||||
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
|
||||
{
|
||||
type EthApi = EthB::EthApi;
|
||||
|
||||
@@ -397,8 +349,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> EngineValidatorAddOn<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EngineValidatorAddOn<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
where
|
||||
N: FullNodeComponents<
|
||||
Types: NodeTypes<
|
||||
@@ -415,7 +367,6 @@ where
|
||||
EthApiError: FromEvmError<N::Evm>,
|
||||
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
|
||||
RpcMiddleware: Send,
|
||||
AuthHttpMiddleware: Send,
|
||||
{
|
||||
type ValidatorBuilder = EVB;
|
||||
|
||||
|
||||
@@ -25,7 +25,7 @@ use reth_evm::{
|
||||
ConfigureEvm, Evm, NextBlockEnvAttributes,
|
||||
};
|
||||
use reth_evm_ethereum::EthEvmConfig;
|
||||
use reth_execution_cache::{CachedStateMetrics, CachedStateMetricsSource, CachedStateProvider};
|
||||
use reth_execution_cache::CachedStateProvider;
|
||||
use reth_payload_builder::{BlobSidecars, EthBuiltPayload};
|
||||
use reth_payload_builder_primitives::PayloadBuilderError;
|
||||
use reth_payload_primitives::PayloadAttributes;
|
||||
@@ -172,19 +172,12 @@ where
|
||||
state_provider = Box::new(CachedStateProvider::new(
|
||||
state_provider,
|
||||
execution_cache.cache().clone(),
|
||||
// It's ok to recreate the cache every time, because it's cheap to do so for a vanilla
|
||||
// Ethereum builder every 12s.
|
||||
CachedStateMetrics::zeroed(CachedStateMetricsSource::Builder),
|
||||
execution_cache.metrics().clone(),
|
||||
));
|
||||
}
|
||||
let state = StateProviderDatabase::new(state_provider.as_ref());
|
||||
let chain_spec = client.chain_spec();
|
||||
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(attributes.timestamp());
|
||||
let mut db = State::builder()
|
||||
.with_database(cached_reads.as_db_mut(state))
|
||||
.with_bundle_update()
|
||||
.with_bal_builder_if(is_amsterdam)
|
||||
.build();
|
||||
let mut db =
|
||||
State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build();
|
||||
|
||||
let mut builder = evm_config
|
||||
.builder_for_next_block(
|
||||
@@ -203,6 +196,8 @@ where
|
||||
)
|
||||
.map_err(PayloadBuilderError::other)?;
|
||||
|
||||
let chain_spec = client.chain_spec();
|
||||
|
||||
debug!(target: "payload_builder", id=%payload_id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload");
|
||||
let mut cumulative_gas_used = 0;
|
||||
let block_gas_limit: u64 = builder.evm_mut().block().gas_limit();
|
||||
@@ -362,25 +357,6 @@ where
|
||||
}
|
||||
continue
|
||||
}
|
||||
// EIP-7778: the executor tracks gas_before_refund while the payload builder's
|
||||
// pre-check uses gas_after_refund. Near-full blocks can pass the pre-check but
|
||||
// fail the executor's check. Skip the tx and continue building.
|
||||
Err(BlockExecutionError::Validation(
|
||||
BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas {
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
},
|
||||
)) => {
|
||||
trace!(target: "payload_builder", %transaction_gas_limit, %block_available_gas, ?tx_hash, "skipping transaction exceeding block gas limit");
|
||||
best_txs.mark_invalid(
|
||||
&pool_tx,
|
||||
&InvalidPoolTransactionError::ExceedsGasLimit(
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
),
|
||||
);
|
||||
continue
|
||||
}
|
||||
// this is an error that we should treat as fatal for this attempt
|
||||
Err(err) => return Err(PayloadBuilderError::evm(err)),
|
||||
};
|
||||
|
||||
@@ -225,7 +225,7 @@ impl Discv5 {
|
||||
bootstrap_lookup_interval,
|
||||
bootstrap_lookup_countdown,
|
||||
metrics.clone(),
|
||||
Arc::downgrade(&discv5),
|
||||
discv5.clone(),
|
||||
);
|
||||
|
||||
Ok((
|
||||
@@ -248,7 +248,7 @@ impl Discv5 {
|
||||
discv5::Event::SocketUpdated(_) | discv5::Event::TalkRequest(_) |
|
||||
// `Discovered` not unique discovered peers
|
||||
discv5::Event::Discovered(_) => None,
|
||||
discv5::Event::NodeInserted { .. } => {
|
||||
discv5::Event::NodeInserted { replaced: _, .. } => {
|
||||
|
||||
// node has been inserted into kbuckets
|
||||
|
||||
@@ -573,19 +573,14 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
bootstrap_lookup_interval: u64,
|
||||
bootstrap_lookup_countdown: u64,
|
||||
metrics: Discv5Metrics,
|
||||
discv5: std::sync::Weak<discv5::Discv5>,
|
||||
discv5: Arc<discv5::Discv5>,
|
||||
) {
|
||||
let local_node_id = discv5.local_enr().node_id();
|
||||
let lookup_interval = Duration::from_secs(lookup_interval);
|
||||
let metrics = metrics.discovered_peers;
|
||||
let mut kbucket_index = MAX_KBUCKET_INDEX;
|
||||
let pulse_lookup_interval = Duration::from_secs(bootstrap_lookup_interval);
|
||||
task::spawn(async move {
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let local_node_id = discv5_handle.local_enr().node_id();
|
||||
drop(discv5_handle);
|
||||
|
||||
// make many fast lookup queries at bootstrap, trying to fill kbuckets at furthest
|
||||
// log2distance from local node
|
||||
for i in (0..bootstrap_lookup_countdown).rev() {
|
||||
@@ -598,12 +593,7 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
"starting bootstrap boost lookup query"
|
||||
);
|
||||
|
||||
{
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
lookup(target, &discv5_handle, &metrics).await;
|
||||
}
|
||||
lookup(target, &discv5, &metrics).await;
|
||||
|
||||
tokio::time::sleep(pulse_lookup_interval).await;
|
||||
}
|
||||
@@ -620,12 +610,7 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
"starting periodic lookup query"
|
||||
);
|
||||
|
||||
{
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
lookup(target, &discv5_handle, &metrics).await;
|
||||
}
|
||||
lookup(target, &discv5, &metrics).await;
|
||||
|
||||
if kbucket_index > DEFAULT_MIN_TARGET_KBUCKET_INDEX {
|
||||
// try to populate bucket one step closer
|
||||
@@ -713,10 +698,8 @@ mod test {
|
||||
use ::enr::{CombinedKey, EnrKey};
|
||||
use rand_08::thread_rng;
|
||||
use reth_chainspec::MAINNET;
|
||||
use std::{
|
||||
net::UdpSocket,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use reth_tracing::init_test_tracing;
|
||||
use std::env;
|
||||
use tracing::trace;
|
||||
|
||||
fn discv5_noop() -> Discv5 {
|
||||
@@ -755,61 +738,6 @@ mod test {
|
||||
Discv5::start(&secret_key, discv5_config).await.expect("should build discv5")
|
||||
}
|
||||
|
||||
async fn start_discovery_node_with_key(
|
||||
secret_key: &SecretKey,
|
||||
udp_port_discv5: u16,
|
||||
) -> Result<(Discv5, mpsc::Receiver<discv5::Event>), Error> {
|
||||
let discv5_addr: SocketAddr = format!("127.0.0.1:{udp_port_discv5}").parse().unwrap();
|
||||
let rlpx_addr: SocketAddr = "127.0.0.1:30303".parse().unwrap();
|
||||
|
||||
let discv5_listen_config = ListenConfig::from(discv5_addr);
|
||||
let discv5_config = Config::builder(rlpx_addr)
|
||||
.discv5_config(discv5::ConfigBuilder::new(discv5_listen_config).build())
|
||||
.build();
|
||||
|
||||
Discv5::start(secret_key, discv5_config).await
|
||||
}
|
||||
|
||||
fn unused_udp_port() -> u16 {
|
||||
UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
|
||||
}
|
||||
|
||||
async fn wait_for_udp_port_release(port: u16, timeout: Duration) {
|
||||
let deadline = Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
match UdpSocket::bind(("127.0.0.1", port)) {
|
||||
Ok(socket) => {
|
||||
drop(socket);
|
||||
return;
|
||||
}
|
||||
Err(err) if Instant::now() < deadline => {
|
||||
trace!(target: "net::discv5::test", %port, %err, "waiting for discv5 port release");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => panic!("discv5 did not release port {port} before timeout: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discv5_releases_port_on_drop() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let secret_key = SecretKey::new(&mut thread_rng());
|
||||
let port = unused_udp_port();
|
||||
|
||||
let (node, updates) =
|
||||
start_discovery_node_with_key(&secret_key, port).await.expect("should start discv5");
|
||||
drop(updates);
|
||||
drop(node);
|
||||
|
||||
wait_for_udp_port_release(port, Duration::from_secs(1)).await;
|
||||
|
||||
let restarted = start_discovery_node_with_key(&secret_key, port).await;
|
||||
assert!(restarted.is_ok(), "discv5 failed to rebind dropped port: {restarted:?}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discv5() {
|
||||
reth_tracing::init_test_tracing();
|
||||
@@ -1009,6 +937,11 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn get_fork_id_with_different_network_stack_ids() {
|
||||
unsafe {
|
||||
env::set_var("RUST_LOG", "net::discv5=trace");
|
||||
}
|
||||
init_test_tracing();
|
||||
|
||||
let fork_id = MAINNET.latest_fork_id();
|
||||
let sk = SecretKey::new(&mut thread_rng());
|
||||
|
||||
|
||||
@@ -20,7 +20,6 @@ reth-primitives-traits.workspace = true
|
||||
|
||||
# ethereum
|
||||
alloy-chains = { workspace = true, features = ["rlp"] }
|
||||
alloy-eip7928 = { workspace = true, features = ["rlp"], optional = true }
|
||||
alloy-eips.workspace = true
|
||||
alloy-primitives = { workspace = true, features = ["map"] }
|
||||
alloy-rlp = { workspace = true, features = ["derive"] }
|
||||
@@ -41,7 +40,6 @@ alloy-hardforks.workspace = true
|
||||
reth-ethereum-primitives = { workspace = true, features = ["arbitrary"] }
|
||||
alloy-primitives = { workspace = true, features = ["arbitrary", "rand"] }
|
||||
alloy-consensus = { workspace = true, features = ["arbitrary"] }
|
||||
alloy-eip7928 = { workspace = true, features = ["arbitrary", "rlp"] }
|
||||
alloy-eips = { workspace = true, features = ["arbitrary"] }
|
||||
alloy-genesis.workspace = true
|
||||
alloy-chains = { workspace = true, features = ["arbitrary"] }
|
||||
@@ -55,7 +53,6 @@ default = ["std"]
|
||||
std = [
|
||||
"alloy-chains/std",
|
||||
"alloy-consensus/std",
|
||||
"alloy-eip7928?/std",
|
||||
"alloy-eips/std",
|
||||
"alloy-genesis/std",
|
||||
"alloy-primitives/std",
|
||||
@@ -71,8 +68,6 @@ std = [
|
||||
arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
"alloy-chains/arbitrary",
|
||||
"dep:alloy-eip7928",
|
||||
"alloy-eip7928/arbitrary",
|
||||
"dep:arbitrary",
|
||||
"dep:proptest",
|
||||
"dep:proptest-arbitrary-interop",
|
||||
@@ -93,5 +88,4 @@ serde = [
|
||||
"reth-primitives-traits/serde",
|
||||
"reth-ethereum-primitives/serde",
|
||||
"alloy-hardforks/serde",
|
||||
"alloy-eip7928?/serde",
|
||||
]
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
use alloc::vec::Vec;
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_rlp::{BufMut, Decodable, Encodable, Header, RlpDecodableWrapper, RlpEncodableWrapper};
|
||||
use alloy_rlp::{RlpDecodableWrapper, RlpEncodableWrapper};
|
||||
use reth_codecs_derive::add_arbitrary_tests;
|
||||
|
||||
/// A request for block access lists from the given block hashes.
|
||||
@@ -16,209 +16,12 @@ pub struct GetBlockAccessLists(
|
||||
);
|
||||
|
||||
/// Response for [`GetBlockAccessLists`] containing one BAL per requested block hash.
|
||||
///
|
||||
/// The inner `Bytes` values store raw BAL RLP payloads and are encoded as nested RLP items, not
|
||||
/// as RLP byte strings.
|
||||
#[derive(Clone, Debug, PartialEq, Eq, Default)]
|
||||
#[derive(Clone, Debug, PartialEq, Eq, RlpEncodableWrapper, RlpDecodableWrapper, Default)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct BlockAccessLists(
|
||||
/// The requested block access lists as raw RLP blobs. Per EIP-8159, unavailable entries are
|
||||
/// represented by an RLP-encoded empty list (`0xc0`).
|
||||
/// The requested block access lists as opaque bytes. Unavailable entries are represented by
|
||||
/// empty byte slices.
|
||||
pub Vec<Bytes>,
|
||||
);
|
||||
|
||||
impl Encodable for BlockAccessLists {
|
||||
fn encode(&self, out: &mut dyn BufMut) {
|
||||
let payload_length = self.0.iter().map(|bytes| bytes.len()).sum();
|
||||
Header { list: true, payload_length }.encode(out);
|
||||
for bal in &self.0 {
|
||||
out.put_slice(bal);
|
||||
}
|
||||
}
|
||||
|
||||
fn length(&self) -> usize {
|
||||
let payload_length = self.0.iter().map(|bytes| bytes.len()).sum();
|
||||
Header { list: true, payload_length }.length_with_payload()
|
||||
}
|
||||
}
|
||||
|
||||
impl Decodable for BlockAccessLists {
|
||||
fn decode(buf: &mut &[u8]) -> alloy_rlp::Result<Self> {
|
||||
let header = Header::decode(buf)?;
|
||||
if !header.list {
|
||||
return Err(alloy_rlp::Error::UnexpectedString)
|
||||
}
|
||||
|
||||
let (mut payload, rest) = buf.split_at(header.payload_length);
|
||||
*buf = rest;
|
||||
let mut bals = Vec::new();
|
||||
|
||||
while !payload.is_empty() {
|
||||
let item_start = payload;
|
||||
let item_header = Header::decode(&mut payload)?;
|
||||
if !item_header.list {
|
||||
return Err(alloy_rlp::Error::UnexpectedString)
|
||||
}
|
||||
|
||||
let item_length = item_header.length_with_payload();
|
||||
bals.push(Bytes::copy_from_slice(&item_start[..item_length]));
|
||||
payload = &payload[item_header.payload_length..];
|
||||
}
|
||||
|
||||
Ok(Self(bals))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(any(test, feature = "arbitrary"))]
|
||||
impl<'a> arbitrary::Arbitrary<'a> for BlockAccessLists {
|
||||
fn arbitrary(u: &mut arbitrary::Unstructured<'a>) -> arbitrary::Result<Self> {
|
||||
let entries = Vec::<Vec<alloy_eip7928::AccountChanges>>::arbitrary(u)?
|
||||
.into_iter()
|
||||
.map(|entry| {
|
||||
let mut out = Vec::new();
|
||||
alloy_rlp::encode_list(&entry, &mut out);
|
||||
Bytes::from(out)
|
||||
})
|
||||
.collect();
|
||||
Ok(Self(entries))
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eip7928::{
|
||||
AccountChanges, BalanceChange, CodeChange, NonceChange, SlotChanges, StorageChange,
|
||||
};
|
||||
use alloy_primitives::{Address, U256};
|
||||
use alloy_rlp::EMPTY_LIST_CODE;
|
||||
|
||||
fn elaborate_account_changes(seed: u8) -> Vec<AccountChanges> {
|
||||
vec![
|
||||
AccountChanges {
|
||||
address: Address::from([seed; 20]),
|
||||
storage_changes: vec![SlotChanges::new(
|
||||
U256::from_be_bytes([seed.wrapping_add(1); 32]),
|
||||
vec![
|
||||
StorageChange::new(1, U256::from_be_bytes([seed.wrapping_add(2); 32])),
|
||||
StorageChange::new(2, U256::from_be_bytes([seed.wrapping_add(3); 32])),
|
||||
],
|
||||
)],
|
||||
storage_reads: vec![
|
||||
U256::from_be_bytes([seed.wrapping_add(4); 32]),
|
||||
U256::from_be_bytes([seed.wrapping_add(5); 32]),
|
||||
],
|
||||
balance_changes: vec![
|
||||
BalanceChange::new(1, U256::from(1_000 + seed as u64)),
|
||||
BalanceChange::new(2, U256::from(2_000 + seed as u64)),
|
||||
],
|
||||
nonce_changes: vec![
|
||||
NonceChange::new(1, seed as u64),
|
||||
NonceChange::new(2, seed as u64 + 1),
|
||||
],
|
||||
code_changes: vec![CodeChange::new(
|
||||
1,
|
||||
Bytes::from(vec![0x60, seed, 0x61, seed.wrapping_add(1), 0x56]),
|
||||
)],
|
||||
},
|
||||
AccountChanges {
|
||||
address: Address::from([seed.wrapping_add(9); 20]),
|
||||
storage_changes: Vec::new(),
|
||||
storage_reads: vec![U256::from_be_bytes([seed.wrapping_add(10); 32])],
|
||||
balance_changes: vec![BalanceChange::new(3, U256::from(3_000 + seed as u64))],
|
||||
nonce_changes: vec![NonceChange::new(3, seed as u64 + 2)],
|
||||
code_changes: vec![CodeChange::new(2, Bytes::from(vec![0x5f, 0x5f, 0xf3]))],
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
fn elaborate_bal_entry(seed: u8) -> Bytes {
|
||||
let account_changes = elaborate_account_changes(seed);
|
||||
let mut out = Vec::new();
|
||||
alloy_rlp::encode_list(&account_changes, &mut out);
|
||||
Bytes::from(out)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_bal_entry_encodes_as_empty_list() {
|
||||
let encoded =
|
||||
alloy_rlp::encode(BlockAccessLists(vec![Bytes::from_static(&[EMPTY_LIST_CODE])]));
|
||||
assert_eq!(encoded, vec![0xc1, EMPTY_LIST_CODE]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn block_access_lists_roundtrip_preserves_raw_bal_items() {
|
||||
let original = BlockAccessLists(vec![
|
||||
Bytes::from_static(&[EMPTY_LIST_CODE]),
|
||||
Bytes::from_static(&[0xc1, EMPTY_LIST_CODE]),
|
||||
Bytes::from_static(&[0xc2, EMPTY_LIST_CODE, EMPTY_LIST_CODE]),
|
||||
]);
|
||||
|
||||
let encoded = alloy_rlp::encode(&original);
|
||||
let decoded = alloy_rlp::decode_exact::<BlockAccessLists>(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_response_roundtrips() {
|
||||
let original = BlockAccessLists(Vec::new());
|
||||
let encoded = alloy_rlp::encode(&original);
|
||||
let decoded = alloy_rlp::decode_exact::<BlockAccessLists>(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded, original);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn rejects_non_list_bal_entries() {
|
||||
let err = alloy_rlp::decode_exact::<BlockAccessLists>(&[0xc1, 0x01]).unwrap_err();
|
||||
assert!(matches!(err, alloy_rlp::Error::UnexpectedString));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elaborate_bal_entry_roundtrips_into_account_changes() {
|
||||
let expected = elaborate_account_changes(0x11);
|
||||
let decoded =
|
||||
alloy_rlp::decode_exact::<Vec<AccountChanges>>(&elaborate_bal_entry(0x11)).unwrap();
|
||||
|
||||
assert_eq!(decoded, expected);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elaborate_block_access_lists_roundtrip_preserves_complex_bal_contents() {
|
||||
let original = BlockAccessLists(vec![
|
||||
elaborate_bal_entry(0x11),
|
||||
Bytes::from_static(&[EMPTY_LIST_CODE]),
|
||||
elaborate_bal_entry(0x77),
|
||||
]);
|
||||
|
||||
let encoded = alloy_rlp::encode(&original);
|
||||
let decoded = alloy_rlp::decode_exact::<BlockAccessLists>(&encoded).unwrap();
|
||||
|
||||
assert_eq!(decoded, original);
|
||||
assert_eq!(
|
||||
alloy_rlp::decode_exact::<Vec<AccountChanges>>(&decoded.0[0]).unwrap(),
|
||||
elaborate_account_changes(0x11)
|
||||
);
|
||||
assert_eq!(alloy_rlp::decode_exact::<Vec<AccountChanges>>(&decoded.0[1]).unwrap(), vec![]);
|
||||
assert_eq!(
|
||||
alloy_rlp::decode_exact::<Vec<AccountChanges>>(&decoded.0[2]).unwrap(),
|
||||
elaborate_account_changes(0x77)
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn elaborate_block_access_lists_embed_raw_bal_payloads_without_reencoding() {
|
||||
let first = elaborate_bal_entry(0x21);
|
||||
let second = elaborate_bal_entry(0x42);
|
||||
let encoded = alloy_rlp::encode(BlockAccessLists(vec![first.clone(), second.clone()]));
|
||||
|
||||
let header = alloy_rlp::Header::decode(&mut &encoded[..]).unwrap();
|
||||
let payload = &encoded[header.length()..];
|
||||
let expected_payload = [first.as_ref(), second.as_ref()].concat();
|
||||
|
||||
assert!(header.list);
|
||||
assert_eq!(payload, expected_payload.as_slice());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! All capability related types
|
||||
|
||||
use crate::{EthMessageID, EthVersion, SnapVersion};
|
||||
use crate::{EthMessageID, EthVersion};
|
||||
use alloc::{borrow::Cow, string::String, vec::Vec};
|
||||
use alloy_primitives::bytes::Bytes;
|
||||
use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
|
||||
@@ -85,11 +85,6 @@ impl Capability {
|
||||
Self::new_static("eth", version as usize)
|
||||
}
|
||||
|
||||
/// Returns the corresponding snap capability for the given version.
|
||||
pub const fn snap(version: SnapVersion) -> Self {
|
||||
Self::new_static("snap", version as usize)
|
||||
}
|
||||
|
||||
/// Returns the [`EthVersion::Eth66`] capability.
|
||||
pub const fn eth_66() -> Self {
|
||||
Self::eth(EthVersion::Eth66)
|
||||
@@ -120,16 +115,6 @@ impl Capability {
|
||||
Self::eth(EthVersion::Eth71)
|
||||
}
|
||||
|
||||
/// Returns the `snap/1` capability.
|
||||
pub const fn snap_1() -> Self {
|
||||
Self::snap(SnapVersion::V1)
|
||||
}
|
||||
|
||||
/// Returns the `snap/2` capability.
|
||||
pub const fn snap_2() -> Self {
|
||||
Self::snap(SnapVersion::V2)
|
||||
}
|
||||
|
||||
/// Whether this is eth v66 protocol.
|
||||
#[inline]
|
||||
pub fn is_eth_v66(&self) -> bool {
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! `RLPx` disconnect reason sent to/received from peer
|
||||
|
||||
use alloc::vec;
|
||||
use alloy_primitives::bytes::{Buf, BufMut};
|
||||
use alloy_rlp::{Decodable, Encodable, Header};
|
||||
use derive_more::Display;
|
||||
@@ -83,10 +84,10 @@ impl Encodable for DisconnectReason {
|
||||
/// The [`Encodable`] implementation for [`DisconnectReason`] encodes the disconnect reason in
|
||||
/// a single-element RLP list.
|
||||
fn encode(&self, out: &mut dyn BufMut) {
|
||||
alloy_rlp::encode_list(&[*self as u8], out);
|
||||
vec![*self as u8].encode(out);
|
||||
}
|
||||
fn length(&self) -> usize {
|
||||
alloy_rlp::list_length(&[*self as u8])
|
||||
vec![*self as u8].length()
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -3,41 +3,13 @@
|
||||
//! facilitating the exchange of Ethereum state snapshots between peers
|
||||
//! Reference: [Ethereum Snapshot Protocol](https://github.com/ethereum/devp2p/blob/master/caps/snap.md#protocol-messages)
|
||||
//!
|
||||
//! This module currently includes snap/1 plus preparatory snap/2 message definitions.
|
||||
//! Current version: snap/1
|
||||
|
||||
use crate::BlockAccessLists;
|
||||
use alloc::vec::Vec;
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
|
||||
use reth_codecs_derive::add_arbitrary_tests;
|
||||
|
||||
/// Supported SNAP protocol versions.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[repr(u8)]
|
||||
pub enum SnapVersion {
|
||||
/// The original snapshot protocol.
|
||||
#[default]
|
||||
V1 = 1,
|
||||
/// BAL-based healing as proposed by EIP-8189.
|
||||
V2 = 2,
|
||||
}
|
||||
|
||||
impl SnapVersion {
|
||||
/// Returns the number of messages supported by this version.
|
||||
pub const fn message_count(self) -> u8 {
|
||||
match self {
|
||||
Self::V1 => 8,
|
||||
Self::V2 => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the highest supported message id for this version.
|
||||
pub const fn max_message_id(self) -> u8 {
|
||||
self.message_count() - 1
|
||||
}
|
||||
}
|
||||
|
||||
/// Message IDs for the snap sync protocol
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SnapMessageId {
|
||||
@@ -55,21 +27,9 @@ pub enum SnapMessageId {
|
||||
/// Response for the number of requested contract codes.
|
||||
ByteCodes = 0x05,
|
||||
/// Request of the number of state (either account or storage) Merkle trie nodes by path.
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
GetTrieNodes = 0x06,
|
||||
/// Response for the number of requested state trie nodes.
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
TrieNodes = 0x07,
|
||||
/// Request BALs for a list of block hashes.
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
GetBlockAccessLists = 0x08,
|
||||
/// Response containing BALs for the requested block hashes.
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
BlockAccessLists = 0x09,
|
||||
}
|
||||
|
||||
/// Request for a range of accounts from the state trie.
|
||||
@@ -227,30 +187,6 @@ pub struct TrieNodesMessage {
|
||||
pub nodes: Vec<Bytes>,
|
||||
}
|
||||
|
||||
/// Request BALs for the given block hashes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct GetBlockAccessListsMessage {
|
||||
/// Request ID to match up responses with.
|
||||
pub request_id: u64,
|
||||
/// Block hashes to retrieve BALs for.
|
||||
pub block_hashes: Vec<B256>,
|
||||
/// Soft limit at which to stop returning data (in bytes).
|
||||
pub response_bytes: u64,
|
||||
}
|
||||
|
||||
/// Response containing one BAL per requested block hash.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct BlockAccessListsMessage {
|
||||
/// ID of the request this is a response for.
|
||||
pub request_id: u64,
|
||||
/// Raw BAL payloads in request order.
|
||||
pub block_access_lists: BlockAccessLists,
|
||||
}
|
||||
|
||||
/// Represents all types of messages in the snap sync protocol.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum SnapProtocolMessage {
|
||||
@@ -267,21 +203,9 @@ pub enum SnapProtocolMessage {
|
||||
/// Response with contract codes - see [`ByteCodesMessage`]
|
||||
ByteCodes(ByteCodesMessage),
|
||||
/// Request for trie nodes - see [`GetTrieNodesMessage`]
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
GetTrieNodes(GetTrieNodesMessage),
|
||||
/// Response with trie nodes - see [`TrieNodesMessage`]
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
TrieNodes(TrieNodesMessage),
|
||||
/// Request for block access lists - see [`GetBlockAccessListsMessage`]
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
GetBlockAccessLists(GetBlockAccessListsMessage),
|
||||
/// Response with block access lists - see [`BlockAccessListsMessage`]
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
BlockAccessLists(BlockAccessListsMessage),
|
||||
}
|
||||
|
||||
impl SnapProtocolMessage {
|
||||
@@ -298,8 +222,6 @@ impl SnapProtocolMessage {
|
||||
Self::ByteCodes(_) => SnapMessageId::ByteCodes,
|
||||
Self::GetTrieNodes(_) => SnapMessageId::GetTrieNodes,
|
||||
Self::TrieNodes(_) => SnapMessageId::TrieNodes,
|
||||
Self::GetBlockAccessLists(_) => SnapMessageId::GetBlockAccessLists,
|
||||
Self::BlockAccessLists(_) => SnapMessageId::BlockAccessLists,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -319,8 +241,6 @@ impl SnapProtocolMessage {
|
||||
Self::ByteCodes(msg) => msg.encode(&mut buf),
|
||||
Self::GetTrieNodes(msg) => msg.encode(&mut buf),
|
||||
Self::TrieNodes(msg) => msg.encode(&mut buf),
|
||||
Self::GetBlockAccessLists(msg) => msg.encode(&mut buf),
|
||||
Self::BlockAccessLists(msg) => msg.encode(&mut buf),
|
||||
}
|
||||
|
||||
Bytes::from(buf)
|
||||
@@ -394,20 +314,6 @@ impl SnapProtocolMessage {
|
||||
TrieNodes,
|
||||
TrieNodesMessage
|
||||
);
|
||||
decode_snap_message_variant!(
|
||||
message_id,
|
||||
buf,
|
||||
SnapMessageId::GetBlockAccessLists,
|
||||
GetBlockAccessLists,
|
||||
GetBlockAccessListsMessage
|
||||
);
|
||||
decode_snap_message_variant!(
|
||||
message_id,
|
||||
buf,
|
||||
SnapMessageId::BlockAccessLists,
|
||||
BlockAccessLists,
|
||||
BlockAccessListsMessage
|
||||
);
|
||||
|
||||
Err(alloy_rlp::Error::Custom("Unknown message ID"))
|
||||
}
|
||||
@@ -438,9 +344,6 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_all_message_roundtrips() {
|
||||
assert_eq!(SnapVersion::V1.message_count(), 8);
|
||||
assert_eq!(SnapVersion::V2.message_count(), 10);
|
||||
|
||||
test_roundtrip(SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage {
|
||||
request_id: 42,
|
||||
root_hash: b256_from_u64(123),
|
||||
@@ -501,20 +404,6 @@ mod tests {
|
||||
request_id: 42,
|
||||
nodes: vec![Bytes::from(vec![1, 2, 3])],
|
||||
}));
|
||||
|
||||
test_roundtrip(SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage {
|
||||
request_id: 42,
|
||||
block_hashes: vec![b256_from_u64(123), b256_from_u64(456)],
|
||||
response_bytes: 4096,
|
||||
}));
|
||||
|
||||
test_roundtrip(SnapProtocolMessage::BlockAccessLists(BlockAccessListsMessage {
|
||||
request_id: 42,
|
||||
block_access_lists: BlockAccessLists(vec![
|
||||
Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]),
|
||||
Bytes::from_static(&[0xc1, alloy_rlp::EMPTY_LIST_CODE]),
|
||||
]),
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -7,7 +7,7 @@ use super::message::MAX_MESSAGE_SIZE;
|
||||
use crate::{
|
||||
message::{EthBroadcastMessage, ProtocolBroadcastMessage},
|
||||
EthMessage, EthMessageID, EthNetworkPrimitives, EthVersion, NetworkPrimitives, ProtocolMessage,
|
||||
RawCapabilityMessage, SnapProtocolMessage, SnapVersion,
|
||||
RawCapabilityMessage, SnapMessageId, SnapProtocolMessage,
|
||||
};
|
||||
use alloy_rlp::{Bytes, BytesMut, Encodable};
|
||||
use core::fmt::Debug;
|
||||
@@ -74,18 +74,6 @@ where
|
||||
Self { eth_snap: EthSnapStreamInner::new(eth_version), inner: stream }
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with an explicit snap version.
|
||||
pub const fn new_with_snap_version(
|
||||
stream: S,
|
||||
eth_version: EthVersion,
|
||||
snap_version: SnapVersion,
|
||||
) -> Self {
|
||||
Self {
|
||||
eth_snap: EthSnapStreamInner::new_with_snap_version(eth_version, snap_version),
|
||||
inner: stream,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size.
|
||||
pub const fn with_max_message_size(
|
||||
stream: S,
|
||||
@@ -98,35 +86,12 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size and snap version.
|
||||
pub const fn with_max_message_size_and_snap_version(
|
||||
stream: S,
|
||||
eth_version: EthVersion,
|
||||
snap_version: SnapVersion,
|
||||
max_message_size: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
eth_snap: EthSnapStreamInner::with_max_message_size_and_snap_version(
|
||||
eth_version,
|
||||
snap_version,
|
||||
max_message_size,
|
||||
),
|
||||
inner: stream,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the eth version
|
||||
#[inline]
|
||||
pub const fn eth_version(&self) -> EthVersion {
|
||||
self.eth_snap.eth_version()
|
||||
}
|
||||
|
||||
/// Returns the snap version.
|
||||
#[inline]
|
||||
pub const fn snap_version(&self) -> SnapVersion {
|
||||
self.eth_snap.snap_version()
|
||||
}
|
||||
|
||||
/// Returns the underlying stream
|
||||
#[inline]
|
||||
pub const fn inner(&self) -> &S {
|
||||
@@ -228,13 +193,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream handling combined eth and snap protocol logic.
|
||||
/// Stream handling combined eth and snap protocol logic
|
||||
/// Snap version is not critical to specify yet,
|
||||
/// Only one version, snap/1, does exist.
|
||||
#[derive(Debug, Clone)]
|
||||
struct EthSnapStreamInner<N> {
|
||||
/// Eth protocol version
|
||||
eth_version: EthVersion,
|
||||
/// Snap protocol version
|
||||
snap_version: SnapVersion,
|
||||
/// Maximum allowed ETH/Snap message size.
|
||||
max_message_size: usize,
|
||||
/// Type marker
|
||||
@@ -247,26 +212,12 @@ where
|
||||
{
|
||||
/// Create a new eth and snap protocol stream
|
||||
const fn new(eth_version: EthVersion) -> Self {
|
||||
Self::new_with_snap_version(eth_version, SnapVersion::V1)
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with an explicit snap version.
|
||||
const fn new_with_snap_version(eth_version: EthVersion, snap_version: SnapVersion) -> Self {
|
||||
Self::with_max_message_size_and_snap_version(eth_version, snap_version, MAX_MESSAGE_SIZE)
|
||||
Self::with_max_message_size(eth_version, MAX_MESSAGE_SIZE)
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size.
|
||||
const fn with_max_message_size(eth_version: EthVersion, max_message_size: usize) -> Self {
|
||||
Self::with_max_message_size_and_snap_version(eth_version, SnapVersion::V1, max_message_size)
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size and snap version.
|
||||
const fn with_max_message_size_and_snap_version(
|
||||
eth_version: EthVersion,
|
||||
snap_version: SnapVersion,
|
||||
max_message_size: usize,
|
||||
) -> Self {
|
||||
Self { eth_version, snap_version, max_message_size, _pd: PhantomData }
|
||||
Self { eth_version, max_message_size, _pd: PhantomData }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -274,11 +225,6 @@ where
|
||||
self.eth_version
|
||||
}
|
||||
|
||||
#[inline]
|
||||
const fn snap_version(&self) -> SnapVersion {
|
||||
self.snap_version
|
||||
}
|
||||
|
||||
/// Decode a message from the stream
|
||||
fn decode_message(&self, bytes: BytesMut) -> Result<EthSnapMessage<N>, EthSnapStreamError> {
|
||||
if bytes.len() > self.max_message_size {
|
||||
@@ -310,9 +256,8 @@ where
|
||||
}
|
||||
}
|
||||
} else if message_id > EthMessageID::max(self.eth_version) &&
|
||||
message_id <
|
||||
EthMessageID::message_count(self.eth_version) +
|
||||
self.snap_version.message_count()
|
||||
message_id <=
|
||||
EthMessageID::message_count(self.eth_version) + SnapMessageId::TrieNodes as u8
|
||||
{
|
||||
// Checks for multiplexed snap message IDs :
|
||||
// - message_id > EthMessageID::max() : ensures it's not an eth message
|
||||
@@ -368,8 +313,8 @@ mod tests {
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::Encodable;
|
||||
use reth_eth_wire_types::{
|
||||
message::RequestPair, BlockAccessLists, BlockAccessListsMessage, GetAccountRangeMessage,
|
||||
GetBlockAccessLists, GetBlockAccessListsMessage, GetBlockHeaders, HeadersDirection,
|
||||
message::RequestPair, GetAccountRangeMessage, GetBlockAccessLists, GetBlockHeaders,
|
||||
HeadersDirection,
|
||||
};
|
||||
|
||||
// Helper to create eth message and its bytes
|
||||
@@ -407,22 +352,6 @@ mod tests {
|
||||
(snap_msg, BytesMut::from(&encoded[..]))
|
||||
}
|
||||
|
||||
fn create_snap2_message() -> (SnapProtocolMessage, BytesMut) {
|
||||
let snap_msg = SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage {
|
||||
request_id: 1,
|
||||
block_hashes: vec![B256::default()],
|
||||
response_bytes: 1000,
|
||||
});
|
||||
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
|
||||
EthVersion::Eth67,
|
||||
SnapVersion::V2,
|
||||
);
|
||||
let encoded = inner.encode_snap_message(snap_msg.clone());
|
||||
|
||||
(snap_msg, BytesMut::from(&encoded[..]))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_eth_message_roundtrip() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
|
||||
@@ -483,25 +412,6 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snap2_protocol() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
|
||||
EthVersion::Eth67,
|
||||
SnapVersion::V2,
|
||||
);
|
||||
let (snap_msg, snap_bytes) = create_snap2_message();
|
||||
|
||||
let encoded_bytes = inner.encode_snap_message(snap_msg.clone());
|
||||
assert!(!encoded_bytes.is_empty());
|
||||
|
||||
let decoded_result = inner.decode_message(snap_bytes.clone());
|
||||
assert!(matches!(decoded_result, Ok(EthSnapMessage::Snap(_))));
|
||||
|
||||
if let Ok(EthSnapMessage::Snap(decoded_msg)) = inner.decode_message(snap_bytes) {
|
||||
assert_eq!(decoded_msg, snap_msg);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_message_id_boundaries() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
|
||||
@@ -565,24 +475,4 @@ mod tests {
|
||||
};
|
||||
assert_eq!(decoded_eth, eth_msg);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snap1_rejects_snap2_message_ids() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
|
||||
let snap2_msg = SnapProtocolMessage::BlockAccessLists(BlockAccessListsMessage {
|
||||
request_id: 1,
|
||||
block_access_lists: BlockAccessLists(vec![alloy_primitives::Bytes::from_static(&[
|
||||
alloy_rlp::EMPTY_LIST_CODE,
|
||||
])]),
|
||||
});
|
||||
|
||||
let encoded = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
|
||||
EthVersion::Eth67,
|
||||
SnapVersion::V2,
|
||||
)
|
||||
.encode_snap_message(snap2_msg);
|
||||
|
||||
let decoded = inner.decode_message(BytesMut::from(&encoded[..]));
|
||||
assert!(matches!(decoded, Err(EthSnapStreamError::UnknownMessageId(_))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,7 @@
|
||||
use crate::{
|
||||
errors::{EthHandshakeError, EthStreamError},
|
||||
handshake::EthereumEthHandshake,
|
||||
message::{EthBroadcastMessage, EthMessageID, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
|
||||
message::{EthBroadcastMessage, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
|
||||
p2pstream::HANDSHAKE_TIMEOUT,
|
||||
CanDisconnect, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, ProtocolMessage,
|
||||
UnifiedStatus,
|
||||
@@ -108,9 +108,6 @@ pub struct EthStreamInner<N> {
|
||||
version: EthVersion,
|
||||
/// Maximum allowed ETH message size.
|
||||
max_message_size: usize,
|
||||
/// When true, `NewBlock` (0x07) and `NewBlockHashes` (0x01) messages are rejected before RLP
|
||||
/// decoding to avoid any memory impact for non-PoW networks.
|
||||
reject_block_announcements: bool,
|
||||
_pd: std::marker::PhantomData<N>,
|
||||
}
|
||||
|
||||
@@ -125,12 +122,7 @@ where
|
||||
|
||||
/// Creates a new [`EthStreamInner`] with the given eth version and message size limit.
|
||||
pub const fn with_max_message_size(version: EthVersion, max_message_size: usize) -> Self {
|
||||
Self {
|
||||
version,
|
||||
max_message_size,
|
||||
reject_block_announcements: false,
|
||||
_pd: std::marker::PhantomData,
|
||||
}
|
||||
Self { version, max_message_size, _pd: std::marker::PhantomData }
|
||||
}
|
||||
|
||||
/// Returns the eth version
|
||||
@@ -139,25 +131,12 @@ where
|
||||
self.version
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding.
|
||||
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
self.reject_block_announcements = reject;
|
||||
}
|
||||
|
||||
/// Decodes incoming bytes into an [`EthMessage`].
|
||||
pub fn decode_message(&self, bytes: BytesMut) -> Result<EthMessage<N>, EthStreamError> {
|
||||
if bytes.len() > self.max_message_size {
|
||||
return Err(EthStreamError::MessageTooBig(bytes.len()));
|
||||
}
|
||||
|
||||
if self.reject_block_announcements &&
|
||||
let Some(&id) = bytes.first() &&
|
||||
(id == EthMessageID::NewBlock.to_u8() || id == EthMessageID::NewBlockHashes.to_u8())
|
||||
{
|
||||
return Err(EthStreamError::UnsupportedMessage { message_id: id });
|
||||
}
|
||||
|
||||
let msg = match ProtocolMessage::decode_message(self.version, &mut bytes.as_ref()) {
|
||||
Ok(m) => m,
|
||||
Err(err) => {
|
||||
@@ -229,12 +208,6 @@ impl<S, N: NetworkPrimitives> EthStream<S, N> {
|
||||
self.eth.version()
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding.
|
||||
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
self.eth.set_reject_block_announcements(reject);
|
||||
}
|
||||
|
||||
/// Returns the underlying stream.
|
||||
#[inline]
|
||||
pub const fn inner(&self) -> &S {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! A Protocol defines a P2P subprotocol in an `RLPx` connection
|
||||
|
||||
use crate::{Capability, EthMessageID, EthVersion, SnapVersion};
|
||||
use crate::{Capability, EthMessageID, EthVersion};
|
||||
|
||||
/// Type that represents a [Capability] and the number of messages it uses.
|
||||
///
|
||||
@@ -30,13 +30,6 @@ impl Protocol {
|
||||
Self::new(cap, messages)
|
||||
}
|
||||
|
||||
/// Returns the corresponding snap capability for the given version.
|
||||
pub const fn snap(version: SnapVersion) -> Self {
|
||||
let cap = Capability::snap(version);
|
||||
let messages = version.message_count();
|
||||
Self::new(cap, messages)
|
||||
}
|
||||
|
||||
/// Returns the [`EthVersion::Eth66`] capability.
|
||||
pub const fn eth_66() -> Self {
|
||||
Self::eth(EthVersion::Eth66)
|
||||
@@ -52,16 +45,6 @@ impl Protocol {
|
||||
Self::eth(EthVersion::Eth68)
|
||||
}
|
||||
|
||||
/// Returns the `snap/1` capability.
|
||||
pub const fn snap_1() -> Self {
|
||||
Self::snap(SnapVersion::V1)
|
||||
}
|
||||
|
||||
/// Returns the `snap/2` capability.
|
||||
pub const fn snap_2() -> Self {
|
||||
Self::snap(SnapVersion::V2)
|
||||
}
|
||||
|
||||
/// Consumes the type and returns a tuple of the [Capability] and number of messages.
|
||||
#[inline]
|
||||
pub(crate) fn split(self) -> (Capability, u8) {
|
||||
@@ -103,7 +86,5 @@ mod tests {
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth69).messages(), 18);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth70).messages(), 18);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth71).messages(), 20);
|
||||
assert_eq!(Protocol::snap(SnapVersion::V1).messages(), 8);
|
||||
assert_eq!(Protocol::snap(SnapVersion::V2).messages(), 10);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,7 +13,6 @@ use crate::{
|
||||
};
|
||||
use reth_eth_wire::{EthNetworkPrimitives, NetworkPrimitives};
|
||||
use reth_network_api::test_utils::PeersHandleProvider;
|
||||
use reth_storage_api::BalProvider;
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -64,10 +63,7 @@ impl<Tx, Eth, N: NetworkPrimitives> NetworkBuilder<Tx, Eth, N> {
|
||||
pub fn request_handler<Client>(
|
||||
self,
|
||||
client: Client,
|
||||
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N>
|
||||
where
|
||||
Client: BalProvider,
|
||||
{
|
||||
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N> {
|
||||
let Self { mut network, transactions, .. } = self;
|
||||
let (tx, rx) = mpsc::channel(ETH_REQUEST_CHANNEL_CAPACITY);
|
||||
network.set_eth_request_handler(tx);
|
||||
|
||||
@@ -20,9 +20,7 @@ use reth_eth_wire_types::message::MAX_MESSAGE_SIZE;
|
||||
use reth_ethereum_forks::{ForkFilter, Head};
|
||||
use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPeer};
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_storage_api::{
|
||||
noop::NoopProvider, BalProvider, BlockNumReader, BlockReader, HeaderProvider,
|
||||
};
|
||||
use reth_storage_api::{noop::NoopProvider, BlockNumReader, BlockReader, HeaderProvider};
|
||||
use reth_tasks::Runtime;
|
||||
use secp256k1::SECP256K1;
|
||||
use std::{collections::HashSet, net::SocketAddr, sync::Arc};
|
||||
@@ -159,8 +157,7 @@ where
|
||||
impl<C, N> NetworkConfig<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider
|
||||
+ BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
|
||||
C: BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
|
||||
+ HeaderProvider
|
||||
+ Clone
|
||||
+ Unpin
|
||||
|
||||
@@ -18,7 +18,7 @@ use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::error::RequestResult;
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_primitives_traits::Block;
|
||||
use reth_storage_api::{BalProvider, BlockReader, GetBlockAccessListLimit, HeaderProvider};
|
||||
use reth_storage_api::{BlockReader, HeaderProvider};
|
||||
use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
@@ -282,6 +282,19 @@ where
|
||||
let _ = response.send(Ok(Receipts70 { last_block_incomplete, receipts }));
|
||||
}
|
||||
|
||||
/// Handles [`GetBlockAccessLists`] queries.
|
||||
///
|
||||
/// For now this returns one empty BAL per requested hash.
|
||||
fn on_block_access_lists_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
request: GetBlockAccessLists,
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
) {
|
||||
let access_lists = request.0.into_iter().map(|_| Bytes::new()).collect();
|
||||
let _ = response.send(Ok(BlockAccessLists(access_lists)));
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_receipts_response<T, F>(&self, request: GetReceipts, transform_fn: F) -> Vec<Vec<T>>
|
||||
where
|
||||
@@ -311,55 +324,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<C, N> EthRequestHandler<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider,
|
||||
{
|
||||
/// Handles [`GetBlockAccessLists`] queries.
|
||||
///
|
||||
/// EIP-8159 defines the final `BlockAccessLists` response semantics:
|
||||
/// <https://eips.ethereum.org/EIPS/eip-8159>
|
||||
fn on_block_access_lists_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
request: GetBlockAccessLists,
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
) {
|
||||
let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit(SOFT_RESPONSE_LIMIT);
|
||||
let access_lists = self
|
||||
.client
|
||||
.bal_store()
|
||||
.get_by_hashes_with_limit(&request.0, limit)
|
||||
.unwrap_or_else(|_| empty_block_access_lists_with_limit(request.0.len(), limit));
|
||||
let _ = response.send(Ok(BlockAccessLists(access_lists)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the error fallback response while still enforcing the BAL response soft limit.
|
||||
fn empty_block_access_lists_with_limit(count: usize, limit: GetBlockAccessListLimit) -> Vec<Bytes> {
|
||||
let mut out = Vec::with_capacity(count);
|
||||
let mut size = 0;
|
||||
for _ in 0..count {
|
||||
let bal = Bytes::from_static(&[0xc0]);
|
||||
size += bal.len();
|
||||
out.push(bal);
|
||||
|
||||
if limit.exceeds(size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// An endless future.
|
||||
///
|
||||
/// This should be spawned or used as part of `tokio::select!`.
|
||||
impl<C, N> Future for EthRequestHandler<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider
|
||||
+ BlockReader<Block = N::Block, Receipt = N::Receipt>
|
||||
C: BlockReader<Block = N::Block, Receipt = N::Receipt>
|
||||
+ HeaderProvider<Header = N::BlockHeader>
|
||||
+ Unpin,
|
||||
{
|
||||
|
||||
@@ -6,7 +6,7 @@ use futures::{future, future::Either};
|
||||
use reth_eth_wire::{BlockAccessLists, EthNetworkPrimitives, NetworkPrimitives};
|
||||
use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::{
|
||||
block_access_lists::client::{BalRequirement, BlockAccessListsClient},
|
||||
block_access_lists::client::BlockAccessListsClient,
|
||||
bodies::client::{BodiesClient, BodiesFut},
|
||||
download::DownloadClient,
|
||||
error::{PeerRequestResult, RequestError},
|
||||
@@ -135,29 +135,11 @@ impl<N: NetworkPrimitives> BlockAccessListsClient for FetchClient<N> {
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
priority: Priority,
|
||||
) -> Self::Output {
|
||||
self.get_block_access_lists_with_priority_and_requirement(
|
||||
hashes,
|
||||
priority,
|
||||
BalRequirement::Mandatory,
|
||||
)
|
||||
}
|
||||
|
||||
fn get_block_access_lists_with_priority_and_requirement(
|
||||
&self,
|
||||
hashes: Vec<B256>,
|
||||
priority: Priority,
|
||||
requirement: BalRequirement,
|
||||
) -> Self::Output {
|
||||
let (response, rx) = oneshot::channel();
|
||||
if self
|
||||
.request_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: hashes,
|
||||
response,
|
||||
priority,
|
||||
requirement,
|
||||
})
|
||||
.send(DownloadRequest::GetBlockAccessLists { request: hashes, response, priority })
|
||||
.is_ok()
|
||||
{
|
||||
Box::pin(FlattenedResponse::from(rx))
|
||||
|
||||
@@ -13,7 +13,6 @@ use reth_eth_wire::{
|
||||
};
|
||||
use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::{
|
||||
block_access_lists::client::BalRequirement,
|
||||
error::{EthResponseValidator, PeerRequestResult, RequestError, RequestResult},
|
||||
headers::client::HeadersRequest,
|
||||
priority::Priority,
|
||||
@@ -160,10 +159,15 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
/// full history available
|
||||
fn next_best_peer(&self, requirement: BestPeerRequirements) -> Option<PeerId> {
|
||||
// filter out peers that aren't idle or don't meet the requirement
|
||||
let mut idle = self
|
||||
.peers
|
||||
.iter()
|
||||
.filter(|(_, peer)| peer.state.is_idle() && peer.satisfies(&requirement));
|
||||
let mut idle = self.peers.iter().filter(|(_, peer)| {
|
||||
peer.state.is_idle() &&
|
||||
match &requirement {
|
||||
BestPeerRequirements::EthVersion(ver) => {
|
||||
peer.capabilities.supports_eth_at_least(ver)
|
||||
}
|
||||
_ => true,
|
||||
}
|
||||
});
|
||||
|
||||
let mut best_peer = idle.next()?;
|
||||
|
||||
@@ -191,14 +195,6 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
Some(*best_peer.0)
|
||||
}
|
||||
|
||||
/// Returns whether any connected peer can serve BAL requests.
|
||||
fn has_eth71_peer(&self) -> bool {
|
||||
self.peers.values().any(|peer| {
|
||||
!matches!(peer.state, PeerState::Closing) &&
|
||||
peer.capabilities.supports_eth_at_least(&EthVersion::Eth71)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the next action to return
|
||||
fn poll_action(&mut self) -> PollAction {
|
||||
// we only check and not pop here since we don't know yet whether a peer is available.
|
||||
@@ -212,15 +208,9 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
|
||||
let request = self.queued_requests.pop_front().expect("not empty");
|
||||
let Some(peer_id) = self.next_best_peer(request.best_peer_requirements()) else {
|
||||
// Optional BAL requests can lose their eth/71 peer while queued; complete them
|
||||
// instead of waiting for future peer churn.
|
||||
if request.is_optional_bal() && !self.has_eth71_peer() {
|
||||
request.send_err_response(RequestError::UnsupportedCapability);
|
||||
} else {
|
||||
// no peer matches this request's requirements; requeue at the back so other
|
||||
// queued requests get a chance on the next poll instead of head-of-line blocking.
|
||||
self.queued_requests.push_back(request);
|
||||
}
|
||||
// no peer matches this request's requirements; requeue at the back so other
|
||||
// queued requests get a chance on the next poll instead of head-of-line blocking.
|
||||
self.queued_requests.push_back(request);
|
||||
return PollAction::NoPeersAvailable
|
||||
};
|
||||
|
||||
@@ -242,30 +232,21 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
loop {
|
||||
// poll incoming requests
|
||||
match self.download_requests_rx.poll_next_unpin(cx) {
|
||||
Poll::Ready(Some(request)) => {
|
||||
// Optional BAL requests should not wait for future peer churn if no
|
||||
// connected peer can serve them right now.
|
||||
if request.is_optional_bal() && !self.has_eth71_peer() {
|
||||
request.send_err_response(RequestError::UnsupportedCapability);
|
||||
continue
|
||||
Poll::Ready(Some(request)) => match request.get_priority() {
|
||||
Priority::High => {
|
||||
// find the first normal request and queue before, add this request to
|
||||
// the back of the high-priority queue
|
||||
let pos = self
|
||||
.queued_requests
|
||||
.iter()
|
||||
.position(|req| req.is_normal_priority())
|
||||
.unwrap_or(0);
|
||||
self.queued_requests.insert(pos, request);
|
||||
}
|
||||
|
||||
match request.get_priority() {
|
||||
Priority::High => {
|
||||
// find first normal request and queue before it; add this request
|
||||
// to the back of the high-priority queue
|
||||
let pos = self
|
||||
.queued_requests
|
||||
.iter()
|
||||
.position(|req| req.is_normal_priority())
|
||||
.unwrap_or(0);
|
||||
self.queued_requests.insert(pos, request);
|
||||
}
|
||||
Priority::Normal => {
|
||||
self.queued_requests.push_back(request);
|
||||
}
|
||||
Priority::Normal => {
|
||||
self.queued_requests.push_back(request);
|
||||
}
|
||||
}
|
||||
},
|
||||
Poll::Ready(None) => {
|
||||
unreachable!("channel can't close")
|
||||
}
|
||||
@@ -288,15 +269,6 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
peer.state = req.peer_state();
|
||||
}
|
||||
|
||||
self.prepare_inflight_block_request(peer_id, req)
|
||||
}
|
||||
|
||||
/// Tracks an inflight request and converts it into a peer request.
|
||||
fn prepare_inflight_block_request(
|
||||
&mut self,
|
||||
peer_id: PeerId,
|
||||
req: DownloadRequest<N>,
|
||||
) -> BlockRequest {
|
||||
match req {
|
||||
DownloadRequest::GetBlockHeaders { request, response, .. } => {
|
||||
let inflight = Request { request: request.clone(), response };
|
||||
@@ -327,23 +299,12 @@ impl<N: NetworkPrimitives> StateFetcher<N> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a queued followup request the peer can serve.
|
||||
///
|
||||
/// This is an immediate scheduling shortcut after a successful response. It skips queued
|
||||
/// requests whose hard requirements do not match this peer, leaving them for the regular peer
|
||||
/// selection path.
|
||||
/// Returns a new followup request for the peer.
|
||||
///
|
||||
/// Caution: this expects that the peer is _not_ closed.
|
||||
fn followup_request(&mut self, peer_id: PeerId) -> Option<BlockResponseOutcome> {
|
||||
let peer = self.peers.get_mut(&peer_id)?;
|
||||
let req_idx = self.queued_requests.iter().position(|req| {
|
||||
// Find the first queued request this peer can serve.
|
||||
peer.satisfies(&req.best_peer_requirements())
|
||||
})?;
|
||||
let req = self.queued_requests.remove(req_idx).expect("valid request index");
|
||||
|
||||
peer.state = req.peer_state();
|
||||
let req = self.prepare_inflight_block_request(peer_id, req);
|
||||
let req = self.queued_requests.pop_front()?;
|
||||
let req = self.prepare_block_request(peer_id, req);
|
||||
Some(BlockResponseOutcome::Request(peer_id, req))
|
||||
}
|
||||
|
||||
@@ -515,16 +476,6 @@ impl Peer {
|
||||
self.range_info.as_ref().map(|info| info.range())
|
||||
}
|
||||
|
||||
/// Returns whether this peer can serve requests with the given hard requirements.
|
||||
fn satisfies(&self, requirement: &BestPeerRequirements) -> bool {
|
||||
match requirement {
|
||||
BestPeerRequirements::EthVersion(ver) => self.capabilities.supports_eth_at_least(ver),
|
||||
BestPeerRequirements::None |
|
||||
BestPeerRequirements::FullBlock |
|
||||
BestPeerRequirements::FullBlockRange(_) => true,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns true if this peer has a better range than the other peer for serving the requested
|
||||
/// range.
|
||||
///
|
||||
@@ -651,7 +602,6 @@ pub(crate) enum DownloadRequest<N: NetworkPrimitives> {
|
||||
request: Vec<B256>,
|
||||
response: oneshot::Sender<PeerRequestResult<BlockAccessLists>>,
|
||||
priority: Priority,
|
||||
requirement: BalRequirement,
|
||||
},
|
||||
/// Download receipts for the given block hashes and send response through channel
|
||||
GetReceipts {
|
||||
@@ -689,21 +639,6 @@ impl<N: NetworkPrimitives> DownloadRequest<N> {
|
||||
self.get_priority().is_normal()
|
||||
}
|
||||
|
||||
/// Returns `true` if this is an optional BAL request.
|
||||
const fn is_optional_bal(&self) -> bool {
|
||||
matches!(self, Self::GetBlockAccessLists { requirement: BalRequirement::Optional, .. })
|
||||
}
|
||||
|
||||
/// Sends an error response to the waiting caller.
|
||||
fn send_err_response(self, err: RequestError) {
|
||||
let _ = match self {
|
||||
Self::GetBlockHeaders { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetBlockBodies { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetBlockAccessLists { response, .. } => response.send(Err(err)).ok(),
|
||||
Self::GetReceipts { response, .. } => response.send(Err(err)).ok(),
|
||||
};
|
||||
}
|
||||
|
||||
/// Returns the best peer requirements for this request.
|
||||
fn best_peer_requirements(&self) -> BestPeerRequirements {
|
||||
match self {
|
||||
@@ -1469,98 +1404,6 @@ mod tests {
|
||||
assert!(matches!(outcome, Some(BlockResponseOutcome::Request(pid, _)) if pid == peer_id));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_followup_skips_request_peer_cannot_serve() {
|
||||
let (mut fetcher, peer_id) = fetcher_with_peer();
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (followup_tx, _followup_rx) = oneshot::channel();
|
||||
fetcher.queued_requests.push_back(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![B256::random()],
|
||||
response: followup_tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
});
|
||||
|
||||
let _rx = insert_inflight_receipts(&mut fetcher, peer_id);
|
||||
|
||||
let resp = ReceiptsResponse::new(vec![vec![]]);
|
||||
assert!(fetcher.on_receipts_response(peer_id, Ok(resp)).is_none());
|
||||
assert!(fetcher.peers[&peer_id].state.is_idle());
|
||||
assert!(!fetcher.inflight_bals_requests.contains_key(&peer_id));
|
||||
assert!(matches!(
|
||||
fetcher.queued_requests.front(),
|
||||
Some(DownloadRequest::GetBlockAccessLists {
|
||||
requirement: BalRequirement::Optional,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_followup_uses_first_satisfiable_request() {
|
||||
let (mut fetcher, peer_id) = fetcher_with_peer();
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (bal_tx, _bal_rx) = oneshot::channel();
|
||||
fetcher.queued_requests.push_back(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![B256::random()],
|
||||
response: bal_tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
});
|
||||
|
||||
let (bodies_tx, _bodies_rx) = oneshot::channel();
|
||||
fetcher.queued_requests.push_back(DownloadRequest::GetBlockBodies {
|
||||
request: vec![B256::random()],
|
||||
response: bodies_tx,
|
||||
priority: Priority::Normal,
|
||||
range_hint: None,
|
||||
});
|
||||
|
||||
let _rx = insert_inflight_receipts(&mut fetcher, peer_id);
|
||||
|
||||
let resp = ReceiptsResponse::new(vec![vec![]]);
|
||||
let outcome = fetcher.on_receipts_response(peer_id, Ok(resp));
|
||||
|
||||
assert!(matches!(
|
||||
outcome,
|
||||
Some(BlockResponseOutcome::Request(pid, BlockRequest::GetBlockBodies(_))) if pid == peer_id
|
||||
));
|
||||
assert!(fetcher.inflight_bodies_requests.contains_key(&peer_id));
|
||||
assert!(matches!(fetcher.peers[&peer_id].state, PeerState::GetBlockBodies));
|
||||
assert_eq!(fetcher.queued_requests.len(), 1);
|
||||
assert!(matches!(
|
||||
fetcher.queued_requests.front(),
|
||||
Some(DownloadRequest::GetBlockAccessLists {
|
||||
requirement: BalRequirement::Optional,
|
||||
..
|
||||
})
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_prepare_block_request_creates_inflight_receipts() {
|
||||
let (mut fetcher, peer_id) = fetcher_with_peer();
|
||||
@@ -1698,7 +1541,6 @@ mod tests {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Mandatory,
|
||||
});
|
||||
|
||||
let waker = noop_waker();
|
||||
@@ -1741,138 +1583,4 @@ mod tests {
|
||||
assert_eq!(peer_id, peer_71);
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optional_bal_request_rejected_without_eth71_peer() {
|
||||
use futures::task::noop_waker;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
let manager = PeersManager::new(PeersConfig::default());
|
||||
let mut fetcher =
|
||||
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
|
||||
|
||||
let peer_old = B512::random();
|
||||
let caps_old = Arc::new(Capabilities::new(vec![]));
|
||||
fetcher.new_active_peer(
|
||||
peer_old,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_old,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
fetcher
|
||||
.download_requests_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert!(fetcher.queued_requests.is_empty());
|
||||
assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_optional_bal_request_waits_for_busy_eth71_peer() {
|
||||
use futures::task::noop_waker;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
let manager = PeersManager::new(PeersConfig::default());
|
||||
let mut fetcher =
|
||||
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (tx, _rx) = oneshot::channel();
|
||||
fetcher
|
||||
.download_requests_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert_eq!(fetcher.queued_requests.len(), 1);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_queued_optional_bal_request_rejected_after_eth71_disconnect() {
|
||||
use futures::task::noop_waker;
|
||||
use std::task::{Context, Poll};
|
||||
|
||||
let manager = PeersManager::new(PeersConfig::default());
|
||||
let mut fetcher =
|
||||
StateFetcher::<EthNetworkPrimitives>::new(manager.handle(), Default::default());
|
||||
|
||||
let peer_old = B512::random();
|
||||
let caps_old = Arc::new(Capabilities::new(vec![]));
|
||||
fetcher.new_active_peer(
|
||||
peer_old,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_old,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
|
||||
let peer_71 = B512::random();
|
||||
let caps_71 = Arc::new(Capabilities::from(vec![Capability::new("eth".into(), 71)]));
|
||||
fetcher.new_active_peer(
|
||||
peer_71,
|
||||
B256::random(),
|
||||
100,
|
||||
caps_71,
|
||||
Arc::new(AtomicU64::new(10)),
|
||||
None,
|
||||
);
|
||||
fetcher.peers.get_mut(&peer_71).expect("peer exists").state = PeerState::GetBlockHeaders;
|
||||
|
||||
let (tx, rx) = oneshot::channel();
|
||||
fetcher
|
||||
.download_requests_tx
|
||||
.send(DownloadRequest::GetBlockAccessLists {
|
||||
request: vec![],
|
||||
response: tx,
|
||||
priority: Priority::Normal,
|
||||
requirement: BalRequirement::Optional,
|
||||
})
|
||||
.unwrap();
|
||||
|
||||
let waker = noop_waker();
|
||||
let mut cx = Context::from_waker(&waker);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert_eq!(fetcher.queued_requests.len(), 1);
|
||||
|
||||
fetcher.on_session_closed(&peer_71);
|
||||
|
||||
assert!(matches!(fetcher.poll(&mut cx), Poll::Pending));
|
||||
assert!(fetcher.queued_requests.is_empty());
|
||||
assert_eq!(rx.await.unwrap().unwrap_err(), RequestError::UnsupportedCapability);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -318,7 +318,6 @@ impl<N: NetworkPrimitives> NetworkManager<N> {
|
||||
extra_protocols,
|
||||
handshake,
|
||||
eth_max_message_size,
|
||||
network_mode.is_stake(),
|
||||
);
|
||||
|
||||
let state = NetworkState::new(
|
||||
|
||||
@@ -577,9 +577,6 @@ pub struct AnnouncedTxTypesMetrics {
|
||||
|
||||
/// Histogram for tracking frequency of EIP-7702 transaction type
|
||||
pub(crate) eip7702: Histogram,
|
||||
|
||||
/// Histogram for tracking frequency of unknown/other transaction types
|
||||
pub(crate) other: Histogram,
|
||||
}
|
||||
|
||||
/// Counts the number of transactions by their type in a block or collection.
|
||||
@@ -602,9 +599,6 @@ pub struct TxTypesCounter {
|
||||
|
||||
/// Count of transactions conforming to EIP-7702 (Restricted Storage Windows).
|
||||
pub(crate) eip7702: usize,
|
||||
|
||||
/// Count of unknown/other transaction types not matching any known EIP.
|
||||
pub(crate) other: usize,
|
||||
}
|
||||
|
||||
impl TxTypesCounter {
|
||||
@@ -627,10 +621,6 @@ impl TxTypesCounter {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn increase_other(&mut self) {
|
||||
self.other += 1;
|
||||
}
|
||||
}
|
||||
|
||||
impl AnnouncedTxTypesMetrics {
|
||||
@@ -642,6 +632,5 @@ impl AnnouncedTxTypesMetrics {
|
||||
self.eip1559.record(tx_types_counter.eip1559 as f64);
|
||||
self.eip4844.record(tx_types_counter.eip4844 as f64);
|
||||
self.eip7702.record(tx_types_counter.eip7702 as f64);
|
||||
self.other.record(tx_types_counter.other as f64);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -79,9 +79,6 @@ const TIMEOUT_SCALING: u32 = 3;
|
||||
/// before reading any more messages from the remote peer, throttling the peer.
|
||||
const MAX_QUEUED_OUTGOING_RESPONSES: usize = 4;
|
||||
|
||||
/// Minimum capacity to retain for buffered incoming requests from the remote peer.
|
||||
const MIN_RECEIVED_REQUESTS_CAPACITY: usize = 1;
|
||||
|
||||
/// Soft limit for the total number of buffered outgoing broadcast items (e.g. transaction hashes).
|
||||
///
|
||||
/// Many small broadcast messages carrying a single tx hash each are equivalent in cost to one
|
||||
@@ -207,8 +204,8 @@ impl<N: NetworkPrimitives> ActiveSession<N> {
|
||||
|
||||
/// Shrinks the capacity of the internal buffers.
|
||||
pub fn shrink_to_fit(&mut self) {
|
||||
self.received_requests_from_remote.shrink_to(MIN_RECEIVED_REQUESTS_CAPACITY);
|
||||
self.queued_outgoing.shrink_to(MAX_QUEUED_OUTGOING_RESPONSES);
|
||||
self.received_requests_from_remote.shrink_to_fit();
|
||||
self.queued_outgoing.shrink_to_fit();
|
||||
}
|
||||
|
||||
/// Returns how many responses we've currently queued up.
|
||||
@@ -1093,8 +1090,8 @@ impl<N: NetworkPrimitives> QueuedOutgoingMessages<N> {
|
||||
self.count.increment(1);
|
||||
}
|
||||
|
||||
pub(crate) fn shrink_to(&mut self, min_capacity: usize) {
|
||||
self.messages.shrink_to(min_capacity);
|
||||
pub(crate) fn shrink_to_fit(&mut self) {
|
||||
self.messages.shrink_to_fit();
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -93,15 +93,6 @@ impl<N: NetworkPrimitives> EthRlpxConnection<N> {
|
||||
Self::Satellite(conn) => conn.primary_mut().start_send_raw(msg),
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding to avoid memory amplification from deserializing blocks that will be discarded.
|
||||
pub fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
match self {
|
||||
Self::EthOnly(conn) => conn.set_reject_block_announcements(reject),
|
||||
Self::Satellite(conn) => conn.primary_mut().set_reject_block_announcements(reject),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<N: NetworkPrimitives> From<EthPeerConnection<N>> for EthRlpxConnection<N> {
|
||||
|
||||
@@ -123,9 +123,6 @@ pub struct SessionManager<N: NetworkPrimitives> {
|
||||
/// Shared local range information that gets propagated to active sessions.
|
||||
/// This represents the range of blocks that this node can serve to other peers.
|
||||
local_range_info: BlockRangeInfo,
|
||||
/// When true, block announcement messages (`NewBlock`, `NewBlockHashes`) are rejected before
|
||||
/// RLP decoding on new sessions to avoid memory amplification.
|
||||
reject_block_announcements: bool,
|
||||
}
|
||||
|
||||
// === impl SessionManager ===
|
||||
@@ -143,7 +140,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
extra_protocols: RlpxSubProtocols,
|
||||
handshake: Arc<dyn EthRlpxHandshake>,
|
||||
eth_max_message_size: usize,
|
||||
reject_block_announcements: bool,
|
||||
) -> Self {
|
||||
let (pending_sessions_tx, pending_sessions_rx) = mpsc::channel(config.session_event_buffer);
|
||||
let (active_session_tx, active_session_rx) = mpsc::channel(config.session_event_buffer);
|
||||
@@ -180,7 +176,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
handshake,
|
||||
eth_max_message_size,
|
||||
local_range_info,
|
||||
reject_block_announcements,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -501,7 +496,7 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
local_addr,
|
||||
peer_id,
|
||||
capabilities,
|
||||
mut conn,
|
||||
conn,
|
||||
status,
|
||||
direction,
|
||||
client_id,
|
||||
@@ -568,10 +563,6 @@ impl<N: NetworkPrimitives> SessionManager<N> {
|
||||
BlockRangeInfo::new(update.earliest, update.latest, update.latest_hash)
|
||||
});
|
||||
|
||||
if self.reject_block_announcements {
|
||||
conn.set_reject_block_announcements(true);
|
||||
}
|
||||
|
||||
let session = ActiveSession {
|
||||
next_id: 0,
|
||||
remote_peer_id: peer_id,
|
||||
|
||||
@@ -27,8 +27,7 @@ use reth_network_api::{
|
||||
};
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_storage_api::{
|
||||
noop::NoopProvider, BalProvider, BlockReader, BlockReaderIdExt, HeaderProvider,
|
||||
StateProviderFactory,
|
||||
noop::NoopProvider, BlockReader, BlockReaderIdExt, HeaderProvider, StateProviderFactory,
|
||||
};
|
||||
use reth_tasks::Runtime;
|
||||
use reth_tokio_util::EventStream;
|
||||
@@ -248,7 +247,6 @@ where
|
||||
Receipt = reth_ethereum_primitives::Receipt,
|
||||
Header = alloy_consensus::Header,
|
||||
> + HeaderProvider
|
||||
+ BalProvider
|
||||
+ Clone
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
@@ -321,7 +319,6 @@ where
|
||||
Receipt = reth_ethereum_primitives::Receipt,
|
||||
Header = alloy_consensus::Header,
|
||||
> + HeaderProvider
|
||||
+ BalProvider
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
Pool: TransactionPool<
|
||||
@@ -465,10 +462,7 @@ where
|
||||
}
|
||||
|
||||
/// Set a new request handler that's connected to the peer's network
|
||||
pub fn install_request_handler(&mut self)
|
||||
where
|
||||
C: BalProvider,
|
||||
{
|
||||
pub fn install_request_handler(&mut self) {
|
||||
let (tx, rx) = channel(ETH_REQUEST_CHANNEL_CAPACITY);
|
||||
self.network.set_eth_request_handler(tx);
|
||||
let peers = self.network.peers_handle();
|
||||
@@ -579,7 +573,6 @@ where
|
||||
Receipt = reth_ethereum_primitives::Receipt,
|
||||
Header = alloy_consensus::Header,
|
||||
> + HeaderProvider
|
||||
+ BalProvider
|
||||
+ Unpin
|
||||
+ 'static,
|
||||
Pool: TransactionPool<
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user