mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
197 Commits
bench/scca
...
main
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fcfa8287f6 | ||
|
|
d25de30050 | ||
|
|
4ffde69d94 | ||
|
|
077e5eecfe | ||
|
|
709485dcb7 | ||
|
|
88505c7fcb | ||
|
|
c14bc59236 | ||
|
|
347c1325cc | ||
|
|
5f85eb7ac8 | ||
|
|
a12454d2e6 | ||
|
|
c194c17a27 | ||
|
|
43a7452b0e | ||
|
|
73ec2c9d56 | ||
|
|
76e886578b | ||
|
|
ad08829288 | ||
|
|
b89288582b | ||
|
|
87d878a979 | ||
|
|
473f85c558 | ||
|
|
a8eee6028f | ||
|
|
671da55884 | ||
|
|
a8fc13dc25 | ||
|
|
4c1f6b6507 | ||
|
|
b850f2a81d | ||
|
|
79144cb430 | ||
|
|
b04346ffe5 | ||
|
|
674623f14e | ||
|
|
97b1b56b2d | ||
|
|
af6d20b5ea | ||
|
|
64cf412aaf | ||
|
|
5b10e03c5c | ||
|
|
91d248e6fa | ||
|
|
344037d04e | ||
|
|
aca6261107 | ||
|
|
d4ca2e2687 | ||
|
|
e5e0abb47e | ||
|
|
225e3ae238 | ||
|
|
345fbbbfdb | ||
|
|
db17c899c3 | ||
|
|
2c86c0b876 | ||
|
|
bd4cd28a8d | ||
|
|
6fa48a497a | ||
|
|
6886cd7742 | ||
|
|
eeb223f0b8 | ||
|
|
f344f5abfb | ||
|
|
68845d1114 | ||
|
|
ecfb6cc089 | ||
|
|
b271694301 | ||
|
|
41c68729ab | ||
|
|
79578e35b8 | ||
|
|
e4f14b2ae1 | ||
|
|
05e6da66e1 | ||
|
|
6be5520e34 | ||
|
|
d29db3b765 | ||
|
|
40c30dbc73 | ||
|
|
5c383818a6 | ||
|
|
cf6ffb1599 | ||
|
|
ba3cd2872a | ||
|
|
4f9af7c16a | ||
|
|
13c5504aa2 | ||
|
|
fa6b44b038 | ||
|
|
6377a957c1 | ||
|
|
378d4052ee | ||
|
|
62d99888d2 | ||
|
|
73f5d77b51 | ||
|
|
b62f71977a | ||
|
|
ad27be67be | ||
|
|
63f80907cc | ||
|
|
a57930481c | ||
|
|
bbcfe354a1 | ||
|
|
7839f3d876 | ||
|
|
e89b4611e4 | ||
|
|
2b7d4b54d4 | ||
|
|
fe7a4c80b6 | ||
|
|
122c5b322b | ||
|
|
f1ed5f0ade | ||
|
|
6364fb87d0 | ||
|
|
d55458479d | ||
|
|
42f49132b7 | ||
|
|
f39c47bd11 | ||
|
|
b1ac264107 | ||
|
|
0195da5b84 | ||
|
|
b964195ef8 | ||
|
|
252fe42c54 | ||
|
|
3edb271183 | ||
|
|
165a80441b | ||
|
|
981e32d4d9 | ||
|
|
d7522904a0 | ||
|
|
e92af360ae | ||
|
|
408ef4657d | ||
|
|
3574ecaaa0 | ||
|
|
d58c6e3d07 | ||
|
|
d577814eb1 | ||
|
|
8b46f1a6d0 | ||
|
|
c527c2e7d6 | ||
|
|
14570f325a | ||
|
|
41fe41f2f2 | ||
|
|
27bfddeada | ||
|
|
981a7ef99b | ||
|
|
8c826a5cd0 | ||
|
|
6465997ea1 | ||
|
|
03a308da63 | ||
|
|
af84b982c3 | ||
|
|
77c3e86ec6 | ||
|
|
98ebc3454f | ||
|
|
c8979d0a1d | ||
|
|
742a7e7a18 | ||
|
|
99bf7a17c0 | ||
|
|
24436ca9f9 | ||
|
|
c26ec53d7d | ||
|
|
3a136fc8c3 | ||
|
|
d215d16a7d | ||
|
|
b36fff0ab8 | ||
|
|
e4d4ba30cb | ||
|
|
7c219fa955 | ||
|
|
0ac36468c6 | ||
|
|
93b2201c76 | ||
|
|
9990670990 | ||
|
|
1b69c9bb42 | ||
|
|
c2e649fc90 | ||
|
|
cff41bb9c2 | ||
|
|
0a9af7907f | ||
|
|
815d8407ce | ||
|
|
6cf6378e36 | ||
|
|
39f078e40f | ||
|
|
37a23ae169 | ||
|
|
8da8f3e4bc | ||
|
|
f97947b5a5 | ||
|
|
199b7460a9 | ||
|
|
41592ef1f8 | ||
|
|
bdbb8df17e | ||
|
|
f451ad5380 | ||
|
|
6e4009eed4 | ||
|
|
cf29b3fffe | ||
|
|
7fe76a83d1 | ||
|
|
b1cff500ad | ||
|
|
0b33057414 | ||
|
|
3891092ee9 | ||
|
|
8784aa45fc | ||
|
|
f1d90612e3 | ||
|
|
03d69f59a5 | ||
|
|
d372c8f5a9 | ||
|
|
dbb8495be1 | ||
|
|
044db3ec95 | ||
|
|
13217d5517 | ||
|
|
0165569bc1 | ||
|
|
84c14fe0a8 | ||
|
|
5b4af55017 | ||
|
|
b8ab2c628e | ||
|
|
766f4317a6 | ||
|
|
c20d897efe | ||
|
|
ad1e8f2cea | ||
|
|
51309ff55c | ||
|
|
e0aac5015f | ||
|
|
3b8290439a | ||
|
|
1a2836ff53 | ||
|
|
bce7368a82 | ||
|
|
1e461ef281 | ||
|
|
a5113622fd | ||
|
|
bfb7ab72f7 | ||
|
|
3d5c29c179 | ||
|
|
a05960ab07 | ||
|
|
6b499151d8 | ||
|
|
a9bd38a43e | ||
|
|
a544d244d8 | ||
|
|
a550b7a7d3 | ||
|
|
7035bbcf3a | ||
|
|
0c278f5fab | ||
|
|
03dd1c3ae2 | ||
|
|
6aa2234d9a | ||
|
|
5ae8f0bc54 | ||
|
|
e3536f768e | ||
|
|
ff1a78e1ce | ||
|
|
fc3f465321 | ||
|
|
b0956b12ae | ||
|
|
a774920b78 | ||
|
|
77d5f86b42 | ||
|
|
e118963b8f | ||
|
|
64f6117dc0 | ||
|
|
53fe0a077a | ||
|
|
828965c39d | ||
|
|
53e1ec81b3 | ||
|
|
608c96791f | ||
|
|
13ae241a0d | ||
|
|
cecbb4cc8c | ||
|
|
0ed4739482 | ||
|
|
76bdfb30ff | ||
|
|
d68dc8306b | ||
|
|
3b5045021d | ||
|
|
20c75ab0e5 | ||
|
|
44bcf5e9d4 | ||
|
|
ac3120703a | ||
|
|
c9866e2c85 | ||
|
|
5f2f4908ae | ||
|
|
f29c83dee9 | ||
|
|
d56b1be103 | ||
|
|
346f0e5851 | ||
|
|
91b8e1a8ae |
@@ -20,11 +20,6 @@
|
||||
# include dist directory, where the reth binary is located after compilation
|
||||
!/dist
|
||||
|
||||
# include PGO build helper used by Dockerfile.depot
|
||||
!/.github
|
||||
!/.github/scripts
|
||||
!/.github/scripts/build_pgo_bolt.sh
|
||||
|
||||
# include licenses
|
||||
!LICENSE-*
|
||||
|
||||
|
||||
4
.github/dependabot.yml
vendored
4
.github/dependabot.yml
vendored
@@ -4,10 +4,14 @@ updates:
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
- package-ecosystem: "cargo"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
labels:
|
||||
- "A-dependencies"
|
||||
commit-message:
|
||||
|
||||
118
.github/scripts/bench-reth-build.sh
vendored
118
.github/scripts/bench-reth-build.sh
vendored
@@ -1,24 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds (or fetches from cache) reth binaries for benchmarking.
|
||||
# Builds reth binaries for benchmarking from local source only.
|
||||
#
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit>
|
||||
#
|
||||
# baseline — build/fetch the baseline binary at <commit> (merge-base)
|
||||
# baseline — build the baseline binary at <commit> (merge-base)
|
||||
# source-dir must be checked out at <commit>
|
||||
# feature — build/fetch the candidate binary + reth-bench at <commit>
|
||||
# feature — build the candidate binary + reth-bench at <commit>
|
||||
# source-dir must be checked out at <commit>
|
||||
# optional branch-sha is the PR head commit for cache key
|
||||
#
|
||||
# Outputs:
|
||||
# baseline: <source-dir>/target/profiling/reth (or reth-bb if BENCH_BIG_BLOCKS=true)
|
||||
# feature: <source-dir>/target/profiling/reth (or reth-bb), reth-bench installed to cargo bin
|
||||
#
|
||||
# Required: mc (MinIO client) with a configured alias
|
||||
# Optional env: BENCH_BIG_BLOCKS (true/false) — build reth-bb instead of reth
|
||||
set -euo pipefail
|
||||
set -euxo pipefail
|
||||
|
||||
MC="mc"
|
||||
MODE="$1"
|
||||
SOURCE_DIR="$2"
|
||||
COMMIT="$3"
|
||||
@@ -42,103 +39,38 @@ if [ "${BENCH_TRACY:-off}" != "off" ]; then
|
||||
EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
|
||||
fi
|
||||
|
||||
# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
|
||||
if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
|
||||
BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
|
||||
else
|
||||
BUILD_SUFFIX=""
|
||||
fi
|
||||
# Build the requested node binary with the benchmark profile.
|
||||
build_node_binary() {
|
||||
local features_arg=""
|
||||
local workspace_arg=""
|
||||
|
||||
# Verify a cached reth binary was built from the expected commit.
|
||||
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
|
||||
verify_binary() {
|
||||
local binary="$1" expected_commit="$2"
|
||||
local version binary_sha
|
||||
version=$("$binary" --version 2>/dev/null) || return 1
|
||||
binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
|
||||
if [ -z "$binary_sha" ]; then
|
||||
echo "Warning: could not extract commit SHA from version output"
|
||||
return 1
|
||||
cd "$SOURCE_DIR"
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
features_arg="--features ${EXTRA_FEATURES}"
|
||||
workspace_arg="--workspace"
|
||||
fi
|
||||
if [ "$binary_sha" = "$expected_commit" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
|
||||
return 1
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
baseline|main)
|
||||
BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat "${BUCKET}/${NODE_BIN}" &>/dev/null; then
|
||||
echo "Cache hit for baseline (${COMMIT}), downloading ${NODE_BIN}..."
|
||||
$MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}"
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}"
|
||||
if verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached baseline binary is stale, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
FEATURES_ARG=""
|
||||
WORKSPACE_ARG=""
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
FEATURES_ARG="--features ${EXTRA_FEATURES}"
|
||||
WORKSPACE_ARG="--workspace"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling $NODE_PKG $WORKSPACE_ARG $FEATURES_ARG
|
||||
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
|
||||
fi
|
||||
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
|
||||
build_node_binary
|
||||
;;
|
||||
|
||||
feature|branch)
|
||||
BRANCH_SHA="${4:-$COMMIT}"
|
||||
BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat "${BUCKET}/${NODE_BIN}" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
|
||||
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
$MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}"
|
||||
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" /home/ubuntu/.cargo/bin/reth-bench
|
||||
if verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached feature binary is stale, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
rustup show active-toolchain || rustup default stable
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# Can't use `make profiling` when adding features; build explicitly
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling --workspace $NODE_PKG --features "${EXTRA_FEATURES}"
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling $NODE_PKG
|
||||
fi
|
||||
make install-reth-bench
|
||||
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
|
||||
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
|
||||
fi
|
||||
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
|
||||
rustup show active-toolchain || rustup default stable
|
||||
build_node_binary
|
||||
make -C "$SOURCE_DIR" install-reth-bench
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit>"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
39
.github/scripts/bench-reth-local.sh
vendored
39
.github/scripts/bench-reth-local.sh
vendored
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# local-reth-bench.sh — Run the reth Engine API benchmark locally.
|
||||
#
|
||||
# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
|
||||
# Replicates the CI bench.yml workflow (build, local snapshot validation, system tuning,
|
||||
# interleaved B-F-F-B execution, summary, charts) without any GitHub
|
||||
# Actions glue (no PR comments, no artifact upload, no Slack).
|
||||
#
|
||||
@@ -21,15 +21,17 @@
|
||||
# Requires: the reth repo at RETH_REPO (default: ~/reth)
|
||||
#
|
||||
# Dependencies (install before first run):
|
||||
# mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
|
||||
# make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
|
||||
# schelk, cpupower, taskset, stdbuf, python3, curl,
|
||||
# make, uv, jq, Rust toolchain (cargo/rustup)
|
||||
# Optional:
|
||||
# mc for Tracy profile upload
|
||||
#
|
||||
# The script delegates to the existing bench-reth-*.sh scripts in the reth
|
||||
# repo for the actual build, snapshot, and run steps.
|
||||
set -euo pipefail
|
||||
set -euxo pipefail
|
||||
|
||||
# ── PATH ──────────────────────────────────────────────────────────────
|
||||
# Ensure cargo and user-local bins (mc, uv) are visible
|
||||
# Ensure cargo and user-local bins (uv) are visible
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────
|
||||
@@ -106,7 +108,7 @@ fi
|
||||
|
||||
# ── Check dependencies ───────────────────────────────────────────────
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq cargo; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -238,19 +240,14 @@ echo " Baseline src : $BASELINE_SRC"
|
||||
echo " Feature src : $FEATURE_SRC"
|
||||
echo
|
||||
|
||||
# ── Step 3: Check / download snapshot ────────────────────────────────
|
||||
echo "▸ Checking snapshot..."
|
||||
# ── Step 3: Validate local snapshot ──────────────────────────────────
|
||||
echo "▸ Validating local snapshot..."
|
||||
cd "$RETH_REPO"
|
||||
SNAPSHOT_NEEDED=false
|
||||
if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
|
||||
SNAPSHOT_NEEDED=true
|
||||
echo " Snapshot needs update."
|
||||
else
|
||||
echo " Snapshot is up-to-date."
|
||||
fi
|
||||
"${SCRIPTS_DIR}/bench-reth-snapshot.sh"
|
||||
echo " Snapshot is ready."
|
||||
echo
|
||||
|
||||
# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
|
||||
# ── Step 4: Build binaries in parallel ───────────────────────────────
|
||||
echo "▸ Building binaries (parallel)..."
|
||||
cd "$RETH_REPO"
|
||||
|
||||
@@ -262,19 +259,11 @@ PID_BASELINE=$!
|
||||
"${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
|
||||
PID_FEATURE=$!
|
||||
|
||||
PID_SNAPSHOT=
|
||||
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
|
||||
echo " Also downloading snapshot in parallel..."
|
||||
"${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
|
||||
PID_SNAPSHOT=$!
|
||||
fi
|
||||
|
||||
wait $PID_BASELINE || FAIL=1
|
||||
wait $PID_FEATURE || FAIL=1
|
||||
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
|
||||
|
||||
if [ $FAIL -ne 0 ]; then
|
||||
echo "Error: one or more parallel tasks failed (builds / snapshot)"
|
||||
echo "Error: one or more build tasks failed"
|
||||
exit 1
|
||||
fi
|
||||
echo " Binaries built successfully."
|
||||
|
||||
54
.github/scripts/bench-reth-run.sh
vendored
54
.github/scripts/bench-reth-run.sh
vendored
@@ -7,13 +7,14 @@
|
||||
#
|
||||
# Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
|
||||
# Optional env: BENCH_BIG_BLOCKS (true/false), BENCH_WORK_DIR (for big blocks path)
|
||||
# BENCH_BAL (false/true/feature/baseline; only used with big blocks)
|
||||
# BENCH_WAIT_TIME (duration like 500ms, default empty)
|
||||
# BENCH_BASELINE_ARGS (extra reth node args for baseline runs)
|
||||
# BENCH_FEATURE_ARGS (extra reth node args for feature runs)
|
||||
# BENCH_OTLP_TRACES_ENDPOINT (OTLP HTTP endpoint for traces, e.g. https://host/insert/opentelemetry/v1/traces)
|
||||
# BENCH_OTLP_LOGS_ENDPOINT (OTLP HTTP endpoint for logs, e.g. https://host/insert/opentelemetry/v1/logs)
|
||||
# BENCH_OTLP_DISABLED (true to skip OTLP export even if endpoints are set)
|
||||
set -euo pipefail
|
||||
set -euxo pipefail
|
||||
|
||||
LABEL="$1"
|
||||
BINARY="$2"
|
||||
@@ -87,10 +88,16 @@ trap cleanup EXIT
|
||||
# Stop any leftover reth process in the scope, then recover schelk state.
|
||||
sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
|
||||
sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
|
||||
sudo schelk recover -y --kill || true
|
||||
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
|
||||
|
||||
# Mount
|
||||
sudo schelk mount -y
|
||||
sudo schelk mount -y || true
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Failed to mount benchmark datadir at ${DATADIR}"
|
||||
ls -la "$SCHELK_MOUNT" || true
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
sync
|
||||
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
|
||||
echo "=== Cache state after drop ==="
|
||||
@@ -249,11 +256,33 @@ fi
|
||||
if [ "$BIG_BLOCKS" = "true" ]; then
|
||||
# Big blocks mode: replay pre-generated payloads
|
||||
BIG_BLOCKS_DIR="${BENCH_BIG_BLOCKS_DIR:-${BENCH_WORK_DIR}/big-blocks}"
|
||||
BENCH_BAL_MODE="${BENCH_BAL:-false}"
|
||||
|
||||
BB_BENCH_ARGS=(--reth-new-payload)
|
||||
if [ -n "${BENCH_WAIT_TIME:-}" ]; then
|
||||
BB_BENCH_ARGS+=(--wait-time "$BENCH_WAIT_TIME")
|
||||
fi
|
||||
case "$BENCH_BAL_MODE" in
|
||||
false)
|
||||
;;
|
||||
true)
|
||||
BB_BENCH_ARGS+=(--bal)
|
||||
;;
|
||||
baseline)
|
||||
if [[ "$LABEL" == baseline* ]]; then
|
||||
BB_BENCH_ARGS+=(--bal)
|
||||
fi
|
||||
;;
|
||||
feature)
|
||||
if [[ "$LABEL" == feature* ]]; then
|
||||
BB_BENCH_ARGS+=(--bal)
|
||||
fi
|
||||
;;
|
||||
*)
|
||||
echo "::error::Unknown BENCH_BAL value: $BENCH_BAL_MODE"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Warmup
|
||||
WARMUP="${BENCH_WARMUP_BLOCKS:-50}"
|
||||
@@ -294,13 +323,18 @@ if [ "$BIG_BLOCKS" = "true" ]; then
|
||||
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
|
||||
else
|
||||
# Standard mode: warmup + new-payload-fcu
|
||||
# Warmup
|
||||
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
|
||||
"${EXTRA_BENCH_ARGS[@]}" 2>&1 | sed -u "s/^/[bench] /"
|
||||
WARMUP="${BENCH_WARMUP_BLOCKS:-50}"
|
||||
if [ "$WARMUP" -gt 0 ] 2>/dev/null; then
|
||||
# Warm up the node before measuring the benchmark window.
|
||||
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "$WARMUP" \
|
||||
"${EXTRA_BENCH_ARGS[@]}" 2>&1 | sed -u "s/^/[bench] /"
|
||||
else
|
||||
echo "Skipping warmup (0 blocks)..."
|
||||
fi
|
||||
|
||||
# Start tracy-capture after warmup so profile only covers the benchmark
|
||||
if [ "${BENCH_TRACY:-off}" != "off" ]; then
|
||||
|
||||
139
.github/scripts/bench-reth-snapshot.sh
vendored
139
.github/scripts/bench-reth-snapshot.sh
vendored
@@ -1,129 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Downloads the latest snapshot into the schelk volume using
|
||||
# `reth download` with progress reporting to the GitHub PR comment.
|
||||
#
|
||||
# Skips the download if the manifest content hasn't changed since
|
||||
# the last successful download (checked via SHA-256 of the manifest).
|
||||
# Validates that the benchmark snapshot has already been populated into the
|
||||
# local schelk volume.
|
||||
#
|
||||
# Usage: bench-reth-snapshot.sh [--check]
|
||||
# --check Only check if a download is needed; exits 0 if up-to-date, 10 if not.
|
||||
# --check Exit 0 if the local snapshot is ready, 10 if it is missing.
|
||||
#
|
||||
# Required env:
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# BENCH_RETH_BINARY – path to the reth binary
|
||||
# GITHUB_TOKEN – token for GitHub API calls (only for download)
|
||||
# BENCH_COMMENT_ID – PR comment ID to update (optional)
|
||||
# BENCH_REPO – owner/repo (e.g. paradigmxyz/reth)
|
||||
# BENCH_JOB_URL – link to the Actions job
|
||||
# BENCH_ACTOR – user who triggered the benchmark
|
||||
# BENCH_CONFIG – config summary line
|
||||
set -euo pipefail
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# Optional env:
|
||||
# BENCH_BIG_BLOCKS – true when validating the big-blocks snapshot datadir
|
||||
# BENCH_SNAPSHOT_NAME – expected snapshot label for log/error output
|
||||
set -euxo pipefail
|
||||
|
||||
: "${SCHELK_MOUNT:?SCHELK_MOUNT must be set}"
|
||||
|
||||
MC="mc"
|
||||
BUCKET="minio/reth-snapshots"
|
||||
# Allow overriding the snapshot name (e.g. for big-blocks mode where the
|
||||
# big-blocks manifest specifies which base snapshot to use).
|
||||
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable}"
|
||||
MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
|
||||
DATADIR_NAME="datadir"
|
||||
HASH_MODE_SUFFIX=""
|
||||
if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
DATADIR_NAME="datadir-big-blocks"
|
||||
HASH_MODE_SUFFIX="-big-blocks"
|
||||
fi
|
||||
DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
|
||||
HASH_FILE="$HOME/.reth-bench-snapshot-hash${HASH_MODE_SUFFIX}"
|
||||
|
||||
# Fetch manifest and compute content hash for reliable freshness check
|
||||
MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
|
||||
exit 2
|
||||
describe_snapshot() {
|
||||
if [ -n "${BENCH_SNAPSHOT_NAME:-}" ]; then
|
||||
printf '%s' "${BENCH_SNAPSHOT_NAME}"
|
||||
elif [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
printf '%s' 'big-block weekly snapshot'
|
||||
else
|
||||
printf '%s' 'benchmark snapshot'
|
||||
fi
|
||||
}
|
||||
REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
|
||||
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
snapshot_ready() {
|
||||
[ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ]
|
||||
}
|
||||
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
EXPECTED_SNAPSHOT="$(describe_snapshot)"
|
||||
|
||||
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
|
||||
sudo schelk mount -y || true
|
||||
|
||||
if snapshot_ready; then
|
||||
echo "Found local ${EXPECTED_SNAPSHOT} at ${DATADIR}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
echo "::error::Missing local ${EXPECTED_SNAPSHOT} at ${DATADIR}. Benchmarks no longer download snapshots; pre-populate the local schelk data first."
|
||||
ls -la "$SCHELK_MOUNT" || true
|
||||
ls -la "$DATADIR" || true
|
||||
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 10
|
||||
fi
|
||||
|
||||
RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
|
||||
if [ ! -x "$RETH" ]; then
|
||||
echo "::error::reth binary not found or not executable at $RETH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve the MinIO HTTP endpoint from the mc alias so reth can
|
||||
# fetch archives over HTTP (the manifest's embedded base_url points
|
||||
# to the cluster-internal address which is unreachable from runners).
|
||||
MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
|
||||
if [ -z "$MINIO_ENDPOINT" ]; then
|
||||
echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
|
||||
exit 1
|
||||
fi
|
||||
BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/${SNAPSHOT_NAME}"
|
||||
|
||||
# Rewrite manifest's base_url with the runner-reachable endpoint
|
||||
MANIFEST_TMP=$(mktemp --suffix=.json)
|
||||
trap 'rm -f -- "$MANIFEST_TMP"' EXIT
|
||||
echo "$MANIFEST_CONTENT" \
|
||||
| jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
|
||||
|
||||
# Prepare mount. If a previous run left the volume mounted, recover first.
|
||||
sudo schelk recover -y --kill || true
|
||||
sudo schelk mount -y
|
||||
sudo rm -rf "$DATADIR"
|
||||
sudo mkdir -p "$DATADIR"
|
||||
# reth download runs as current user (not root), needs write access
|
||||
sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
|
||||
|
||||
update_comment() {
|
||||
local status="$1"
|
||||
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
|
||||
local body
|
||||
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
|
||||
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
|
||||
curl -sf -X PATCH \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
|
||||
-d "$(jq -nc --arg body "$body" '{body: $body}')" \
|
||||
> /dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
update_comment "Downloading snapshot…"
|
||||
|
||||
# Download using reth download (manifest-path with rewritten base_url)
|
||||
"$RETH" download \
|
||||
--manifest-path "$MANIFEST_TMP" \
|
||||
-y \
|
||||
--minimal \
|
||||
--datadir "$DATADIR"
|
||||
|
||||
update_comment "Downloading snapshot… done"
|
||||
echo "Snapshot download complete"
|
||||
|
||||
# Sanity check: verify expected directories exist
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Promote the new snapshot to become the schelk baseline (virgin volume).
|
||||
# This copies changed blocks from scratch → virgin so that future
|
||||
# `schelk recover` calls restore to this new state.
|
||||
sync
|
||||
sudo schelk promote -y
|
||||
|
||||
# Save manifest hash
|
||||
echo "$REMOTE_HASH" > "$HASH_FILE"
|
||||
echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
exit 1
|
||||
|
||||
49
.github/scripts/bench-reth-summary.py
vendored
49
.github/scripts/bench-reth-summary.py
vendored
@@ -111,6 +111,14 @@ def compute_stats(combined: list[dict]) -> dict:
|
||||
wall_clock_s = sum(total_latencies_ms) / 1_000
|
||||
mean_total_lat_ms = sum(total_latencies_ms) / n
|
||||
|
||||
# Persistence wait mean (for main table)
|
||||
persist_values_ms = []
|
||||
for r in combined:
|
||||
v = r.get("persistence_wait_us")
|
||||
if v is not None:
|
||||
persist_values_ms.append(v / 1_000)
|
||||
mean_persist_ms = sum(persist_values_ms) / len(persist_values_ms) if persist_values_ms else 0.0
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_ms": mean_lat,
|
||||
@@ -121,6 +129,7 @@ def compute_stats(combined: list[dict]) -> dict:
|
||||
"mean_mgas_s": mean_mgas_s,
|
||||
"wall_clock_s": wall_clock_s,
|
||||
"mean_total_lat_ms": mean_total_lat_ms,
|
||||
"mean_persist_ms": mean_persist_ms,
|
||||
}
|
||||
|
||||
|
||||
@@ -145,7 +154,7 @@ def compute_wait_stats(combined: list[dict], field: str) -> dict:
|
||||
|
||||
def _paired_data(
|
||||
baseline: list[dict], feature: list[dict]
|
||||
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float]]:
|
||||
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float], list[float]]:
|
||||
"""Match blocks and return paired latencies and per-block diffs.
|
||||
|
||||
Returns:
|
||||
@@ -153,6 +162,7 @@ def _paired_data(
|
||||
lat_diffs_ms: list of feature − baseline latency diffs in ms
|
||||
mgas_diffs: list of feature − baseline Mgas/s diffs
|
||||
total_lat_diffs_ms: list of feature − baseline total latency diffs in ms
|
||||
persist_diffs_ms: list of feature − baseline persistence wait diffs in ms
|
||||
"""
|
||||
baseline_by_block = {r["block_number"]: r for r in baseline}
|
||||
feature_by_block = {r["block_number"]: r for r in feature}
|
||||
@@ -162,6 +172,7 @@ def _paired_data(
|
||||
lat_diffs_ms = []
|
||||
mgas_diffs = []
|
||||
total_lat_diffs_ms = []
|
||||
persist_diffs_ms = []
|
||||
for bn in common_blocks:
|
||||
b = baseline_by_block[bn]
|
||||
f = feature_by_block[bn]
|
||||
@@ -179,7 +190,10 @@ def _paired_data(
|
||||
total_lat_diffs_ms.append(
|
||||
f["total_latency_us"] / 1_000 - b["total_latency_us"] / 1_000
|
||||
)
|
||||
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms
|
||||
b_persist = (b.get("persistence_wait_us") or 0) / 1_000
|
||||
f_persist = (f.get("persistence_wait_us") or 0) / 1_000
|
||||
persist_diffs_ms.append(f_persist - b_persist)
|
||||
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms, persist_diffs_ms
|
||||
|
||||
|
||||
def compute_paired_stats(
|
||||
@@ -195,13 +209,15 @@ def compute_paired_stats(
|
||||
all_lat_diffs = []
|
||||
all_mgas_diffs = []
|
||||
all_total_lat_diffs = []
|
||||
all_persist_diffs = []
|
||||
blocks_per_pair = []
|
||||
for baseline, feature in zip(baseline_runs, feature_runs):
|
||||
pairs, lat_diffs, mgas_diffs, total_lat_diffs = _paired_data(baseline, feature)
|
||||
pairs, lat_diffs, mgas_diffs, total_lat_diffs, persist_diffs = _paired_data(baseline, feature)
|
||||
all_pairs.extend(pairs)
|
||||
all_lat_diffs.extend(lat_diffs)
|
||||
all_mgas_diffs.extend(mgas_diffs)
|
||||
all_total_lat_diffs.extend(total_lat_diffs)
|
||||
all_persist_diffs.extend(persist_diffs)
|
||||
blocks_per_pair.append(len(pairs))
|
||||
|
||||
if not all_lat_diffs:
|
||||
@@ -245,6 +261,11 @@ def compute_paired_stats(
|
||||
total_se = std_total_diff / math.sqrt(len(all_total_lat_diffs)) if all_total_lat_diffs else 0.0
|
||||
wall_clock_ci_ms = T_CRITICAL * total_se
|
||||
|
||||
mean_persist_diff = sum(all_persist_diffs) / len(all_persist_diffs) if all_persist_diffs else 0.0
|
||||
std_persist_diff = stddev(all_persist_diffs, mean_persist_diff) if len(all_persist_diffs) > 1 else 0.0
|
||||
persist_se = std_persist_diff / math.sqrt(len(all_persist_diffs)) if all_persist_diffs else 0.0
|
||||
persist_ci_ms = T_CRITICAL * persist_se
|
||||
|
||||
return {
|
||||
"n": n,
|
||||
"mean_diff_ms": mean_diff,
|
||||
@@ -258,6 +279,7 @@ def compute_paired_stats(
|
||||
"mean_mgas_diff": mean_mgas_diff,
|
||||
"mgas_ci": mgas_ci,
|
||||
"wall_clock_ci_ms": wall_clock_ci_ms,
|
||||
"persist_ci_ms": persist_ci_ms,
|
||||
"blocks": max(blocks_per_pair),
|
||||
}
|
||||
|
||||
@@ -290,6 +312,14 @@ def fmt_s(v: float) -> str:
|
||||
return f"{v:.2f}s"
|
||||
|
||||
|
||||
def display_bal_mode(bal_mode: str | None) -> str | None:
|
||||
if not bal_mode or bal_mode == "false":
|
||||
return None
|
||||
if bal_mode == "both":
|
||||
return "true"
|
||||
return bal_mode
|
||||
|
||||
|
||||
def significance(pct: float, ci_pct: float, lower_is_better: bool) -> str:
|
||||
"""Return significance label: 'good', 'bad', or 'neutral'."""
|
||||
significant = abs(pct) > ci_pct
|
||||
@@ -328,6 +358,7 @@ def compute_changes(
|
||||
("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
|
||||
("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
|
||||
("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True),
|
||||
("persist_wait", "mean_persist_ms", "persist_ci_ms", "mean_persist_ms", True),
|
||||
]
|
||||
changes = {}
|
||||
for name, stat_key, ci_key, base_key, lower_is_better in metrics:
|
||||
@@ -353,6 +384,7 @@ def generate_comparison_table(
|
||||
big_blocks: bool = False,
|
||||
warmup_blocks: str | None = None,
|
||||
wait_time: str | None = None,
|
||||
bal_mode: str | None = None,
|
||||
) -> str:
|
||||
"""Generate a markdown comparison table between baseline and feature."""
|
||||
n = paired["blocks"]
|
||||
@@ -368,6 +400,8 @@ def generate_comparison_table(
|
||||
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
|
||||
p99_pct = pct(run1["p99_ms"], run2["p99_ms"])
|
||||
|
||||
persist_pct = pct(run1["mean_persist_ms"], run2["mean_persist_ms"])
|
||||
|
||||
# Bootstrap CIs as % of baseline percentile
|
||||
p50_ci_pct = paired["p50_ci_ms"] / run1["p50_ms"] * 100.0 if run1["p50_ms"] > 0 else 0.0
|
||||
p90_ci_pct = paired["p90_ci_ms"] / run1["p90_ms"] * 100.0 if run1["p90_ms"] > 0 else 0.0
|
||||
@@ -377,6 +411,7 @@ def generate_comparison_table(
|
||||
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
|
||||
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
|
||||
wall_ci_pct = paired["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0
|
||||
persist_ci_pct = paired["persist_ci_ms"] / run1["mean_persist_ms"] * 100.0 if run1["mean_persist_ms"] > 0 else 0.0
|
||||
|
||||
base_url = f"https://github.com/{repo}/commit"
|
||||
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
|
||||
@@ -392,6 +427,7 @@ def generate_comparison_table(
|
||||
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
|
||||
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
|
||||
f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, lower_is_better=True)} |",
|
||||
f"| Persist Wait | {fmt_ms(run1['mean_persist_ms'])} | {fmt_ms(run2['mean_persist_ms'])} | {change_str(persist_pct, persist_ci_pct, lower_is_better=True)} |",
|
||||
"",
|
||||
]
|
||||
meta_parts = [f"{n} {'big blocks' if big_blocks else 'blocks'}"]
|
||||
@@ -399,6 +435,9 @@ def generate_comparison_table(
|
||||
meta_parts.append(f"{warmup_blocks} warmup")
|
||||
if wait_time:
|
||||
meta_parts.append(f"wait time: {wait_time}")
|
||||
display_mode = display_bal_mode(bal_mode)
|
||||
if big_blocks and display_mode:
|
||||
meta_parts.append(f"BAL: {display_mode}")
|
||||
lines.append(f"*{', '.join(meta_parts)}*")
|
||||
return "\n".join(lines)
|
||||
|
||||
@@ -481,6 +520,7 @@ def main():
|
||||
parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode")
|
||||
parser.add_argument("--warmup-blocks", default=None, help="Number of warmup blocks")
|
||||
parser.add_argument("--wait-time", default=None, help="Wait time interval used between blocks")
|
||||
parser.add_argument("--bal-mode", default=None, help="BAL mode (true, feature, baseline)")
|
||||
parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL for this benchmark run")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -520,6 +560,7 @@ def main():
|
||||
baseline_name = args.baseline_name or "baseline"
|
||||
feature_name = args.feature_name or "feature"
|
||||
feature_sha = args.feature_ref or "unknown"
|
||||
bal_mode = display_bal_mode(args.bal_mode)
|
||||
|
||||
comparison_table = generate_comparison_table(
|
||||
baseline_stats,
|
||||
@@ -533,6 +574,7 @@ def main():
|
||||
big_blocks=args.big_blocks,
|
||||
warmup_blocks=args.warmup_blocks,
|
||||
wait_time=args.wait_time,
|
||||
bal_mode=bal_mode,
|
||||
)
|
||||
print(f"Generated comparison ({paired_stats['n']} paired blocks, "
|
||||
f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")
|
||||
@@ -566,6 +608,7 @@ def main():
|
||||
"big_blocks": args.big_blocks,
|
||||
"warmup_blocks": args.warmup_blocks,
|
||||
"wait_time": args.wait_time,
|
||||
"bal_mode": bal_mode,
|
||||
"baseline": {
|
||||
"name": baseline_name,
|
||||
"ref": baseline_ref,
|
||||
|
||||
111
.github/scripts/bench-scheduled-refs.sh
vendored
111
.github/scripts/bench-scheduled-refs.sh
vendored
@@ -2,17 +2,20 @@
|
||||
#
|
||||
# Resolves baseline and feature refs for scheduled benchmark runs.
|
||||
#
|
||||
# Supports two modes:
|
||||
# Supports three modes:
|
||||
# nightly — Queries the latest successful scheduled docker.yml run via
|
||||
# GitHub API to find the nightly Docker image commit. Compares
|
||||
# with the last successful feature ref to detect staleness.
|
||||
# hourly — Compares origin/main HEAD against the last successfully
|
||||
# benchmarked commit (falls back to HEAD~1 on first run).
|
||||
# Checks for in-progress sibling runs to avoid overlap.
|
||||
# release — Compares the latest GitHub release tag against the current
|
||||
# nightly Docker build. Baseline is the release tag commit,
|
||||
# feature is the nightly commit.
|
||||
#
|
||||
# Usage: bench-scheduled-refs.sh <force> <mode>
|
||||
# force — "true" to run even if no new commit (bypass skip logic)
|
||||
# mode — "nightly" or "hourly"
|
||||
# mode — "nightly", "hourly", or "release"
|
||||
#
|
||||
# Outputs (via GITHUB_OUTPUT):
|
||||
# baseline-ref — commit SHA for baseline
|
||||
@@ -21,13 +24,15 @@
|
||||
# is-stale — "true" if latest nightly build is >24h old (nightly only)
|
||||
# stale-age-hours — age of the nightly build in hours (nightly only)
|
||||
# nightly-created — ISO timestamp of the nightly build (nightly only)
|
||||
# release-tag — release tag name (release mode only, e.g. "v2.0.0")
|
||||
#
|
||||
# Reads:
|
||||
# state/nightly-last-feature-ref (nightly, from decofe/reth-bench-charts repo)
|
||||
# state/hourly-last-feature-ref (hourly, from decofe/reth-bench-charts repo)
|
||||
# state/release-last-feature-ref (release, from decofe/reth-bench-charts repo)
|
||||
#
|
||||
# Requires: gh (GitHub CLI), jq, date, git (hourly mode), curl, DEREK_TOKEN env
|
||||
set -euo pipefail
|
||||
set -euxo pipefail
|
||||
|
||||
FORCE="${1:-false}"
|
||||
MODE="${2:-nightly}"
|
||||
@@ -121,6 +126,106 @@ if [ "$MODE" = "hourly" ]; then
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==========================================================================
|
||||
# Release mode: compare latest GitHub release tag vs current nightly build
|
||||
# ==========================================================================
|
||||
if [ "$MODE" = "release" ]; then
|
||||
|
||||
# --- Step 1: Resolve feature ref from latest nightly Docker build ---
|
||||
echo "::group::Querying latest nightly docker build"
|
||||
RUNS_JSON=$(gh run list \
|
||||
-R "$REPO" \
|
||||
--workflow=docker.yml \
|
||||
--event=schedule \
|
||||
--status=completed \
|
||||
--limit 5 \
|
||||
--json headSha,createdAt,conclusion)
|
||||
|
||||
LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
|
||||
if [ -z "$LATEST" ]; then
|
||||
echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
|
||||
echo "Nightly commit (feature): $FEATURE_REF"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 2: Resolve baseline ref from latest GitHub release ---
|
||||
echo "::group::Resolving latest release tag"
|
||||
RELEASE_JSON=$(gh release view --repo "$REPO" --json tagName,targetCommitish,publishedAt 2>/dev/null || echo "{}")
|
||||
RELEASE_TAG=$(echo "$RELEASE_JSON" | jq -r '.tagName // empty')
|
||||
|
||||
if [ -z "$RELEASE_TAG" ]; then
|
||||
echo "::error::No release found on $REPO"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve the tag to a commit SHA
|
||||
BASELINE_REF=$(gh api "repos/$REPO/git/ref/tags/$RELEASE_TAG" --jq '.object.sha' 2>/dev/null || true)
|
||||
|
||||
# If tag points to an annotated tag object, dereference to the commit
|
||||
if [ -n "$BASELINE_REF" ]; then
|
||||
OBJ_TYPE=$(gh api "repos/$REPO/git/tags/$BASELINE_REF" --jq '.object.type' 2>/dev/null || echo "commit")
|
||||
if [ "$OBJ_TYPE" = "commit" ]; then
|
||||
BASELINE_REF=$(gh api "repos/$REPO/git/tags/$BASELINE_REF" --jq '.object.sha' 2>/dev/null || echo "$BASELINE_REF")
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ -z "$BASELINE_REF" ]; then
|
||||
echo "::error::Could not resolve release tag $RELEASE_TAG to a commit"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Release tag: $RELEASE_TAG"
|
||||
echo "Release commit (baseline): $BASELINE_REF"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 3: Read last successful feature ref from charts repo ---
|
||||
echo "::group::Reading persisted state"
|
||||
LAST_FEATURE_REF=""
|
||||
STATE_URL="https://raw.githubusercontent.com/decofe/reth-bench-charts/state/state/release-last-feature-ref"
|
||||
if RAW=$(curl -sfL -H "Authorization: token ${DEREK_TOKEN}" "$STATE_URL"); then
|
||||
LAST_FEATURE_REF=$(echo "$RAW" | tr -d '[:space:]')
|
||||
echo "Previous feature ref: $LAST_FEATURE_REF"
|
||||
else
|
||||
echo "No persisted state found (first run)"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 4: Skip logic ---
|
||||
echo "::group::Resolving skip logic"
|
||||
SHOULD_SKIP="false"
|
||||
if [ -n "$LAST_FEATURE_REF" ] && [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
|
||||
if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
|
||||
echo "No new nightly, but force=true — running anyway"
|
||||
else
|
||||
SHOULD_SKIP="true"
|
||||
echo "No new nightly since last release regression run — will skip"
|
||||
fi
|
||||
else
|
||||
echo "New nightly detected or first run"
|
||||
fi
|
||||
|
||||
echo "Baseline: $BASELINE_REF ($RELEASE_TAG)"
|
||||
echo "Feature: $FEATURE_REF"
|
||||
echo "Skip: $SHOULD_SKIP"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 5: Write outputs ---
|
||||
{
|
||||
echo "baseline-ref=$BASELINE_REF"
|
||||
echo "feature-ref=$FEATURE_REF"
|
||||
echo "should-skip=$SHOULD_SKIP"
|
||||
echo "is-stale=false"
|
||||
echo "stale-age-hours=0"
|
||||
echo "nightly-created="
|
||||
echo "long-running=false"
|
||||
echo "release-tag=$RELEASE_TAG"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# ==========================================================================
|
||||
# Nightly mode: query latest Docker nightly build (original logic)
|
||||
# ==========================================================================
|
||||
|
||||
10
.github/scripts/bench-slack-notify.js
vendored
10
.github/scripts/bench-slack-notify.js
vendored
@@ -250,6 +250,8 @@ async function success({ core, context }) {
|
||||
}
|
||||
}
|
||||
|
||||
const slackMode = process.env.BENCH_SLACK || 'always';
|
||||
|
||||
// Post to public channel if any metric shows significant improvement or regression
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
let postedToChannel = false;
|
||||
@@ -264,6 +266,14 @@ async function success({ core, context }) {
|
||||
}
|
||||
}
|
||||
|
||||
// In on-win mode, only notify on improvement — skip DM fallback entirely
|
||||
if (slackMode === 'on-win') {
|
||||
if (!postedToChannel) {
|
||||
core.info('on-win mode: no improvement detected, skipping all notifications');
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// DM the actor only when results were not posted to the public channel
|
||||
if (!postedToChannel) {
|
||||
if (actorSlackId) {
|
||||
|
||||
16
.github/scripts/bench-utils.js
vendored
16
.github/scripts/bench-utils.js
vendored
@@ -39,10 +39,25 @@ function loadSamplyUrls(workDir) {
|
||||
return urls;
|
||||
}
|
||||
|
||||
function balModeLabel(mode) {
|
||||
switch (mode) {
|
||||
case 'true':
|
||||
case 'feature':
|
||||
case 'baseline':
|
||||
return mode;
|
||||
case 'both':
|
||||
return 'true';
|
||||
default:
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
function blocksLabel(summary) {
|
||||
const parts = [];
|
||||
if (summary.big_blocks) {
|
||||
parts.push({ key: 'Big Blocks', value: summary.blocks });
|
||||
const balMode = balModeLabel(summary.bal_mode || summary.bal || process.env.BENCH_BAL || 'false');
|
||||
if (balMode) parts.push({ key: 'BAL', value: balMode });
|
||||
} else {
|
||||
const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
|
||||
if (warmup) parts.push({ key: 'Warmup', value: warmup });
|
||||
@@ -68,6 +83,7 @@ function metricRows(summary) {
|
||||
{ label: 'P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) },
|
||||
{ label: 'Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
|
||||
{ label: 'Wall Clock', baseline: fmtS(b.wall_clock_s), feature: fmtS(f.wall_clock_s), change: fmtChange(c.wall_clock) },
|
||||
{ label: 'Persist Wait', baseline: fmtMs(b.mean_persist_ms || 0), feature: fmtMs(f.mean_persist_ms || 0), change: fmtChange(c.persist_wait) },
|
||||
];
|
||||
}
|
||||
|
||||
|
||||
414
.github/scripts/build_pgo_bolt.sh
vendored
414
.github/scripts/build_pgo_bolt.sh
vendored
@@ -1,414 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Full PGO+BOLT optimized build for reth using real reth-bench workloads.
|
||||
#
|
||||
# Phases:
|
||||
# 1. Build PGO-instrumented reth, run reth-bench → collect PGO profiles
|
||||
# 2. Build BOLT-instrumented reth (with PGO), run reth-bench → collect BOLT profiles
|
||||
# 3. Build final PGO+BOLT optimized binary
|
||||
#
|
||||
# Required environment variables:
|
||||
# DATADIR - Path to reth datadir (must already contain chain data)
|
||||
# RPC_URL - Source RPC URL for reth-bench to fetch payloads from
|
||||
#
|
||||
# Optional environment variables:
|
||||
# PGO_BLOCKS - Number of blocks for PGO profiling (default: 20)
|
||||
# BOLT_BLOCKS - Number of blocks for BOLT profiling (default: 20)
|
||||
# SKIP_BOLT - Temporarily skip BOLT phases (default: false)
|
||||
# STRIP_SYMBOLS - Strip debug symbols from output binary (default: true)
|
||||
# COLLECT_PGO_ONLY - Stop after producing merged.profdata (default: false)
|
||||
# PGO_PROFDATA - Path to pre-collected merged.profdata (optional)
|
||||
# PROFILE - Cargo profile (default: maxperf-symbols)
|
||||
# FEATURES - Cargo features (default: jemalloc,asm-keccak,min-debug-logs)
|
||||
# TARGET - Target triple (default: auto-detected)
|
||||
# EXTRA_RUSTFLAGS - Additional RUSTFLAGS (e.g. -C target-cpu=x86-64-v3)
|
||||
#
|
||||
# Output:
|
||||
# target/$PROFILE_DIR/reth — final optimized binary
|
||||
set -euo pipefail
|
||||
|
||||
gha_section_start() {
|
||||
local title="$1"
|
||||
if [ -n "${GITHUB_ACTIONS:-}" ]; then
|
||||
echo "::group::$title"
|
||||
else
|
||||
echo ""
|
||||
echo "=== $title ==="
|
||||
fi
|
||||
}
|
||||
|
||||
gha_section_end() {
|
||||
if [ -n "${GITHUB_ACTIONS:-}" ]; then
|
||||
echo "::endgroup::"
|
||||
fi
|
||||
}
|
||||
|
||||
cd "$(dirname "$0")/../.."
|
||||
|
||||
# ── Configuration ──────────────────────────────────────────────────────────────
|
||||
PGO_BLOCKS="${PGO_BLOCKS:-20}"
|
||||
BOLT_BLOCKS="${BOLT_BLOCKS:-20}"
|
||||
SKIP_BOLT="${SKIP_BOLT:-false}"
|
||||
STRIP_SYMBOLS="${STRIP_SYMBOLS:-true}"
|
||||
COLLECT_PGO_ONLY="${COLLECT_PGO_ONLY:-false}"
|
||||
PROFILE="${PROFILE:-maxperf-symbols}"
|
||||
FEATURES="${FEATURES:-jemalloc,asm-keccak,min-debug-logs}"
|
||||
TARGET="${TARGET:-$(rustc -Vv | grep host | cut -d' ' -f2)}"
|
||||
BASE_RUSTFLAGS="${RUSTFLAGS:-}"
|
||||
EXTRA_RUSTFLAGS="${EXTRA_RUSTFLAGS:-}"
|
||||
COMBINED_RUSTFLAGS="$BASE_RUSTFLAGS $EXTRA_RUSTFLAGS"
|
||||
PGO_PROFDATA="${PGO_PROFDATA:-}"
|
||||
DATADIR="${DATADIR:-}"
|
||||
RPC_URL="${RPC_URL:-}"
|
||||
|
||||
SKIP_BOLT_BOOL=false
|
||||
if [[ "${SKIP_BOLT,,}" == "true" || "$SKIP_BOLT" == "1" ]]; then
|
||||
SKIP_BOLT_BOOL=true
|
||||
fi
|
||||
|
||||
STRIP_SYMBOLS_BOOL=false
|
||||
if [[ "${STRIP_SYMBOLS,,}" == "true" || "$STRIP_SYMBOLS" == "1" ]]; then
|
||||
STRIP_SYMBOLS_BOOL=true
|
||||
fi
|
||||
|
||||
COLLECT_PGO_ONLY_BOOL=false
|
||||
if [[ "${COLLECT_PGO_ONLY,,}" == "true" || "$COLLECT_PGO_ONLY" == "1" ]]; then
|
||||
COLLECT_PGO_ONLY_BOOL=true
|
||||
fi
|
||||
|
||||
USE_PRECOLLECTED_PGO=false
|
||||
if [ -n "$PGO_PROFDATA" ]; then
|
||||
if [ ! -f "$PGO_PROFDATA" ]; then
|
||||
echo "error: PGO_PROFDATA points to a missing file: $PGO_PROFDATA"
|
||||
exit 1
|
||||
fi
|
||||
USE_PRECOLLECTED_PGO=true
|
||||
fi
|
||||
|
||||
NEEDS_BENCH_WORKLOAD=true
|
||||
if [ "$USE_PRECOLLECTED_PGO" = true ] && [ "$SKIP_BOLT_BOOL" = true ]; then
|
||||
NEEDS_BENCH_WORKLOAD=false
|
||||
fi
|
||||
|
||||
if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
|
||||
: "${DATADIR:?DATADIR must be set to the reth data directory}"
|
||||
: "${RPC_URL:?RPC_URL must be set}"
|
||||
fi
|
||||
|
||||
if [[ "$PROFILE" == dev ]]; then
|
||||
PROFILE_DIR=debug
|
||||
else
|
||||
PROFILE_DIR=$PROFILE
|
||||
fi
|
||||
|
||||
MANIFEST_PATH="bin/reth"
|
||||
|
||||
LLVM_VERSION=$(rustc -Vv | grep -oP 'LLVM version: \K\d+')
|
||||
PGO_DIR="$PWD/target/pgo-profiles"
|
||||
BOLT_DIR="$PWD/target/bolt-profiles"
|
||||
CARGO_ARGS=(--profile "$PROFILE" --features "$FEATURES" --manifest-path "$MANIFEST_PATH/Cargo.toml" --bin "reth" --locked)
|
||||
|
||||
# Enable debug symbols for BOLT (requires symbols to reorder code).
|
||||
# Strip them at the end.
|
||||
PROFILE_UPPER=$(echo "$PROFILE" | tr '[:lower:]-' '[:upper:]_')
|
||||
export "CARGO_PROFILE_${PROFILE_UPPER}_STRIP=debuginfo"
|
||||
|
||||
gha_section_start "Full PGO+BOLT Build"
|
||||
echo "Binary: reth"
|
||||
echo "Manifest: $MANIFEST_PATH"
|
||||
echo "Target: $TARGET"
|
||||
echo "Profile: $PROFILE"
|
||||
echo "Features: $FEATURES"
|
||||
echo "LLVM: $LLVM_VERSION"
|
||||
echo "PGO blocks: $PGO_BLOCKS"
|
||||
echo "BOLT blocks: $BOLT_BLOCKS"
|
||||
echo "Skip BOLT: $SKIP_BOLT"
|
||||
echo "Strip symbols: $STRIP_SYMBOLS"
|
||||
echo "Collect only: $COLLECT_PGO_ONLY"
|
||||
echo "PGO profdata: ${PGO_PROFDATA:-<collect with reth-bench>}"
|
||||
echo "RUSTFLAGS: ${BASE_RUSTFLAGS:-<unset>}"
|
||||
echo "EXTRA_RUSTFLAGS: ${EXTRA_RUSTFLAGS:-<unset>}"
|
||||
if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
|
||||
echo "Datadir: $DATADIR"
|
||||
echo "RPC URL: $RPC_URL"
|
||||
else
|
||||
echo "Datadir: <not required>"
|
||||
echo "RPC URL: <not required>"
|
||||
fi
|
||||
gha_section_end
|
||||
|
||||
# ── Prerequisites ──────────────────────────────────────────────────────────────
|
||||
gha_section_start "Installing prerequisites"
|
||||
rustup component add llvm-tools-preview
|
||||
|
||||
LLVM_PROFDATA=$(find "$(rustc --print sysroot)" -name llvm-profdata -type f | head -1)
|
||||
if [ -z "$LLVM_PROFDATA" ]; then
|
||||
echo "error: llvm-profdata not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
install_bolt() {
|
||||
if command -v llvm-bolt &>/dev/null; then
|
||||
echo "BOLT already installed"
|
||||
return
|
||||
fi
|
||||
echo "Installing BOLT from apt.llvm.org..."
|
||||
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc >/dev/null
|
||||
CODENAME=$(lsb_release -cs)
|
||||
echo "deb http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-$LLVM_VERSION main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y -qq "bolt-$LLVM_VERSION"
|
||||
sudo ln -sf "/usr/bin/llvm-bolt-$LLVM_VERSION" /usr/local/bin/llvm-bolt
|
||||
sudo ln -sf "/usr/bin/merge-fdata-$LLVM_VERSION" /usr/local/bin/merge-fdata
|
||||
}
|
||||
if [ "$SKIP_BOLT_BOOL" = true ]; then
|
||||
echo "Skipping BOLT installation (SKIP_BOLT=$SKIP_BOLT)"
|
||||
else
|
||||
install_bolt
|
||||
fi
|
||||
gha_section_end
|
||||
|
||||
if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
|
||||
# Build reth-bench once (non-instrumented) — reused for both phases.
|
||||
gha_section_start "Building reth-bench"
|
||||
RUSTFLAGS="$COMBINED_RUSTFLAGS" \
|
||||
cargo build --profile "$PROFILE" --features "$FEATURES" \
|
||||
--manifest-path bin/reth-bench/Cargo.toml --bin reth-bench --locked
|
||||
RETH_BENCH_BIN="$(find target -name reth-bench -type f -executable | head -1)"
|
||||
echo "reth-bench: $RETH_BENCH_BIN"
|
||||
gha_section_end
|
||||
else
|
||||
gha_section_start "Building reth-bench"
|
||||
echo "Skipping reth-bench build (pre-collected PGO with SKIP_BOLT=true)"
|
||||
gha_section_end
|
||||
fi
|
||||
|
||||
# ── Helpers ────────────────────────────────────────────────────────────────────
|
||||
RETH_PID=
|
||||
cleanup() {
|
||||
if [ -n "${RETH_PID:-}" ] && kill -0 "$RETH_PID" 2>/dev/null; then
|
||||
echo "Stopping reth (pid $RETH_PID)..."
|
||||
sudo kill "$RETH_PID" 2>/dev/null || true
|
||||
for i in $(seq 1 60); do
|
||||
sudo kill -0 "$RETH_PID" 2>/dev/null || break
|
||||
if [ $((i % 10)) -eq 0 ]; then
|
||||
echo " waiting... (${i}s)"
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
sudo kill -9 "$RETH_PID" 2>/dev/null || true
|
||||
fi
|
||||
}
|
||||
trap cleanup EXIT
|
||||
|
||||
# Start reth, wait for RPC, run reth-bench, then stop reth.
|
||||
# Arguments: $1 = reth binary path, $2 = number of blocks, $3 = log label
|
||||
run_bench_workload() {
|
||||
local reth_bin="$1" blocks="$2" label="$3"
|
||||
local http_port=8545 authrpc_port=8551
|
||||
|
||||
echo "--- Starting reth ($label) ---"
|
||||
sudo "$reth_bin" node \
|
||||
--datadir "$DATADIR" \
|
||||
--log.file.directory "/tmp/reth-${label}-logs" \
|
||||
--engine.accept-execution-requests-hash \
|
||||
--http --http.port "$http_port" \
|
||||
--authrpc.port "$authrpc_port" \
|
||||
--disable-discovery --no-persist-peers \
|
||||
> "/tmp/reth-${label}.log" 2>&1 &
|
||||
RETH_PID=$!
|
||||
|
||||
echo "Waiting for reth RPC..."
|
||||
for i in $(seq 1 120); do
|
||||
if curl -sf "http://127.0.0.1:$http_port" -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
> /dev/null 2>&1; then
|
||||
echo "reth is ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 120 ]; then
|
||||
echo "error: reth failed to start within 120s"
|
||||
cat "/tmp/reth-${label}.log"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
|
||||
echo "Running reth-bench ($blocks blocks)..."
|
||||
"$RETH_BENCH_BIN" new-payload-fcu \
|
||||
--rpc-url "$RPC_URL" \
|
||||
--engine-rpc-url "http://127.0.0.1:$authrpc_port" \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "$blocks" \
|
||||
--reth-new-payload 2>&1 | sed -u "s/^/[$label] /"
|
||||
|
||||
echo "Stopping reth ($label)..."
|
||||
sudo kill "$RETH_PID" 2>/dev/null || true
|
||||
for i in $(seq 1 60); do
|
||||
sudo kill -0 "$RETH_PID" 2>/dev/null || break
|
||||
sleep 1
|
||||
done
|
||||
sudo kill -9 "$RETH_PID" 2>/dev/null || true
|
||||
RETH_PID=
|
||||
}
|
||||
|
||||
publish_binary() {
|
||||
local source_bin="$1"
|
||||
for out in "target/$TARGET/$PROFILE_DIR" "target/$PROFILE_DIR"; do
|
||||
local destination="$out/reth"
|
||||
mkdir -p "$out"
|
||||
# Skip copying when source and destination resolve to the same inode.
|
||||
if [ -e "$destination" ] && [ "$source_bin" -ef "$destination" ]; then
|
||||
continue
|
||||
fi
|
||||
cp "$source_bin" "$destination"
|
||||
done
|
||||
}
|
||||
|
||||
if [ "$USE_PRECOLLECTED_PGO" = true ]; then
|
||||
gha_section_start "Phase 1: Using Pre-Collected PGO Profile"
|
||||
rm -rf "$PGO_DIR"
|
||||
mkdir -p "$PGO_DIR"
|
||||
cp "$PGO_PROFDATA" "$PGO_DIR/merged.profdata"
|
||||
echo "Using pre-collected profile: $PGO_PROFDATA"
|
||||
echo "PGO profile: $PGO_DIR/merged.profdata ($(ls -lh "$PGO_DIR/merged.profdata" | awk '{print $5}'))"
|
||||
gha_section_end
|
||||
else
|
||||
# ── Phase 1: PGO profile collection ───────────────────────────────────────
|
||||
gha_section_start "Phase 1: PGO Profile Collection"
|
||||
|
||||
rm -rf "$PGO_DIR"
|
||||
mkdir -p "$PGO_DIR"
|
||||
|
||||
echo "Building PGO-instrumented binary..."
|
||||
RUSTFLAGS="-Cprofile-generate=$PGO_DIR -Crelocation-model=pic $COMBINED_RUSTFLAGS" \
|
||||
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
|
||||
|
||||
PGO_RETH_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
|
||||
echo "Instrumented binary: $PGO_RETH_BIN ($(ls -lh "$PGO_RETH_BIN" | awk '{print $5}'))"
|
||||
|
||||
run_bench_workload "$PGO_RETH_BIN" "$PGO_BLOCKS" "pgo"
|
||||
|
||||
# Fix ownership if reth ran as root.
|
||||
sudo chown -R "$(id -un):$(id -gn)" "$PGO_DIR" 2>/dev/null || true
|
||||
|
||||
# Merge PGO profiles.
|
||||
echo "Merging PGO profiles..."
|
||||
PROFRAW_COUNT=$(find "$PGO_DIR" -name '*.profraw' | wc -l)
|
||||
echo "Found $PROFRAW_COUNT .profraw files"
|
||||
if [ "$PROFRAW_COUNT" -eq 0 ]; then
|
||||
echo "error: no .profraw files — instrumented binary did not produce profiles"
|
||||
exit 1
|
||||
fi
|
||||
"$LLVM_PROFDATA" merge -o "$PGO_DIR/merged.profdata" "$PGO_DIR"/*.profraw
|
||||
echo "PGO profile: $PGO_DIR/merged.profdata ($(ls -lh "$PGO_DIR/merged.profdata" | awk '{print $5}'))"
|
||||
gha_section_end
|
||||
fi
|
||||
|
||||
if [ "$COLLECT_PGO_ONLY_BOOL" = true ]; then
|
||||
gha_section_start "PGO Collection Complete"
|
||||
echo "COLLECT_PGO_ONLY=true, skipping PGO/BOLT optimized binary build"
|
||||
echo "Profile: $PGO_DIR/merged.profdata"
|
||||
gha_section_end
|
||||
exit 0
|
||||
fi
|
||||
|
||||
if [ "$SKIP_BOLT_BOOL" = true ]; then
|
||||
gha_section_start "BOLT Phase Skipped"
|
||||
echo "SKIP_BOLT=$SKIP_BOLT, building PGO-only binary"
|
||||
echo "Building PGO-optimized binary..."
|
||||
RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata $COMBINED_RUSTFLAGS" \
|
||||
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
|
||||
|
||||
BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
|
||||
if [ "$STRIP_SYMBOLS_BOOL" = true ]; then
|
||||
echo "Stripping debug symbols..."
|
||||
strip "$BUILT_BIN"
|
||||
else
|
||||
echo "Skipping strip (STRIP_SYMBOLS=$STRIP_SYMBOLS)"
|
||||
fi
|
||||
publish_binary "$BUILT_BIN"
|
||||
gha_section_end
|
||||
else
|
||||
# ── Phase 2: BOLT profile collection (with PGO) ──────────────────────────
|
||||
gha_section_start "Phase 2: BOLT Profile Collection (with PGO)"
|
||||
|
||||
rm -rf "$BOLT_DIR"
|
||||
mkdir -p "$BOLT_DIR"
|
||||
|
||||
echo "Building BOLT-instrumented binary with PGO..."
|
||||
# --emit-relocs preserves relocation entries in the binary, required by llvm-bolt -instrument
|
||||
RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata -Clink-arg=-Wl,--emit-relocs $COMBINED_RUSTFLAGS" \
|
||||
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
|
||||
|
||||
# Instrument with BOLT
|
||||
BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
|
||||
BOLT_INSTRUMENTED_BIN="$BUILT_BIN-bolt-instrumented"
|
||||
|
||||
echo "Instrumenting binary with BOLT..."
|
||||
# --skip-funcs: skip compiler-generated drop_in_place functions that BOLT can't handle
|
||||
# as split functions in relocation mode (triggered by --emit-relocs)
|
||||
llvm-bolt "$BUILT_BIN" \
|
||||
-instrument \
|
||||
--instrumentation-file-append-pid \
|
||||
--instrumentation-file="$BOLT_DIR/prof" \
|
||||
--skip-funcs='.*drop_in_place.*' \
|
||||
-o "$BOLT_INSTRUMENTED_BIN"
|
||||
echo "BOLT-instrumented binary: $BOLT_INSTRUMENTED_BIN ($(ls -lh "$BOLT_INSTRUMENTED_BIN" | awk '{print $5}'))"
|
||||
|
||||
run_bench_workload "$BOLT_INSTRUMENTED_BIN" "$BOLT_BLOCKS" "bolt"
|
||||
|
||||
# Fix ownership for BOLT profiles
|
||||
sudo chown -R "$(id -un):$(id -gn)" "$BOLT_DIR" 2>/dev/null || true
|
||||
|
||||
# Merge BOLT profiles
|
||||
echo "Merging BOLT profiles..."
|
||||
FDATA_COUNT=$(find "$BOLT_DIR" -name '*.fdata' | wc -l)
|
||||
echo "Found $FDATA_COUNT .fdata files"
|
||||
if [ "$FDATA_COUNT" -eq 0 ]; then
|
||||
echo "error: no .fdata files — BOLT-instrumented binary did not produce profiles"
|
||||
exit 1
|
||||
fi
|
||||
merge-fdata "$BOLT_DIR"/*.fdata > "$BOLT_DIR/merged.fdata"
|
||||
echo "BOLT profile: $BOLT_DIR/merged.fdata ($(ls -lh "$BOLT_DIR/merged.fdata" | awk '{print $5}'))"
|
||||
gha_section_end
|
||||
|
||||
# ── Phase 3: Final optimized build ───────────────────────────────────────
|
||||
gha_section_start "Phase 3: Final PGO+BOLT Optimized Build"
|
||||
|
||||
echo "Building PGO-optimized binary..."
|
||||
# --emit-relocs preserves relocation entries in the binary, required by llvm-bolt for code reordering
|
||||
RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata -Clink-arg=-Wl,--emit-relocs $COMBINED_RUSTFLAGS" \
|
||||
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
|
||||
|
||||
BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
|
||||
OPTIMIZED_BIN="$BUILT_BIN-bolt-optimized"
|
||||
|
||||
echo "Optimizing with BOLT..."
|
||||
llvm-bolt "$BUILT_BIN" \
|
||||
-o "$OPTIMIZED_BIN" \
|
||||
--data "$BOLT_DIR/merged.fdata" \
|
||||
-reorder-blocks=ext-tsp \
|
||||
-reorder-functions=cdsort \
|
||||
-split-functions \
|
||||
-split-all-cold \
|
||||
-dyno-stats \
|
||||
-icf=1 \
|
||||
-use-gnu-stack \
|
||||
--skip-funcs='.*drop_in_place.*'
|
||||
|
||||
if [ "$STRIP_SYMBOLS_BOOL" = true ]; then
|
||||
echo "Stripping debug symbols..."
|
||||
strip "$OPTIMIZED_BIN"
|
||||
else
|
||||
echo "Skipping strip (STRIP_SYMBOLS=$STRIP_SYMBOLS)"
|
||||
fi
|
||||
publish_binary "$OPTIMIZED_BIN"
|
||||
gha_section_end
|
||||
fi
|
||||
|
||||
gha_section_start "Build Complete"
|
||||
ls -lh "target/$PROFILE_DIR/reth"
|
||||
echo "Output: target/$PROFILE_DIR/reth"
|
||||
gha_section_end
|
||||
2
.github/scripts/check_rv32imac.sh
vendored
2
.github/scripts/check_rv32imac.sh
vendored
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
set -uxo pipefail
|
||||
|
||||
crates_to_check=(
|
||||
reth-network-peers
|
||||
|
||||
2
.github/scripts/check_wasm.sh
vendored
2
.github/scripts/check_wasm.sh
vendored
@@ -1,5 +1,5 @@
|
||||
#!/usr/bin/env bash
|
||||
set -uo pipefail
|
||||
set -uxo pipefail
|
||||
|
||||
readarray -t crates < <(
|
||||
cargo metadata --format-version=1 --no-deps | jq -r '.packages[].name' | grep '^reth' | sort
|
||||
|
||||
244
.github/scripts/fetch-grafana-dashboard.py
vendored
Normal file
244
.github/scripts/fetch-grafana-dashboard.py
vendored
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch a Grafana dashboard and convert it to the portable import format.
|
||||
|
||||
Fetches the dashboard via API, replaces internal datasource/variable references
|
||||
with template variables, and adds __inputs/__requires/__elements so the JSON is
|
||||
importable on any Grafana instance.
|
||||
|
||||
Usage:
|
||||
export FETCH_GRAFANA_DASHBOARD_URL=https://<NAMESPACE>.grafana.net
|
||||
export FETCH_GRAFANA_DASHBOARD_TOKEN=glsa_...
|
||||
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py <dashboard-uid> > output.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
PANEL_TYPE_NAMES = {
|
||||
"bargauge": "Bar gauge",
|
||||
"gauge": "Gauge",
|
||||
"heatmap": "Heatmap",
|
||||
"piechart": "Pie chart",
|
||||
"stat": "Stat",
|
||||
"table": "Table",
|
||||
"timeseries": "Time series",
|
||||
"barchart": "Bar chart",
|
||||
"text": "Text",
|
||||
"dashlist": "Dashboard list",
|
||||
"logs": "Logs",
|
||||
"nodeGraph": "Node Graph",
|
||||
"histogram": "Histogram",
|
||||
"candlestick": "Candlestick",
|
||||
"state-timeline": "State timeline",
|
||||
"status-history": "Status history",
|
||||
"geomap": "Geomap",
|
||||
"canvas": "Canvas",
|
||||
"news": "News",
|
||||
"xychart": "XY Chart",
|
||||
"trend": "Trend",
|
||||
"datagrid": "Datagrid",
|
||||
"flamegraph": "Flame Graph",
|
||||
"traces": "Traces",
|
||||
}
|
||||
|
||||
|
||||
def fetch_json(base_url: str, token: str, path: str) -> dict:
|
||||
url = f"{base_url}{path}"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def fetch_dashboard(base_url: str, token: str, uid: str) -> dict:
|
||||
return fetch_json(base_url, token, f"/api/dashboards/uid/{uid}")
|
||||
|
||||
|
||||
def fetch_grafana_version(base_url: str) -> str:
|
||||
req = urllib.request.Request(f"{base_url}/api/health")
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
data = json.loads(resp.read())
|
||||
# version string like "13.0.0-23940615780.patch2" -> take just the semver part
|
||||
version = data.get("version", "")
|
||||
# strip build metadata after the first hyphen if it looks like a pre-release
|
||||
parts = version.split("-")
|
||||
return parts[0] if parts else version
|
||||
|
||||
|
||||
def collect_panel_types(panels: list) -> set[str]:
|
||||
types = set()
|
||||
for panel in panels:
|
||||
ptype = panel.get("type", "")
|
||||
if ptype and ptype != "row":
|
||||
types.add(ptype)
|
||||
# nested panels inside collapsed rows
|
||||
for sub in panel.get("panels", []):
|
||||
sub_type = sub.get("type", "")
|
||||
if sub_type and sub_type != "row":
|
||||
types.add(sub_type)
|
||||
return types
|
||||
|
||||
|
||||
def has_expression_datasource(dashboard: dict) -> bool:
|
||||
return "__expr__" in json.dumps(dashboard)
|
||||
|
||||
|
||||
def make_exportable(dashboard: dict, grafana_version: str = "") -> dict:
|
||||
dash = json.loads(json.dumps(dashboard)) # deep copy
|
||||
|
||||
# --- Strip internal fields ---
|
||||
dash.pop("id", None)
|
||||
|
||||
# --- Rewrite links: point to the public repo instead of internal ---
|
||||
dash["links"] = [
|
||||
{
|
||||
"asDropdown": False,
|
||||
"icon": "external link",
|
||||
"includeVars": False,
|
||||
"keepTime": False,
|
||||
"tags": [],
|
||||
"targetBlank": True,
|
||||
"title": "Source (GitHub)",
|
||||
"tooltip": "View source file in repository",
|
||||
"type": "link",
|
||||
"url": "https://github.com/paradigmxyz/reth/tree/main/etc/grafana/dashboards",
|
||||
}
|
||||
]
|
||||
|
||||
# --- Datasource: victoriametrics -> prometheus ---
|
||||
dash_str = json.dumps(dash)
|
||||
dash_str = dash_str.replace("victoriametrics-metrics-datasource", "prometheus")
|
||||
dash = json.loads(dash_str)
|
||||
|
||||
# --- Templating: instance_label constant -> ${VAR_INSTANCE_LABEL} ---
|
||||
# Also strip default-value fields the API returns that are not needed for import
|
||||
STRIP_VAR_DEFAULTS = {"allowCustomValue", "regexApplyTo"}
|
||||
for var in dash.get("templating", {}).get("list", []):
|
||||
if var.get("name") == "instance_label" and var.get("type") == "constant":
|
||||
var["query"] = "${VAR_INSTANCE_LABEL}"
|
||||
var["current"] = {
|
||||
"value": "${VAR_INSTANCE_LABEL}",
|
||||
"text": "${VAR_INSTANCE_LABEL}",
|
||||
"selected": False,
|
||||
}
|
||||
var["options"] = [
|
||||
{
|
||||
"value": "${VAR_INSTANCE_LABEL}",
|
||||
"text": "${VAR_INSTANCE_LABEL}",
|
||||
"selected": False,
|
||||
}
|
||||
]
|
||||
# Clear current values for query/datasource vars (not meaningful for import)
|
||||
elif var.get("type") in ("query", "datasource"):
|
||||
var["current"] = {}
|
||||
# Remove noisy default fields
|
||||
for field in STRIP_VAR_DEFAULTS:
|
||||
var.pop(field, None)
|
||||
# Strip falsy defaults on query/datasource vars (API returns them, export omits them)
|
||||
if var.get("type") in ("query", "datasource"):
|
||||
for field in ("hide", "multi", "skipUrlSync"):
|
||||
if not var.get(field):
|
||||
var.pop(field, None)
|
||||
|
||||
# --- Build __inputs ---
|
||||
inputs = [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus",
|
||||
},
|
||||
]
|
||||
|
||||
if has_expression_datasource(dash):
|
||||
inputs.append(
|
||||
{
|
||||
"name": "DS_EXPRESSION",
|
||||
"label": "Expression",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "__expr__",
|
||||
}
|
||||
)
|
||||
|
||||
inputs.append(
|
||||
{
|
||||
"name": "VAR_INSTANCE_LABEL",
|
||||
"type": "constant",
|
||||
"label": "Instance Label",
|
||||
"value": "job",
|
||||
"description": "",
|
||||
}
|
||||
)
|
||||
|
||||
# --- Build __requires ---
|
||||
requires = []
|
||||
|
||||
if has_expression_datasource(dash):
|
||||
requires.append({"type": "datasource", "id": "__expr__", "version": "1.0.0"})
|
||||
|
||||
panel_types = collect_panel_types(dash.get("panels", []))
|
||||
for pt in sorted(panel_types):
|
||||
requires.append(
|
||||
{
|
||||
"type": "panel",
|
||||
"id": pt,
|
||||
"name": PANEL_TYPE_NAMES.get(pt, pt),
|
||||
"version": "",
|
||||
}
|
||||
)
|
||||
|
||||
requires.append(
|
||||
{"type": "grafana", "id": "grafana", "name": "Grafana", "version": grafana_version}
|
||||
)
|
||||
requires.append(
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "1.0.0",
|
||||
}
|
||||
)
|
||||
|
||||
# --- Assemble output (with __inputs/__requires/__elements first) ---
|
||||
output = {
|
||||
"__inputs": inputs,
|
||||
"__elements": {},
|
||||
"__requires": requires,
|
||||
}
|
||||
output.update(dash)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <dashboard-uid>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
uid = sys.argv[1]
|
||||
base_url = os.environ.get("FETCH_GRAFANA_DASHBOARD_URL", "").rstrip("/")
|
||||
token = os.environ.get("FETCH_GRAFANA_DASHBOARD_TOKEN", "")
|
||||
|
||||
if not base_url or not token:
|
||||
print(
|
||||
"Error: FETCH_GRAFANA_DASHBOARD_URL and FETCH_GRAFANA_DASHBOARD_TOKEN env vars required",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
resp = fetch_dashboard(base_url, token, uid)
|
||||
dashboard = resp["dashboard"]
|
||||
|
||||
grafana_version = fetch_grafana_version(base_url)
|
||||
exported = make_exportable(dashboard, grafana_version)
|
||||
print(json.dumps(exported, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
25
.github/scripts/hive/build_simulators.sh
vendored
25
.github/scripts/hive/build_simulators.sh
vendored
@@ -1,6 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
fixture_variant="${1:-osaka}"
|
||||
|
||||
case "${fixture_variant}" in
|
||||
amsterdam)
|
||||
eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/snobal-devnet-5@v8037.0.0/fixtures_snobal-devnet-5.tar.gz"
|
||||
eels_branch="devnets/snobal/5"
|
||||
;;
|
||||
osaka)
|
||||
eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz"
|
||||
eels_branch="forks/osaka"
|
||||
;;
|
||||
*)
|
||||
echo "unknown hive fixture variant: ${fixture_variant}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Create the hive_assets directory
|
||||
mkdir hive_assets/
|
||||
|
||||
@@ -12,12 +29,12 @@ go build .
|
||||
# Run each hive command in the background for each simulator and wait
|
||||
echo "Building images"
|
||||
./hive -client reth --sim "ethereum/eels/consume-engine" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
|
||||
--sim.buildarg branch=forks/osaka \
|
||||
--sim.buildarg fixtures="${eels_fixtures}" \
|
||||
--sim.buildarg branch="${eels_branch}" \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/eels/consume-rlp" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
|
||||
--sim.buildarg branch=forks/osaka \
|
||||
--sim.buildarg fixtures="${eels_fixtures}" \
|
||||
--sim.buildarg branch="${eels_branch}" \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
|
||||
|
||||
78
.github/scripts/hive/expected_failures.yaml
vendored
78
.github/scripts/hive/expected_failures.yaml
vendored
@@ -13,15 +13,15 @@ rpc-compat:
|
||||
# syncing mode, the test expects syncing to be false on start
|
||||
- eth_syncing/check-syncing (reth)
|
||||
|
||||
engine-withdrawals: [ ]
|
||||
engine-withdrawals: []
|
||||
|
||||
engine-api: [ ]
|
||||
engine-api: []
|
||||
|
||||
engine-cancun: [ ]
|
||||
engine-cancun: []
|
||||
|
||||
sync: [ ]
|
||||
sync: []
|
||||
|
||||
engine-auth: [ ]
|
||||
engine-auth: []
|
||||
|
||||
# EIP-7610 related tests (Revert creation in case of non-empty storage):
|
||||
#
|
||||
@@ -99,6 +99,40 @@ eels/consume-engine:
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Prague-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Shanghai-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Paris-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Cancun-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Osaka-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
|
||||
# Blob limit tests:
|
||||
#
|
||||
@@ -193,3 +227,37 @@ eels/consume-rlp:
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Prague-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Shanghai-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Paris-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Cancun-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Osaka-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
|
||||
3
.github/scripts/hive/ignored_tests.yaml
vendored
3
.github/scripts/hive/ignored_tests.yaml
vendored
@@ -21,7 +21,6 @@ engine-cancun:
|
||||
# Affects all clients, not just reth. Tracked: https://github.com/ethereum/hive/issues/1382
|
||||
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
|
||||
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
|
||||
- Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Cancun) (reth)
|
||||
engine-api:
|
||||
- Transaction Re-Org, Re-Org Out (Paris) (reth)
|
||||
- Transaction Re-Org, Re-Org to Different Block (Paris) (reth)
|
||||
@@ -31,5 +30,3 @@ engine-api:
|
||||
- Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=True, Invalid P9 (Paris) (reth)
|
||||
- Invalid Missing Ancestor Syncing ReOrg, Transaction Signature, EmptyTxs=False, CanonicalReOrg=False, Invalid P9 (Paris) (reth)
|
||||
- Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=True, Invalid P10 (Paris) (reth)
|
||||
- Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Paris) (reth)
|
||||
- Multiple New Payloads Extending Canonical Chain, Set Head to First Payload Received (Paris) (reth)
|
||||
|
||||
6
.github/scripts/hive/run_simulator.sh
vendored
6
.github/scripts/hive/run_simulator.sh
vendored
@@ -5,6 +5,12 @@ cd hivetests/
|
||||
|
||||
sim="${1}"
|
||||
limit="${2}"
|
||||
fixture_variant="${3:-}"
|
||||
|
||||
if [[ "${fixture_variant}" == "osaka" && "${sim}" == *"eels"* && "${limit}" == *"tests/amsterdam"* ]]; then
|
||||
echo "osaka fixtures do not support amsterdam tests"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use lower parallelism for eels tests to avoid OOM-killing the runner
|
||||
parallelism=16
|
||||
|
||||
2
.github/scripts/install_geth.sh
vendored
2
.github/scripts/install_geth.sh
vendored
@@ -2,7 +2,7 @@
|
||||
|
||||
# Installs Geth (https://geth.ethereum.org) in $HOME/bin for x86_64 Linux.
|
||||
|
||||
set -eo pipefail
|
||||
set -exo pipefail
|
||||
|
||||
GETH_BUILD=${GETH_BUILD:-"1.13.4-3f907d6a"}
|
||||
|
||||
|
||||
2
.github/scripts/verify_image_arch.sh
vendored
2
.github/scripts/verify_image_arch.sh
vendored
@@ -7,7 +7,7 @@
|
||||
# Environment:
|
||||
# DRY_RUN=true - Skip actual verification, just print what would be checked.
|
||||
|
||||
set -euo pipefail
|
||||
set -euxo pipefail
|
||||
|
||||
TARGETS="${1:-}"
|
||||
REGISTRY="${2:-}"
|
||||
|
||||
199
.github/workflows/bench-scheduled.yml
vendored
199
.github/workflows/bench-scheduled.yml
vendored
@@ -1,11 +1,13 @@
|
||||
# Scheduled regression benchmarks (nightly + hourly).
|
||||
# Scheduled regression benchmarks (nightly + hourly + release).
|
||||
#
|
||||
# Two modes:
|
||||
# Three modes:
|
||||
# nightly — Compares the previous nightly Docker build against the current one.
|
||||
# Runs daily after docker.yml produces a new nightly image.
|
||||
# hourly — Compares main HEAD against the last benchmarked commit to catch
|
||||
# regressions quickly. Falls back to HEAD~1 on first run.
|
||||
# Skips if no new commits or if a previous run is still in progress.
|
||||
# release — Compares the latest GitHub release tag against the current nightly
|
||||
# Docker build. Runs daily to track nightly vs release performance.
|
||||
#
|
||||
# State is persisted between runs via the decofe/reth-bench-charts repo: each
|
||||
# successful run saves the feature commit SHA so the next run knows what to
|
||||
@@ -17,6 +19,8 @@ on:
|
||||
- cron: "30 5 * * *"
|
||||
# Hourly: compares main HEAD vs last benchmarked commit, skips if no new commits
|
||||
- cron: "0 * * * *"
|
||||
# Release: compares latest GitHub release tag vs current nightly Docker build
|
||||
- cron: "0 9 * * *"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force:
|
||||
@@ -24,11 +28,16 @@ on:
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
no_slack:
|
||||
description: "Suppress Slack notifications"
|
||||
slack:
|
||||
description: "Slack notification policy"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
default: "never"
|
||||
type: choice
|
||||
options:
|
||||
- always
|
||||
- on-win
|
||||
- on-error
|
||||
- never
|
||||
mode:
|
||||
description: "Benchmark mode"
|
||||
required: false
|
||||
@@ -37,6 +46,7 @@ on:
|
||||
options:
|
||||
- nightly
|
||||
- hourly
|
||||
- release
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
@@ -44,9 +54,7 @@ env:
|
||||
|
||||
name: bench-scheduled
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -55,6 +63,9 @@ jobs:
|
||||
resolve-refs:
|
||||
name: resolve-refs
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
mode: ${{ steps.mode.outputs.mode }}
|
||||
baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
|
||||
@@ -64,21 +75,29 @@ jobs:
|
||||
stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
|
||||
nightly-created: ${{ steps.refs.outputs.nightly-created }}
|
||||
long-running: ${{ steps.refs.outputs.long-running }}
|
||||
release-tag: ${{ steps.refs.outputs.release-tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: .github/scripts
|
||||
sparse-checkout-cone-mode: true
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Detect mode
|
||||
id: mode
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
INPUT_MODE: ${{ inputs.mode }}
|
||||
SCHEDULE: ${{ github.event.schedule }}
|
||||
run: |
|
||||
# Maps cron schedules to modes (must match the schedule entries above)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
MODE="${{ inputs.mode || 'nightly' }}"
|
||||
elif [ "${{ github.event.schedule }}" = "30 5 * * *" ]; then
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
MODE="${INPUT_MODE:-nightly}"
|
||||
elif [ "$SCHEDULE" = "30 5 * * *" ]; then
|
||||
MODE="nightly"
|
||||
elif [ "$SCHEDULE" = "0 9 * * *" ]; then
|
||||
MODE="release"
|
||||
else
|
||||
MODE="hourly"
|
||||
fi
|
||||
@@ -92,14 +111,15 @@ jobs:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
INPUT_FORCE: ${{ inputs.force || 'false' }}
|
||||
run: |
|
||||
FORCE="${{ inputs.force || 'false' }}"
|
||||
FORCE="${INPUT_FORCE:-false}"
|
||||
MODE="${{ steps.mode.outputs.mode }}"
|
||||
.github/scripts/bench-scheduled-refs.sh "$FORCE" "$MODE"
|
||||
|
||||
- name: Alert on long-running hourly
|
||||
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true'
|
||||
uses: actions/github-script@v8
|
||||
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -140,8 +160,8 @@ jobs:
|
||||
});
|
||||
|
||||
- name: Alert on stale nightly
|
||||
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true'
|
||||
uses: actions/github-script@v8
|
||||
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -229,6 +249,9 @@ jobs:
|
||||
needs.resolve-refs.outputs.is-stale != 'true'
|
||||
name: bench-scheduled
|
||||
runs-on: [self-hosted, Linux, X64, available]
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
@@ -248,7 +271,7 @@ jobs:
|
||||
BENCH_FEATURE_ARGS: ""
|
||||
BENCH_ABBA: "true"
|
||||
BENCH_COMMENT_ID: ""
|
||||
BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
|
||||
BENCH_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.slack || 'always' }}
|
||||
BENCH_METRICS_ADDR: "127.0.0.1:9100"
|
||||
BENCH_OTLP_DISABLED: "true"
|
||||
BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
|
||||
@@ -257,15 +280,16 @@ jobs:
|
||||
- name: Clean up previous bench-work
|
||||
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
|
||||
- name: Resolve job URL
|
||||
id: job-url
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
@@ -278,8 +302,9 @@ jobs:
|
||||
core.exportVariable('BENCH_JOB_URL', jobUrl);
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
continue-on-error: true
|
||||
- uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
@@ -294,12 +319,6 @@ jobs:
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
@@ -327,7 +346,7 @@ jobs:
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -338,43 +357,45 @@ jobs:
|
||||
|
||||
- name: Resolve display names
|
||||
id: refs
|
||||
env:
|
||||
RELEASE_TAG: ${{ needs.resolve-refs.outputs.release-tag }}
|
||||
run: |
|
||||
BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
|
||||
FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
|
||||
echo "baseline-name=${BENCH_MODE}-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
if [ "$BENCH_MODE" = "release" ] && [ -n "$RELEASE_TAG" ]; then
|
||||
echo "baseline-name=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline-ref=${RELEASE_TAG}" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
|
||||
echo "baseline-name=${BENCH_MODE}-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
- name: Validate local snapshot
|
||||
id: snapshot-check
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
@@ -398,15 +419,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -559,11 +571,12 @@ jobs:
|
||||
env:
|
||||
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
||||
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
||||
BASELINE_REF_DISPLAY: ${{ steps.refs.outputs.baseline-ref }}
|
||||
run: |
|
||||
SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF_DISPLAY}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
|
||||
@@ -590,7 +603,11 @@ jobs:
|
||||
CLICKHOUSE_USER: ${{ secrets.CLICKHOUSE_USER }}
|
||||
CLICKHOUSE_PASSWORD: ${{ secrets.CLICKHOUSE_PASSWORD }}
|
||||
run: |
|
||||
WORKFLOW_NAME="workflows-nightly-regression-${{ github.run_id }}"
|
||||
if [ "$BENCH_MODE" = "release" ]; then
|
||||
WORKFLOW_NAME="workflows-release-regression-${{ github.run_id }}"
|
||||
else
|
||||
WORKFLOW_NAME="workflows-nightly-regression-${{ github.run_id }}"
|
||||
fi
|
||||
DIFF_URL="https://github.com/${{ github.repository }}/compare/${BASELINE_REF}...${FEATURE_REF}"
|
||||
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
|
||||
JOB_URL="${BENCH_JOB_URL:-${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}}"
|
||||
@@ -623,7 +640,7 @@ jobs:
|
||||
|
||||
- name: Upload results
|
||||
if: "!cancelled()"
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: bench-scheduled-results
|
||||
path: ${{ env.BENCH_WORK_DIR }}
|
||||
@@ -631,10 +648,12 @@ jobs:
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success() && env.BENCH_MODE != 'hourly'
|
||||
env:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="nightly/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
|
||||
CHART_DIR="${BENCH_MODE}/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
|
||||
@@ -655,7 +674,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
@@ -733,8 +752,8 @@ jobs:
|
||||
await core.summary.addRaw(md).write();
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && env.BENCH_NO_SLACK != 'true'
|
||||
uses: actions/github-script@v8
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -761,8 +780,15 @@ jobs:
|
||||
// Filter notifications based on mode
|
||||
const changes = summary.changes || {};
|
||||
const mode = process.env.BENCH_MODE || 'nightly';
|
||||
const slackMode = process.env.BENCH_SLACK || 'always';
|
||||
const hasRegression = Object.values(changes).some(c => c.sig === 'bad');
|
||||
const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
|
||||
const hasImprovement = Object.values(changes).some(c => c.sig === 'good');
|
||||
|
||||
// on-win mode: only notify on improvements
|
||||
if (slackMode === 'on-win' && !hasImprovement) {
|
||||
core.info('on-win mode: no improvement detected, skipping Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
// Hourly mode: only notify on regressions
|
||||
if (mode === 'hourly' && !hasRegression) {
|
||||
@@ -770,11 +796,7 @@ jobs:
|
||||
return;
|
||||
}
|
||||
|
||||
// Nightly mode: notify on any significant change (regression or improvement)
|
||||
if (!hasSignificant) {
|
||||
core.info('No significant changes detected, skipping nightly Slack notification');
|
||||
return;
|
||||
}
|
||||
// Nightly mode: always notify (report every run regardless of significance)
|
||||
|
||||
const SLACK_VERDICT = {
|
||||
'⚠️': ':warning:',
|
||||
@@ -794,7 +816,7 @@ jobs:
|
||||
|
||||
function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
|
||||
|
||||
const modeLabel = mode === 'hourly' ? 'Hourly Regression' : 'Nightly Regression';
|
||||
const modeLabel = mode === 'release' ? 'Release Regression' : mode === 'hourly' ? 'Hourly Regression' : 'Nightly Regression';
|
||||
const sectionText = [
|
||||
`*${modeLabel}*`,
|
||||
'',
|
||||
@@ -885,8 +907,8 @@ jobs:
|
||||
}
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure()
|
||||
uses: actions/github-script@v8
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -897,8 +919,8 @@ jobs:
|
||||
if (!token || !channel) return;
|
||||
|
||||
const steps_status = [
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
|
||||
@@ -911,7 +933,7 @@ jobs:
|
||||
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const mode = process.env.BENCH_MODE || 'nightly';
|
||||
const modeLabel = mode === 'hourly' ? 'Hourly' : 'Nightly';
|
||||
const modeLabel = mode === 'release' ? 'Release' : mode === 'hourly' ? 'Hourly' : 'Nightly';
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
@@ -920,7 +942,7 @@ jobs:
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>` },
|
||||
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n<@U0AAA8F0JEM> investigate this` },
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
@@ -947,6 +969,11 @@ jobs:
|
||||
}),
|
||||
});
|
||||
|
||||
- name: Clean build outputs
|
||||
if: always()
|
||||
run: |
|
||||
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
|
||||
424
.github/workflows/bench.yml
vendored
424
.github/workflows/bench.yml
vendored
@@ -21,10 +21,20 @@ on:
|
||||
required: false
|
||||
default: "false"
|
||||
type: boolean
|
||||
bal:
|
||||
description: "Replay block access lists during big-block benchmarks"
|
||||
required: false
|
||||
default: "false"
|
||||
type: choice
|
||||
options:
|
||||
- "false"
|
||||
- "true"
|
||||
- "feature"
|
||||
- "baseline"
|
||||
warmup:
|
||||
description: "Number of warmup blocks"
|
||||
required: false
|
||||
default: "100"
|
||||
default: "200"
|
||||
type: string
|
||||
baseline:
|
||||
description: "Baseline git ref (default: merge-base)"
|
||||
@@ -61,13 +71,18 @@ on:
|
||||
required: false
|
||||
default: "0"
|
||||
type: string
|
||||
no_slack:
|
||||
description: "Suppress Slack notifications for benchmark results"
|
||||
slack:
|
||||
description: "Slack notification policy"
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
default: "never"
|
||||
type: choice
|
||||
options:
|
||||
- always
|
||||
- on-win
|
||||
- on-error
|
||||
- never
|
||||
abba:
|
||||
description: "Run ABBA (BFFB) interleaved order; false = single AB pass"
|
||||
description: "Run ABBA (FBBF) interleaved order; false = single FB pass"
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
@@ -84,9 +99,7 @@ env:
|
||||
|
||||
name: bench
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
reth-bench-ack:
|
||||
@@ -95,6 +108,9 @@ jobs:
|
||||
github.event_name == 'workflow_dispatch'
|
||||
name: reth-bench-ack
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
outputs:
|
||||
pr: ${{ steps.args.outputs.pr }}
|
||||
actor: ${{ steps.args.outputs.actor }}
|
||||
@@ -105,9 +121,10 @@ jobs:
|
||||
baseline-name: ${{ steps.args.outputs.baseline-name }}
|
||||
feature-name: ${{ steps.args.outputs.feature-name }}
|
||||
samply: ${{ steps.args.outputs.samply }}
|
||||
no-slack: ${{ steps.args.outputs.no-slack }}
|
||||
slack: ${{ steps.args.outputs.slack }}
|
||||
cores: ${{ steps.args.outputs.cores }}
|
||||
big-blocks: ${{ steps.args.outputs.big-blocks }}
|
||||
bal: ${{ steps.args.outputs.bal }}
|
||||
wait-time: ${{ steps.args.outputs.wait-time }}
|
||||
baseline-args: ${{ steps.args.outputs.baseline-args }}
|
||||
feature-args: ${{ steps.args.outputs.feature-args }}
|
||||
@@ -117,7 +134,7 @@ jobs:
|
||||
steps:
|
||||
- name: Check org membership
|
||||
if: github.event_name == 'issue_comment'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
@@ -136,22 +153,28 @@ jobs:
|
||||
|
||||
- name: Parse arguments
|
||||
id: args
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
let pr, actor, blocks, warmup, baseline, feature, samply, cores, bigBlocks;
|
||||
const validBalModes = new Set(['false', 'true', 'feature', 'baseline']);
|
||||
const validSlackModes = new Set(['always', 'on-win', 'on-error', 'never']);
|
||||
const usage = '`@decofe bench [blocks=N] [big-blocks[=true|false]] [bal=true|false|feature|baseline] [warmup=N] [baseline=REF] [feature=REF] [samply] [slack=always|on-win|on-error|never] [cores=N] [abba=true|false] [otlp=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]`';
|
||||
let pr, actor, blocks, warmup, baseline, feature, samply, cores, bigBlocks, bal;
|
||||
let explicitWarmup = false;
|
||||
|
||||
if (context.eventName === 'workflow_dispatch') {
|
||||
actor = '${{ github.actor }}';
|
||||
blocks = '${{ github.event.inputs.blocks }}' || '500';
|
||||
warmup = '${{ github.event.inputs.warmup }}' || '100';
|
||||
warmup = '${{ github.event.inputs.warmup }}' || '200';
|
||||
if (warmup !== '200') explicitWarmup = true;
|
||||
baseline = '${{ github.event.inputs.baseline }}';
|
||||
feature = '${{ github.event.inputs.feature }}';
|
||||
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
|
||||
var noSlack = '${{ github.event.inputs.no_slack }}' !== 'false' ? 'true' : 'false';
|
||||
var slack = '${{ github.event.inputs.slack }}' || 'never';
|
||||
cores = '${{ github.event.inputs.cores }}' || '0';
|
||||
bigBlocks = '${{ github.event.inputs.big_blocks }}' === 'true' ? 'true' : 'false';
|
||||
bal = '${{ github.event.inputs.bal }}' || 'false';
|
||||
var abba = '${{ github.event.inputs.abba }}' !== 'false' ? 'true' : 'false';
|
||||
var otlp = '${{ github.event.inputs.otlp }}' !== 'false' ? 'true' : 'false';
|
||||
var waitTime = '${{ github.event.inputs.wait_time }}' || '';
|
||||
@@ -178,11 +201,12 @@ jobs:
|
||||
const body = context.payload.comment.body.trim();
|
||||
const intArgs = new Set(['warmup', 'cores', 'blocks']);
|
||||
const refArgs = new Set(['baseline', 'feature']);
|
||||
const boolArgs = new Set(['samply', 'no-slack', 'big-blocks']);
|
||||
const boolArgs = new Set(['samply', 'big-blocks']);
|
||||
const boolDefaultTrue = new Set(['abba', 'otlp']);
|
||||
const enumArgs = new Map([['bal', validBalModes], ['slack', validSlackModes]]);
|
||||
const durationArgs = new Set(['wait-time']);
|
||||
const stringArgs = new Set(['baseline-args', 'feature-args']);
|
||||
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', 'big-blocks': 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const defaults = { blocks: '500', warmup: '200', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const unknown = [];
|
||||
const invalid = [];
|
||||
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
|
||||
@@ -209,7 +233,7 @@ jobs:
|
||||
if ((value.startsWith('"') && value.endsWith('"')) || (value.startsWith("'") && value.endsWith("'"))) {
|
||||
value = value.slice(1, -1);
|
||||
}
|
||||
if (boolDefaultTrue.has(key)) {
|
||||
if (boolArgs.has(key) || boolDefaultTrue.has(key)) {
|
||||
if (value === 'true' || value === 'false') {
|
||||
defaults[key] = value;
|
||||
} else {
|
||||
@@ -221,11 +245,18 @@ jobs:
|
||||
} else {
|
||||
invalid.push(`\`${key}=${value}\` (must be a duration like 500ms, 1s, 2m)`);
|
||||
}
|
||||
} else if (enumArgs.has(key)) {
|
||||
if (enumArgs.get(key).has(value)) {
|
||||
defaults[key] = value;
|
||||
} else {
|
||||
invalid.push(`\`${key}=${value}\` (must be true, false, feature, or baseline)`);
|
||||
}
|
||||
} else if (intArgs.has(key)) {
|
||||
if (!/^\d+$/.test(value)) {
|
||||
invalid.push(`\`${key}=${value}\` (must be a positive integer)`);
|
||||
} else {
|
||||
defaults[key] = value;
|
||||
if (key === 'warmup') explicitWarmup = true;
|
||||
}
|
||||
} else if (refArgs.has(key)) {
|
||||
if (!value) {
|
||||
@@ -243,7 +274,7 @@ jobs:
|
||||
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
|
||||
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
|
||||
if (errors.length) {
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N] [big-blocks] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [abba=true|false] [otlp=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** ${usage}`;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
@@ -258,9 +289,10 @@ jobs:
|
||||
baseline = defaults.baseline;
|
||||
feature = defaults.feature;
|
||||
samply = defaults.samply;
|
||||
var noSlack = defaults['no-slack'];
|
||||
var slack = defaults.slack;
|
||||
cores = defaults.cores;
|
||||
bigBlocks = defaults['big-blocks'];
|
||||
bal = defaults.bal;
|
||||
var abba = defaults.abba;
|
||||
var otlp = defaults.otlp;
|
||||
var waitTime = defaults['wait-time'];
|
||||
@@ -268,6 +300,29 @@ jobs:
|
||||
var featureNodeArgs = defaults['feature-args'];
|
||||
}
|
||||
|
||||
// Default warmup to 20 for big-blocks mode unless explicitly set
|
||||
if (bigBlocks === 'true' && !explicitWarmup) {
|
||||
warmup = '20';
|
||||
}
|
||||
|
||||
if (!validBalModes.has(bal)) {
|
||||
core.setFailed(`Invalid bal mode: ${bal}`);
|
||||
return;
|
||||
}
|
||||
if (bal !== 'false' && bigBlocks !== 'true') {
|
||||
const msg = `❌ **Invalid bench command**\n\n\`bal\` requires \`big-blocks=true\`.\n\n**Usage:** ${usage}`;
|
||||
if (context.eventName === 'issue_comment') {
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
issue_number: context.issue.number,
|
||||
body: msg,
|
||||
});
|
||||
}
|
||||
core.setFailed(msg);
|
||||
return;
|
||||
}
|
||||
|
||||
// Resolve display names for baseline/feature
|
||||
let baselineName = baseline || 'main';
|
||||
let featureName = feature;
|
||||
@@ -293,9 +348,10 @@ jobs:
|
||||
core.setOutput('baseline-name', baselineName);
|
||||
core.setOutput('feature-name', featureName);
|
||||
core.setOutput('samply', samply);
|
||||
core.setOutput('no-slack', noSlack);
|
||||
core.setOutput('slack', slack);
|
||||
core.setOutput('cores', cores);
|
||||
core.setOutput('big-blocks', bigBlocks);
|
||||
core.setOutput('bal', bal);
|
||||
core.setOutput('wait-time', waitTime);
|
||||
core.setOutput('baseline-args', baselineNodeArgs);
|
||||
core.setOutput('feature-args', featureNodeArgs);
|
||||
@@ -304,7 +360,7 @@ jobs:
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -358,10 +414,12 @@ jobs:
|
||||
const baseline = '${{ steps.args.outputs.baseline-name }}';
|
||||
const feature = '${{ steps.args.outputs.feature-name }}';
|
||||
const samply = '${{ steps.args.outputs.samply }}' === 'true';
|
||||
const noSlack = '${{ steps.args.outputs.no-slack }}' === 'true';
|
||||
const slack = '${{ steps.args.outputs.slack }}' || 'always';
|
||||
const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true';
|
||||
const bal = '${{ steps.args.outputs.bal }}' || 'false';
|
||||
const samplyNote = samply ? ', samply: `enabled`' : '';
|
||||
const noSlackNote = noSlack ? ', no-slack' : '';
|
||||
const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : '';
|
||||
const balNote = bigBlocks && bal !== 'false' ? `, BAL: \`${bal}\`` : '';
|
||||
const cores = '${{ steps.args.outputs.cores }}';
|
||||
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
|
||||
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
|
||||
@@ -375,7 +433,7 @@ jobs:
|
||||
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${slackNote}${balNote}${coresNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
|
||||
const { data: comment } = await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
@@ -388,7 +446,7 @@ jobs:
|
||||
|
||||
- name: Poll queue position
|
||||
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -400,10 +458,12 @@ jobs:
|
||||
const baseline = '${{ steps.args.outputs.baseline-name }}';
|
||||
const feature = '${{ steps.args.outputs.feature-name }}';
|
||||
const samply = '${{ steps.args.outputs.samply }}' === 'true';
|
||||
const noSlack = '${{ steps.args.outputs.no-slack }}' === 'true';
|
||||
const slack = '${{ steps.args.outputs.slack }}' || 'always';
|
||||
const bigBlocks = '${{ steps.args.outputs.big-blocks }}' === 'true';
|
||||
const bal = '${{ steps.args.outputs.bal }}' || 'false';
|
||||
const samplyNote = samply ? ', samply: `enabled`' : '';
|
||||
const noSlackNote = noSlack ? ', no-slack' : '';
|
||||
const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : '';
|
||||
const balNote = bigBlocks && bal !== 'false' ? `, BAL: \`${bal}\`` : '';
|
||||
const cores = '${{ steps.args.outputs.cores }}';
|
||||
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
|
||||
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
|
||||
@@ -417,7 +477,7 @@ jobs:
|
||||
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${slackNote}${balNote}${coresNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
|
||||
@@ -470,6 +530,9 @@ jobs:
|
||||
needs: reth-bench-ack
|
||||
name: reth-bench
|
||||
runs-on: [self-hosted, Linux, X64, available]
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
@@ -483,13 +546,14 @@ jobs:
|
||||
BENCH_SAMPLY: ${{ needs.reth-bench-ack.outputs.samply }}
|
||||
BENCH_CORES: ${{ needs.reth-bench-ack.outputs.cores }}
|
||||
BENCH_BIG_BLOCKS: ${{ needs.reth-bench-ack.outputs.big-blocks }}
|
||||
BENCH_BAL: ${{ needs.reth-bench-ack.outputs.bal }}
|
||||
BENCH_WAIT_TIME: ${{ needs.reth-bench-ack.outputs.wait-time }}
|
||||
BENCH_BASELINE_ARGS: ${{ needs.reth-bench-ack.outputs.baseline-args }}
|
||||
BENCH_FEATURE_ARGS: ${{ needs.reth-bench-ack.outputs.feature-args }}
|
||||
BENCH_ABBA: ${{ needs.reth-bench-ack.outputs.abba }}
|
||||
BENCH_OTLP: ${{ needs.reth-bench-ack.outputs.otlp }}
|
||||
BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
|
||||
BENCH_NO_SLACK: ${{ needs.reth-bench-ack.outputs.no-slack }}
|
||||
BENCH_SLACK: ${{ needs.reth-bench-ack.outputs.slack }}
|
||||
BENCH_NODE_BIN: ${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
|
||||
BENCH_METRICS_ADDR: "127.0.0.1:9100"
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_TRACES_ENDPOINT || '' }}
|
||||
@@ -500,7 +564,7 @@ jobs:
|
||||
|
||||
- name: Resolve checkout ref
|
||||
id: checkout-ref
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
if (!process.env.BENCH_PR) {
|
||||
@@ -518,15 +582,16 @@ jobs:
|
||||
core.info(`PR #${process.env.BENCH_PR} (${pr.state}), using head SHA ${pr.head.sha}`);
|
||||
core.setOutput('ref', pr.head.sha);
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ steps.checkout-ref.outputs.ref }}
|
||||
|
||||
- name: Resolve job URL and update status
|
||||
if: env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -544,10 +609,12 @@ jobs:
|
||||
const baseline = '${{ needs.reth-bench-ack.outputs.baseline-name }}';
|
||||
const feature = '${{ needs.reth-bench-ack.outputs.feature-name }}';
|
||||
const samply = process.env.BENCH_SAMPLY === 'true';
|
||||
const noSlack = process.env.BENCH_NO_SLACK === 'true';
|
||||
const slack = process.env.BENCH_SLACK || 'always';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const bal = process.env.BENCH_BAL || 'false';
|
||||
const samplyNote = samply ? ', samply: `enabled`' : '';
|
||||
const noSlackNote = noSlack ? ', no-slack' : '';
|
||||
const slackNote = slack !== 'always' ? `, slack: \`${slack}\`` : '';
|
||||
const balNote = bigBlocks && bal !== 'false' ? `, BAL: \`${bal}\`` : '';
|
||||
const cores = process.env.BENCH_CORES || '0';
|
||||
const coresNote = cores && cores !== '0' ? `, cores: \`${cores}\`` : '';
|
||||
const abbaEnabled = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
@@ -561,7 +628,7 @@ jobs:
|
||||
const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
|
||||
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${slackNote}${balNote}${coresNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
|
||||
|
||||
const { buildBody } = require('./.github/scripts/bench-update-status.js');
|
||||
await github.rest.issues.updateComment({
|
||||
@@ -572,8 +639,9 @@ jobs:
|
||||
});
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
continue-on-error: true
|
||||
- uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
@@ -588,12 +656,6 @@ jobs:
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
@@ -628,7 +690,7 @@ jobs:
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -640,7 +702,7 @@ jobs:
|
||||
# Build binaries
|
||||
- name: Resolve PR head branch
|
||||
id: pr-info
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
if (process.env.BENCH_PR) {
|
||||
@@ -658,7 +720,7 @@ jobs:
|
||||
|
||||
- name: Resolve refs
|
||||
id: refs
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { execSync } = require('child_process');
|
||||
@@ -704,79 +766,68 @@ jobs:
|
||||
core.setOutput('feature-ref', featureRef);
|
||||
core.setOutput('feature-name', featureName);
|
||||
|
||||
- name: Check big-blocks freshness
|
||||
- name: Validate local big blocks
|
||||
if: env.BENCH_BIG_BLOCKS == 'true'
|
||||
id: big-blocks-check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
|
||||
HASH_FILE="$HOME/.reth-bench-big-blocks-hash"
|
||||
echo "Fetching big-blocks manifest from $MANIFEST..."
|
||||
BB_MANIFEST=$($MC cat "$MANIFEST" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch big-blocks manifest from $MANIFEST"
|
||||
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
|
||||
PAYLOAD_DIR="$BIG_BLOCKS_DIR/payloads"
|
||||
MANIFEST="$BIG_BLOCKS_DIR/manifest.json"
|
||||
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
|
||||
|
||||
if [ ! -f "$MANIFEST" ]; then
|
||||
echo "::error::Missing local big-blocks manifest at $MANIFEST"
|
||||
exit 1
|
||||
}
|
||||
BASE_SNAPSHOT=$(echo "$BB_MANIFEST" | jq -r '.base_snapshot // empty')
|
||||
fi
|
||||
|
||||
BASE_SNAPSHOT=$(jq -r '.base_snapshot // empty' "$MANIFEST")
|
||||
if [ -z "$BASE_SNAPSHOT" ]; then
|
||||
echo "::error::Big-blocks manifest missing base_snapshot field"
|
||||
exit 1
|
||||
fi
|
||||
echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
|
||||
echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
|
||||
|
||||
REMOTE_HASH=$(echo "$BB_MANIFEST" | sha256sum | awk '{print $1}')
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Big blocks up-to-date (hash: ${REMOTE_HASH:0:16}…)"
|
||||
echo "needed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Big blocks need update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
echo "needed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "remote-hash=${REMOTE_HASH}" >> "$GITHUB_OUTPUT"
|
||||
if [ ! -d "$PAYLOAD_DIR" ]; then
|
||||
echo "::error::Missing local big-block payload directory at $PAYLOAD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
PAYLOAD_COUNT=$(find "$PAYLOAD_DIR" -name '*.json' | wc -l)
|
||||
if [ "$PAYLOAD_COUNT" -eq 0 ]; then
|
||||
echo "::error::No payload files found in $PAYLOAD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Update status (snapshot needed)
|
||||
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const s = require('./.github/scripts/bench-update-status.js');
|
||||
await s({github, context, status: 'Building binaries (snapshot update pending)...'});
|
||||
echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
|
||||
echo "Payload files: $PAYLOAD_COUNT"
|
||||
echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Validate local snapshot
|
||||
id: snapshot-check
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
|
||||
FEATURE_REF="${{ steps.refs.outputs.feature-ref }}"
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
@@ -800,15 +851,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -848,8 +890,11 @@ jobs:
|
||||
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
||||
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
||||
done || true
|
||||
# Replace any stale PM QoS holders left behind by earlier benchmark jobs.
|
||||
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
|
||||
# Prevent deep C-states (avoids wake-up latency jitter)
|
||||
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
||||
sudo bash -c 'exec 3<>/dev/cpu_dma_latency; printf "\0\0\0\0" >&3; exec -a bench-cpu-dma-latency sleep infinity' &
|
||||
echo "BENCH_CPU_DMA_LATENCY_PID=$!" >> "$GITHUB_ENV"
|
||||
# Move all IRQs to core 0 (housekeeping core)
|
||||
for irq in /proc/irq/*/smp_affinity_list; do
|
||||
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
||||
@@ -876,45 +921,6 @@ jobs:
|
||||
rm -rf "$BENCH_WORK_DIR"
|
||||
mkdir -p "$BENCH_WORK_DIR"
|
||||
|
||||
- name: Download big blocks
|
||||
if: env.BENCH_BIG_BLOCKS == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
|
||||
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
|
||||
if [ "${{ steps.big-blocks-check.outputs.needed }}" = "false" ]; then
|
||||
echo "Big blocks cached at $BIG_BLOCKS_DIR, skipping download"
|
||||
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
|
||||
exit 0
|
||||
fi
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
|
||||
rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
|
||||
|
||||
# Download and parse manifest
|
||||
echo "Downloading manifest from $MANIFEST..."
|
||||
$MC cat "$MANIFEST" > "$BIG_BLOCKS_DIR/manifest.json"
|
||||
UPLOAD_PATH=$(jq -r '.upload_path' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
COUNT=$(jq -r '.count' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
TARGET_GAS=$(jq -r '.target_gas' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
echo "Manifest: count=$COUNT, target_gas=$TARGET_GAS, archive=$UPLOAD_PATH"
|
||||
|
||||
# Download and extract archive
|
||||
ARCHIVE="minio/$UPLOAD_PATH"
|
||||
echo "Downloading big blocks from $ARCHIVE..."
|
||||
$MC cat "$ARCHIVE" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
|
||||
echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
|
||||
|
||||
# Verify expected directory structure
|
||||
if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
|
||||
echo "::error::Big blocks archive missing expected payloads/ directory"
|
||||
ls -laR "$BIG_BLOCKS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
|
||||
# Save manifest hash for freshness check on next run
|
||||
echo "${{ steps.big-blocks-check.outputs.remote-hash }}" > "$HOME/.reth-bench-big-blocks-hash"
|
||||
|
||||
- name: Start metrics proxy
|
||||
run: |
|
||||
BENCH_ID="ci-${{ github.run_id }}"
|
||||
@@ -937,28 +943,15 @@ jobs:
|
||||
|
||||
- name: Update status (running benchmarks)
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const s = require('./.github/scripts/bench-update-status.js');
|
||||
await s({github, context, status: 'Running benchmarks...'});
|
||||
|
||||
# Interleaved run order (B-F-F-B) to reduce systematic bias from
|
||||
# Interleaved run order (F-B-B-F) to reduce systematic bias from
|
||||
# thermal drift and cache warming.
|
||||
- name: "Run benchmark: baseline (1/2)"
|
||||
id: run-baseline-1
|
||||
env:
|
||||
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=baseline-1,run_type=baseline,git_ref=${{ steps.refs.outputs.baseline-ref }}"
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-1"
|
||||
|
||||
- name: "Run benchmark: feature (1/2)"
|
||||
id: run-feature-1
|
||||
env:
|
||||
@@ -972,19 +965,18 @@ jobs:
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-1"
|
||||
|
||||
- name: "Run benchmark: feature (2/2)"
|
||||
if: env.BENCH_ABBA != 'false'
|
||||
id: run-feature-2
|
||||
- name: "Run benchmark: baseline (1/2)"
|
||||
id: run-baseline-1
|
||||
env:
|
||||
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=feature-2,run_type=feature,git_ref=${{ steps.refs.outputs.feature-ref }}"
|
||||
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=baseline-1,run_type=baseline,git_ref=${{ steps.refs.outputs.baseline-ref }}"
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-2"
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-1"
|
||||
|
||||
- name: "Run benchmark: baseline (2/2)"
|
||||
if: env.BENCH_ABBA != 'false'
|
||||
@@ -1000,6 +992,20 @@ jobs:
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-2"
|
||||
|
||||
- name: "Run benchmark: feature (2/2)"
|
||||
if: env.BENCH_ABBA != 'false'
|
||||
id: run-feature-2
|
||||
env:
|
||||
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=feature-2,run_type=feature,git_ref=${{ steps.refs.outputs.feature-ref }}"
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-2"
|
||||
|
||||
- name: Stop metrics proxy & generate Grafana URL
|
||||
id: metrics
|
||||
if: "!cancelled()"
|
||||
@@ -1134,6 +1140,9 @@ jobs:
|
||||
if [ -n "${BENCH_WAIT_TIME:-}" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --wait-time $BENCH_WAIT_TIME"
|
||||
fi
|
||||
if [ -n "${BENCH_BAL:-}" ] && [ "${BENCH_BAL}" != "false" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --bal-mode $BENCH_BAL"
|
||||
fi
|
||||
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
|
||||
if [ -n "$GRAFANA_URL" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
|
||||
@@ -1163,7 +1172,7 @@ jobs:
|
||||
|
||||
- name: Upload results
|
||||
if: "!cancelled()"
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: bench-reth-results
|
||||
path: ${{ env.BENCH_WORK_DIR }}
|
||||
@@ -1171,11 +1180,13 @@ jobs:
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success()
|
||||
env:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
PR_NUMBER="${BENCH_PR:-0}"
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
|
||||
CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
|
||||
@@ -1188,15 +1199,35 @@ jobs:
|
||||
mkdir -p "${TMP_DIR}/${CHART_DIR}"
|
||||
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
|
||||
git -C "${TMP_DIR}" add "${CHART_DIR}"
|
||||
if git -C "${TMP_DIR}" diff --cached --quiet; then
|
||||
echo "Charts for ${CHART_DIR} are already present, skipping push"
|
||||
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
rm -rf "${TMP_DIR}"
|
||||
exit 0
|
||||
fi
|
||||
git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
||||
commit -m "bench charts for PR #${PR_NUMBER} run ${RUN_ID}"
|
||||
git -C "${TMP_DIR}" push origin HEAD:main
|
||||
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if git -C "${TMP_DIR}" push origin HEAD:main; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -eq 5 ]; then
|
||||
echo "::error::Failed to push charts after ${attempt} attempts"
|
||||
rm -rf "${TMP_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
sleep "$attempt"
|
||||
git -C "${TMP_DIR}" fetch origin main
|
||||
git -C "${TMP_DIR}" rebase origin/main
|
||||
done
|
||||
|
||||
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
rm -rf "${TMP_DIR}"
|
||||
|
||||
- name: Compare & comment
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1232,7 +1263,7 @@ jobs:
|
||||
// Samply profile links (URLs point directly to Firefox Profiler)
|
||||
if (process.env.BENCH_SAMPLY === 'true') {
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const runs = abba ? ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'] : ['baseline-1', 'feature-1'];
|
||||
const runs = abba ? ['feature-1', 'baseline-1', 'baseline-2', 'feature-2'] : ['feature-1', 'baseline-1'];
|
||||
const links = [];
|
||||
for (const run of runs) {
|
||||
try {
|
||||
@@ -1276,7 +1307,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const jobSummary = require('./.github/scripts/bench-job-summary.js');
|
||||
@@ -1289,8 +1320,8 @@ jobs:
|
||||
});
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && env.BENCH_NO_SLACK != 'true'
|
||||
uses: actions/github-script@v8
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -1301,18 +1332,20 @@ jobs:
|
||||
|
||||
- name: Update status (failed)
|
||||
if: failure() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const steps_status = [
|
||||
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
...(abba ? [['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}']] : []),
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
];
|
||||
const failed = steps_status.find(([, o]) => o === 'failure');
|
||||
const failedStep = failed ? failed[0] : 'unknown step';
|
||||
@@ -1334,21 +1367,23 @@ jobs:
|
||||
});
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure()
|
||||
uses: actions/github-script@v8
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const steps_status = [
|
||||
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
...(abba ? [['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}']] : []),
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
];
|
||||
const failed = steps_status.find(([, o]) => o === 'failure');
|
||||
const failedStep = failed ? failed[0] : 'unknown step';
|
||||
@@ -1357,7 +1392,7 @@ jobs:
|
||||
|
||||
- name: Update status (cancelled)
|
||||
if: cancelled() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1368,6 +1403,11 @@ jobs:
|
||||
body: `cc @${process.env.BENCH_ACTOR}\n\n⚠️ Benchmark cancelled. [View logs](${jobUrl})`,
|
||||
});
|
||||
|
||||
- name: Clean build outputs
|
||||
if: always()
|
||||
run: |
|
||||
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
@@ -1380,5 +1420,9 @@ jobs:
|
||||
done
|
||||
# Restore amd-pstate to active (EPP) mode with powersave governor
|
||||
echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
|
||||
if [ -n "${BENCH_CPU_DMA_LATENCY_PID:-}" ]; then
|
||||
sudo kill "$BENCH_CPU_DMA_LATENCY_PID" 2>/dev/null || true
|
||||
fi
|
||||
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
|
||||
sudo cpupower frequency-set -g powersave 2>/dev/null || true
|
||||
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|
||||
|
||||
19
.github/workflows/book.yml
vendored
19
.github/workflows/book.yml
vendored
@@ -10,19 +10,22 @@ on:
|
||||
types: [opened, reopened, synchronize, closed]
|
||||
merge_group:
|
||||
|
||||
env:
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version: v1.2.23
|
||||
|
||||
@@ -36,8 +39,6 @@ jobs:
|
||||
- name: Install Rust nightly
|
||||
uses: dtolnay/rust-toolchain@nightly
|
||||
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
|
||||
- name: Build docs
|
||||
run: cd docs/vocs && bash scripts/build-cargo-docs.sh
|
||||
|
||||
@@ -47,10 +48,10 @@ jobs:
|
||||
echo "Vocs Build Complete"
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v6
|
||||
uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
|
||||
with:
|
||||
path: "./docs/vocs/docs/dist"
|
||||
|
||||
@@ -74,4 +75,4 @@ jobs:
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v4
|
||||
uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0
|
||||
|
||||
26
.github/workflows/check-alloy.yml
vendored
26
.github/workflows/check-alloy.yml
vendored
@@ -22,31 +22,41 @@ on:
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Apply alloy patches
|
||||
env:
|
||||
ALLOY_BRANCH: ${{ inputs.alloy_branch }}
|
||||
ALLOY_EVM_BRANCH: ${{ inputs.alloy_evm_branch }}
|
||||
OP_ALLOY_BRANCH: ${{ inputs.op_alloy_branch }}
|
||||
run: |
|
||||
ARGS=""
|
||||
if [ -n "${{ inputs.alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
|
||||
if [ -n "$ALLOY_BRANCH" ]; then
|
||||
ARGS="$ARGS --alloy $ALLOY_BRANCH"
|
||||
fi
|
||||
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
|
||||
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
|
||||
if [ -n "$ALLOY_EVM_BRANCH" ]; then
|
||||
ARGS="$ARGS --evm $ALLOY_EVM_BRANCH"
|
||||
fi
|
||||
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
|
||||
if [ -n "$OP_ALLOY_BRANCH" ]; then
|
||||
ARGS="$ARGS --op $OP_ALLOY_BRANCH"
|
||||
fi
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
|
||||
16
.github/workflows/compact.yml
vendored
16
.github/workflows/compact.yml
vendored
@@ -16,32 +16,38 @@ env:
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
name: compact-codec
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
compact-codec:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
bin:
|
||||
- cargo run --bin reth --features "dev"
|
||||
steps:
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Checkout base
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.base_ref || 'main' }}
|
||||
persist-credentials: false
|
||||
# On `main` branch, generates test vectors and serializes them to disk using `Compact`.
|
||||
- name: Generate compact vectors
|
||||
run: |
|
||||
${{ matrix.bin }} -- test-vectors compact --write
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
clean: false
|
||||
persist-credentials: false
|
||||
# On incoming merge try to read and decode previously generated vectors with `Compact`
|
||||
- name: Read vectors
|
||||
run: ${{ matrix.bin }} -- test-vectors compact --read
|
||||
|
||||
9
.github/workflows/dependencies.yml
vendored
9
.github/workflows/dependencies.yml
vendored
@@ -9,13 +9,14 @@ on:
|
||||
workflow_dispatch:
|
||||
# Needed so we can run it manually
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
update:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
secrets:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
14
.github/workflows/docker-tag-latest.yml
vendored
14
.github/workflows/docker-tag-latest.yml
vendored
@@ -17,6 +17,8 @@ on:
|
||||
env:
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
tag-reth-latest:
|
||||
name: Tag reth as latest
|
||||
@@ -27,16 +29,22 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Log in to Docker
|
||||
env:
|
||||
DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
echo "$DOCKER_PASSWORD" | docker login ghcr.io --username "${DOCKER_USERNAME}" --password-stdin
|
||||
|
||||
- name: Pull reth release image
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/reth:${{ inputs.version }}
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/reth:${VERSION}
|
||||
|
||||
- name: Tag reth as latest
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
docker tag ghcr.io/${{ github.repository_owner }}/reth:${{ inputs.version }} ghcr.io/${{ github.repository_owner }}/reth:latest
|
||||
docker tag ghcr.io/${{ github.repository_owner }}/reth:${VERSION} ghcr.io/${{ github.repository_owner }}/reth:latest
|
||||
|
||||
- name: Push reth latest tag
|
||||
run: |
|
||||
|
||||
22
.github/workflows/docker-test.yml
vendored
22
.github/workflows/docker-test.yml
vendored
@@ -13,6 +13,8 @@ on:
|
||||
default: "artifacts"
|
||||
description: "Name for the uploaded artifact"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
timeout-minutes: 45
|
||||
@@ -21,7 +23,9 @@ jobs:
|
||||
id-token: write
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: mkdir -p artifacts
|
||||
|
||||
- name: Get git info
|
||||
@@ -32,8 +36,12 @@ jobs:
|
||||
|
||||
- name: Detect fork
|
||||
id: fork
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
|
||||
if [ "$EVENT_NAME" = "pull_request" ] && [ "$HEAD_REPO" != "$REPO" ]; then
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -42,11 +50,11 @@ jobs:
|
||||
# Depot build (upstream only)
|
||||
- name: Set up Depot CLI
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/setup-action@v1
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
|
||||
|
||||
- name: Build reth image (Depot)
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/bake-action@v1
|
||||
uses: depot/bake-action@1d58c2668346981089b088b7ef36755b206b20e9 # v1.13.0
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
@@ -60,11 +68,11 @@ jobs:
|
||||
# Docker build (forks)
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build reth image (Docker)
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/bake-action@v6
|
||||
uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7.0.0
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
@@ -76,7 +84,7 @@ jobs:
|
||||
*.dockerfile=Dockerfile
|
||||
|
||||
- name: Upload reth image
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: ${{ inputs.artifact_name }}
|
||||
path: ./artifacts
|
||||
|
||||
70
.github/workflows/docker.yml
vendored
70
.github/workflows/docker.yml
vendored
@@ -28,42 +28,28 @@ on:
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
pgo:
|
||||
description: "Enable PGO profiling"
|
||||
required: false
|
||||
type: boolean
|
||||
default: false
|
||||
pgo_blocks:
|
||||
description: "Number of blocks to execute for PGO profiling"
|
||||
required: false
|
||||
type: string
|
||||
default: "20"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
collect-pgo-profile:
|
||||
if: github.repository == 'paradigmxyz/reth' && github.event_name == 'workflow_dispatch' && inputs.pgo
|
||||
uses: ./.github/workflows/pgo-profile.yml
|
||||
with:
|
||||
pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
|
||||
secrets: inherit
|
||||
|
||||
build:
|
||||
if: github.repository == 'paradigmxyz/reth' && !failure() && !cancelled()
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: Build Docker images
|
||||
runs-on: ubuntu-24.04
|
||||
needs: collect-pgo-profile
|
||||
permissions:
|
||||
packages: write
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Depot CLI
|
||||
uses: depot/setup-action@v1
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -76,36 +62,15 @@ jobs:
|
||||
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
|
||||
echo "dirty=false" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Download pre-collected PGO profile
|
||||
if: ${{ github.event_name == 'workflow_dispatch' && inputs.pgo }}
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: pgo-profdata
|
||||
path: dist
|
||||
|
||||
- name: Configure PGO build args
|
||||
id: pgo
|
||||
run: |
|
||||
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ "${{ inputs.pgo }}" == "true" ]]; then
|
||||
if [ ! -f dist/merged.profdata ]; then
|
||||
echo "::error::Expected dist/merged.profdata from collect-pgo-profile job"
|
||||
exit 1
|
||||
fi
|
||||
echo "use_pgo_bolt=true" >> "$GITHUB_OUTPUT"
|
||||
echo "pgo_profdata=dist/merged.profdata" >> "$GITHUB_OUTPUT"
|
||||
echo "Using pre-collected PGO profile from collect-pgo-profile job"
|
||||
else
|
||||
echo "use_pgo_bolt=false" >> "$GITHUB_OUTPUT"
|
||||
echo "pgo_profdata=" >> "$GITHUB_OUTPUT"
|
||||
echo "PGO disabled"
|
||||
fi
|
||||
|
||||
- name: Determine build parameters
|
||||
id: params
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BUILD_TYPE: ${{ inputs.build_type }}
|
||||
run: |
|
||||
REGISTRY="ghcr.io/${{ github.repository_owner }}"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "push" ]]; then
|
||||
if [[ "${EVENT_NAME}" == "push" ]]; then
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
|
||||
|
||||
@@ -123,7 +88,7 @@ jobs:
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
elif [[ "${EVENT_NAME}" == "schedule" ]] || [[ "${BUILD_TYPE}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
@@ -136,7 +101,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
uses: depot/bake-action@v1
|
||||
uses: depot/bake-action@1d58c2668346981089b088b7ef36755b206b20e9 # v1.13.0
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
@@ -147,11 +112,10 @@ jobs:
|
||||
files: docker-bake.hcl
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
save: false
|
||||
load: false
|
||||
set: |
|
||||
${{ steps.params.outputs.ethereum_set }}
|
||||
*.args.USE_PGO_BOLT=${{ steps.pgo.outputs.use_pgo_bolt }}
|
||||
*.args.PGO_PROFDATA=${{ steps.pgo.outputs.pgo_profdata }}
|
||||
*.args.STRIP_SYMBOLS=false
|
||||
|
||||
- name: Verify image architectures
|
||||
env:
|
||||
@@ -169,7 +133,7 @@ jobs:
|
||||
if: failure() && github.event_name == 'schedule'
|
||||
steps:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3
|
||||
env:
|
||||
SLACK_COLOR: danger
|
||||
SLACK_ICON_EMOJI: ":rotating_light:"
|
||||
|
||||
30
.github/workflows/e2e.yml
vendored
30
.github/workflows/e2e.yml
vendored
@@ -17,19 +17,27 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: e2e-testsuite
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run e2e tests
|
||||
@@ -48,15 +56,21 @@ jobs:
|
||||
rocksdb:
|
||||
name: e2e-rocksdb
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run RocksDB e2e tests
|
||||
|
||||
72
.github/workflows/fetch-grafana-dashboard.yml
vendored
Normal file
72
.github/workflows/fetch-grafana-dashboard.yml
vendored
Normal file
@@ -0,0 +1,72 @@
|
||||
name: Fetch Grafana Dashboard
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dashboard_uid:
|
||||
description: "Grafana dashboard UID to export"
|
||||
required: true
|
||||
default: "2k8BXz24x"
|
||||
target_path:
|
||||
description: "Target file path in the repo (e.g. etc/grafana/dashboards/overview.json)"
|
||||
required: true
|
||||
default: "etc/grafana/dashboards/overview.json"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
fetch:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Fetch dashboard from Grafana
|
||||
env:
|
||||
FETCH_GRAFANA_DASHBOARD_URL: ${{ secrets.FETCH_GRAFANA_DASHBOARD_URL }}
|
||||
FETCH_GRAFANA_DASHBOARD_TOKEN: ${{ secrets.FETCH_GRAFANA_DASHBOARD_TOKEN }}
|
||||
DASHBOARD_UID: ${{ inputs.dashboard_uid }}
|
||||
TARGET_PATH: ${{ inputs.target_path }}
|
||||
run: |
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py "${DASHBOARD_UID}" \
|
||||
> "${TARGET_PATH}"
|
||||
|
||||
- name: Check for changes
|
||||
id: diff
|
||||
env:
|
||||
TARGET_PATH: ${{ inputs.target_path }}
|
||||
run: |
|
||||
if git diff --quiet "${TARGET_PATH}"; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No changes detected."
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create pull request
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DASHBOARD_UID: ${{ inputs.dashboard_uid }}
|
||||
TARGET_PATH: ${{ inputs.target_path }}
|
||||
run: |
|
||||
TARGET="${TARGET_PATH}"
|
||||
FILENAME="$(basename "$TARGET")"
|
||||
BRANCH="chore/sync-grafana-${FILENAME%.*}-$(date +%Y%m%d-%H%M%S)"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$BRANCH"
|
||||
git add "$TARGET"
|
||||
git commit -m "chore: update Grafana dashboard ${FILENAME}"
|
||||
git push origin "$BRANCH"
|
||||
gh pr create \
|
||||
--title "chore: update Grafana dashboard ${FILENAME}" \
|
||||
--body "Automated export from Grafana (dashboard UID: \`${DASHBOARD_UID}\`, target: \`${TARGET}\`)."
|
||||
33
.github/workflows/grafana.yml
vendored
33
.github/workflows/grafana.yml
vendored
@@ -6,16 +6,33 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-dashboard:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Check for ${DS_PROMETHEUS} in overview.json
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Validate dashboard format
|
||||
run: |
|
||||
if grep -Fn '${DS_PROMETHEUS}' etc/grafana/dashboards/overview.json; then
|
||||
echo "Error: overview.json contains '\${DS_PROMETHEUS}' placeholder"
|
||||
echo "Please replace it with '\${datasource}'"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ overview.json does not contain '\${DS_PROMETHEUS}' placeholder"
|
||||
python3 -c "
|
||||
import json, sys
|
||||
with open('etc/grafana/dashboards/overview.json') as f:
|
||||
d = json.load(f)
|
||||
errors = []
|
||||
if '__inputs' not in d:
|
||||
errors.append('missing __inputs')
|
||||
if '__requires' not in d:
|
||||
errors.append('missing __requires')
|
||||
if d.get('id') is not None:
|
||||
errors.append('contains internal id field — use export-dashboard.py')
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f'Error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print('✓ overview.json is a valid exported dashboard')
|
||||
"
|
||||
|
||||
252
.github/workflows/hive.yml
vendored
252
.github/workflows/hive.yml
vendored
@@ -14,8 +14,13 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build-reth:
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/docker-test.yml
|
||||
with:
|
||||
hive_target: hive
|
||||
@@ -25,34 +30,46 @@ jobs:
|
||||
prepare-hive:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant:
|
||||
- amsterdam
|
||||
- osaka
|
||||
name: Prepare Hive - ${{ matrix.variant == 'amsterdam' && 'Amsterdam' || 'Osaka' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
path: hivetests
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get hive commit hash
|
||||
id: hive-commit
|
||||
run: echo "hash=$(cd hivetests && git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "^1.13.1"
|
||||
- run: go version
|
||||
|
||||
- name: Restore hive assets cache
|
||||
id: cache-hive
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
with:
|
||||
path: ./hive_assets
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
key: hive-assets-${{ matrix.variant }}-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
|
||||
- name: Build hive assets
|
||||
if: steps.cache-hive.outputs.cache-hit != 'true'
|
||||
run: .github/scripts/hive/build_simulators.sh
|
||||
run: .github/scripts/hive/build_simulators.sh ${{ matrix.variant }}
|
||||
|
||||
- name: Load cached Docker images
|
||||
if: steps.cache-hive.outputs.cache-hit == 'true'
|
||||
@@ -68,11 +85,195 @@ jobs:
|
||||
chmod +x hive
|
||||
|
||||
- name: Upload hive assets
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: hive_assets
|
||||
name: hive_assets_${{ matrix.variant }}
|
||||
path: ./hive_assets
|
||||
test:
|
||||
test-amsterdam:
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# ethereum/rpc to be deprecated:
|
||||
# https://github.com/ethereum/hive/pull/1117
|
||||
scenario:
|
||||
- sim: smoke/genesis
|
||||
- sim: smoke/network
|
||||
- sim: ethereum/sync
|
||||
- sim: devp2p
|
||||
limit: discv4
|
||||
# started failing after https://github.com/ethereum/go-ethereum/pull/31843, no
|
||||
# action on our side, remove from here when we get unexpected passes on these tests
|
||||
# - sim: devp2p
|
||||
# limit: eth
|
||||
# include:
|
||||
# - MaliciousHandshake
|
||||
# # failures tracked in https://github.com/paradigmxyz/reth/issues/14825
|
||||
# - Status
|
||||
# - GetBlockHeaders
|
||||
# - ZeroRequestID
|
||||
# - GetBlockBodies
|
||||
# - Transaction
|
||||
# - NewPooledTxs
|
||||
- sim: devp2p
|
||||
limit: discv5
|
||||
include:
|
||||
# failures tracked at https://github.com/paradigmxyz/reth/issues/14825
|
||||
- PingLargeRequestID
|
||||
- sim: ethereum/engine
|
||||
limit: engine-exchange-capabilities
|
||||
- sim: ethereum/engine
|
||||
limit: engine-withdrawals
|
||||
- sim: ethereum/engine
|
||||
limit: engine-auth
|
||||
- sim: ethereum/engine
|
||||
limit: engine-api
|
||||
- sim: ethereum/engine
|
||||
limit: cancun
|
||||
# eth_ rpc methods
|
||||
- sim: ethereum/rpc-compat
|
||||
include:
|
||||
- eth_blockNumber
|
||||
- eth_call
|
||||
- eth_chainId
|
||||
- eth_createAccessList
|
||||
- eth_estimateGas
|
||||
- eth_feeHistory
|
||||
- eth_getBalance
|
||||
- eth_getBlockBy
|
||||
- eth_getBlockTransactionCountBy
|
||||
- eth_getCode
|
||||
- eth_getProof
|
||||
- eth_getStorage
|
||||
- eth_getTransactionBy
|
||||
- eth_getTransactionCount
|
||||
- eth_getTransactionReceipt
|
||||
- eth_sendRawTransaction
|
||||
- eth_syncing
|
||||
# debug_ rpc methods
|
||||
- debug_
|
||||
|
||||
# consume-engine
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/prague.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/cancun.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/shanghai.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/berlin.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/istanbul.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/homestead.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/frontier.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/paris.*
|
||||
|
||||
# consume-rlp
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/prague.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/cancun.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/shanghai.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/berlin.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/istanbul.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/homestead.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/frontier.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/paris.*
|
||||
needs:
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: Hive-Amsterdam / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download hive assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: hive_assets_amsterdam
|
||||
path: /tmp
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reth
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker images
|
||||
run: .github/scripts/hive/load_images.sh
|
||||
|
||||
- name: Move hive binary
|
||||
run: |
|
||||
mv /tmp/hive /usr/local/bin
|
||||
chmod +x /usr/local/bin/hive
|
||||
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
ref: master
|
||||
path: hivetests
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run simulator
|
||||
env:
|
||||
SCENARIO_SIM: ${{ matrix.scenario.sim }}
|
||||
SCENARIO_LIMIT: ${{ matrix.scenario.limit }}
|
||||
SCENARIO_TESTS: ${{ join(matrix.scenario.include, '|') }}
|
||||
run: |
|
||||
LIMIT="$SCENARIO_LIMIT"
|
||||
TESTS="$SCENARIO_TESTS"
|
||||
if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then
|
||||
FILTER="$LIMIT/$TESTS"
|
||||
elif [ -n "$LIMIT" ]; then
|
||||
FILTER="$LIMIT"
|
||||
elif [ -n "$TESTS" ]; then
|
||||
FILTER="/$TESTS"
|
||||
else
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "$SCENARIO_SIM" "$FILTER" "amsterdam"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
|
||||
|
||||
- name: Print simulator output
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
cat hivetests/workspace/logs/*simulator*.log
|
||||
|
||||
- name: Print reth client logs
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
cat hivetests/workspace/logs/reth/client-*.log
|
||||
|
||||
test-osaka:
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -178,24 +379,26 @@ jobs:
|
||||
needs:
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
name: Hive-Osaka / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download hive assets
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: hive_assets
|
||||
name: hive_assets_osaka
|
||||
path: /tmp
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reth
|
||||
path: /tmp
|
||||
@@ -209,16 +412,21 @@ jobs:
|
||||
chmod +x /usr/local/bin/hive
|
||||
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
ref: master
|
||||
path: hivetests
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run simulator
|
||||
env:
|
||||
SCENARIO_SIM: ${{ matrix.scenario.sim }}
|
||||
SCENARIO_LIMIT: ${{ matrix.scenario.limit }}
|
||||
SCENARIO_TESTS: ${{ join(matrix.scenario.include, '|') }}
|
||||
run: |
|
||||
LIMIT="${{ matrix.scenario.limit }}"
|
||||
TESTS="${{ join(matrix.scenario.include, '|') }}"
|
||||
LIMIT="$SCENARIO_LIMIT"
|
||||
TESTS="$SCENARIO_TESTS"
|
||||
if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then
|
||||
FILTER="$LIMIT/$TESTS"
|
||||
elif [ -n "$LIMIT" ]; then
|
||||
@@ -229,7 +437,7 @@ jobs:
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "$SCENARIO_SIM" "$FILTER" "osaka"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
@@ -245,12 +453,14 @@ jobs:
|
||||
run: |
|
||||
cat hivetests/workspace/logs/reth/client-*.log
|
||||
notify-on-error:
|
||||
needs: test
|
||||
needs:
|
||||
- test-amsterdam
|
||||
- test-osaka
|
||||
if: failure()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3
|
||||
env:
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
|
||||
36
.github/workflows/integration.yml
vendored
36
.github/workflows/integration.yml
vendored
@@ -20,11 +20,15 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.network }}
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
strategy:
|
||||
@@ -32,14 +36,18 @@ jobs:
|
||||
network: ["ethereum"]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Install Geth
|
||||
run: .github/scripts/install_geth.sh
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run tests
|
||||
@@ -58,7 +66,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Decide whether the needed jobs succeeded or failed
|
||||
uses: re-actors/alls-green@release/v1
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
@@ -66,13 +74,19 @@ jobs:
|
||||
name: era1 file integration tests once a day
|
||||
if: github.event_name == 'schedule' && github.repository == 'paradigmxyz/reth'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: run era1 files integration tests
|
||||
|
||||
16
.github/workflows/kurtosis.yml
vendored
16
.github/workflows/kurtosis.yml
vendored
@@ -18,9 +18,14 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build-reth:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/docker-test.yml
|
||||
with:
|
||||
hive_target: kurtosis
|
||||
@@ -32,15 +37,18 @@ jobs:
|
||||
fail-fast: false
|
||||
name: run kurtosis
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
needs:
|
||||
- build-reth
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: artifacts
|
||||
path: /tmp
|
||||
@@ -52,7 +60,7 @@ jobs:
|
||||
docker image ls -a
|
||||
|
||||
- name: Run kurtosis
|
||||
uses: ethpandaops/kurtosis-assertoor-github-action@v1
|
||||
uses: ethpandaops/kurtosis-assertoor-github-action@f64942cbc780df731a731ea9f45765b161d2c8df # v1.0.1
|
||||
with:
|
||||
ethereum_package_args: ".github/assets/kurtosis_network_params.yaml"
|
||||
|
||||
@@ -62,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3
|
||||
env:
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
|
||||
8
.github/workflows/label-pr.yml
vendored
8
.github/workflows/label-pr.yml
vendored
@@ -4,19 +4,23 @@ on:
|
||||
pull_request:
|
||||
types: [opened]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
label_prs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Label PRs
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const label_pr = require('./.github/scripts/label_pr.js')
|
||||
|
||||
8
.github/workflows/lint-actions.yml
vendored
8
.github/workflows/lint-actions.yml
vendored
@@ -8,11 +8,17 @@ on:
|
||||
paths:
|
||||
- '.github/**'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
actionlint:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Download actionlint
|
||||
id: get_actionlint
|
||||
run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
|
||||
|
||||
192
.github/workflows/lint.yml
vendored
192
.github/workflows/lint.yml
vendored
@@ -10,10 +10,14 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
clippy-binaries:
|
||||
name: clippy binaries / ${{ matrix.type }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -22,17 +26,19 @@ jobs:
|
||||
args: --workspace --lib --examples --tests --benches --locked
|
||||
features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@clippy
|
||||
with:
|
||||
components: clippy
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- if: "${{ matrix.type == 'book' }}"
|
||||
uses: arduino/setup-protoc@v3
|
||||
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Run clippy on binaries
|
||||
@@ -43,15 +49,19 @@ jobs:
|
||||
clippy:
|
||||
name: clippy
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: clippy
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
@@ -60,19 +70,25 @@ jobs:
|
||||
|
||||
wasm:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: wasm32-wasip1
|
||||
- uses: taiki-e/install-action@cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: dcarbone/install-jq-action@v3
|
||||
- uses: dcarbone/install-jq-action@b7ef57d46ece78760b4019dbc4080a1ba2a40b45 # v3.2.0
|
||||
- name: Run Wasm checks
|
||||
run: |
|
||||
sudo apt update && sudo apt install gcc-multilib
|
||||
@@ -80,53 +96,69 @@ jobs:
|
||||
|
||||
riscv:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: riscv32imac-unknown-none-elf
|
||||
- uses: taiki-e/install-action@cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: dcarbone/install-jq-action@v3
|
||||
- uses: dcarbone/install-jq-action@b7ef57d46ece78760b4019dbc4080a1ba2a40b45 # v3.2.0
|
||||
- name: Run RISC-V checks
|
||||
run: .github/scripts/check_rv32imac.sh
|
||||
|
||||
crate-checks:
|
||||
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2, 3]
|
||||
total_partitions: [3]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo hack check --workspace --partition ${{ matrix.partition }}/${{ matrix.total_partitions }}
|
||||
|
||||
msrv:
|
||||
name: MSRV
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: "1.93" # MSRV
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo build --bin reth --workspace
|
||||
@@ -136,13 +168,17 @@ jobs:
|
||||
docs:
|
||||
name: docs
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo docs --document-private-items
|
||||
@@ -154,42 +190,56 @@ jobs:
|
||||
fmt:
|
||||
name: fmt
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: rustfmt
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- name: Run fmt
|
||||
run: cargo fmt --all --check
|
||||
|
||||
udeps:
|
||||
name: udeps
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: taiki-e/install-action@cargo-udeps
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-udeps
|
||||
- run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked
|
||||
|
||||
book:
|
||||
name: book
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo build --bin reth --workspace
|
||||
@@ -201,38 +251,54 @@ jobs:
|
||||
|
||||
typos:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: crate-ci/typos@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: crate-ci/typos@02ea592e44b3a53c302f697cddca7641cd051c3d # v1.45.0
|
||||
|
||||
check-toml:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Run dprint
|
||||
uses: dprint/check@v2.3
|
||||
uses: dprint/check@9cb3a2b17a8e606d37aae341e49df3654933fc23 # v2.3
|
||||
with:
|
||||
config-path: dprint.json
|
||||
|
||||
grafana:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check dashboard JSON with jq
|
||||
uses: sergeysova/jq-action@v2
|
||||
uses: sergeysova/jq-action@a3f0d4ff59cc1dddf023fc0b325dd75b10deec58 # v2.3.0
|
||||
with:
|
||||
cmd: jq empty etc/grafana/dashboards/overview.json
|
||||
|
||||
no-test-deps:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Ensure no arbitrary or proptest dependency on default build
|
||||
run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0
|
||||
@@ -240,13 +306,17 @@ jobs:
|
||||
# Check crates correctly propagate features
|
||||
feature-propagation:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: taiki-e/cache-cargo-install-action@v3
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: taiki-e/cache-cargo-install-action@a8b9ecf8e0c0ea09d7481cfc583a5203ecd585b5 # v3.0.5
|
||||
with:
|
||||
tool: zepter
|
||||
- name: Eagerly pull dependencies
|
||||
@@ -254,6 +324,8 @@ jobs:
|
||||
- run: zepter run check
|
||||
|
||||
deny:
|
||||
permissions:
|
||||
contents: read
|
||||
uses: tempoxyz/ci/.github/workflows/deny.yml@main
|
||||
|
||||
lint-success:
|
||||
@@ -277,6 +349,6 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Decide whether the needed jobs succeeded or failed
|
||||
uses: re-actors/alls-green@release/v1
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
99
.github/workflows/pgo-profile.yml
vendored
99
.github/workflows/pgo-profile.yml
vendored
@@ -1,99 +0,0 @@
|
||||
name: pgo-profile
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
pgo_blocks:
|
||||
description: "Number of blocks to execute for PGO profiling"
|
||||
required: false
|
||||
type: string
|
||||
default: "20"
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
pgo_blocks:
|
||||
description: "Number of blocks to execute for PGO profiling"
|
||||
required: false
|
||||
type: string
|
||||
default: "20"
|
||||
|
||||
jobs:
|
||||
collect:
|
||||
name: collect PGO profiles
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
timeout-minutes: 180
|
||||
env:
|
||||
SCHELK_MOUNT: /reth-bench
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
continue-on-error: true
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
dmsetup lsb-release wget linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
- name: Download snapshot if needed
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
if ! .github/scripts/bench-reth-snapshot.sh --check; then
|
||||
echo "Snapshot outdated or missing, downloading..."
|
||||
.github/scripts/bench-reth-snapshot.sh
|
||||
fi
|
||||
|
||||
- name: Mount snapshot
|
||||
run: |
|
||||
sudo schelk recover -y --kill || true
|
||||
sudo schelk mount -y
|
||||
sync
|
||||
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
|
||||
|
||||
- name: Collect PGO profile
|
||||
run: |
|
||||
DATADIR="$SCHELK_MOUNT/datadir" \
|
||||
RPC_URL="$BENCH_RPC_URL" \
|
||||
PGO_BLOCKS="${{ inputs.pgo_blocks || '20' }}" \
|
||||
BOLT_BLOCKS="${{ inputs.pgo_blocks || '20' }}" \
|
||||
COLLECT_PGO_ONLY=true \
|
||||
SKIP_BOLT=true \
|
||||
PROFILE=maxperf-symbols \
|
||||
FEATURES="jemalloc,asm-keccak,min-debug-logs" \
|
||||
TARGET=x86_64-unknown-linux-gnu \
|
||||
EXTRA_RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" \
|
||||
.github/scripts/build_pgo_bolt.sh
|
||||
|
||||
- name: Show PGO profile stats
|
||||
run: |
|
||||
LLVM_PROFDATA=$(find "$(rustc --print sysroot)" -name llvm-profdata -type f | head -1)
|
||||
if [ -z "$LLVM_PROFDATA" ]; then
|
||||
echo "::error::llvm-profdata not found in rust toolchain"
|
||||
exit 1
|
||||
fi
|
||||
"$LLVM_PROFDATA" show --detailed-summary --topn=20 target/pgo-profiles/merged.profdata
|
||||
|
||||
- name: Upload PGO profile
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: pgo-profdata
|
||||
path: target/pgo-profiles/merged.profdata
|
||||
retention-days: 1
|
||||
|
||||
- name: Recover snapshot
|
||||
if: always()
|
||||
run: |
|
||||
sudo schelk recover -y --kill || true
|
||||
23
.github/workflows/pr-audit.yml
vendored
23
.github/workflows/pr-audit.yml
vendored
@@ -4,27 +4,36 @@ on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.label.name == 'cyclops'
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Publish event
|
||||
env:
|
||||
EVENTS_KEY: ${{ secrets.EVENTS_KEY }}
|
||||
EVENTS_CERT: ${{ secrets.EVENTS_CERT }}
|
||||
EVENTS_ARGS: ${{ secrets.EVENTS_ARGS }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "${{ secrets.EVENTS_KEY }}" > ${{ runner.temp }}/key
|
||||
echo "${{ secrets.EVENTS_CERT }}" > ${{ runner.temp }}/cert
|
||||
echo "$EVENTS_KEY" > "${{ runner.temp }}/key"
|
||||
echo "$EVENTS_CERT" > "${{ runner.temp }}/cert"
|
||||
|
||||
curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \
|
||||
curl -sf -o /dev/null -X POST $EVENTS_ARGS \
|
||||
-H "Content-Type: application/json" \
|
||||
--key ${{ runner.temp }}/key \
|
||||
--cert ${{ runner.temp }}/cert \
|
||||
--key "${{ runner.temp }}/key" \
|
||||
--cert "${{ runner.temp }}/cert" \
|
||||
-d '{
|
||||
"repository": "${{ github.repository }}",
|
||||
"event": "pr_audit",
|
||||
"data": {
|
||||
"pr_number": ${{ github.event.pull_request.number }},
|
||||
"sha": "${{ github.event.pull_request.head.sha }}"
|
||||
"pr_number": '"$PR_NUMBER"',
|
||||
"sha": "'"$PR_SHA"'"
|
||||
}
|
||||
}'
|
||||
|
||||
11
.github/workflows/pr-title.yml
vendored
11
.github/workflows/pr-title.yml
vendored
@@ -8,20 +8,19 @@ on:
|
||||
- edited
|
||||
- synchronize
|
||||
|
||||
permissions:
|
||||
pull-requests: read
|
||||
contents: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
conventional-title:
|
||||
name: Validate PR title is Conventional Commit
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Check title
|
||||
id: lint_pr_title
|
||||
uses: amannn/action-semantic-pull-request@v6
|
||||
uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
@@ -40,7 +39,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
- name: Add PR Comment for Invalid Title
|
||||
if: steps.lint_pr_title.outcome == 'failure'
|
||||
uses: marocchino/sticky-pull-request-comment@v2
|
||||
uses: marocchino/sticky-pull-request-comment@d4d6b0936434b21bc8345ad45a440c5f7d2c40ff # v3.0.3
|
||||
with:
|
||||
header: pr-title-lint-error
|
||||
message: |
|
||||
@@ -76,7 +75,7 @@ jobs:
|
||||
|
||||
- name: Remove Comment for Valid Title
|
||||
if: steps.lint_pr_title.outcome == 'success'
|
||||
uses: marocchino/sticky-pull-request-comment@v2
|
||||
uses: marocchino/sticky-pull-request-comment@d4d6b0936434b21bc8345ad45a440c5f7d2c40ff # v3.0.3
|
||||
with:
|
||||
header: pr-title-lint-error
|
||||
delete: true
|
||||
|
||||
5
.github/workflows/release-dist.yml
vendored
5
.github/workflows/release-dist.yml
vendored
@@ -7,12 +7,15 @@ on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
release-homebrew:
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Update Homebrew formula
|
||||
uses: dawidd6/action-homebrew-bump-formula@v7
|
||||
uses: dawidd6/action-homebrew-bump-formula@1446dca236b0440c6f02723a3f14f13be2c04ab0 # v7
|
||||
with:
|
||||
token: ${{ secrets.HOMEBREW }}
|
||||
no_fork: true
|
||||
|
||||
24
.github/workflows/release-reproducible.yml
vendored
24
.github/workflows/release-reproducible.yml
vendored
@@ -2,6 +2,8 @@
|
||||
|
||||
name: release-reproducible
|
||||
|
||||
permissions: {}
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [release]
|
||||
@@ -15,20 +17,23 @@ jobs:
|
||||
name: extract version
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Extract version from triggering tag
|
||||
id: extract_version
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
run: |
|
||||
# Get the tag that points to the head SHA of the triggering workflow
|
||||
TAG=$(gh api /repos/${{ github.repository }}/git/refs/tags \
|
||||
--jq '.[] | select(.object.sha == "${{ github.event.workflow_run.head_sha }}") | .ref' \
|
||||
--jq ".[] | select(.object.sha == \"${HEAD_SHA}\") | .ref" \
|
||||
| head -1 \
|
||||
| sed 's|refs/tags/||')
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "No tag found for SHA ${{ github.event.workflow_run.head_sha }}"
|
||||
echo "No tag found for SHA ${HEAD_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -44,15 +49,16 @@ jobs:
|
||||
packages: write
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.extract-version.outputs.VERSION }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -65,7 +71,7 @@ jobs:
|
||||
echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build reproducible artifacts
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
id: docker_build
|
||||
with:
|
||||
context: .
|
||||
@@ -75,13 +81,11 @@ jobs:
|
||||
VERSION=${{ needs.extract-version.outputs.VERSION }}
|
||||
target: artifacts
|
||||
outputs: type=local,dest=./docker-artifacts
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
env:
|
||||
DOCKER_BUILD_RECORD_UPLOAD: false
|
||||
|
||||
- name: Build and push final image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.reproducible
|
||||
@@ -92,8 +96,6 @@ jobs:
|
||||
tags: |
|
||||
${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}
|
||||
${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
provenance: false
|
||||
env:
|
||||
DOCKER_BUILD_RECORD_UPLOAD: false
|
||||
|
||||
151
.github/workflows/release.yml
vendored
151
.github/workflows/release.yml
vendored
@@ -3,6 +3,8 @@
|
||||
|
||||
name: release
|
||||
|
||||
permissions: {}
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
@@ -13,14 +15,6 @@ on:
|
||||
description: "Enable dry run mode (builds artifacts but skips uploads and release creation)"
|
||||
type: boolean
|
||||
default: false
|
||||
pgo:
|
||||
description: "Enable PGO profiling"
|
||||
type: boolean
|
||||
default: false
|
||||
pgo_blocks:
|
||||
description: "Number of blocks to execute for PGO profiling on self-hosted runner"
|
||||
type: string
|
||||
default: "20"
|
||||
|
||||
env:
|
||||
REPO_NAME: ${{ github.repository_owner }}/reth
|
||||
@@ -28,21 +22,24 @@ env:
|
||||
REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/reth-reproducible
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/reth
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
jobs:
|
||||
dry-run:
|
||||
name: check dry run
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- run: |
|
||||
echo "Dry run value: ${{ github.event.inputs.dry_run }}"
|
||||
echo "Dry run enabled: ${{ github.event.inputs.dry_run == 'true'}}"
|
||||
echo "Dry run disabled: ${{ github.event.inputs.dry_run != 'true'}}"
|
||||
- env:
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run }}
|
||||
run: |
|
||||
echo "Dry run value: ${DRY_RUN}"
|
||||
echo "Dry run enabled: $( [ "${DRY_RUN}" = 'true' ] && echo true || echo false )"
|
||||
echo "Dry run disabled: $( [ "${DRY_RUN}" != 'true' ] && echo true || echo false )"
|
||||
|
||||
extract-version:
|
||||
name: extract version
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Extract version
|
||||
run: echo "VERSION=${GITHUB_REF_NAME//\//-}" >> $GITHUB_OUTPUT
|
||||
@@ -53,12 +50,15 @@ jobs:
|
||||
check-version:
|
||||
name: check version
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
needs: extract-version
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- name: Verify crate version matches tag
|
||||
# Check that the Cargo version starts with the tag,
|
||||
# so that Cargo version 1.4.8 can be matched against both v1.4.8 and v1.4.8-rc.1
|
||||
@@ -71,12 +71,20 @@ jobs:
|
||||
build:
|
||||
name: build release
|
||||
runs-on: ${{ matrix.configs.os }}
|
||||
permissions:
|
||||
contents: read
|
||||
needs: extract-version
|
||||
continue-on-error: ${{ matrix.configs.allow_fail }}
|
||||
strategy:
|
||||
fail-fast: true
|
||||
matrix:
|
||||
configs:
|
||||
- target: x86_64-unknown-linux-gnu
|
||||
os: ubuntu-24.04
|
||||
profile: maxperf
|
||||
allow_fail: false
|
||||
rustflags: "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
|
||||
native: true
|
||||
- target: aarch64-unknown-linux-gnu
|
||||
os: ubuntu-24.04-arm
|
||||
profile: maxperf
|
||||
@@ -97,20 +105,20 @@ jobs:
|
||||
- command: build
|
||||
binary: reth
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: ${{ matrix.configs.target }}
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- name: Install cross main
|
||||
if: ${{ !matrix.configs.native }}
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --locked --git https://github.com/cross-rs/cross
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
cargo install cross --locked \
|
||||
--git https://github.com/cross-rs/cross \
|
||||
--rev 65fe72b0cdb1e7e0cc0652517498d4389cc8f5cf
|
||||
|
||||
- name: Apple M1 setup
|
||||
if: matrix.configs.target == 'aarch64-apple-darwin'
|
||||
@@ -119,6 +127,8 @@ jobs:
|
||||
echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
|
||||
|
||||
- name: Build Reth
|
||||
env:
|
||||
CC: clang
|
||||
run: |
|
||||
if [ "${{ matrix.configs.native }}" = "true" ]; then
|
||||
make PROFILE=${{ matrix.configs.profile }} EXTRA_RUSTFLAGS="${{ matrix.configs.rustflags }}" ${{ matrix.build.command }}-native-${{ matrix.configs.target }}
|
||||
@@ -145,105 +155,24 @@ jobs:
|
||||
|
||||
- name: Upload artifact
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
|
||||
path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
|
||||
|
||||
- name: Upload signature
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
|
||||
path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
|
||||
|
||||
collect-pgo-profile:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.pgo
|
||||
uses: ./.github/workflows/pgo-profile.yml
|
||||
with:
|
||||
pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
|
||||
secrets: inherit
|
||||
|
||||
build-pgo:
|
||||
if: github.event_name == 'workflow_dispatch' && inputs.pgo
|
||||
name: build release (x86_64-linux PGO+BOLT)
|
||||
runs-on: [self-hosted, Linux, X64]
|
||||
needs: [extract-version, collect-pgo-profile]
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-linux-gnu
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
continue-on-error: true
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Download pre-collected PGO profile
|
||||
uses: actions/download-artifact@v7
|
||||
with:
|
||||
name: pgo-profdata
|
||||
path: dist
|
||||
|
||||
- name: Verify PGO profile artifact
|
||||
run: |
|
||||
test -f dist/merged.profdata
|
||||
ls -lh dist/merged.profdata
|
||||
|
||||
- name: Build Reth with PGO+BOLT
|
||||
run: |
|
||||
SKIP_BOLT=true \
|
||||
PGO_PROFDATA="$PWD/dist/merged.profdata" \
|
||||
PROFILE=maxperf-symbols \
|
||||
FEATURES="jemalloc,asm-keccak,min-debug-logs" \
|
||||
TARGET=x86_64-unknown-linux-gnu \
|
||||
EXTRA_RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" \
|
||||
.github/scripts/build_pgo_bolt.sh
|
||||
|
||||
- name: Move binary
|
||||
run: |
|
||||
mkdir artifacts
|
||||
mv target/maxperf-symbols/reth ./artifacts
|
||||
|
||||
- name: Configure GPG and create artifacts
|
||||
env:
|
||||
GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
|
||||
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
|
||||
run: |
|
||||
export GPG_TTY=$(tty)
|
||||
echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import
|
||||
cd artifacts
|
||||
tar -czf reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz reth*
|
||||
echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
|
||||
mv *tar.gz* ..
|
||||
shell: bash
|
||||
|
||||
- name: Upload artifact
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
|
||||
path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
|
||||
|
||||
- name: Upload signature
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
|
||||
path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
|
||||
|
||||
draft-release:
|
||||
name: draft release
|
||||
runs-on: ubuntu-latest
|
||||
needs: [build, build-pgo, extract-version]
|
||||
if: ${{ !failure() && !cancelled() && github.event.inputs.dry_run != 'true' }}
|
||||
needs: [build, extract-version]
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
env:
|
||||
VERSION: ${{ needs.extract-version.outputs.VERSION }}
|
||||
permissions:
|
||||
@@ -252,11 +181,12 @@ jobs:
|
||||
steps:
|
||||
# This is necessary for generating the changelog.
|
||||
# It has to come before "Download Artifacts" or else it deletes the artifacts.
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
- name: Generate full changelog
|
||||
id: changelog
|
||||
run: |
|
||||
@@ -276,7 +206,7 @@ jobs:
|
||||
fi
|
||||
|
||||
body=$(cat <<- "ENDBODY"
|
||||

|
||||

|
||||
|
||||
## Testing Checklist (DELETE ME)
|
||||
|
||||
@@ -342,6 +272,7 @@ jobs:
|
||||
dry-run-summary:
|
||||
name: dry run summary
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
needs: [build, extract-version]
|
||||
if: ${{ github.event.inputs.dry_run == 'true' }}
|
||||
env:
|
||||
|
||||
16
.github/workflows/reproducible-build.yml
vendored
16
.github/workflows/reproducible-build.yml
vendored
@@ -5,11 +5,15 @@ on:
|
||||
schedule:
|
||||
- cron: "0 1 */2 * *"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: build reproducible binaries
|
||||
runs-on: ${{ matrix.runner }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -18,14 +22,16 @@ jobs:
|
||||
- runner: ubuntu-22.04
|
||||
machine: machine-2
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-linux-gnu
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build reproducible binary with Docker
|
||||
run: |
|
||||
@@ -43,7 +49,7 @@ jobs:
|
||||
echo "Binaries SHA256 on ${{ matrix.machine }}: $(cat checksum.sha256)"
|
||||
|
||||
- name: Upload the hash
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: checksum-${{ matrix.machine }}
|
||||
path: |
|
||||
@@ -56,12 +62,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts from machine-1
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: checksum-machine-1
|
||||
path: machine-1/
|
||||
- name: Download artifacts from machine-2
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: checksum-machine-2
|
||||
path: machine-2/
|
||||
|
||||
14
.github/workflows/stage.yml
vendored
14
.github/workflows/stage.yml
vendored
@@ -18,22 +18,28 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
stage:
|
||||
name: stage-run-test
|
||||
# Only run stage commands test in merge groups
|
||||
if: github.event_name == 'merge_group'
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Build reth
|
||||
|
||||
4
.github/workflows/stale.yml
vendored
4
.github/workflows/stale.yml
vendored
@@ -7,6 +7,8 @@ on:
|
||||
schedule:
|
||||
- cron: "30 1 * * *"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
close-issues:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
@@ -15,7 +17,7 @@ jobs:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@v10
|
||||
- uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
|
||||
with:
|
||||
days-before-stale: 21
|
||||
days-before-close: 7
|
||||
|
||||
14
.github/workflows/sync-era.yml
vendored
14
.github/workflows/sync-era.yml
vendored
@@ -15,11 +15,15 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -34,11 +38,13 @@ jobs:
|
||||
block: 100000
|
||||
unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Build ${{ matrix.chain.bin }}
|
||||
|
||||
14
.github/workflows/sync.yml
vendored
14
.github/workflows/sync.yml
vendored
@@ -15,11 +15,15 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -34,11 +38,13 @@ jobs:
|
||||
block: 100000
|
||||
unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Build ${{ matrix.chain.bin }}
|
||||
|
||||
53
.github/workflows/unit.yml
vendored
53
.github/workflows/unit.yml
vendored
@@ -17,10 +17,14 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.type }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
strategy:
|
||||
@@ -32,16 +36,20 @@ jobs:
|
||||
exclude_args: ""
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- if: "${{ matrix.type == 'book' }}"
|
||||
uses: arduino/setup-protoc@v3
|
||||
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Run tests
|
||||
@@ -56,20 +64,25 @@ jobs:
|
||||
state:
|
||||
name: Ethereum state tests
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Checkout ethereum/tests
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/tests
|
||||
ref: 81862e4848585a438d64f911a19b3825f0f4cd95
|
||||
path: testing/ef-tests/ethereum-tests
|
||||
submodules: recursive
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
- name: Download & extract EEST fixtures (public)
|
||||
shell: bash
|
||||
env:
|
||||
@@ -79,11 +92,13 @@ jobs:
|
||||
mkdir -p testing/ef-tests/execution-spec-tests
|
||||
URL="https://github.com/ethereum/execution-spec-tests/releases/download/${EEST_TESTS_TAG}/fixtures_stable.tar.gz"
|
||||
curl -L "$URL" | tar -xz --strip-components=1 -C testing/ef-tests/execution-spec-tests
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo nextest run --no-fail-fast --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
|
||||
@@ -91,15 +106,19 @@ jobs:
|
||||
doc:
|
||||
name: doc tests
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run doctests
|
||||
@@ -113,6 +132,6 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Decide whether the needed jobs succeeded or failed
|
||||
uses: re-actors/alls-green@release/v1
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
1561
Cargo.lock
generated
1561
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
137
Cargo.toml
137
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "2.0.0"
|
||||
version = "2.2.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.93"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -129,12 +129,12 @@ members = [
|
||||
"examples/custom-node-components/",
|
||||
"examples/custom-payload-builder/",
|
||||
"examples/custom-rlpx-subprotocol",
|
||||
"examples/custom-auth-http-middleware",
|
||||
"examples/custom-rpc-middleware",
|
||||
"examples/db-access",
|
||||
"examples/exex-subscription",
|
||||
"examples/exex-test",
|
||||
"examples/full-contract-state",
|
||||
"examples/migrate-trie-to-packed",
|
||||
"examples/manual-p2p/",
|
||||
"examples/network-txpool/",
|
||||
"examples/network/",
|
||||
@@ -326,8 +326,8 @@ reth-cli = { path = "crates/cli/cli" }
|
||||
reth-cli-commands = { path = "crates/cli/commands" }
|
||||
reth-cli-runner = { path = "crates/cli/runner" }
|
||||
reth-cli-util = { path = "crates/cli/util" }
|
||||
reth-codecs = { version = "0.1.0", default-features = false }
|
||||
reth-codecs-derive = "0.1.0"
|
||||
reth-codecs = { version = "0.3.1", default-features = false }
|
||||
reth-codecs-derive = "0.3.1"
|
||||
reth-config = { path = "crates/config", default-features = false }
|
||||
reth-consensus = { path = "crates/consensus/consensus", default-features = false }
|
||||
reth-consensus-common = { path = "crates/consensus/common", default-features = false }
|
||||
@@ -395,7 +395,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
|
||||
reth-payload-primitives = { path = "crates/payload/primitives" }
|
||||
reth-payload-validator = { path = "crates/payload/validator" }
|
||||
reth-payload-util = { path = "crates/payload/util" }
|
||||
reth-primitives-traits = { version = "0.1.0", default-features = false }
|
||||
reth-primitives-traits = { version = "0.3.1", default-features = false }
|
||||
reth-provider = { path = "crates/storage/provider" }
|
||||
reth-prune = { path = "crates/prune/prune" }
|
||||
reth-prune-types = { path = "crates/prune/types", default-features = false }
|
||||
@@ -411,7 +411,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal
|
||||
reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
|
||||
reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
|
||||
reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
|
||||
reth-rpc-traits = { version = "0.1.0", default-features = false }
|
||||
reth-rpc-traits = { version = "0.3.1", default-features = false }
|
||||
reth-stages = { path = "crates/stages/stages" }
|
||||
reth-stages-api = { path = "crates/stages/api" }
|
||||
reth-stages-types = { path = "crates/stages/types", default-features = false }
|
||||
@@ -430,17 +430,17 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
|
||||
reth-trie-db = { path = "crates/trie/db" }
|
||||
reth-trie-parallel = { path = "crates/trie/parallel" }
|
||||
reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.1.0", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.1", default-features = false }
|
||||
|
||||
# revm
|
||||
revm = { version = "36.0.0", default-features = false }
|
||||
revm-bytecode = { version = "9.0.0", default-features = false }
|
||||
revm-database = { version = "12.0.0", default-features = false }
|
||||
revm-state = { version = "10.0.0", default-features = false }
|
||||
revm-primitives = { version = "22.1.0", default-features = false }
|
||||
revm-interpreter = { version = "34.0.0", default-features = false }
|
||||
revm-database-interface = { version = "10.0.0", default-features = false }
|
||||
revm-inspectors = "0.36.0"
|
||||
revm = { version = "38.0.0", default-features = false }
|
||||
revm-bytecode = { version = "10.0.0", default-features = false }
|
||||
revm-database = { version = "13.0.0", default-features = false }
|
||||
revm-state = { version = "11.0.0", default-features = false }
|
||||
revm-primitives = { version = "23.0.0", default-features = false }
|
||||
revm-interpreter = { version = "35.0.0", default-features = false }
|
||||
revm-database-interface = { version = "11.0.0", default-features = false }
|
||||
revm-inspectors = "0.39.0"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.6"
|
||||
@@ -449,40 +449,40 @@ alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.33", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.30.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.4", default-features = false }
|
||||
alloy-evm = { version = "0.34.0", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
alloy-hardforks = "0.4.7"
|
||||
|
||||
alloy-consensus = { version = "1.8.2", default-features = false }
|
||||
alloy-contract = { version = "1.8.2", default-features = false }
|
||||
alloy-eips = { version = "1.8.2", default-features = false }
|
||||
alloy-genesis = { version = "1.8.2", default-features = false }
|
||||
alloy-json-rpc = { version = "1.8.2", default-features = false }
|
||||
alloy-network = { version = "1.8.2", default-features = false }
|
||||
alloy-network-primitives = { version = "1.8.2", default-features = false }
|
||||
alloy-provider = { version = "1.8.2", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-client = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types = { version = "1.8.2", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "1.8.2", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "1.8.2", default-features = false }
|
||||
alloy-serde = { version = "1.8.2", default-features = false }
|
||||
alloy-signer = { version = "1.8.2", default-features = false }
|
||||
alloy-signer-local = { version = "1.8.2", default-features = false }
|
||||
alloy-transport = { version = "1.8.2" }
|
||||
alloy-transport-http = { version = "1.8.2", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.8.2", default-features = false }
|
||||
alloy-transport-ws = { version = "1.8.2", default-features = false }
|
||||
alloy-consensus = { version = "2.0.4", default-features = false }
|
||||
alloy-contract = { version = "2.0.4", default-features = false }
|
||||
alloy-eips = { version = "2.0.4", default-features = false }
|
||||
alloy-genesis = { version = "2.0.4", default-features = false }
|
||||
alloy-json-rpc = { version = "2.0.4", default-features = false }
|
||||
alloy-network = { version = "2.0.4", default-features = false }
|
||||
alloy-network-primitives = { version = "2.0.4", default-features = false }
|
||||
alloy-provider = { version = "2.0.4", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-client = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types = { version = "2.0.4", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "2.0.4", default-features = false }
|
||||
alloy-serde = { version = "2.0.4", default-features = false }
|
||||
alloy-signer = { version = "2.0.4", default-features = false }
|
||||
alloy-signer-local = { version = "2.0.4", default-features = false }
|
||||
alloy-transport = { version = "2.0.4" }
|
||||
alloy-transport-http = { version = "2.0.4", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "2.0.4", default-features = false }
|
||||
alloy-transport-ws = { version = "2.0.4", default-features = false }
|
||||
|
||||
# misc
|
||||
either = { version = "1.15.0", default-features = false }
|
||||
@@ -507,6 +507,7 @@ eyre = "0.6"
|
||||
fdlimit = "0.3.0"
|
||||
fixed-map = { version = "0.9", default-features = false }
|
||||
humantime = "2.1"
|
||||
imbl = "7"
|
||||
humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
@@ -559,7 +560,7 @@ proc-macro2 = "1.0"
|
||||
quote = "1.0"
|
||||
|
||||
# tokio
|
||||
tokio = { version = "1.44.2", default-features = false }
|
||||
tokio = { version = "1.51.1", default-features = false }
|
||||
tokio-stream = "0.1.11"
|
||||
tokio-tungstenite = "0.28.0"
|
||||
tokio-util = { version = "0.7.4", features = ["codec"] }
|
||||
@@ -580,7 +581,7 @@ tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
|
||||
# p2p
|
||||
discv5 = "0.10"
|
||||
discv5 = { git = "https://github.com/sigp/discv5", rev = "7663c00" }
|
||||
if-addrs = "0.14"
|
||||
|
||||
# rpc
|
||||
@@ -672,7 +673,6 @@ indexmap = "2"
|
||||
interprocess = "2.2.0"
|
||||
lz4_flex = { version = "0.12", default-features = false }
|
||||
memmap2 = "0.9.4"
|
||||
mev-share-sse = { version = "0.5.0", default-features = false }
|
||||
num-traits = "0.2.15"
|
||||
page_size = "0.6.0"
|
||||
plain_hasher = "0.2"
|
||||
@@ -700,44 +700,3 @@ vergen-git2 = "9.1.0"
|
||||
|
||||
# networking
|
||||
ipnet = "2.11"
|
||||
|
||||
[patch.crates-io]
|
||||
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-genesis = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-network = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-provider = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-serde = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-signer = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-transport = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
# alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
|
||||
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "1207e33" }
|
||||
#
|
||||
# jsonrpsee = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-core = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-server = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "9bc2dba" }
|
||||
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
|
||||
|
||||
13
Dockerfile
13
Dockerfile
@@ -33,15 +33,24 @@ ENV FEATURES=$FEATURES
|
||||
RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json
|
||||
|
||||
# Build application
|
||||
# Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
|
||||
#
|
||||
# TARGETPLATFORM is set by BuildKit: https://docs.docker.com/reference/dockerfile#automatic-platform-args-in-the-global-scope
|
||||
ARG TARGETPLATFORM
|
||||
COPY --exclude=dist . .
|
||||
RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
|
||||
RUN if [ -n "$RUSTFLAGS" ]; then \
|
||||
export RUSTFLAGS="$RUSTFLAGS"; \
|
||||
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
||||
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
|
||||
fi && \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
|
||||
|
||||
# ARG is not resolved in COPY so we have to hack around it by copying the
|
||||
# binary to a temporary location
|
||||
RUN cp /app/target/$BUILD_PROFILE/reth /app/reth
|
||||
|
||||
# Use Ubuntu as the release image
|
||||
FROM ubuntu AS runtime
|
||||
FROM ubuntu:24.04 AS runtime
|
||||
WORKDIR /app
|
||||
|
||||
# Copy reth over from the build stage
|
||||
|
||||
@@ -1,10 +1,8 @@
|
||||
# syntax=docker/dockerfile:1
|
||||
|
||||
# Dockerfile for reth, optimized for Depot builds
|
||||
# Supports PGO+BOLT optimization for maximum performance
|
||||
# Usage:
|
||||
# reth: --build-arg BINARY=reth
|
||||
# PGO+BOLT: --build-arg USE_PGO_BOLT=true (Linux x86_64/aarch64 only)
|
||||
|
||||
FROM rust:1.93 AS builder
|
||||
WORKDIR /app
|
||||
@@ -45,18 +43,6 @@ ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
|
||||
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
|
||||
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
|
||||
|
||||
# Enable PGO+BOLT optimization (Linux only)
|
||||
ARG USE_PGO_BOLT=false
|
||||
ENV USE_PGO_BOLT=$USE_PGO_BOLT
|
||||
|
||||
# Optional path to a pre-collected merged.profdata file in build context.
|
||||
ARG PGO_PROFDATA=""
|
||||
ENV PGO_PROFDATA=$PGO_PROFDATA
|
||||
|
||||
# Whether to strip debug symbols from PGO-built binaries.
|
||||
ARG STRIP_SYMBOLS=true
|
||||
ENV STRIP_SYMBOLS=$STRIP_SYMBOLS
|
||||
|
||||
# Build application
|
||||
# Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
|
||||
ARG TARGETPLATFORM
|
||||
@@ -67,21 +53,12 @@ RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
|
||||
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
|
||||
export RUSTC_WRAPPER=sccache SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev SCCACHE_DIR=/sccache && \
|
||||
sccache --start-server && \
|
||||
if [ "$USE_PGO_BOLT" = "true" ] && [ "$TARGETPLATFORM" = "linux/amd64" ] && [ -n "$PGO_PROFDATA" ] && [ -f "$PGO_PROFDATA" ]; then \
|
||||
apt-get update && apt-get install -y -qq lsb-release wget sudo && \
|
||||
BINARY="$BINARY" PROFILE="$BUILD_PROFILE" FEATURES="$FEATURES" SKIP_BOLT=true STRIP_SYMBOLS="$STRIP_SYMBOLS" PGO_PROFDATA="$PGO_PROFDATA" \
|
||||
./.github/scripts/build_pgo_bolt.sh; \
|
||||
else \
|
||||
if [ "$USE_PGO_BOLT" = "true" ]; then \
|
||||
echo "PGO requested but pre-collected profile missing at '${PGO_PROFDATA:-<unset>}' - falling back to non-PGO build"; \
|
||||
fi; \
|
||||
if [ -n "$RUSTFLAGS" ]; then \
|
||||
export RUSTFLAGS="$RUSTFLAGS"; \
|
||||
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
||||
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
|
||||
fi && \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml; \
|
||||
if [ -n "$RUSTFLAGS" ]; then \
|
||||
export RUSTFLAGS="$RUSTFLAGS"; \
|
||||
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
||||
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
|
||||
fi && \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml && \
|
||||
sccache --show-stats
|
||||
|
||||
# Copy binary to a known location (ARG not resolved in COPY)
|
||||
|
||||
44
README.md
44
README.md
@@ -7,9 +7,9 @@
|
||||
|
||||
**Modular, contributor-friendly and blazing-fast implementation of the Ethereum protocol**
|
||||
|
||||

|
||||

|
||||
|
||||
**[Install](https://paradigmxyz.github.io/reth/installation/installation.html)**
|
||||
**[Install](https://reth.rs/installation/installation)**
|
||||
| [User Docs](https://reth.rs)
|
||||
| [Developer Docs](./docs)
|
||||
| [Crate Docs](https://reth.rs/docs)
|
||||
@@ -18,51 +18,43 @@
|
||||
[gh-lint]: https://github.com/paradigmxyz/reth/actions/workflows/lint.yml
|
||||
[tg-badge]: https://img.shields.io/endpoint?color=neon&logo=telegram&label=chat&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Fparadigm%5Freth
|
||||
|
||||
> **Note: OP-Reth has moved**
|
||||
>
|
||||
> The Optimism (op-reth) crates have been moved to [ethereum-optimism/optimism](https://github.com/ethereum-optimism/optimism).
|
||||
> Git contribution history has been preserved. If you are looking for op-reth, please see the new repository.
|
||||
|
||||
## What is Reth?
|
||||
|
||||
Reth (short for Rust Ethereum, [pronunciation](https://x.com/kelvinfichter/status/1597653609411268608)) is a new Ethereum full node implementation that is focused on being user-friendly, highly modular, as well as being fast and efficient. Reth is an Execution Layer (EL) and is compatible with all Ethereum Consensus Layer (CL) implementations that support the [Engine API](https://github.com/ethereum/execution-apis/tree/a0d03086564ab1838b462befbc083f873dcf0c0f/src/engine). It is originally built and driven forward by [Paradigm](https://paradigm.xyz/), and is licensed under the Apache and MIT licenses.
|
||||
Reth (short for Rust Ethereum, [pronunciation](https://x.com/kelvinfichter/status/1597653609411268608)) is a production-ready Ethereum execution layer client focused on modularity, performance, and user-friendliness. Reth is compatible with all Ethereum Consensus Layer (CL) implementations that support the [Engine API](https://github.com/ethereum/execution-apis/tree/a0d03086564ab1838b462befbc083f873dcf0c0f/src/engine). It is built and driven forward by [Paradigm](https://paradigm.xyz/), and is licensed under the Apache and MIT licenses.
|
||||
|
||||
> **Note:** OP-Reth has moved to [ethereum-optimism/optimism](https://github.com/ethereum-optimism/optimism). Git history has been preserved.
|
||||
|
||||
## Goals
|
||||
|
||||
As a full Ethereum node, Reth allows users to connect to the Ethereum network and interact with the Ethereum blockchain. This includes sending and receiving transactions/logs/traces, as well as accessing and interacting with smart contracts. Building a successful Ethereum node requires creating a high-quality implementation that is both secure and efficient, as well as being easy to use on consumer hardware. It also requires building a strong community of contributors who can help support and improve the software.
|
||||
1. **Modularity**: Every component is built to be used as a library: well-tested, documented and benchmarked. Import crates, mix and match, and innovate on top of them. Learn more about the project's components [here](./docs/repo/layout.md).
|
||||
2. **Performance**: Built with Rust, [Alloy](https://github.com/alloy-rs/alloy/), [revm](https://github.com/bluealloy/revm/), and [Foundry](https://github.com/foundry-rs/foundry/) — battle-tested and optimized for speed. Check the [ethPandaOps Lab Dashboard](https://lab.ethpandaops.io/ethereum/execution/timings) for a third-party comparison against other Ethereum clients.
|
||||
Here's what that looks like in practice, measured with [reth-bench](https://github.com/paradigmxyz/reth/tree/main/bin/reth-bench) on Ethereum Mainnet:
|
||||
|
||||
More concretely, our goals are:
|
||||

|
||||
|
||||
1. **Modularity**: Every component of Reth is built to be used as a library: well-tested, heavily documented and benchmarked. We envision that developers will import the node's crates, mix and match, and innovate on top of them. Examples of such usage include but are not limited to spinning up standalone P2P networks, talking directly to a node's database, or "unbundling" the node into the components you need. To achieve that, we are licensing Reth under the Apache/MIT permissive license. You can learn more about the project's components [here](./docs/repo/layout.md).
|
||||
2. **Performance**: Reth aims to be fast, so we use Rust and the [Erigon staged-sync](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) node architecture. We also use our Ethereum libraries (including [Alloy](https://github.com/alloy-rs/alloy/) and [revm](https://github.com/bluealloy/revm/)) which we've battle-tested and optimized via [Foundry](https://github.com/foundry-rs/foundry/).
|
||||
3. **Free for anyone to use any way they want**: Reth is free open source software, built for the community, by the community. By licensing the software under the Apache/MIT license, we want developers to use it without being bound by business licenses, or having to think about the implications of GPL-like licenses.
|
||||
4. **Client Diversity**: The Ethereum protocol becomes more antifragile when no node implementation dominates. This ensures that if there's a software bug, the network does not finalize a bad block. By building a new client, we hope to contribute to Ethereum's antifragility.
|
||||
5. **Support as many EVM chains as possible**: We aspire that Reth can full-sync not only Ethereum, but also other chains like Optimism, Polygon, BNB Smart Chain, and more. If you're working on any of these projects, please reach out. Note: OP-Reth has moved to [ethereum-optimism/optimism](https://github.com/ethereum-optimism/optimism).
|
||||
6. **Configurability**: We want to solve for node operators that care about fast historical queries, but also for hobbyists who cannot operate on large hardware. We also want to support teams and individuals who want both sync from genesis and via "fast sync". We envision that Reth will be configurable enough and provide configurable "profiles" for the tradeoffs that each team faces.
|
||||
3. **Free for anyone to use any way they want**: Apache/MIT licensed, no business license restrictions.
|
||||
4. **Client Diversity**: More client implementations make Ethereum more antifragile.
|
||||
5. **Support as many EVM chains as possible**: Reth can sync Ethereum and other EVM chains. If you're building one, reach out.
|
||||
6. **Configurability**: Profiles for different use cases — from high-performance RPC operators to hobbyists on consumer hardware.
|
||||
|
||||
## Status
|
||||
|
||||
Reth is production ready, and suitable for usage in mission-critical environments such as staking or high-uptime services. We also actively recommend professional node operators to switch to Reth in production for performance and cost reasons in use cases where high performance with great margins is required such as RPC, MEV, Indexing, Simulations, and P2P activities.
|
||||
|
||||
More historical context below:
|
||||
|
||||
- We released **Reth 2.0** in April 2026. See the [release notes](https://github.com/paradigmxyz/reth/releases/tag/v2.0.0) and [blog post](https://www.paradigm.xyz/2026/04/releasing-reth-2-0).
|
||||
- We released 1.0 "production-ready" stable Reth in June 2024.
|
||||
- Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf).
|
||||
- Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon.
|
||||
- Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)).
|
||||
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024, the last beta release.
|
||||
- We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4, 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives.
|
||||
- We shipped iterative improvements until the last alpha release on February 28, 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21).
|
||||
- We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) on June 20, 2023.
|
||||
|
||||
### Database compatibility
|
||||
### Storage compatibility
|
||||
|
||||
We do not have any breaking database changes since beta.1, and we do not plan any in the near future.
|
||||
Storage V2 is the default for new nodes in Reth 2.0. Existing V1 nodes continue to work, but V1 support will be removed in a future release — all users are encouraged to migrate. V2 snapshots are available at [snapshots.reth.rs](https://snapshots.reth.rs/).
|
||||
|
||||
Reth [v0.2.0-beta.1](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) includes
|
||||
a [set of breaking database changes](https://github.com/paradigmxyz/reth/pull/5191) that makes it impossible to use database files produced by earlier versions.
|
||||
|
||||
If you had a database produced by alpha versions of Reth, you need to drop it with `reth db drop`
|
||||
(using the same arguments such as `--config` or `--datadir` that you passed to `reth node`), and resync using the same `reth node` command you've used before.
|
||||

|
||||
|
||||
## For Users
|
||||
|
||||
|
||||
BIN
assets/reth-perf.png
Normal file
BIN
assets/reth-perf.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 77 KiB |
BIN
assets/reth-storage.png
Normal file
BIN
assets/reth-storage.png
Normal file
Binary file not shown.
|
After Width: | Height: | Size: 50 KiB |
@@ -69,6 +69,7 @@ default = [
|
||||
"jemalloc",
|
||||
"reth-cli-util/jemalloc",
|
||||
"asm-keccak",
|
||||
"keccak-cache-global",
|
||||
"min-debug-logs",
|
||||
]
|
||||
|
||||
@@ -89,6 +90,12 @@ asm-keccak = [
|
||||
"revm-primitives/asm-keccak",
|
||||
]
|
||||
|
||||
keccak-cache-global = [
|
||||
"reth-node-core/keccak-cache-global",
|
||||
"reth-node-ethereum/keccak-cache-global",
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
]
|
||||
|
||||
min-debug-logs = [
|
||||
"tracing/release_max_level_debug",
|
||||
"reth-ethereum-cli/min-debug-logs",
|
||||
|
||||
@@ -7,15 +7,16 @@
|
||||
//! `execute_transaction` to apply segment-boundary changes.
|
||||
|
||||
use crate::evm_config::BigBlockSegment;
|
||||
use alloy_consensus::TransactionEnvelope;
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_evm::{
|
||||
block::{
|
||||
BlockExecutionError, BlockExecutionResult, BlockExecutor, BlockExecutorFactory,
|
||||
BlockExecutorFor, ExecutableTx, OnStateHook, StateChangeSource, StateDB,
|
||||
ExecutableTx, GasOutput, OnStateHook, StateChangeSource, StateDB,
|
||||
},
|
||||
eth::{EthBlockExecutionCtx, EthBlockExecutor, EthEvmContext, EthTxResult},
|
||||
precompiles::PrecompilesMap,
|
||||
Database, EthEvm, EthEvmFactory, Evm, FromRecoveredTx, FromTxWithEncoded,
|
||||
Database, EthEvm, EthEvmFactory, Evm, EvmFactory, FromRecoveredTx, FromTxWithEncoded,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
@@ -36,6 +37,7 @@ use tracing::{debug, trace};
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Runtime state for segment boundary tracking.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BbEvmPlan {
|
||||
/// The segment boundaries and environments.
|
||||
pub(crate) segments: Vec<BigBlockSegment>,
|
||||
@@ -73,6 +75,10 @@ impl BbEvmPlan {
|
||||
.filter(|(n, _)| *n >= min && *n < block_number)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn segment_index_for_tx(&self, tx_index: usize) -> usize {
|
||||
self.segments.partition_point(|segment| segment.start_tx <= tx_index).saturating_sub(1)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BbEvmPlan {
|
||||
@@ -97,6 +103,9 @@ impl std::fmt::Debug for BbEvmPlan {
|
||||
/// segment boundaries without requiring additional trait bounds on `DB`.
|
||||
pub(crate) type BlockHashSeeder<DB> = fn(&mut DB, &[(u64, B256)]);
|
||||
|
||||
/// Function pointer that reads the BAL index from the DB.
|
||||
pub(crate) type BalIndexReader<DB> = fn(&DB) -> u64;
|
||||
|
||||
/// Block executor that wraps [`EthBlockExecutor`] and handles segment-boundary
|
||||
/// changes for big-block execution.
|
||||
///
|
||||
@@ -108,7 +117,8 @@ pub(crate) type BlockHashSeeder<DB> = fn(&mut DB, &[(u64, B256)]);
|
||||
/// Gas counters reset at each boundary so that each segment's real gas limit
|
||||
/// is used (preserving correct GASLIMIT opcode behavior). Accumulated offsets
|
||||
/// are applied to receipts and totals in `finish()`.
|
||||
pub(crate) struct BbBlockExecutor<'a, DB, I, P, Spec>
|
||||
#[expect(missing_debug_implementations)]
|
||||
pub struct BbBlockExecutor<'a, DB, I, P, Spec>
|
||||
where
|
||||
DB: Database,
|
||||
{
|
||||
@@ -131,6 +141,10 @@ where
|
||||
/// Callback to reseed block hashes into the DB's cache at segment
|
||||
/// boundaries. See [`BlockHashSeeder`].
|
||||
block_hash_seeder: Option<BlockHashSeeder<DB>>,
|
||||
/// Callback to read the BAL index from the DB.
|
||||
bal_index_reader: Option<BalIndexReader<DB>>,
|
||||
/// Whether the executor has selected its starting segment.
|
||||
initialized: bool,
|
||||
}
|
||||
|
||||
impl<'a, DB, I, P, Spec> BbBlockExecutor<'a, DB, I, P, Spec>
|
||||
@@ -156,6 +170,7 @@ where
|
||||
receipt_builder: RethReceiptBuilder,
|
||||
plan: Option<BbEvmPlan>,
|
||||
block_hash_seeder: Option<BlockHashSeeder<DB>>,
|
||||
bal_index_reader: Option<BalIndexReader<DB>>,
|
||||
) -> Self {
|
||||
let inner = EthBlockExecutor::new(evm, ctx, spec, receipt_builder);
|
||||
Self {
|
||||
@@ -166,9 +181,63 @@ where
|
||||
blob_gas_used_offset: 0,
|
||||
shared_hook: Arc::new(Mutex::new(None)),
|
||||
block_hash_seeder,
|
||||
bal_index_reader,
|
||||
initialized: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn initialize(&mut self) -> Result<(), BlockExecutionError> {
|
||||
if self.initialized {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let plan = match &self.plan {
|
||||
Some(plan) => plan,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
self.initialized = true;
|
||||
|
||||
let bal_index =
|
||||
self.bal_index_reader.map(|reader| reader(self.inner().evm().db())).unwrap_or(0);
|
||||
let segment_idx =
|
||||
if bal_index == 0 { 0 } else { plan.segment_index_for_tx((bal_index - 1) as usize) };
|
||||
let segment = &plan.segments[segment_idx];
|
||||
|
||||
// Swap the EVM's block_env and executor ctx to the selected segment's
|
||||
// values so that EIP-2935/EIP-4788 system calls use the correct block
|
||||
// number and parent hash. Without this, the outer big block header's
|
||||
// block_number (which is synthetic) would be used, writing to wrong
|
||||
// EIP-2935 slots and corrupting state.
|
||||
let block_env = segment.evm_env.block_env.clone();
|
||||
let block_number = block_env.number.saturating_to::<u64>();
|
||||
let mut cfg_env = segment.evm_env.cfg_env.clone();
|
||||
cfg_env.disable_base_fee = true;
|
||||
let ctx = EthBlockExecutionCtx {
|
||||
parent_hash: segment.ctx.parent_hash,
|
||||
parent_beacon_block_root: segment.ctx.parent_beacon_block_root,
|
||||
ommers: segment.ctx.ommers,
|
||||
withdrawals: segment.ctx.withdrawals.clone(),
|
||||
extra_data: segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: segment.ctx.tx_count_hint,
|
||||
slot_number: segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
let inner = self.inner_mut();
|
||||
let evm_ctx = inner.evm.ctx_mut();
|
||||
evm_ctx.block = block_env;
|
||||
evm_ctx.cfg = cfg_env;
|
||||
inner.ctx = ctx;
|
||||
|
||||
self.reseed_block_hashes_for(block_number);
|
||||
|
||||
if bal_index > 0 {
|
||||
self.plan = None;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a forwarding `OnStateHook` that delegates to the shared hook.
|
||||
fn forwarding_hook(&self) -> Option<Box<dyn OnStateHook>> {
|
||||
let shared = self.shared_hook.clone();
|
||||
@@ -238,6 +307,7 @@ where
|
||||
withdrawals: prev_segment.ctx.withdrawals.clone(),
|
||||
extra_data: prev_segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: prev_segment.ctx.tx_count_hint,
|
||||
slot_number: prev_segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
// Clone the next segment's data before we consume inner.
|
||||
@@ -252,6 +322,7 @@ where
|
||||
withdrawals: new_segment.ctx.withdrawals.clone(),
|
||||
extra_data: new_segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: new_segment.ctx.tx_count_hint,
|
||||
slot_number: new_segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
plan.next_segment += 1;
|
||||
@@ -347,34 +418,9 @@ where
|
||||
type Result = EthTxResult<HaltReason, alloy_consensus::TxType>;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
// Swap the EVM's block_env and executor ctx to the first segment's
|
||||
// values so that the initial EIP-2935/EIP-4788 system calls use the
|
||||
// correct block number and parent hash. Without this, the outer big
|
||||
// block header's block_number (which is synthetic) would be used,
|
||||
// writing to wrong EIP-2935 slots and corrupting state.
|
||||
if let Some(seg0) = self.plan.as_ref().map(|p| &p.segments[0]) {
|
||||
let block_env = seg0.evm_env.block_env.clone();
|
||||
let block_number = block_env.number.saturating_to::<u64>();
|
||||
let mut cfg_env = seg0.evm_env.cfg_env.clone();
|
||||
cfg_env.disable_base_fee = true;
|
||||
let seg0_ctx = EthBlockExecutionCtx {
|
||||
parent_hash: seg0.ctx.parent_hash,
|
||||
parent_beacon_block_root: seg0.ctx.parent_beacon_block_root,
|
||||
ommers: seg0.ctx.ommers,
|
||||
withdrawals: seg0.ctx.withdrawals.clone(),
|
||||
extra_data: seg0.ctx.extra_data.clone(),
|
||||
tx_count_hint: seg0.ctx.tx_count_hint,
|
||||
};
|
||||
|
||||
let inner = self.inner_mut();
|
||||
let evm_ctx = inner.evm.ctx_mut();
|
||||
evm_ctx.block = block_env;
|
||||
evm_ctx.cfg = cfg_env;
|
||||
inner.ctx = seg0_ctx;
|
||||
|
||||
self.reseed_block_hashes_for(block_number);
|
||||
}
|
||||
|
||||
// The outer big-block header uses a synthetic block number, so start
|
||||
// system calls must run against the selected real segment env.
|
||||
self.initialize()?;
|
||||
self.inner_mut().apply_pre_execution_changes()
|
||||
}
|
||||
|
||||
@@ -382,12 +428,16 @@ where
|
||||
&mut self,
|
||||
tx: impl ExecutableTx<Self>,
|
||||
) -> Result<Self::Result, BlockExecutionError> {
|
||||
self.initialize()?;
|
||||
self.maybe_apply_boundary()?;
|
||||
self.inner_mut().execute_transaction_without_commit(tx)
|
||||
}
|
||||
|
||||
fn commit_transaction(&mut self, output: Self::Result) -> Result<u64, BlockExecutionError> {
|
||||
let gas_used = self.inner_mut().commit_transaction(output)?;
|
||||
fn commit_transaction(&mut self, output: Self::Result) -> GasOutput {
|
||||
self.maybe_apply_boundary()
|
||||
.expect("segment boundary application must succeed before committing transaction");
|
||||
|
||||
let gas_used = self.inner_mut().commit_transaction(output);
|
||||
|
||||
// Fix up cumulative_gas_used on the just-committed receipt so that
|
||||
// the receipt root task (which reads receipts incrementally) sees
|
||||
@@ -402,7 +452,7 @@ where
|
||||
if let Some(plan) = &mut self.plan {
|
||||
plan.tx_counter += 1;
|
||||
}
|
||||
Ok(gas_used)
|
||||
gas_used
|
||||
}
|
||||
|
||||
fn finish(
|
||||
@@ -419,6 +469,7 @@ where
|
||||
withdrawals: last_seg.ctx.withdrawals.clone(),
|
||||
extra_data: last_seg.ctx.extra_data.clone(),
|
||||
tx_count_hint: last_seg.ctx.tx_count_hint,
|
||||
slot_number: last_seg.ctx.slot_number,
|
||||
};
|
||||
self.inner_mut().ctx = last_ctx;
|
||||
}
|
||||
@@ -492,7 +543,7 @@ pub struct BbBlockExecutorFactory<Spec> {
|
||||
receipt_builder: RethReceiptBuilder,
|
||||
spec: Spec,
|
||||
evm_factory: EthEvmFactory,
|
||||
/// Staged plan consumed by the next [`BbBlockExecutor`].
|
||||
/// Staged plan cloned into each [`BbBlockExecutor`].
|
||||
pub(crate) staged_plan: Arc<Mutex<Option<BbEvmPlan>>>,
|
||||
}
|
||||
|
||||
@@ -521,8 +572,12 @@ impl<Spec> BbBlockExecutorFactory<Spec> {
|
||||
*self.staged_plan.lock().unwrap() = Some(plan);
|
||||
}
|
||||
|
||||
fn take_plan(&self) -> Option<BbEvmPlan> {
|
||||
self.staged_plan.lock().unwrap().take()
|
||||
pub(crate) fn clear_staged_plan(&self) {
|
||||
*self.staged_plan.lock().unwrap() = None;
|
||||
}
|
||||
|
||||
fn peek_plan(&self) -> Option<BbEvmPlan> {
|
||||
self.staged_plan.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
pub(crate) fn create_executor_with_seeder<'a, DB, I>(
|
||||
@@ -530,14 +585,23 @@ impl<Spec> BbBlockExecutorFactory<Spec> {
|
||||
evm: EthEvm<DB, I, PrecompilesMap>,
|
||||
ctx: EthBlockExecutionCtx<'a>,
|
||||
block_hash_seeder: Option<BlockHashSeeder<DB>>,
|
||||
bal_index_reader: Option<BalIndexReader<DB>>,
|
||||
) -> BbBlockExecutor<'a, DB, I, PrecompilesMap, &'a Spec>
|
||||
where
|
||||
Spec: alloy_evm::eth::spec::EthExecutorSpec,
|
||||
DB: StateDB + 'a,
|
||||
I: Inspector<EthEvmContext<DB>> + 'a,
|
||||
{
|
||||
let plan = self.take_plan();
|
||||
BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, block_hash_seeder)
|
||||
let plan = self.peek_plan();
|
||||
BbBlockExecutor::new(
|
||||
evm,
|
||||
ctx,
|
||||
&self.spec,
|
||||
self.receipt_builder,
|
||||
plan,
|
||||
block_hash_seeder,
|
||||
bal_index_reader,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -550,6 +614,12 @@ where
|
||||
type ExecutionCtx<'a> = EthBlockExecutionCtx<'a>;
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
type TxExecutionResult = EthTxResult<
|
||||
<EthEvmFactory as EvmFactory>::HaltReason,
|
||||
<TransactionSigned as TransactionEnvelope>::TxType,
|
||||
>;
|
||||
type Executor<'a, DB: StateDB, I: Inspector<EthEvmContext<DB>>> =
|
||||
BbBlockExecutor<'a, DB, I, PrecompilesMap, &'a Spec>;
|
||||
|
||||
fn evm_factory(&self) -> &Self::EvmFactory {
|
||||
&self.evm_factory
|
||||
@@ -559,12 +629,12 @@ where
|
||||
&'a self,
|
||||
evm: EthEvm<DB, I, PrecompilesMap>,
|
||||
ctx: EthBlockExecutionCtx<'a>,
|
||||
) -> impl BlockExecutorFor<'a, Self, DB, I>
|
||||
) -> Self::Executor<'a, DB, I>
|
||||
where
|
||||
DB: StateDB + 'a,
|
||||
I: Inspector<EthEvmContext<DB>> + 'a,
|
||||
DB: StateDB,
|
||||
I: Inspector<EthEvmContext<DB>>,
|
||||
{
|
||||
let plan = self.take_plan();
|
||||
BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, None)
|
||||
let plan = self.peek_plan();
|
||||
BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, None, None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,11 +8,14 @@
|
||||
pub(crate) use reth_engine_primitives::BigBlockData;
|
||||
|
||||
use crate::{
|
||||
evm::{BbBlockExecutorFactory, BbEvmPlan},
|
||||
evm::{BalIndexReader, BbBlockExecutorFactory, BbEvmPlan},
|
||||
BigBlockMap,
|
||||
};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_evm::eth::EthBlockExecutionCtx;
|
||||
use alloy_evm::{
|
||||
eth::{spec::EthExecutorSpec, EthBlockExecutionCtx},
|
||||
EthEvmFactory,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types::engine::ExecutionData;
|
||||
use core::convert::Infallible;
|
||||
@@ -20,8 +23,8 @@ use reth_chainspec::{ChainSpec, EthChainSpec};
|
||||
use reth_ethereum_forks::Hardforks;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_evm::{
|
||||
ConfigureEngineEvm, ConfigureEvm, Database, EvmEnv, ExecutableTxIterator,
|
||||
NextBlockEnvAttributes,
|
||||
ConfigureEngineEvm, ConfigureEvm, Database, EvmEnv, EvmEnvFor, ExecutableTxIterator,
|
||||
ExecutionCtxFor, NextBlockEnvAttributes,
|
||||
};
|
||||
use reth_evm_ethereum::{EthBlockAssembler, EthEvmConfig, RethReceiptBuilder};
|
||||
use reth_primitives_traits::{SealedBlock, SealedHeader};
|
||||
@@ -29,9 +32,6 @@ use revm::primitives::hardfork::SpecId;
|
||||
use std::sync::Arc;
|
||||
use tracing::debug;
|
||||
|
||||
use alloy_evm::{eth::spec::EthExecutorSpec, EthEvmFactory};
|
||||
use reth_evm::{EvmEnvFor, ExecutionCtxFor};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Execution plan types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -55,7 +55,7 @@ pub(crate) struct BigBlockSegment {
|
||||
///
|
||||
/// Wraps [`EthEvmConfig`] and a shared [`BigBlockMap`]. When a big-block
|
||||
/// payload is received, the plan is staged on the [`BbBlockExecutorFactory`]
|
||||
/// and consumed when the executor is created. Block hashes for inter-segment
|
||||
/// and cloned when executors are created. Block hashes for inter-segment
|
||||
/// BLOCKHASH resolution are reseeded into `State::block_hashes` at each
|
||||
/// segment boundary via a [`BlockHashSeeder`](crate::evm::BlockHashSeeder)
|
||||
/// callback injected in [`ConfigureEvm::create_executor`].
|
||||
@@ -106,6 +106,10 @@ fn seed_state_block_hashes<DB>(state: &mut &mut revm::database::State<DB>, hashe
|
||||
}
|
||||
}
|
||||
|
||||
fn read_bal_index<DB>(state: &&mut revm::database::State<DB>) -> u64 {
|
||||
state.bal_state.bal_index()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ConfigureEvm
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -144,6 +148,12 @@ where
|
||||
&self,
|
||||
block: &'a SealedBlock<reth_ethereum_primitives::Block>,
|
||||
) -> Result<EthBlockExecutionCtx<'a>, Self::Error> {
|
||||
if let Some(plan) = self.plan_for_payload_hash(&block.hash()) {
|
||||
self.executor_factory.stage_plan(plan);
|
||||
} else {
|
||||
self.executor_factory.clear_staged_plan();
|
||||
}
|
||||
|
||||
self.inner.context_for_block(block)
|
||||
}
|
||||
|
||||
@@ -159,7 +169,7 @@ where
|
||||
&'a self,
|
||||
evm: reth_evm::EvmFor<Self, &'a mut revm::database::State<DB>, I>,
|
||||
ctx: EthBlockExecutionCtx<'a>,
|
||||
) -> impl alloy_evm::block::BlockExecutorFor<
|
||||
) -> alloy_evm::block::BlockExecutorFor<
|
||||
'a,
|
||||
Self::BlockExecutorFactory,
|
||||
&'a mut revm::database::State<DB>,
|
||||
@@ -169,15 +179,16 @@ where
|
||||
DB: Database,
|
||||
I: reth_evm::InspectorFor<Self, &'a mut revm::database::State<DB>> + 'a,
|
||||
{
|
||||
// Use create_executor_with_seeder to inject a concrete seeder that
|
||||
// can reseed State::block_hashes at segment boundaries. The seeder
|
||||
// is a function pointer that knows the concrete State<DB> type,
|
||||
// allowing the generic BbBlockExecutor to reseed without additional
|
||||
// trait bounds on DB.
|
||||
let bal_index_reader: Option<BalIndexReader<&'a mut revm::database::State<DB>>> =
|
||||
Some(read_bal_index::<DB>);
|
||||
|
||||
// Inject concrete function pointers that know the `State<DB>` type so
|
||||
// the generic executor can reseed block hashes and read `bal_index`.
|
||||
self.executor_factory.create_executor_with_seeder(
|
||||
evm,
|
||||
ctx,
|
||||
Some(seed_state_block_hashes::<DB>),
|
||||
bal_index_reader,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -214,6 +225,7 @@ where
|
||||
|
||||
Ok(env)
|
||||
} else {
|
||||
self.executor_factory.clear_staged_plan();
|
||||
self.inner.evm_env_for_payload(payload)
|
||||
}
|
||||
}
|
||||
@@ -248,10 +260,12 @@ where
|
||||
/// In practice, this is called from `evm_env_for_payload` in the
|
||||
/// engine pipeline.
|
||||
pub fn stage_plan_for_payload(&self, payload_hash: &B256) {
|
||||
let bb = match self.pending.lock().unwrap().remove(payload_hash) {
|
||||
Some(bb) => bb,
|
||||
None => return,
|
||||
};
|
||||
let Some(plan) = self.plan_for_payload_hash(payload_hash) else { return };
|
||||
self.executor_factory.stage_plan(plan);
|
||||
}
|
||||
|
||||
fn plan_for_payload_hash(&self, payload_hash: &B256) -> Option<BbEvmPlan> {
|
||||
let bb = self.pending.lock().unwrap().remove(payload_hash)?;
|
||||
|
||||
let segments: Vec<_> = bb
|
||||
.env_switches
|
||||
@@ -266,6 +280,7 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: ctx.withdrawals.map(|w| std::borrow::Cow::Owned(w.into_owned())),
|
||||
extra_data: ctx.extra_data,
|
||||
slot_number: ctx.slot_number,
|
||||
};
|
||||
BigBlockSegment { start_tx, evm_env, ctx }
|
||||
})
|
||||
@@ -286,6 +301,6 @@ where
|
||||
|
||||
plan.block_hashes_to_seed.sort_unstable_by_key(|(n, _)| *n);
|
||||
|
||||
self.executor_factory.stage_plan(plan);
|
||||
Some(plan)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -176,6 +176,7 @@ impl BbAddOns {
|
||||
BasicEngineApiBuilder::default(),
|
||||
BasicEngineValidatorBuilder::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -31,8 +31,10 @@ reth-tracing.workspace = true
|
||||
|
||||
# alloy
|
||||
alloy-consensus.workspace = true
|
||||
alloy-eip7928 = { workspace = true, features = ["rlp"] }
|
||||
alloy-eips.workspace = true
|
||||
alloy-json-rpc.workspace = true
|
||||
alloy-rlp.workspace = true
|
||||
|
||||
alloy-primitives = { workspace = true, features = ["rand"] }
|
||||
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
|
||||
|
||||
@@ -21,6 +21,8 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) auth_provider: RootProvider<AnyNetwork>,
|
||||
/// The block provider is used for block queries.
|
||||
pub(crate) block_provider: RootProvider<AnyNetwork>,
|
||||
/// The local regular RPC provider is used for non-authenticated node RPCs like `testing_*`.
|
||||
pub(crate) local_rpc_provider: RootProvider<AnyNetwork>,
|
||||
/// The benchmark mode, which defines whether the benchmark should run for a closed or open
|
||||
/// range of blocks.
|
||||
pub(crate) benchmark_mode: BenchMode,
|
||||
@@ -54,13 +56,8 @@ impl BenchContext {
|
||||
}
|
||||
}
|
||||
|
||||
// set up alloy client for blocks, retrying on 429/503 (default) and 502
|
||||
let retry_policy =
|
||||
RateLimitRetryPolicy::default().or(|err: &alloy_transport::TransportError| -> bool {
|
||||
err.as_transport_err()
|
||||
.and_then(|t| t.as_http_error())
|
||||
.is_some_and(|e| e.status == 502)
|
||||
});
|
||||
// set up alloy client for blocks, retrying on any errors, whether HTTP or OS
|
||||
let retry_policy = RateLimitRetryPolicy::default().or(|_| true);
|
||||
let max_retries = bench_args.rpc_block_fetch_retries.as_max_retries();
|
||||
let client = ClientBuilder::default()
|
||||
.layer(RetryBackoffLayer::new_with_policy(max_retries, 800, u64::MAX, retry_policy))
|
||||
@@ -88,6 +85,11 @@ impl BenchContext {
|
||||
let client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let auth_provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
let local_rpc_url = Url::parse(&bench_args.local_rpc_url)?;
|
||||
info!(target: "reth-bench", "Connecting to local regular RPC at {} for testing namespace calls", local_rpc_url);
|
||||
let local_rpc_provider =
|
||||
RootProvider::<AnyNetwork>::new(ClientBuilder::default().http(local_rpc_url));
|
||||
|
||||
// Computes the block range for the benchmark.
|
||||
//
|
||||
// - If `--advance` is provided, fetches the latest block from the engine and sets:
|
||||
@@ -164,6 +166,7 @@ impl BenchContext {
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
local_rpc_provider,
|
||||
benchmark_mode,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
|
||||
@@ -6,7 +6,12 @@
|
||||
//! [`ExecutionData`] and environment switches at each block boundary.
|
||||
|
||||
use alloy_consensus::{TxEnvelope, TxReceipt};
|
||||
use alloy_eips::{eip1559::BaseFeeParams, eip7840::BlobParams, Typed2718};
|
||||
use alloy_eips::{
|
||||
eip1559::BaseFeeParams,
|
||||
eip7840::BlobParams,
|
||||
eip7928::{AccountChanges, BlockAccessList, SlotChanges},
|
||||
Typed2718,
|
||||
};
|
||||
use alloy_primitives::{Bloom, Bytes, B256};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
@@ -16,6 +21,7 @@ use alloy_rpc_types_engine::{
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use futures::{stream, StreamExt};
|
||||
use reth_chainspec::EthChainSpec;
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
@@ -24,9 +30,14 @@ use reth_ethereum_cli::chainspec::EthereumChainSpecParser;
|
||||
use reth_ethereum_primitives::Receipt;
|
||||
use reth_primitives_traits::proofs;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::future::Future;
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
future::Future,
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
|
||||
/// A single transaction with its gas used and raw encoded bytes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RawTransaction {
|
||||
@@ -215,6 +226,9 @@ pub struct BigBlockPayload {
|
||||
/// Big block data containing environment switches and prior block hashes.
|
||||
#[serde(default)]
|
||||
pub big_block_data: BigBlockData<ExecutionData>,
|
||||
/// Flattened BAL across all constituent blocks, if requested during generation.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub block_access_list: Option<BlockAccessList>,
|
||||
}
|
||||
|
||||
/// `reth bench generate-big-block` command
|
||||
@@ -252,6 +266,20 @@ pub struct Command {
|
||||
/// Output directory for generated payloads.
|
||||
#[arg(long, value_name = "OUTPUT_DIR")]
|
||||
output_dir: std::path::PathBuf,
|
||||
|
||||
/// Query `eth_getBlockAccessListByBlockNumber` for each fetched block and persist
|
||||
/// the flattened BAL on the stored payload.
|
||||
#[arg(long, default_value_t = false)]
|
||||
bal: bool,
|
||||
|
||||
/// Maximum number of in-flight RPC fetches to keep buffered ahead of the merger.
|
||||
///
|
||||
/// Each entry is one full per-block fetch (block + receipts, plus BAL when `--bal` is
|
||||
/// set). Larger values absorb RPC latency at the cost of more concurrent connections
|
||||
/// and memory; the buffer persists across `--num-big-blocks` so prefetching continues
|
||||
/// across big-block boundaries.
|
||||
#[arg(long, value_name = "PREFETCH_BUFFER", default_value_t = 32)]
|
||||
prefetch_buffer: usize,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -273,6 +301,7 @@ impl Command {
|
||||
from_block = self.from_block,
|
||||
target_gas = self.target_gas,
|
||||
num_big_blocks = self.num_big_blocks,
|
||||
include_bal = self.bal,
|
||||
chain = %chain_spec.chain(),
|
||||
output_dir = %self.output_dir.display(),
|
||||
"Generating big block payloads"
|
||||
@@ -303,30 +332,40 @@ impl Command {
|
||||
}
|
||||
let mut prev_big_block_header: Option<PrevBigBlockHeader> = None;
|
||||
|
||||
// Track the next block to fetch across big blocks so they don't overlap.
|
||||
// Persistent prefetch stream: keeps `prefetch_buffer` per-block fetches in flight
|
||||
// ahead of the merger across all big blocks. Each item is a fully materialized
|
||||
// `FetchedBlock` (or `None` once the chain tip is reached on this fetch).
|
||||
let prefetch_buffer = self.prefetch_buffer.max(1);
|
||||
let bal_enabled = self.bal;
|
||||
let block_stream = stream::iter(self.from_block..)
|
||||
.map(|block_number| {
|
||||
let provider = provider.clone();
|
||||
async move { fetch_one_block(provider, block_number, bal_enabled).await }
|
||||
})
|
||||
.buffered(prefetch_buffer);
|
||||
let mut block_stream = Box::pin(block_stream);
|
||||
|
||||
// Track the next block number we expect from the stream (purely for logging /
|
||||
// big-block range bookkeeping; the stream produces blocks in `from_block..` order).
|
||||
let mut next_block = self.from_block;
|
||||
|
||||
for big_block_idx in 0..self.num_big_blocks {
|
||||
let range_start = next_block;
|
||||
|
||||
// Fetch consecutive blocks until the gas target is reached.
|
||||
// Drain the prefetch stream until the gas target is reached for this big block.
|
||||
let mut blocks = Vec::new();
|
||||
let mut block_receipts: Vec<Vec<Receipt>> = Vec::new();
|
||||
let mut block_access_lists: Vec<Option<BlockAccessList>> = Vec::new();
|
||||
let mut accumulated_block_gas: u64 = 0;
|
||||
|
||||
let mut reached_chain_tip = false;
|
||||
while accumulated_block_gas < self.target_gas {
|
||||
let block_number = next_block;
|
||||
info!(target: "reth-bench", block_number, big_block = big_block_idx, "Fetching block");
|
||||
info!(target: "reth-bench", block_number, big_block = big_block_idx, "Awaiting prefetched block");
|
||||
|
||||
let fetch_result = tokio::try_join!(
|
||||
provider.get_block_by_number(block_number.into()).full(),
|
||||
provider.get_block_receipts(block_number.into()),
|
||||
);
|
||||
|
||||
let (rpc_block, receipts) = match fetch_result {
|
||||
Ok((Some(block), Some(receipts))) => (block, receipts),
|
||||
Ok((None, _) | (_, None)) => {
|
||||
let fetched = match block_stream.next().await {
|
||||
Some(Ok(Some(fetched))) => fetched,
|
||||
Some(Ok(None)) => {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
@@ -335,50 +374,26 @@ impl Command {
|
||||
reached_chain_tip = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
Some(Err(e)) => return Err(e),
|
||||
// The block-number stream is open-ended; this only fires if the
|
||||
// upstream `iter(from..)` is somehow exhausted.
|
||||
None => {
|
||||
reached_chain_tip = true;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
// Convert RPC receipts to consensus receipts
|
||||
let consensus_receipts: Vec<Receipt> = receipts
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let inner = &r.inner.inner.inner;
|
||||
let tx_type = r.inner.inner.r#type.try_into().unwrap_or_default();
|
||||
Receipt {
|
||||
tx_type,
|
||||
success: inner.receipt.status.coerce_status(),
|
||||
cumulative_gas_used: inner.receipt.cumulative_gas_used,
|
||||
logs: inner
|
||||
.receipt
|
||||
.logs
|
||||
.iter()
|
||||
.map(|log| alloy_primitives::Log {
|
||||
address: log.inner.address,
|
||||
data: log.inner.data.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Convert to consensus block
|
||||
let block = rpc_block
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
.try_map_transactions(|tx| -> eyre::Result<TxEnvelope> {
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
// Convert to ExecutionData
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
let execution_data = ExecutionData { payload, sidecar };
|
||||
let FetchedBlock { execution_data, consensus_receipts, block_access_list } =
|
||||
fetched;
|
||||
|
||||
let block_gas = execution_data.payload.as_v1().gas_used;
|
||||
let block_blob_gas =
|
||||
execution_data.payload.as_v3().map(|v3| v3.blob_gas_used).unwrap_or(0);
|
||||
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
gas_used = block_gas,
|
||||
blob_gas_used = block_blob_gas,
|
||||
tx_count = execution_data.payload.transactions().len(),
|
||||
receipts = consensus_receipts.len(),
|
||||
"Fetched block"
|
||||
@@ -387,6 +402,7 @@ impl Command {
|
||||
accumulated_block_gas += block_gas;
|
||||
blocks.push(execution_data);
|
||||
block_receipts.push(consensus_receipts);
|
||||
block_access_lists.push(block_access_list);
|
||||
next_block += 1;
|
||||
}
|
||||
|
||||
@@ -404,6 +420,7 @@ impl Command {
|
||||
// Block 0 is the base
|
||||
let mut base = blocks.remove(0);
|
||||
let base_receipts = block_receipts.remove(0);
|
||||
let mut merged_block_access_list = block_access_lists.remove(0);
|
||||
let mut env_switches = Vec::new();
|
||||
|
||||
// Accumulate all receipts with corrected cumulative_gas_used.
|
||||
@@ -439,12 +456,22 @@ impl Command {
|
||||
let mut total_gas_limit = base.payload.as_v1().gas_limit;
|
||||
|
||||
// Concatenate transactions from subsequent blocks and build env_switches
|
||||
for (block_data, receipts) in blocks.into_iter().zip(block_receipts) {
|
||||
for ((block_data, receipts), block_access_list) in
|
||||
blocks.into_iter().zip(block_receipts).zip(block_access_lists)
|
||||
{
|
||||
let block_v1 = block_data.payload.as_v1();
|
||||
let block_gas = block_v1.gas_used;
|
||||
total_gas_used += block_gas;
|
||||
total_gas_limit += block_v1.gas_limit;
|
||||
|
||||
if let Some(block_access_list) = block_access_list {
|
||||
merge_block_access_list(
|
||||
merged_block_access_list.get_or_insert_with(Default::default),
|
||||
block_access_list,
|
||||
cumulative_tx_count as u64,
|
||||
);
|
||||
}
|
||||
|
||||
// Accumulate receipts with corrected cumulative_gas_used
|
||||
all_receipts.extend(receipts.into_iter().map(|mut r| {
|
||||
r.cumulative_gas_used += cumulative_gas_offset;
|
||||
@@ -579,6 +606,7 @@ impl Command {
|
||||
env_switches,
|
||||
prior_block_hashes: accumulated_block_hashes.clone(),
|
||||
},
|
||||
block_access_list: merged_block_access_list,
|
||||
};
|
||||
|
||||
// Accumulate real block hashes from this big block's env_switches for
|
||||
@@ -610,6 +638,7 @@ impl Command {
|
||||
total_gas_used = big_block.execution_data.payload.as_v1().gas_used,
|
||||
env_switches = big_block.big_block_data.env_switches.len(),
|
||||
prior_block_hashes = big_block.big_block_data.prior_block_hashes.len(),
|
||||
bal_accounts = big_block.block_access_list.as_ref().map_or(0, Vec::len),
|
||||
"Big block payload saved"
|
||||
);
|
||||
|
||||
@@ -628,6 +657,155 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
/// One fully-materialized block fetched by the prefetcher.
|
||||
struct FetchedBlock {
|
||||
/// Execution payload with sidecar derived from the RPC block.
|
||||
execution_data: ExecutionData,
|
||||
/// Consensus-format receipts (`cumulative_gas_used` is still per-block, callers offset
|
||||
/// it when merging).
|
||||
consensus_receipts: Vec<Receipt>,
|
||||
/// `eth_getBlockAccessListByBlockNumber` result when `--bal` is enabled.
|
||||
block_access_list: Option<BlockAccessList>,
|
||||
}
|
||||
|
||||
/// Fetches one block + receipts (and optionally its BAL) from the RPC. Returns `Ok(None)`
|
||||
/// when the block doesn't exist yet (chain-tip reached).
|
||||
async fn fetch_one_block(
|
||||
provider: RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
bal_enabled: bool,
|
||||
) -> eyre::Result<Option<FetchedBlock>> {
|
||||
let (rpc_block, receipts) = tokio::try_join!(
|
||||
provider.get_block_by_number(block_number.into()).full(),
|
||||
provider.get_block_receipts(block_number.into()),
|
||||
)?;
|
||||
let (rpc_block, receipts) = match (rpc_block, receipts) {
|
||||
(Some(b), Some(r)) => (b, r),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
let block_access_list = if bal_enabled {
|
||||
Some(
|
||||
fetch_block_access_list(&provider, block_number)
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch BAL for block {block_number}"))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let consensus_receipts: Vec<Receipt> = receipts
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let inner = &r.inner.inner.inner;
|
||||
let tx_type = r.inner.inner.r#type.try_into().unwrap_or_default();
|
||||
Receipt {
|
||||
tx_type,
|
||||
success: inner.receipt.status.coerce_status(),
|
||||
cumulative_gas_used: inner.receipt.cumulative_gas_used,
|
||||
logs: inner
|
||||
.receipt
|
||||
.logs
|
||||
.iter()
|
||||
.map(|log| alloy_primitives::Log {
|
||||
address: log.inner.address,
|
||||
data: log.inner.data.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let block = rpc_block
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
.try_map_transactions(|tx| -> eyre::Result<TxEnvelope> {
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
let execution_data = ExecutionData { payload, sidecar };
|
||||
|
||||
Ok(Some(FetchedBlock { execution_data, consensus_receipts, block_access_list }))
|
||||
}
|
||||
|
||||
fn merge_block_access_list(
|
||||
merged: &mut BlockAccessList,
|
||||
incoming: BlockAccessList,
|
||||
tx_index_offset: u64,
|
||||
) {
|
||||
let mut account_positions = merged
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, account)| (account.address, idx))
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
for mut account_changes in incoming {
|
||||
shift_account_changes(&mut account_changes, tx_index_offset);
|
||||
|
||||
if let Some(&idx) = account_positions.get(&account_changes.address) {
|
||||
merge_account_changes(&mut merged[idx], account_changes);
|
||||
} else {
|
||||
account_positions.insert(account_changes.address, merged.len());
|
||||
merged.push(account_changes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn shift_account_changes(account_changes: &mut AccountChanges, tx_index_offset: u64) {
|
||||
for slot_changes in &mut account_changes.storage_changes {
|
||||
for change in &mut slot_changes.changes {
|
||||
change.block_access_index += tx_index_offset;
|
||||
}
|
||||
}
|
||||
for change in &mut account_changes.balance_changes {
|
||||
change.block_access_index += tx_index_offset;
|
||||
}
|
||||
for change in &mut account_changes.nonce_changes {
|
||||
change.block_access_index += tx_index_offset;
|
||||
}
|
||||
for change in &mut account_changes.code_changes {
|
||||
change.block_access_index += tx_index_offset;
|
||||
}
|
||||
}
|
||||
|
||||
fn merge_account_changes(existing: &mut AccountChanges, incoming: AccountChanges) {
|
||||
merge_slot_changes(&mut existing.storage_changes, incoming.storage_changes);
|
||||
existing.storage_reads.extend(incoming.storage_reads);
|
||||
existing.balance_changes.extend(incoming.balance_changes);
|
||||
existing.nonce_changes.extend(incoming.nonce_changes);
|
||||
existing.code_changes.extend(incoming.code_changes);
|
||||
|
||||
// EIP-7928 invariant: a slot must appear in either storage_changes or storage_reads,
|
||||
// not both. Per-block BALs respect this, but merging blocks can produce a slot
|
||||
// that is read in one block and changed in another. Without this normalization,
|
||||
// an empty read entry can shadow the real writes during BAL deserialization,
|
||||
// making reads of that slot fall through to stale snapshot state.
|
||||
let written: HashSet<_> =
|
||||
existing.storage_changes.iter().map(|slot_changes| slot_changes.slot).collect();
|
||||
existing.storage_reads.retain(|slot| !written.contains(slot));
|
||||
let mut seen = HashSet::with_capacity(existing.storage_reads.len());
|
||||
existing.storage_reads.retain(|slot| seen.insert(*slot));
|
||||
}
|
||||
|
||||
fn merge_slot_changes(existing: &mut Vec<SlotChanges>, incoming: Vec<SlotChanges>) {
|
||||
let mut slot_positions = existing
|
||||
.iter()
|
||||
.enumerate()
|
||||
.map(|(idx, slot_changes)| (slot_changes.slot, idx))
|
||||
.collect::<HashMap<_, _>>();
|
||||
|
||||
for slot_changes in incoming {
|
||||
if let Some(&idx) = slot_positions.get(&slot_changes.slot) {
|
||||
existing[idx].changes.extend(slot_changes.changes);
|
||||
} else {
|
||||
slot_positions.insert(slot_changes.slot, existing.len());
|
||||
existing.push(slot_changes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the block hash for an [`ExecutionData`] by converting it to a raw block
|
||||
/// and hashing the header.
|
||||
pub fn compute_payload_block_hash(data: &ExecutionData) -> eyre::Result<B256> {
|
||||
@@ -638,3 +816,144 @@ pub fn compute_payload_block_hash(data: &ExecutionData) -> eyre::Result<B256> {
|
||||
.wrap_err("failed to convert payload to block for hash computation")?;
|
||||
Ok(block.header.hash_slow())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_eips::eip7928::{BalanceChange, CodeChange, NonceChange, StorageChange};
|
||||
use alloy_primitives::{Address, U256};
|
||||
|
||||
#[test]
|
||||
fn merge_block_access_list_offsets_and_merges_accounts() {
|
||||
let shared = Address::repeat_byte(0x11);
|
||||
let other = Address::repeat_byte(0x22);
|
||||
|
||||
let mut merged = vec![AccountChanges {
|
||||
address: shared,
|
||||
storage_changes: vec![SlotChanges::new(
|
||||
U256::from(1),
|
||||
vec![StorageChange::new(0, U256::from(10))],
|
||||
)],
|
||||
storage_reads: vec![U256::from(3)],
|
||||
balance_changes: vec![BalanceChange::new(1, U256::from(100))],
|
||||
nonce_changes: vec![NonceChange::new(2, 7)],
|
||||
code_changes: vec![],
|
||||
}];
|
||||
|
||||
let incoming = vec![
|
||||
AccountChanges {
|
||||
address: shared,
|
||||
storage_changes: vec![
|
||||
SlotChanges::new(U256::from(1), vec![StorageChange::new(1, U256::from(20))]),
|
||||
SlotChanges::new(U256::from(2), vec![StorageChange::new(2, U256::from(30))]),
|
||||
],
|
||||
storage_reads: vec![U256::from(4)],
|
||||
balance_changes: vec![BalanceChange::new(0, U256::from(150))],
|
||||
nonce_changes: vec![NonceChange::new(2, 8)],
|
||||
code_changes: vec![CodeChange::new(1, Bytes::from_static(&[0xaa]))],
|
||||
},
|
||||
AccountChanges {
|
||||
address: other,
|
||||
storage_changes: vec![SlotChanges::new(
|
||||
U256::from(9),
|
||||
vec![StorageChange::new(0, U256::from(90))],
|
||||
)],
|
||||
storage_reads: vec![],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
},
|
||||
];
|
||||
|
||||
merge_block_access_list(&mut merged, incoming, 3);
|
||||
|
||||
assert_eq!(merged.len(), 2);
|
||||
|
||||
let shared = &merged[0];
|
||||
assert_eq!(shared.storage_reads, vec![U256::from(3), U256::from(4)]);
|
||||
assert_eq!(
|
||||
shared
|
||||
.balance_changes
|
||||
.iter()
|
||||
.map(|change| change.block_access_index)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![1, 3]
|
||||
);
|
||||
assert_eq!(
|
||||
shared.nonce_changes.iter().map(|change| change.block_access_index).collect::<Vec<_>>(),
|
||||
vec![2, 5]
|
||||
);
|
||||
assert_eq!(shared.code_changes[0].block_access_index, 4);
|
||||
|
||||
let slot_one = shared
|
||||
.storage_changes
|
||||
.iter()
|
||||
.find(|slot_changes| slot_changes.slot == U256::from(1))
|
||||
.unwrap();
|
||||
assert_eq!(
|
||||
slot_one.changes.iter().map(|change| change.block_access_index).collect::<Vec<_>>(),
|
||||
vec![0, 4]
|
||||
);
|
||||
|
||||
let slot_two = shared
|
||||
.storage_changes
|
||||
.iter()
|
||||
.find(|slot_changes| slot_changes.slot == U256::from(2))
|
||||
.unwrap();
|
||||
assert_eq!(slot_two.changes[0].block_access_index, 5);
|
||||
|
||||
let other = &merged[1];
|
||||
assert_eq!(other.address, Address::repeat_byte(0x22));
|
||||
assert_eq!(other.storage_changes[0].changes[0].block_access_index, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_account_changes_normalizes_storage_reads_after_cross_block_merge() {
|
||||
let address = Address::repeat_byte(0x33);
|
||||
const A: U256 = U256::from_limbs([1, 0, 0, 0]);
|
||||
const B: U256 = U256::from_limbs([2, 0, 0, 0]);
|
||||
const C: U256 = U256::from_limbs([3, 0, 0, 0]);
|
||||
const D: U256 = U256::from_limbs([4, 0, 0, 0]);
|
||||
|
||||
// Each AccountChanges value is valid on its own: storage slots only appear in
|
||||
// either reads or changes. The invalid read/change overlap is introduced when
|
||||
// these per-block BAL entries are merged for a standalone big block.
|
||||
let mut existing = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![SlotChanges::new(A, vec![StorageChange::new(0, U256::from(10))])],
|
||||
storage_reads: vec![B, C],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
// B is read before it is written by the incoming block, and A is written before
|
||||
// it appears as a read in the incoming block. C is read in both blocks, so the
|
||||
// merge should also dedupe it. D remains read-only.
|
||||
let incoming = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![SlotChanges::new(B, vec![StorageChange::new(1, U256::from(20))])],
|
||||
storage_reads: vec![A, C, D],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
merge_account_changes(&mut existing, incoming);
|
||||
|
||||
// Written slots remain represented by storage_changes, while storage_reads only
|
||||
// keeps unique read-only slots in first-seen order.
|
||||
assert_eq!(
|
||||
existing
|
||||
.storage_changes
|
||||
.iter()
|
||||
.map(|slot_changes| slot_changes.slot)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![A, B]
|
||||
);
|
||||
assert_eq!(existing.storage_reads, vec![C, D]);
|
||||
assert!(existing.storage_reads.iter().all(|read_slot| {
|
||||
!existing.storage_changes.iter().any(|slot_changes| slot_changes.slot == *read_slot)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use alloy_eips::{eip7928::BlockAccessList, BlockNumberOrTag};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use eyre::Result;
|
||||
use std::{
|
||||
io::{BufReader, Read},
|
||||
@@ -69,6 +71,21 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches the block access list for a given block number using the provided provider.
|
||||
pub(crate) async fn fetch_block_access_list(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
) -> eyre::Result<BlockAccessList> {
|
||||
provider
|
||||
.client()
|
||||
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.and_then(|block_access_list: Option<BlockAccessList>| {
|
||||
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::parse_duration,
|
||||
helpers::{fetch_block_access_list, parse_duration},
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
@@ -14,14 +14,24 @@ use crate::{
|
||||
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
|
||||
},
|
||||
};
|
||||
use alloy_provider::{ext::DebugApi, Provider};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::Encodable2718;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{
|
||||
ext::DebugApi,
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
Provider, RootProvider,
|
||||
};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionData, ExecutionPayloadEnvelopeV5, ForkchoiceState, PayloadAttributes,
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::{Context, OptionExt};
|
||||
use eyre::{bail, ensure, Context, OptionExt};
|
||||
use futures::{stream, StreamExt, TryStreamExt};
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use reth_rpc_api::{RethNewPayloadInput, TestingBuildBlockRequestV1};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -32,6 +42,22 @@ pub struct Command {
|
||||
#[arg(long, value_name = "RPC_URL", verbatim_doc_comment)]
|
||||
rpc_url: String,
|
||||
|
||||
/// Build a separate fork with `testing_buildBlockV1` and alternate forkchoice updates between
|
||||
/// the canonical chain and that fork on every block while the fork grows up to the configured
|
||||
/// depth.
|
||||
///
|
||||
/// This requires enabling the hidden `testing` RPC module on the target node,
|
||||
/// for example with `reth node --http --http.api eth,testing`.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "DEPTH",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "8",
|
||||
value_parser = parse_reorg_depth,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
reorg: Option<usize>,
|
||||
|
||||
/// How long to wait after a forkchoice update before sending the next payload.
|
||||
///
|
||||
/// Accepts a duration string (e.g. `100ms`, `2s`) or a bare integer treated as
|
||||
@@ -75,22 +101,87 @@ pub struct Command {
|
||||
)]
|
||||
rpc_block_buffer_size: usize,
|
||||
|
||||
/// Weather to enable bal by default or not.
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
enable_bal: bool,
|
||||
|
||||
#[command(flatten)]
|
||||
benchmark: BenchmarkArgs,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PreparedBuiltBlock {
|
||||
block_hash: B256,
|
||||
params: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct QueuedForkBlock {
|
||||
block_number: u64,
|
||||
prepared: PreparedBuiltBlock,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ReorgState {
|
||||
depth: usize,
|
||||
fork_length: usize,
|
||||
branch_point_hash: Option<B256>,
|
||||
fork_parent_hash: Option<B256>,
|
||||
}
|
||||
|
||||
impl ReorgState {
|
||||
const fn new(depth: usize) -> Self {
|
||||
Self { depth, fork_length: 0, branch_point_hash: None, fork_parent_hash: None }
|
||||
}
|
||||
|
||||
const fn push_fork_head(&mut self, canonical_parent_hash: B256, fork_head_hash: B256) {
|
||||
if self.fork_length == 0 {
|
||||
self.branch_point_hash = Some(canonical_parent_hash);
|
||||
}
|
||||
self.fork_length += 1;
|
||||
self.fork_parent_hash = Some(fork_head_hash);
|
||||
}
|
||||
|
||||
fn forkchoice_state(&self, fork_head_hash: B256) -> eyre::Result<ForkchoiceState> {
|
||||
let branch_point_hash = self.branch_point_hash.ok_or_eyre("missing reorg branch point")?;
|
||||
|
||||
Ok(ForkchoiceState {
|
||||
head_block_hash: fork_head_hash,
|
||||
safe_block_hash: branch_point_hash,
|
||||
finalized_block_hash: branch_point_hash,
|
||||
})
|
||||
}
|
||||
|
||||
const fn reset(&mut self) {
|
||||
self.fork_length = 0;
|
||||
self.branch_point_hash = None;
|
||||
self.fork_parent_hash = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark new-payload-fcu` command
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.reorg.is_some() && self.benchmark.rlp_blocks {
|
||||
bail!("--reorg cannot be combined with --rlp-blocks")
|
||||
}
|
||||
if self.reorg.is_some() && self.enable_bal {
|
||||
bail!("--reorg cannot be combined with --enable-bal")
|
||||
}
|
||||
|
||||
// Log mode configuration
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!(target: "reth-bench", "Using wait-time mode with {}ms minimum interval between blocks", duration.as_millis());
|
||||
}
|
||||
if let Some(depth) = self.reorg {
|
||||
info!(target: "reth-bench", depth, "Using testing_buildBlockV1 reorg mode");
|
||||
}
|
||||
|
||||
let BenchContext {
|
||||
benchmark_mode,
|
||||
block_provider,
|
||||
auth_provider,
|
||||
local_rpc_provider,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
@@ -178,7 +269,8 @@ impl Command {
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
let mut reorg_state = self.reorg.map(ReorgState::new);
|
||||
let mut queued_fork_block = None;
|
||||
while let Some((block, head, safe, finalized, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = blocks.try_next().await?;
|
||||
@@ -188,27 +280,39 @@ impl Command {
|
||||
let gas_used = block.header.gas_used;
|
||||
let gas_limit = block.header.gas_limit;
|
||||
let block_number = block.header.number;
|
||||
let canonical_parent_hash = block.header.parent_hash;
|
||||
let transaction_count = block.transactions.len() as u64;
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
let deferred_branch_start_block = reorg_state
|
||||
.as_ref()
|
||||
.filter(|state| state.fork_length == 0 && queued_fork_block.is_none())
|
||||
.map(|_| block.clone());
|
||||
let canonical_forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: head,
|
||||
safe_block_hash: safe,
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let bal = if rlp.is_none() &&
|
||||
(block.header.block_access_list_hash.is_some() || self.enable_bal)
|
||||
{
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
@@ -229,17 +333,12 @@ impl Command {
|
||||
};
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, canonical_forkchoice_state)
|
||||
.await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency = if server_timings.is_some() {
|
||||
// When using server-side latency for newPayload, derive total from the
|
||||
// independently measured components to avoid mixing server-side and
|
||||
// client-side (network-inclusive) timings.
|
||||
np_latency + fcu_latency
|
||||
} else {
|
||||
start.elapsed()
|
||||
};
|
||||
let total_latency =
|
||||
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
@@ -249,6 +348,88 @@ impl Command {
|
||||
total_latency,
|
||||
};
|
||||
|
||||
if let Some(reorg_state) = reorg_state.as_mut() {
|
||||
if queued_fork_block.is_none() && reorg_state.fork_length == 0 {
|
||||
// A branch start uses a canonical parent, so it can be built lazily here
|
||||
// instead of being queued ahead of time.
|
||||
let block = deferred_branch_start_block
|
||||
.as_ref()
|
||||
.ok_or_eyre("missing deferred fork block for reorg branch start")?;
|
||||
queued_fork_block = Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
&local_rpc_provider,
|
||||
block,
|
||||
canonical_parent_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
});
|
||||
}
|
||||
|
||||
let queued = queued_fork_block
|
||||
.take()
|
||||
.ok_or_eyre("missing queued fork block for reorg replay")?;
|
||||
ensure!(
|
||||
queued.block_number == block_number,
|
||||
"queued fork block {} does not match source block {}",
|
||||
queued.block_number,
|
||||
block_number
|
||||
);
|
||||
let prepared = queued.prepared;
|
||||
|
||||
call_new_payload_with_reth(&auth_provider, None, prepared.params).await?;
|
||||
|
||||
reorg_state.push_fork_head(canonical_parent_hash, prepared.block_hash);
|
||||
let forkchoice_state = reorg_state.forkchoice_state(prepared.block_hash)?;
|
||||
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
branch_point = %forkchoice_state.safe_block_hash,
|
||||
fork_head = %prepared.block_hash,
|
||||
fork_depth = reorg_state.fork_length,
|
||||
max_reorg_depth = reorg_state.depth,
|
||||
"Switching forkchoice to reorg branch"
|
||||
);
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, None, forkchoice_state).await?;
|
||||
let _fork_fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let next_fork_block_number = block_number + 1;
|
||||
if reorg_state.fork_length < reorg_state.depth {
|
||||
queued_fork_block = queue_fork_block(
|
||||
&block_provider,
|
||||
&local_rpc_provider,
|
||||
&benchmark_mode,
|
||||
next_fork_block_number,
|
||||
Some(prepared.block_hash),
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
reorg_depth = reorg_state.depth,
|
||||
"Resetting reorg branch after reaching max depth"
|
||||
);
|
||||
|
||||
// `testing_buildBlockV1` resolves the parent from canonical state, so switch
|
||||
// back to the source chain before reseeding the next queued fork block.
|
||||
call_forkchoice_updated_with_reth(
|
||||
&auth_provider,
|
||||
version,
|
||||
canonical_forkchoice_state,
|
||||
)
|
||||
.await?;
|
||||
|
||||
reorg_state.reset();
|
||||
queued_fork_block = None;
|
||||
}
|
||||
}
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
@@ -305,3 +486,155 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_built_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<PreparedBuiltBlock> {
|
||||
const MAX_BUILD_ATTEMPTS: usize = 10;
|
||||
const BUILD_RETRY_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
let request = build_block_request(block, parent_block_hash)?;
|
||||
let built_payload: ExecutionPayloadEnvelopeV5 = {
|
||||
let mut attempts_remaining = MAX_BUILD_ATTEMPTS;
|
||||
|
||||
loop {
|
||||
match block_provider.client().request("testing_buildBlockV1", [request.clone()]).await {
|
||||
Ok(payload) => break payload,
|
||||
Err(err) if attempts_remaining > 1 && is_retryable_build_block_error(&err) => {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
block_number = block.header.number,
|
||||
%parent_block_hash,
|
||||
attempts_remaining,
|
||||
error = %err,
|
||||
"Retrying testing_buildBlockV1 after transient fork build failure"
|
||||
);
|
||||
attempts_remaining -= 1;
|
||||
tokio::time::sleep(BUILD_RETRY_INTERVAL).await;
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(err).wrap_err_with(|| {
|
||||
format!(
|
||||
"Failed to build block {} via testing_buildBlockV1",
|
||||
block.header.number
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let payload = &built_payload.execution_payload.payload_inner.payload_inner;
|
||||
let block_hash = payload.block_hash;
|
||||
let (payload, sidecar) = built_payload
|
||||
.into_payload_and_sidecar(block.header.parent_beacon_block_root.unwrap_or_default());
|
||||
// Fork payloads are built immediately before the next `testing_buildBlockV1` call. Leaving
|
||||
// reth's default persistence wait enabled here gives the regular RPC side a consistent base
|
||||
// state for the next synthetic fork block build.
|
||||
let params = serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(ExecutionData { payload, sidecar }),
|
||||
None::<bool>,
|
||||
no_wait_for_caches.then_some(false),
|
||||
))?;
|
||||
|
||||
Ok(PreparedBuiltBlock { block_hash, params })
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn queue_fork_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
local_rpc_provider: &RootProvider<AnyNetwork>,
|
||||
benchmark_mode: &crate::bench_mode::BenchMode,
|
||||
block_number: u64,
|
||||
parent_block_hash: Option<B256>,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<Option<QueuedForkBlock>> {
|
||||
if !benchmark_mode.contains(block_number) {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
let future_block = block_provider
|
||||
.get_block_by_number(alloy_eips::BlockNumberOrTag::Number(block_number))
|
||||
.full()
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch block by number {block_number}"))?
|
||||
.ok_or_eyre("Block not found")?;
|
||||
let parent_block_hash = parent_block_hash.unwrap_or(future_block.header.parent_hash);
|
||||
|
||||
Ok(Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
local_rpc_provider,
|
||||
&future_block,
|
||||
parent_block_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
}))
|
||||
}
|
||||
|
||||
fn is_retryable_build_block_error(err: &alloy_transport::TransportError) -> bool {
|
||||
let message = err.to_string();
|
||||
message.contains("block not found: hash") ||
|
||||
message.contains("block hash not found for block number")
|
||||
}
|
||||
|
||||
fn build_block_request(
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
) -> eyre::Result<TestingBuildBlockRequestV1> {
|
||||
let mut transactions = block
|
||||
.clone()
|
||||
.try_into_transactions()
|
||||
.map_err(|_| eyre::eyre!("Block transactions must be fetched in full for --reorg"))?
|
||||
.into_iter()
|
||||
.map(|tx| {
|
||||
let tx: TxEnvelope =
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type in RPC block"))?;
|
||||
if tx.is_eip4844() {
|
||||
return Ok(None)
|
||||
}
|
||||
Ok(Some(tx.encoded_2718().into()))
|
||||
})
|
||||
.filter_map(|tx| tx.transpose())
|
||||
.collect::<eyre::Result<Vec<_>>>()?;
|
||||
|
||||
// `testing_buildBlockV1` only takes raw transaction bytes, so we exclude blob transactions
|
||||
// from the synthetic fork blocks rather than trying to reconstruct their sidecars.
|
||||
// Keep only 90% of the remaining transactions so the alternate branch produces a materially
|
||||
// different post-state instead of only differing by header data.
|
||||
let keep = transactions.len().saturating_mul(9) / 10;
|
||||
transactions.truncate(keep);
|
||||
|
||||
let rpc_block = block.clone().into_inner();
|
||||
|
||||
Ok(TestingBuildBlockRequestV1 {
|
||||
parent_block_hash,
|
||||
payload_attributes: PayloadAttributes {
|
||||
timestamp: block.header.timestamp,
|
||||
prev_randao: block.header.mix_hash.unwrap_or_default(),
|
||||
suggested_fee_recipient: block.header.beneficiary,
|
||||
withdrawals: rpc_block.withdrawals.map(|withdrawals| withdrawals.into_inner()),
|
||||
parent_beacon_block_root: block.header.parent_beacon_block_root,
|
||||
slot_number: block.header.slot_number,
|
||||
},
|
||||
transactions,
|
||||
extra_data: Some(block.header.extra_data.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_reorg_depth(value: &str) -> Result<usize, String> {
|
||||
let depth = value
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("invalid reorg depth {value:?}, expected a positive integer"))?;
|
||||
|
||||
if depth == 0 {
|
||||
return Err("reorg depth must be greater than 0".to_string())
|
||||
}
|
||||
|
||||
Ok(depth)
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::fetch_block_access_list,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
|
||||
@@ -53,6 +54,7 @@ impl Command {
|
||||
rlp_blocks,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
@@ -69,7 +71,9 @@ impl Command {
|
||||
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
|
||||
|
||||
let block_provider_clone = block_provider.clone();
|
||||
tokio::task::spawn(async move {
|
||||
let block_provider = block_provider_clone;
|
||||
while benchmark_mode.contains(next_block) {
|
||||
let block_res = block_provider
|
||||
.get_block_by_number(next_block.into())
|
||||
@@ -123,12 +127,19 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
generate_big_block::BigBlockPayload,
|
||||
generate_big_block::{compute_payload_block_hash, BigBlockPayload},
|
||||
helpers::parse_duration,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
@@ -12,12 +12,14 @@ use crate::{
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
|
||||
};
|
||||
use alloy_eip7928::bal::Bal;
|
||||
use alloy_eips::eip7928::BlockAccessList;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV4, ExecutionPayloadSidecar,
|
||||
ForkchoiceState, JwtSecret, PraguePayloadFields,
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV6,
|
||||
ExecutionPayloadSidecar, ExecutionPayloadV4, ForkchoiceState, JwtSecret, PraguePayloadFields,
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
@@ -83,6 +85,14 @@ pub struct Command {
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
|
||||
/// Forward embedded block access lists to `reth_newPayload` when payload files contain them.
|
||||
///
|
||||
/// Disabled by default so the same payload set can be replayed with or without BALs.
|
||||
///
|
||||
/// Requires `--reth-new-payload`.
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
|
||||
bal: bool,
|
||||
|
||||
/// Control when `reth_newPayload` waits for in-flight persistence.
|
||||
///
|
||||
/// Accepts `always` (default — wait on every block), `never`, or a number N
|
||||
@@ -126,6 +136,8 @@ struct LoadedPayload {
|
||||
block_hash: B256,
|
||||
/// Big block data containing environment switches and prior block hashes.
|
||||
big_block_data: BigBlockData<ExecutionData>,
|
||||
/// Optional BAL flattened into the payload file.
|
||||
block_access_list: Option<BlockAccessList>,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -139,6 +151,9 @@ impl Command {
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
|
||||
if self.bal {
|
||||
info!(target: "reth-bench", "Forwarding embedded block_access_list data");
|
||||
}
|
||||
}
|
||||
|
||||
let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
|
||||
@@ -185,10 +200,13 @@ impl Command {
|
||||
}
|
||||
info!(target: "reth-bench", count = payloads.len(), "Loaded main payloads from disk");
|
||||
|
||||
let has_env_switches = payloads.iter().any(|p| !p.big_block_data.env_switches.is_empty());
|
||||
let has_block_access_lists = payloads.iter().any(|p| {
|
||||
p.block_access_list.as_ref().is_some_and(|bal: &BlockAccessList| !bal.is_empty())
|
||||
});
|
||||
|
||||
// If any payload has env_switches but we're not using reth_newPayload, warn the user
|
||||
if !self.reth_new_payload {
|
||||
let has_env_switches =
|
||||
payloads.iter().any(|p| !p.big_block_data.env_switches.is_empty());
|
||||
if has_env_switches {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
@@ -196,6 +214,18 @@ impl Command {
|
||||
env_switches are only supported with reth_newPayload and will be ignored."
|
||||
);
|
||||
}
|
||||
if has_block_access_lists {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
"Payloads contain block_access_list data but --reth-new-payload is not set. \
|
||||
BALs are only forwarded with reth_newPayload and will be ignored."
|
||||
);
|
||||
}
|
||||
} else if has_block_access_lists && !self.bal {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
"Payloads contain block_access_list data but --bal is not set. BALs will be ignored."
|
||||
);
|
||||
}
|
||||
|
||||
let mut parent_hash = initial_parent_hash;
|
||||
@@ -205,7 +235,7 @@ impl Command {
|
||||
|
||||
for (i, payload) in payloads.iter().enumerate() {
|
||||
let execution_data = &payload.execution_data;
|
||||
let block_hash = payload.block_hash;
|
||||
let mut block_hash = payload.block_hash;
|
||||
let v1 = execution_data.payload.as_v1();
|
||||
|
||||
let gas_used = v1.gas_used;
|
||||
@@ -243,10 +273,39 @@ impl Command {
|
||||
.wait_for_persistence
|
||||
.unwrap_or(WaitForPersistence::Never)
|
||||
.rpc_value(block_number);
|
||||
|
||||
// Inject sidecar BAL into the inline V4 payload field when --bal is set.
|
||||
// If the payload is not already V4 we upgrade it (V3→V4) so the BAL
|
||||
// can be carried inline. This changes the block hash, so we recompute
|
||||
// it and patch parent_hash to maintain the chain.
|
||||
let mut execution_data = execution_data.clone();
|
||||
if self.bal &&
|
||||
let Some(bal) = &payload.block_access_list
|
||||
{
|
||||
let encoded_bal: alloy_primitives::Bytes =
|
||||
alloy_rlp::encode(Bal::from(bal.clone())).into();
|
||||
|
||||
// Upgrade to V4 if necessary, then set the BAL field.
|
||||
if execution_data.payload.as_v4().is_none() {
|
||||
execution_data.payload = upgrade_to_v4(execution_data.payload, encoded_bal);
|
||||
} else {
|
||||
execution_data.payload.as_v4_mut().unwrap().block_access_list = encoded_bal;
|
||||
}
|
||||
|
||||
// Patch parent_hash so this block chains off the (possibly
|
||||
// rehashed) previous block.
|
||||
execution_data.payload.as_v1_mut().parent_hash = parent_hash;
|
||||
|
||||
// Recompute block hash after payload modification and update
|
||||
// the hash stored in the payload itself.
|
||||
block_hash = compute_payload_block_hash(&execution_data)?;
|
||||
execution_data.payload.as_v1_mut().block_hash = block_hash;
|
||||
}
|
||||
|
||||
(
|
||||
None,
|
||||
serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(execution_data.clone()),
|
||||
RethNewPayloadInput::ExecutionData(execution_data),
|
||||
wait_for_persistence,
|
||||
self.no_wait_for_caches.then_some(false),
|
||||
big_block_data_param,
|
||||
@@ -256,7 +315,7 @@ impl Command {
|
||||
let requests =
|
||||
execution_data.sidecar.requests().cloned().unwrap_or_default().to_vec();
|
||||
(
|
||||
Some(EngineApiMessageVersion::V4),
|
||||
Some(EngineApiMessageVersion::V6),
|
||||
serde_json::to_value((
|
||||
execution_data.payload.clone(),
|
||||
Vec::<B256>::new(),
|
||||
@@ -364,7 +423,7 @@ impl Command {
|
||||
/// Load and parse all payload files from the directory.
|
||||
///
|
||||
/// Tries to load each file as a [`BigBlockPayload`] first (which includes `env_switches`),
|
||||
/// falling back to [`ExecutionPayloadEnvelopeV4`] for backwards compatibility.
|
||||
/// falling back to [`ExecutionPayloadEnvelopeV6`] for backwards compatibility.
|
||||
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
|
||||
let mut payloads = Vec::new();
|
||||
|
||||
@@ -391,12 +450,11 @@ impl Command {
|
||||
let name_str = name.to_string_lossy();
|
||||
let index = if let Some(rest) = name_str.strip_prefix("payload_block_") {
|
||||
rest.strip_suffix(".json")?.parse::<u64>().ok()?
|
||||
} else if let Some(rest) = name_str.strip_prefix("big_block_") {
|
||||
} else {
|
||||
let rest = name_str.strip_prefix("big_block_")?;
|
||||
// "big_block_FROM_to_TO.json" — use FROM as the index
|
||||
let rest = rest.strip_suffix(".json")?;
|
||||
rest.split("_to_").next()?.parse::<u64>().ok()?
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
Some((index, e.path()))
|
||||
})
|
||||
@@ -417,26 +475,27 @@ impl Command {
|
||||
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
|
||||
|
||||
// Try BigBlockPayload first, then fall back to legacy ExecutionPayloadEnvelopeV4
|
||||
let (execution_data, big_block_data) =
|
||||
if let Ok(big_block) = serde_json::from_str::<BigBlockPayload>(&content) {
|
||||
(big_block.execution_data, big_block.big_block_data)
|
||||
} else {
|
||||
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
let execution_data = ExecutionData {
|
||||
payload: envelope.envelope_inner.execution_payload.clone().into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
parent_beacon_block_root: B256::ZERO,
|
||||
},
|
||||
PraguePayloadFields {
|
||||
requests: envelope.execution_requests.clone().into(),
|
||||
},
|
||||
),
|
||||
};
|
||||
(execution_data, BigBlockData::default())
|
||||
let (execution_data, big_block_data, block_access_list) = if let Ok(big_block) =
|
||||
serde_json::from_str::<BigBlockPayload>(&content)
|
||||
{
|
||||
(big_block.execution_data, big_block.big_block_data, big_block.block_access_list)
|
||||
} else {
|
||||
let envelope: ExecutionPayloadEnvelopeV6 = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
let execution_data = ExecutionData {
|
||||
payload: envelope.execution_payload.clone().into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
parent_beacon_block_root: B256::ZERO,
|
||||
},
|
||||
PraguePayloadFields {
|
||||
requests: envelope.execution_requests.clone().into(),
|
||||
},
|
||||
),
|
||||
};
|
||||
(execution_data, BigBlockData::default(), None)
|
||||
};
|
||||
|
||||
let block_hash = execution_data.payload.as_v1().block_hash;
|
||||
|
||||
@@ -446,13 +505,48 @@ impl Command {
|
||||
block_hash = %block_hash,
|
||||
env_switches = big_block_data.env_switches.len(),
|
||||
prior_block_hashes = big_block_data.prior_block_hashes.len(),
|
||||
bal_accounts = block_access_list.as_ref().map_or(0, Vec::len),
|
||||
path = %path.display(),
|
||||
"Loaded payload"
|
||||
);
|
||||
|
||||
payloads.push(LoadedPayload { index, execution_data, block_hash, big_block_data });
|
||||
payloads.push(LoadedPayload {
|
||||
index,
|
||||
execution_data,
|
||||
block_hash,
|
||||
big_block_data,
|
||||
block_access_list,
|
||||
});
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
}
|
||||
|
||||
/// Upgrades an [`ExecutionPayload`] to V4 by wrapping the inner V3 payload (constructing
|
||||
/// default V2/V3 layers for V1 payloads if needed) and setting the provided BAL bytes.
|
||||
fn upgrade_to_v4(
|
||||
payload: ExecutionPayload,
|
||||
block_access_list: alloy_primitives::Bytes,
|
||||
) -> ExecutionPayload {
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV2, ExecutionPayloadV3};
|
||||
|
||||
let v3 = match payload {
|
||||
ExecutionPayload::V4(_) => unreachable!("caller checks as_v4().is_none()"),
|
||||
ExecutionPayload::V3(v3) => v3,
|
||||
ExecutionPayload::V2(v2) => {
|
||||
ExecutionPayloadV3 { payload_inner: v2, blob_gas_used: 0, excess_blob_gas: 0 }
|
||||
}
|
||||
ExecutionPayload::V1(v1) => ExecutionPayloadV3 {
|
||||
payload_inner: ExecutionPayloadV2 { payload_inner: v1, withdrawals: Vec::new() },
|
||||
blob_gas_used: 0,
|
||||
excess_blob_gas: 0,
|
||||
},
|
||||
};
|
||||
|
||||
ExecutionPayload::V4(ExecutionPayloadV4 {
|
||||
payload_inner: v3,
|
||||
block_access_list,
|
||||
slot_number: 0,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4,
|
||||
};
|
||||
|
||||
/// Configuration for invalidating payload fields
|
||||
#[derive(Debug, Default)]
|
||||
@@ -21,6 +23,7 @@ pub(super) struct InvalidationConfig {
|
||||
pub(super) block_hash: Option<B256>,
|
||||
pub(super) blob_gas_used: Option<u64>,
|
||||
pub(super) excess_blob_gas: Option<u64>,
|
||||
pub(super) slot_number: Option<u64>,
|
||||
|
||||
// Auto-invalidation flags
|
||||
pub(super) invalidate_parent_hash: bool,
|
||||
@@ -35,6 +38,8 @@ pub(super) struct InvalidationConfig {
|
||||
pub(super) invalidate_withdrawals: bool,
|
||||
pub(super) invalidate_blob_gas_used: bool,
|
||||
pub(super) invalidate_excess_blob_gas: bool,
|
||||
pub(super) invalidate_block_access_list: bool,
|
||||
pub(super) invalidate_slot_number: bool,
|
||||
}
|
||||
|
||||
impl InvalidationConfig {
|
||||
@@ -216,4 +221,30 @@ impl InvalidationConfig {
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V4 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v4(&self, payload: &mut ExecutionPayloadV4) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v3(&mut payload.payload_inner);
|
||||
|
||||
// Explicit override for slot_number
|
||||
if let Some(slot_number) = self.slot_number {
|
||||
payload.slot_number = slot_number;
|
||||
changes.push(format!("slot_number = {slot_number}"));
|
||||
}
|
||||
|
||||
// Handle block access list invalidation (V4+)
|
||||
if self.invalidate_block_access_list {
|
||||
let fake_bal = Bytes::from_static(&[0x01, 0x02, 0x03]);
|
||||
payload.block_access_list = fake_bal.clone();
|
||||
changes.push(format!("block_access_list = {fake_bal} (auto-invalidated)"));
|
||||
}
|
||||
|
||||
// Handle slot number invalidation (V4+)
|
||||
if self.invalidate_slot_number {
|
||||
payload.slot_number = u64::MAX;
|
||||
changes.push("slot_number = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
//! Command for sending invalid payloads to test Engine API rejection.
|
||||
|
||||
mod invalidation;
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use invalidation::InvalidationConfig;
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use alloy_provider::{
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
RootProvider,
|
||||
};
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
@@ -105,6 +111,9 @@ pub struct Command {
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
requests_hash: Option<B256>,
|
||||
|
||||
/// Override the slot number with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
slot_number: Option<u64>,
|
||||
// ==================== Auto-Invalidation Flags ====================
|
||||
/// Invalidate the parent hash by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
@@ -158,6 +167,14 @@ pub struct Command {
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_requests_hash: bool,
|
||||
|
||||
/// Invalidate the block access list by setting it to a random value (EIP-7928).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_access_list: bool,
|
||||
|
||||
/// Invalidate the slot number by setting it to an random value.(EIP-7843).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_slot_number: bool,
|
||||
|
||||
// ==================== Meta Flags ====================
|
||||
/// Skip block hash recalculation after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
@@ -199,6 +216,7 @@ impl Command {
|
||||
block_hash: self.block_hash,
|
||||
blob_gas_used: self.blob_gas_used,
|
||||
excess_blob_gas: self.excess_blob_gas,
|
||||
slot_number: self.slot_number,
|
||||
invalidate_parent_hash: self.invalidate_parent_hash,
|
||||
invalidate_state_root: self.invalidate_state_root,
|
||||
invalidate_receipts_root: self.invalidate_receipts_root,
|
||||
@@ -211,6 +229,8 @@ impl Command {
|
||||
invalidate_withdrawals: self.invalidate_withdrawals,
|
||||
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
|
||||
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
|
||||
invalidate_block_access_list: self.invalidate_block_access_list,
|
||||
invalidate_slot_number: self.invalidate_slot_number,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,14 +254,21 @@ impl Command {
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let use_v5 = block.header.block_access_list_hash.is_some();
|
||||
let requests_hash = self.requests_hash.or(block.header.requests_hash);
|
||||
|
||||
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
let mut execution_payload = if use_v5 {
|
||||
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
|
||||
} else {
|
||||
ExecutionPayload::from_block_slow(&block).0
|
||||
};
|
||||
|
||||
let changes = match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
|
||||
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
|
||||
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
|
||||
ExecutionPayload::V4(p) => config.apply_to_payload_v4(p),
|
||||
};
|
||||
|
||||
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
|
||||
@@ -256,6 +283,9 @@ impl Command {
|
||||
ExecutionPayload::V1(p) => p.block_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
|
||||
ExecutionPayload::V4(p) => {
|
||||
p.payload_inner.payload_inner.payload_inner.block_hash
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -264,6 +294,9 @@ impl Command {
|
||||
ExecutionPayload::V1(p) => p.block_hash = new_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
|
||||
ExecutionPayload::V4(p) => {
|
||||
p.payload_inner.payload_inner.payload_inner.block_hash = new_hash
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -305,7 +338,13 @@ impl Command {
|
||||
match self.mode {
|
||||
Mode::Execute => {
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
let method = if use_v5 {
|
||||
"engine_newPayloadV5"
|
||||
} else if use_v4 {
|
||||
"engine_newPayloadV4"
|
||||
} else {
|
||||
"engine_newPayloadV3"
|
||||
};
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
@@ -346,4 +385,17 @@ impl Command {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
|
||||
let rpc_url = self
|
||||
.rpc_url
|
||||
.as_deref()
|
||||
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
|
||||
let client = ClientBuilder::default()
|
||||
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
let bal = fetch_block_access_list(&provider, block_number).await?;
|
||||
Ok(alloy_rlp::encode(bal).into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use super::helpers::{fetch_block_access_list, load_jwt_secret, read_input};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_primitives::Bytes;
|
||||
use alloy_provider::{
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
RootProvider,
|
||||
};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
@@ -69,6 +74,9 @@ impl Command {
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let use_v5 = block.header.block_access_list_hash.is_some();
|
||||
|
||||
// Extract parent beacon block root
|
||||
let parent_beacon_block_root = block.header.parent_beacon_block_root;
|
||||
|
||||
@@ -76,10 +84,14 @@ impl Command {
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
|
||||
// Convert to execution payload
|
||||
let execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
// V5 payloads must carry the full RLP-encoded block access list, not just the hash stored
|
||||
// in the header.
|
||||
let execution_payload = if use_v5 {
|
||||
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
|
||||
} else {
|
||||
ExecutionPayload::from_block_slow(&block).0
|
||||
};
|
||||
|
||||
// Create JSON request data
|
||||
let json_request = if use_v4 {
|
||||
@@ -102,7 +114,13 @@ impl Command {
|
||||
Mode::Execute => {
|
||||
// Create cast command
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
let method = if use_v5 {
|
||||
"engine_newPayloadV5"
|
||||
} else if use_v4 {
|
||||
"engine_newPayloadV4"
|
||||
} else {
|
||||
"engine_newPayloadV3"
|
||||
};
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
@@ -146,4 +164,17 @@ impl Command {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
|
||||
let rpc_url = self
|
||||
.rpc_url
|
||||
.as_deref()
|
||||
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
|
||||
let client = ClientBuilder::default()
|
||||
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
let bal = fetch_block_access_list(&provider, block_number).await?;
|
||||
Ok(alloy_rlp::encode(bal).into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! before sending additional calls.
|
||||
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::eip7928::BlockAccessList;
|
||||
use alloy_primitives::Bytes;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
|
||||
use alloy_rpc_types_engine::{
|
||||
@@ -43,6 +44,14 @@ pub trait EngineApiValidWaitExt<N>: Send + Sync {
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated>;
|
||||
|
||||
/// Calls `engine_forkChoiceUpdatedV4` with the given [`ForkchoiceState`] and optional
|
||||
/// [`PayloadAttributes`], and waits until the response is VALID.
|
||||
async fn fork_choice_updated_v4_wait(
|
||||
&self,
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -162,6 +171,40 @@ where
|
||||
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
async fn fork_choice_updated_v4_wait(
|
||||
&self,
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
method = "engine_forkchoiceUpdatedV3",
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Sending forkchoiceUpdated"
|
||||
);
|
||||
|
||||
let mut status =
|
||||
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(
|
||||
target: "reth-bench",
|
||||
?status,
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Invalid forkchoiceUpdatedV4 message",
|
||||
);
|
||||
panic!("Invalid forkchoiceUpdatedV4: {status:?}");
|
||||
}
|
||||
status =
|
||||
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
|
||||
}
|
||||
|
||||
Ok(status)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
|
||||
@@ -176,6 +219,7 @@ pub(crate) fn block_to_new_payload(
|
||||
reth_new_payload: bool,
|
||||
wait_for_persistence: WaitForPersistence,
|
||||
no_wait_for_caches: bool,
|
||||
bal: Option<BlockAccessList>,
|
||||
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
|
||||
let block_number = block.header.number;
|
||||
let wait_for_persistence = wait_for_persistence.rpc_value(block_number);
|
||||
@@ -198,7 +242,11 @@ pub(crate) fn block_to_new_payload(
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
|
||||
let block_access_list = alloy_rlp::encode(bal.unwrap_or_default());
|
||||
|
||||
let (payload, sidecar) =
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, block_access_list.into());
|
||||
let (version, params, execution_data) = payload_to_new_payload(payload, sidecar, None)?;
|
||||
|
||||
if reth_new_payload {
|
||||
@@ -227,6 +275,22 @@ pub(crate) fn payload_to_new_payload(
|
||||
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
|
||||
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V4(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
|
||||
let version = target_version.unwrap_or(EngineApiMessageVersion::V6);
|
||||
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
|
||||
(
|
||||
version,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
@@ -370,6 +434,9 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
|
||||
match message_version {
|
||||
EngineApiMessageVersion::V6 => {
|
||||
provider.fork_choice_updated_v4_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
|
||||
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
|
||||
@@ -70,7 +70,7 @@ aquamarine.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
alloy-node-bindings = "1.6.3"
|
||||
alloy-node-bindings = "2.0.0"
|
||||
alloy-provider = { workspace = true, features = ["reqwest"] }
|
||||
alloy-rpc-types-eth.workspace = true
|
||||
backon.workspace = true
|
||||
|
||||
@@ -18,7 +18,7 @@ reth-errors.workspace = true
|
||||
reth-execution-types.workspace = true
|
||||
reth-metrics.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["dashmap"] }
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie.workspace = true
|
||||
|
||||
|
||||
@@ -274,8 +274,54 @@ impl DeferredTrieData {
|
||||
/// In normal operation, the parent always has a cached overlay and this
|
||||
/// function is never called.
|
||||
///
|
||||
/// Iterates ancestors oldest -> newest, then extends with current block's data,
|
||||
/// so later state takes precedence.
|
||||
/// When the `rayon` feature is enabled:
|
||||
/// 1. Collects ancestor data (states and updates)
|
||||
/// 2. Merges states and trie updates in parallel using k-way merge
|
||||
#[cfg(feature = "rayon")]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
sorted_trie_updates: &TrieUpdatesSorted,
|
||||
) -> TrieInputSorted {
|
||||
// Early exit: no ancestors means just wrap current block's data
|
||||
if ancestors.is_empty() {
|
||||
return TrieInputSorted::new(
|
||||
Arc::new(sorted_trie_updates.clone()),
|
||||
Arc::new(sorted_hashed_state.clone()),
|
||||
Default::default(),
|
||||
);
|
||||
}
|
||||
|
||||
// Collect ancestor data in reverse (newest to oldest) for merge_slice
|
||||
let (states, updates): (Vec<_>, Vec<_>) = ancestors
|
||||
.iter()
|
||||
.rev()
|
||||
.map(|a| {
|
||||
// Note: we can assume that this trie data has already been computed
|
||||
let data = a.wait_cloned();
|
||||
(data.hashed_state, data.trie_updates)
|
||||
})
|
||||
.unzip();
|
||||
|
||||
// Merge state and nodes in parallel using k-way merge
|
||||
let (state, nodes) = rayon::join(
|
||||
|| {
|
||||
let mut merged = HashedPostStateSorted::merge_slice(&states);
|
||||
merged.extend_ref_and_sort(sorted_hashed_state);
|
||||
merged
|
||||
},
|
||||
|| {
|
||||
let mut merged = TrieUpdatesSorted::merge_slice(&updates);
|
||||
merged.extend_ref_and_sort(sorted_trie_updates);
|
||||
merged
|
||||
},
|
||||
);
|
||||
|
||||
TrieInputSorted::new(Arc::new(nodes), Arc::new(state), Default::default())
|
||||
}
|
||||
|
||||
/// Sequential fallback when rayon is not available.
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
fn merge_ancestors_into_overlay(
|
||||
ancestors: &[Self],
|
||||
sorted_hashed_state: &HashedPostStateSorted,
|
||||
@@ -293,18 +339,8 @@ impl DeferredTrieData {
|
||||
nodes_mut.extend_ref_and_sort(ancestor_data.trie_updates.as_ref());
|
||||
}
|
||||
|
||||
// Extend with current block's sorted data last (takes precedence)
|
||||
#[cfg(feature = "rayon")]
|
||||
rayon::join(
|
||||
|| state_mut.extend_ref_and_sort(sorted_hashed_state),
|
||||
|| nodes_mut.extend_ref_and_sort(sorted_trie_updates),
|
||||
);
|
||||
|
||||
#[cfg(not(feature = "rayon"))]
|
||||
{
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
}
|
||||
state_mut.extend_ref_and_sort(sorted_hashed_state);
|
||||
nodes_mut.extend_ref_and_sort(sorted_trie_updates);
|
||||
|
||||
overlay
|
||||
}
|
||||
|
||||
@@ -1169,6 +1169,7 @@ mod tests {
|
||||
&self,
|
||||
_input: TrieInput,
|
||||
_target: HashedPostState,
|
||||
_mode: reth_trie::ExecutionWitnessMode,
|
||||
) -> ProviderResult<Vec<Bytes>> {
|
||||
Ok(Vec::default())
|
||||
}
|
||||
|
||||
@@ -4,26 +4,32 @@
|
||||
//! lazily on first access. This allows execution to start before the trie overlay
|
||||
//! is fully computed.
|
||||
|
||||
use crate::DeferredTrieData;
|
||||
use crate::{EthPrimitives, ExecutedBlock};
|
||||
use alloy_primitives::B256;
|
||||
use reth_primitives_traits::{
|
||||
dashmap::{self, DashMap},
|
||||
AlloyBlockHeader, NodePrimitives,
|
||||
};
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Inputs captured for lazy overlay computation.
|
||||
#[derive(Clone)]
|
||||
struct LazyOverlayInputs {
|
||||
/// The persisted ancestor hash (anchor) this overlay should be built on.
|
||||
anchor_hash: B256,
|
||||
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
|
||||
blocks: Vec<DeferredTrieData>,
|
||||
struct LazyOverlayInputs<N: NodePrimitives = EthPrimitives> {
|
||||
/// In-memory blocks from tip to anchor child.
|
||||
///
|
||||
/// Blocks must be provided in reverse chain order (newest to oldest).
|
||||
blocks: Vec<ExecutedBlock<N>>,
|
||||
}
|
||||
|
||||
/// Lazily computed trie overlay.
|
||||
///
|
||||
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
|
||||
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
|
||||
/// but for overlay computation.
|
||||
/// computation until first access.
|
||||
///
|
||||
/// Blocks must be provided in reverse chain order (newest to oldest), so the first block is the
|
||||
/// chain tip and the last block is the oldest in-memory block in the chain segment.
|
||||
///
|
||||
/// # Fast Path vs Slow Path
|
||||
///
|
||||
@@ -31,37 +37,41 @@ struct LazyOverlayInputs {
|
||||
/// matches our expected anchor, we can reuse it directly (O(1)).
|
||||
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
|
||||
#[derive(Clone)]
|
||||
pub struct LazyOverlay {
|
||||
/// Computed result, cached after first access.
|
||||
inner: Arc<OnceLock<TrieInputSorted>>,
|
||||
pub struct LazyOverlay<N: NodePrimitives = EthPrimitives> {
|
||||
/// Computed results, cached by requested anchor hash.
|
||||
inner: Arc<DashMap<B256, Arc<TrieInputSorted>>>,
|
||||
/// Inputs for lazy computation.
|
||||
inputs: LazyOverlayInputs,
|
||||
inputs: LazyOverlayInputs<N>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LazyOverlay {
|
||||
impl<N: NodePrimitives> std::fmt::Debug for LazyOverlay<N> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LazyOverlay")
|
||||
.field("anchor_hash", &self.inputs.anchor_hash)
|
||||
.field(
|
||||
"oldest_block_parent_hash",
|
||||
&self.inputs.blocks.last().map(|block| block.recovered_block().parent_hash()),
|
||||
)
|
||||
.field("num_blocks", &self.inputs.blocks.len())
|
||||
.field("computed", &self.inner.get().is_some())
|
||||
.field("cached_anchors", &self.inner.len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LazyOverlay {
|
||||
/// Create a new lazy overlay with the given anchor hash and block handles.
|
||||
impl<N: NodePrimitives> LazyOverlay<N> {
|
||||
/// Create a new lazy overlay from in-memory blocks.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
|
||||
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
|
||||
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
|
||||
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
|
||||
}
|
||||
/// * `blocks` - Executed blocks in reverse chain order (newest to oldest)
|
||||
pub fn new(blocks: Vec<ExecutedBlock<N>>) -> Self {
|
||||
debug_assert!(
|
||||
blocks.windows(2).all(|window| {
|
||||
window[0].recovered_block().parent_hash() == window[1].recovered_block().hash()
|
||||
}),
|
||||
"LazyOverlay blocks must be ordered newest to oldest along a single chain"
|
||||
);
|
||||
|
||||
/// Returns the anchor hash this overlay is built on.
|
||||
pub const fn anchor_hash(&self) -> B256 {
|
||||
self.inputs.anchor_hash
|
||||
Self { inner: Default::default(), inputs: LazyOverlayInputs { blocks } }
|
||||
}
|
||||
|
||||
/// Returns the number of in-memory blocks this overlay covers.
|
||||
@@ -69,43 +79,75 @@ impl LazyOverlay {
|
||||
self.inputs.blocks.len()
|
||||
}
|
||||
|
||||
/// Returns true if the overlay has already been computed.
|
||||
pub fn is_computed(&self) -> bool {
|
||||
self.inner.get().is_some()
|
||||
/// Returns the oldest anchor hash this overlay can serve.
|
||||
///
|
||||
/// This is the parent hash of the oldest block in the stored newest-to-oldest chain segment.
|
||||
pub fn anchor_hash(&self) -> Option<B256> {
|
||||
self.inputs.blocks.last().map(|block| block.recovered_block().parent_hash())
|
||||
}
|
||||
|
||||
/// Returns the computed trie input, computing it if necessary.
|
||||
/// Returns true if there are no blocks in the overlay, or if one of the blocks has the given
|
||||
/// hash as a parent hash.
|
||||
pub fn has_anchor_hash(&self, hash: B256) -> bool {
|
||||
self.inputs.blocks.is_empty() ||
|
||||
self.inputs.blocks.iter().any(|b| b.recovered_block().parent_hash() == hash)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Returns true if the overlay has already been computed for the requested anchor.
|
||||
pub fn is_computed(&self, anchor_hash: B256) -> bool {
|
||||
self.inner.contains_key(&anchor_hash)
|
||||
}
|
||||
|
||||
/// Returns the computed trie input for the requested anchor, computing it if necessary.
|
||||
///
|
||||
/// The first call triggers computation (which may block waiting for deferred data).
|
||||
/// Subsequent calls return the cached result immediately.
|
||||
pub fn get(&self) -> &TrieInputSorted {
|
||||
self.inner.get_or_init(|| self.compute())
|
||||
/// Subsequent calls for the same anchor return the cached result immediately.
|
||||
pub fn get(&self, anchor_hash: B256) -> Arc<TrieInputSorted> {
|
||||
match self.inner.entry(anchor_hash) {
|
||||
dashmap::Entry::Occupied(entry) => Arc::clone(entry.get()),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
let input = self.compute(anchor_hash);
|
||||
entry.insert(Arc::clone(&input));
|
||||
input
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
|
||||
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get();
|
||||
pub fn as_overlay(
|
||||
&self,
|
||||
anchor_hash: B256,
|
||||
) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get(anchor_hash);
|
||||
(Arc::clone(&input.nodes), Arc::clone(&input.state))
|
||||
}
|
||||
|
||||
/// Compute the trie input overlay.
|
||||
fn compute(&self) -> TrieInputSorted {
|
||||
let anchor_hash = self.inputs.anchor_hash;
|
||||
fn compute(&self, anchor_hash: B256) -> Arc<TrieInputSorted> {
|
||||
let blocks = &self.inputs.blocks;
|
||||
|
||||
if blocks.is_empty() {
|
||||
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
|
||||
return TrieInputSorted::default();
|
||||
return Default::default()
|
||||
}
|
||||
|
||||
let Some(last_index) =
|
||||
blocks.iter().position(|block| block.recovered_block().parent_hash() == anchor_hash)
|
||||
else {
|
||||
panic!(
|
||||
"LazyOverlay does not contain a block whose parent hash matches requested anchor {anchor_hash}"
|
||||
);
|
||||
};
|
||||
let blocks = &blocks[..=last_index];
|
||||
|
||||
// Fast path: Check if tip block's overlay is ready and anchor matches.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors up to the
|
||||
// requested anchor.
|
||||
if let Some(tip) = blocks.first() {
|
||||
let data = tip.wait_cloned();
|
||||
let data = tip.trie_data();
|
||||
if let Some(anchored) = &data.anchored_trie_input {
|
||||
if anchored.anchor_hash == anchor_hash {
|
||||
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
|
||||
return (*anchored.trie_input).clone();
|
||||
return Arc::clone(&anchored.trie_input);
|
||||
}
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
@@ -116,23 +158,30 @@ impl LazyOverlay {
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: Merge all blocks' trie data into a new overlay.
|
||||
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
|
||||
Self::merge_blocks(blocks)
|
||||
// Slow path: Merge the prefix of blocks from the tip back to the requested anchor.
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
%anchor_hash,
|
||||
num_blocks = blocks.len(),
|
||||
"Merging blocks (slow path)"
|
||||
);
|
||||
Arc::new(Self::merge_blocks(blocks))
|
||||
}
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
fn merge_blocks(blocks: &[ExecutedBlock<N>]) -> TrieInputSorted {
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
let state = HashedPostStateSorted::merge_batch(
|
||||
blocks.iter().map(|block| block.trie_data().hashed_state),
|
||||
);
|
||||
let nodes = TrieUpdatesSorted::merge_batch(
|
||||
blocks.iter().map(|block| block.trie_data().trie_updates),
|
||||
);
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
@@ -141,46 +190,138 @@ impl LazyOverlay {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState};
|
||||
use crate::{test_utils::TestBlockBuilder, ComputedTrieData, EthPrimitives, ExecutedBlock};
|
||||
use alloy_primitives::U256;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostState, HashedStorage};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn empty_deferred(anchor: B256) -> DeferredTrieData {
|
||||
DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
Vec::new(),
|
||||
fn with_unique_state(
|
||||
block: &ExecutedBlock<EthPrimitives>,
|
||||
id: u8,
|
||||
) -> ExecutedBlock<EthPrimitives> {
|
||||
let hashed_address = B256::with_last_byte(id);
|
||||
let hashed_slot = B256::with_last_byte(id.saturating_add(32));
|
||||
let hashed_state = HashedPostState::default()
|
||||
.with_accounts([(hashed_address, Some(Account::default()))])
|
||||
.with_storages([(
|
||||
hashed_address,
|
||||
HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]),
|
||||
)])
|
||||
.into_sorted();
|
||||
|
||||
ExecutedBlock::new(
|
||||
Arc::clone(&block.recovered_block),
|
||||
Arc::clone(&block.execution_output),
|
||||
ComputedTrieData::without_trie_input(
|
||||
Arc::new(hashed_state),
|
||||
Arc::new(TrieUpdatesSorted::default()),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_blocks_returns_default() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
let result = overlay.get();
|
||||
assert!(result.state.is_empty());
|
||||
assert!(result.nodes.is_empty());
|
||||
fn test_blocks() -> Vec<ExecutedBlock<EthPrimitives>> {
|
||||
TestBlockBuilder::eth()
|
||||
.get_executed_blocks(1..4)
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.enumerate()
|
||||
.map(|(index, block)| with_unique_state(&block, index as u8 + 1))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_uses_data_directly() {
|
||||
let anchor = B256::random();
|
||||
let deferred = empty_deferred(anchor);
|
||||
let overlay = LazyOverlay::new(anchor, vec![deferred]);
|
||||
let block = TestBlockBuilder::eth().get_executed_block_with_number(1, B256::random());
|
||||
let anchor_hash = block.recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(vec![block]);
|
||||
|
||||
assert!(!overlay.is_computed());
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
assert!(!overlay.is_computed(anchor_hash));
|
||||
let _ = overlay.get(anchor_hash);
|
||||
assert!(overlay.is_computed(anchor_hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_after_first_access() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
fn caches_results_per_anchor() {
|
||||
let blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let full_anchor = blocks[2].recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
// First access computes
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
let prefix = overlay.get(prefix_anchor);
|
||||
let full = overlay.get(full_anchor);
|
||||
|
||||
// Second access uses cache
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
assert!(overlay.is_computed(prefix_anchor));
|
||||
assert!(overlay.is_computed(full_anchor));
|
||||
assert!(!Arc::ptr_eq(&prefix, &full));
|
||||
assert!(Arc::ptr_eq(&prefix, &overlay.get(prefix_anchor)));
|
||||
assert!(Arc::ptr_eq(&full, &overlay.get(full_anchor)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requested_anchor_limits_the_merged_prefix() {
|
||||
let blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let expected = LazyOverlay::merge_blocks(&blocks[..2]);
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
let actual = overlay.get(prefix_anchor);
|
||||
|
||||
assert_eq!(actual.nodes.as_ref(), expected.nodes.as_ref());
|
||||
assert_eq!(actual.state.as_ref(), expected.state.as_ref());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anchor_hash_returns_oldest_served_anchor() {
|
||||
let blocks = test_blocks();
|
||||
let expected_anchor = blocks.last().unwrap().recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
assert_eq!(overlay.anchor_hash(), Some(expected_anchor));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reuses_tip_overlay_when_anchor_matches() {
|
||||
let mut blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let tip_overlay = Arc::new(LazyOverlay::merge_blocks(&blocks[..2]));
|
||||
let tip_data = blocks[0].trie_data();
|
||||
|
||||
blocks[0] = ExecutedBlock::new(
|
||||
Arc::clone(&blocks[0].recovered_block),
|
||||
Arc::clone(&blocks[0].execution_output),
|
||||
ComputedTrieData::with_trie_input(
|
||||
tip_data.hashed_state,
|
||||
tip_data.trie_updates,
|
||||
prefix_anchor,
|
||||
Arc::clone(&tip_overlay),
|
||||
),
|
||||
);
|
||||
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
let actual = overlay.get(prefix_anchor);
|
||||
|
||||
assert!(Arc::ptr_eq(&actual, &tip_overlay));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "LazyOverlay does not contain a block whose parent hash matches requested anchor"
|
||||
)]
|
||||
fn missing_anchor_panics() {
|
||||
let blocks = test_blocks();
|
||||
let missing_anchor = blocks[0].recovered_block().hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
let _ = overlay.get(missing_anchor);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "LazyOverlay blocks must be ordered newest to oldest along a single chain"
|
||||
)]
|
||||
fn misordered_blocks_panic() {
|
||||
let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect();
|
||||
let _ = LazyOverlay::new(blocks);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -197,9 +197,14 @@ impl<N: NodePrimitives> StateProofProvider for MemoryOverlayStateProviderRef<'_,
|
||||
self.historical.multiproof(input, targets)
|
||||
}
|
||||
|
||||
fn witness(&self, mut input: TrieInput, target: HashedPostState) -> ProviderResult<Vec<Bytes>> {
|
||||
fn witness(
|
||||
&self,
|
||||
mut input: TrieInput,
|
||||
target: HashedPostState,
|
||||
mode: reth_trie::ExecutionWitnessMode,
|
||||
) -> ProviderResult<Vec<Bytes>> {
|
||||
input.prepend_self(self.trie_input().clone());
|
||||
self.historical.witness(input, target)
|
||||
self.historical.witness(input, target, mode)
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -28,7 +28,7 @@ use alloy_consensus::{
|
||||
};
|
||||
use alloy_eips::{
|
||||
eip1559::INITIAL_BASE_FEE, eip7685::EMPTY_REQUESTS_HASH, eip7840::BlobParams,
|
||||
eip7892::BlobScheduleBlobParams,
|
||||
eip7892::BlobScheduleBlobParams, eip7928::EMPTY_BLOCK_ACCESS_LIST_HASH,
|
||||
};
|
||||
use alloy_genesis::{ChainConfig, Genesis};
|
||||
use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256};
|
||||
@@ -76,6 +76,18 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
|
||||
.active_at_timestamp(genesis.timestamp)
|
||||
.then_some(EMPTY_REQUESTS_HASH);
|
||||
|
||||
// If Amsterdam is activated at genesis we set block access list hash to an empty bal hash
|
||||
let block_access_list_hash = hardforks
|
||||
.fork(EthereumHardfork::Amsterdam)
|
||||
.active_at_timestamp(genesis.timestamp)
|
||||
.then_some(EMPTY_BLOCK_ACCESS_LIST_HASH);
|
||||
|
||||
// If Amsterdam is activated at genesis we set slot number to 0
|
||||
let slot_number = hardforks
|
||||
.fork(EthereumHardfork::Amsterdam)
|
||||
.active_at_timestamp(genesis.timestamp)
|
||||
.then_some(0);
|
||||
|
||||
Header {
|
||||
number: genesis.number.unwrap_or_default(),
|
||||
parent_hash: genesis.parent_hash.unwrap_or_default(),
|
||||
@@ -93,6 +105,8 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
|
||||
blob_gas_used,
|
||||
excess_blob_gas,
|
||||
requests_hash,
|
||||
block_access_list_hash,
|
||||
slot_number,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -275,7 +289,6 @@ pub fn create_chain_config(
|
||||
// Check if DAO fork is supported (it has an activation block)
|
||||
let dao_fork_support = hardforks.fork(EthereumHardfork::Dao) != ForkCondition::Never;
|
||||
|
||||
#[expect(clippy::needless_update)]
|
||||
ChainConfig {
|
||||
chain_id: chain.map(|c| c.id()).unwrap_or(0),
|
||||
homestead_block: block_num(EthereumHardfork::Homestead),
|
||||
@@ -298,6 +311,7 @@ pub fn create_chain_config(
|
||||
cancun_time: timestamp(EthereumHardfork::Cancun),
|
||||
prague_time: timestamp(EthereumHardfork::Prague),
|
||||
osaka_time: timestamp(EthereumHardfork::Osaka),
|
||||
amsterdam_time: timestamp(EthereumHardfork::Amsterdam),
|
||||
bpo1_time: timestamp(EthereumHardfork::Bpo1),
|
||||
bpo2_time: timestamp(EthereumHardfork::Bpo2),
|
||||
bpo3_time: timestamp(EthereumHardfork::Bpo3),
|
||||
@@ -305,10 +319,6 @@ pub fn create_chain_config(
|
||||
bpo5_time: timestamp(EthereumHardfork::Bpo5),
|
||||
terminal_total_difficulty,
|
||||
terminal_total_difficulty_passed,
|
||||
ethash: None,
|
||||
clique: None,
|
||||
parlia: None,
|
||||
extra_fields: Default::default(),
|
||||
deposit_contract_address,
|
||||
blob_schedule,
|
||||
..Default::default()
|
||||
@@ -885,6 +895,7 @@ impl From<Genesis> for ChainSpec {
|
||||
(EthereumHardfork::Bpo3.boxed(), genesis.config.bpo3_time),
|
||||
(EthereumHardfork::Bpo4.boxed(), genesis.config.bpo4_time),
|
||||
(EthereumHardfork::Bpo5.boxed(), genesis.config.bpo5_time),
|
||||
(EthereumHardfork::Amsterdam.boxed(), genesis.config.amsterdam_time),
|
||||
];
|
||||
|
||||
let mut time_hardforks = time_hardfork_opts
|
||||
@@ -1191,6 +1202,19 @@ impl ChainSpecBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable Amsterdam at genesis.
|
||||
pub fn amsterdam_activated(mut self) -> Self {
|
||||
self = self.osaka_activated();
|
||||
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(0));
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable Amsterdam at the given timestamp.
|
||||
pub fn with_amsterdam_at(mut self, timestamp: u64) -> Self {
|
||||
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(timestamp));
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the resulting [`ChainSpec`].
|
||||
///
|
||||
/// # Panics
|
||||
|
||||
@@ -150,16 +150,22 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
// commands can proceed.
|
||||
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
|
||||
reth_fs_util::create_dir_all(&rocksdb_path)?;
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.build()?
|
||||
.with_database_log_level(self.db.log_level);
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
} else {
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?
|
||||
.with_read_only(!access.is_read_write());
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
};
|
||||
|
||||
let provider_factory =
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{hex, BlockHash};
|
||||
use alloy_primitives::{hex, Address, BlockHash, B256};
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
static_file::{
|
||||
@@ -10,16 +10,20 @@ use reth_db::{
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbDupCursorRO},
|
||||
database::Database,
|
||||
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
|
||||
table::{Compress, Decompress, DupSort, Table},
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
RawKey, RawTable, Receipts, TableViewer, Transactions,
|
||||
RawKey, RawTable, TableViewer,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, ChangeSetReader, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageChangeSetReader;
|
||||
use tracing::error;
|
||||
@@ -73,6 +77,55 @@ enum Subcommand {
|
||||
#[arg(long)]
|
||||
raw: bool,
|
||||
},
|
||||
/// Gets the content of a RocksDB table for the given key
|
||||
///
|
||||
/// For history tables (accounts-history, storages-history), you can pass a plain address
|
||||
/// instead of a full JSON ShardedKey. Use --block to query a specific block number
|
||||
/// (seeks to the shard containing that block), or --all-shards to list all shards for
|
||||
/// the address.
|
||||
///
|
||||
/// Examples:
|
||||
/// reth db get rocksdb accounts-history 0xdBBE3D8c2d2b22A2611c5A94A9a12C2fCD49Eb29
|
||||
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --block 1000000
|
||||
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --all-shards
|
||||
/// reth db get rocksdb storages-history 0xdBBE...Eb29 --storage-key 0x0000...0003
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: RocksDbTable,
|
||||
|
||||
/// The key to get content for. For history tables, this can be a plain address.
|
||||
#[arg(value_parser = maybe_json_value_parser)]
|
||||
key: String,
|
||||
|
||||
/// Target block number for history tables. Seeks to the shard containing this block.
|
||||
/// Defaults to the latest shard if not specified.
|
||||
#[arg(long)]
|
||||
block: Option<u64>,
|
||||
|
||||
/// Storage key for storages-history table lookups.
|
||||
#[arg(long)]
|
||||
storage_key: Option<String>,
|
||||
|
||||
/// List all shards for the given key (history tables only).
|
||||
#[arg(long)]
|
||||
all_shards: bool,
|
||||
|
||||
/// Output bytes instead of human-readable decoded value
|
||||
#[arg(long)]
|
||||
raw: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// RocksDB tables that can be queried.
|
||||
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -82,6 +135,9 @@ impl Command {
|
||||
Subcommand::Mdbx { table, key, subkey, end_key, end_subkey, raw } => {
|
||||
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
|
||||
}
|
||||
Subcommand::Rocksdb { table, key, block, storage_key, all_shards, raw } => {
|
||||
get_rocksdb(tool, table, &key, block, storage_key.as_deref(), all_shards, raw)?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, key, subkey, raw } => {
|
||||
if let StaticFileSegment::StorageChangeSets = segment {
|
||||
let storage_key =
|
||||
@@ -208,15 +264,12 @@ impl Command {
|
||||
);
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
let transaction = <<Transactions as Table>::Value>::decompress(
|
||||
content[0].as_slice(),
|
||||
)?;
|
||||
let transaction = TxTy::<N>::decompress(content[0].as_slice())?;
|
||||
println!("{}", serde_json::to_string_pretty(&transaction)?);
|
||||
}
|
||||
StaticFileSegment::Receipts => {
|
||||
let receipt = <<Receipts as Table>::Value>::decompress(
|
||||
content[0].as_slice(),
|
||||
)?;
|
||||
let receipt =
|
||||
ReceiptTy::<N>::decompress(content[0].as_slice())?;
|
||||
println!("{}", serde_json::to_string_pretty(&receipt)?);
|
||||
}
|
||||
StaticFileSegment::TransactionSenders => {
|
||||
@@ -246,6 +299,208 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a value from a RocksDB table by key.
|
||||
fn get_rocksdb<N: ProviderNodeTypes>(
|
||||
tool: &DbTool<N>,
|
||||
table: RocksDbTable,
|
||||
key: &str,
|
||||
block: Option<u64>,
|
||||
storage_key: Option<&str>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
if block.is_some() || all_shards || storage_key.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block, --all-shards, and --storage-key are only supported for history tables"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, key, raw)
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
if storage_key.is_some() {
|
||||
return Err(eyre::eyre!("--storage-key is only supported for storages-history"));
|
||||
}
|
||||
get_rocksdb_account_history(&rocksdb, key, block, all_shards, raw)
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
get_rocksdb_storage_history(&rocksdb, key, storage_key, block, all_shards, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to parse a key string as a plain address, falling back to JSON `ShardedKey` parsing.
|
||||
fn parse_address(key: &str) -> eyre::Result<Address> {
|
||||
// Strip surrounding quotes that `maybe_json_value_parser` may have added
|
||||
let stripped = key.trim_matches('"');
|
||||
stripped.parse::<Address>().map_err(|e| eyre::eyre!("failed to parse address: {e}"))
|
||||
}
|
||||
|
||||
/// Gets account history from RocksDB with ergonomic key parsing.
|
||||
///
|
||||
/// Accepts a plain address and uses seek to find the relevant shard.
|
||||
fn get_rocksdb_account_history(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key: &str,
|
||||
block: Option<u64>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
// Try parsing as a plain address first, fall back to full JSON ShardedKey
|
||||
match parse_address(key) {
|
||||
Ok(address) => {
|
||||
let block_number = block.unwrap_or(u64::MAX);
|
||||
let seek_key = ShardedKey::new(address, block_number);
|
||||
|
||||
if all_shards {
|
||||
// Iterate all shards: seek from (address, 0) until address changes
|
||||
let start = ShardedKey::new(address, 0);
|
||||
let iter = rocksdb.iter_from::<tables::AccountsHistory>(start)?;
|
||||
for result in iter {
|
||||
let (k, v) = result?;
|
||||
if k.key != address {
|
||||
break;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"highest_block_number": k.highest_block_number,
|
||||
"value": v,
|
||||
}))?
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Seek to the first shard with highest_block_number >= target
|
||||
let mut iter = rocksdb.iter_from::<tables::AccountsHistory>(seek_key)?;
|
||||
match iter.next() {
|
||||
Some(Ok((k, v))) if k.key == address => {
|
||||
if raw {
|
||||
let raw_val = rocksdb.get_raw::<tables::AccountsHistory>(k)?;
|
||||
if let Some(bytes) = raw_val {
|
||||
println!("{}", hex::encode_prefixed(&bytes));
|
||||
}
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&v)?);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
// Fall back to full JSON key parsing (e.g.
|
||||
// `{"key":"0x...","highest_block_number":...}`)
|
||||
if all_shards || block.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block and --all-shards require a plain address, not a JSON key"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::AccountsHistory>(rocksdb, key, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets storage history from RocksDB with ergonomic key parsing.
|
||||
///
|
||||
/// Accepts a plain address + optional `--storage-key` and uses seek.
|
||||
fn get_rocksdb_storage_history(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key: &str,
|
||||
storage_key: Option<&str>,
|
||||
block: Option<u64>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
match parse_address(key) {
|
||||
Ok(address) => {
|
||||
let storage_key = storage_key
|
||||
.map(|s| s.trim_matches('"').parse::<B256>())
|
||||
.transpose()
|
||||
.map_err(|e| eyre::eyre!("failed to parse storage key: {e}"))?
|
||||
.unwrap_or_default();
|
||||
let block_number = block.unwrap_or(u64::MAX);
|
||||
let seek_key = StorageShardedKey::new(address, storage_key, block_number);
|
||||
|
||||
if all_shards {
|
||||
let start = StorageShardedKey::new(address, storage_key, 0);
|
||||
let iter = rocksdb.iter_from::<tables::StoragesHistory>(start)?;
|
||||
for result in iter {
|
||||
let (k, v) = result?;
|
||||
if k.address != address || k.sharded_key.key != storage_key {
|
||||
break;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"highest_block_number": k.sharded_key.highest_block_number,
|
||||
"value": v,
|
||||
}))?
|
||||
);
|
||||
}
|
||||
} else {
|
||||
let mut iter = rocksdb.iter_from::<tables::StoragesHistory>(seek_key)?;
|
||||
match iter.next() {
|
||||
Some(Ok((k, v)))
|
||||
if k.address == address && k.sharded_key.key == storage_key =>
|
||||
{
|
||||
if raw {
|
||||
let raw_val = rocksdb.get_raw::<tables::StoragesHistory>(k)?;
|
||||
if let Some(bytes) = raw_val {
|
||||
println!("{}", hex::encode_prefixed(&bytes));
|
||||
}
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&v)?);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
if all_shards || block.is_some() || storage_key.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block, --all-shards, and --storage-key require a plain address, not a JSON key"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::StoragesHistory>(rocksdb, key, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a value from a specific RocksDB table by exact key and prints it.
|
||||
fn get_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key_str: &str,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
let key = table_key::<T>(key_str)?;
|
||||
|
||||
if raw {
|
||||
let content = rocksdb.get_raw::<T>(key)?;
|
||||
match content {
|
||||
Some(bytes) => println!("{}", hex::encode_prefixed(&bytes)),
|
||||
None => error!(target: "reth::cli", "No content for the given table key."),
|
||||
}
|
||||
} else {
|
||||
let content = rocksdb.get::<T>(key)?;
|
||||
match content {
|
||||
Some(value) => println!("{}", serde_json::to_string_pretty(&value)?),
|
||||
None => error!(target: "reth::cli", "No content for the given table key."),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get an instance of key for given table
|
||||
pub(crate) fn table_key<T: Table>(key: &str) -> Result<T::Key, eyre::Error> {
|
||||
serde_json::from_str(key).map_err(|e| eyre::eyre!(e))
|
||||
|
||||
386
crates/cli/commands/src/db/migrate_v2.rs
Normal file
386
crates/cli/commands/src/db/migrate_v2.rs
Normal file
@@ -0,0 +1,386 @@
|
||||
//! `reth db migrate-v2` command for migrating v1 storage layout to v2.
|
||||
//!
|
||||
//! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to
|
||||
//! static files, clears recomputable tables (senders, indices, trie, plain
|
||||
//! state), compacts MDBX, then runs the pipeline to rebuild them.
|
||||
|
||||
use crate::common::CliNodeTypes;
|
||||
use alloy_primitives::Address;
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
mdbx::{self, ffi},
|
||||
models::StorageBeforeTx,
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
database::Database,
|
||||
table::Table,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider,
|
||||
MetadataWriter, ProviderFactory, PruneCheckpointReader, StageCheckpointWriter,
|
||||
StaticFileProviderFactory, StaticFileWriter, StorageSettings,
|
||||
};
|
||||
use reth_prune_types::PruneSegment;
|
||||
use reth_stages_types::{StageCheckpoint, StageId};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StageCheckpointReader;
|
||||
use tracing::info;
|
||||
|
||||
/// `reth db migrate-v2` command
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command;
|
||||
|
||||
impl Command {
|
||||
/// Execute the full v1 → v2 migration:
|
||||
///
|
||||
/// 1. Migrate changesets + receipts to static files
|
||||
/// 2. Flip `StorageSettings` to v2
|
||||
/// 3. Clear recomputable MDBX tables + reset stage checkpoints
|
||||
/// 4. Compact MDBX
|
||||
pub async fn execute<N: CliNodeTypes>(
|
||||
self,
|
||||
provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N::Primitives: reth_primitives_traits::NodePrimitives<
|
||||
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
|
||||
>,
|
||||
{
|
||||
// === Phase 0: Preflight ===
|
||||
info!(target: "reth::cli", "Starting v1 → v2 storage migration");
|
||||
|
||||
let provider = provider_factory.provider()?;
|
||||
let current_settings = provider.storage_settings()?;
|
||||
|
||||
if current_settings.is_some_and(|s| s.is_v2()) {
|
||||
info!(target: "reth::cli", "Storage is already v2, nothing to do");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let tip =
|
||||
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number).unwrap_or(0);
|
||||
|
||||
info!(target: "reth::cli", tip, "Chain tip block number");
|
||||
|
||||
let sf_provider = provider_factory.static_file_provider();
|
||||
|
||||
for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets]
|
||||
{
|
||||
if sf_provider.get_highest_static_file_block(segment).is_some() {
|
||||
eyre::bail!(
|
||||
"Static file segment {segment:?} already contains data. \
|
||||
Cannot migrate — target must be empty."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
drop(provider);
|
||||
|
||||
// === Phase 1: Migrate changesets → static files ===
|
||||
Self::migrate_account_changesets(&provider_factory, tip)?;
|
||||
Self::migrate_storage_changesets(&provider_factory, tip)?;
|
||||
|
||||
// === Phase 2: Migrate receipts → static files ===
|
||||
Self::migrate_receipts::<NodeTypesWithDBAdapter<N, DatabaseEnv>>(&provider_factory, tip)?;
|
||||
|
||||
// === Phase 3: Flip metadata to v2 ===
|
||||
info!(target: "reth::cli", "Writing StorageSettings v2 metadata");
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
provider_rw.write_storage_settings(StorageSettings::v2())?;
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
info!(target: "reth::cli", "Storage settings updated to v2");
|
||||
|
||||
// === Phase 4: Clear recomputable tables ===
|
||||
Self::clear_recomputable_tables(&provider_factory)?;
|
||||
|
||||
// === Phase 5: Compact MDBX (before pipeline, so it runs on a smaller DB) ===
|
||||
let db_path = provider_factory.db_ref().path();
|
||||
Self::compact_mdbx(provider_factory.db_ref())?;
|
||||
|
||||
// Drop to release DB handle for swap
|
||||
drop(provider_factory);
|
||||
|
||||
let compact_path = db_path.with_file_name("db_compact");
|
||||
Self::swap_compacted_db(&db_path, &compact_path)?;
|
||||
|
||||
// === Phase 6: Reopen DB and run pipeline ===
|
||||
// The caller will reopen the environment and run the pipeline.
|
||||
// We return here — the pipeline step is handled in mod.rs after
|
||||
// reopening the database with the compacted copy.
|
||||
info!(target: "reth::cli", "Migration complete. You should now restart the node and let it run the pipeline to rebuild the remaining data.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_account_changesets<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Migrating AccountChangeSets → static files");
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let sf_provider = factory.static_file_provider();
|
||||
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::AccountChangeSets>()?;
|
||||
|
||||
let first_block = provider
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::AccountChangeSets)?;
|
||||
if first_block > 0 {
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
}
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(first_block))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Some(Ok((block_number, _))) = walker.peek() {
|
||||
if *block_number != block {
|
||||
break;
|
||||
}
|
||||
let (_, entry) = walker.next().expect("peeked")?;
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
count += entries.len() as u64;
|
||||
writer.append_account_changeset(entries, block)?;
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
|
||||
info!(target: "reth::cli", count, "AccountChangeSets migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_storage_changesets<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Migrating StorageChangeSets → static files");
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let sf_provider = factory.static_file_provider();
|
||||
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::StorageChangeSets>()?;
|
||||
|
||||
let first_block = provider
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::StorageChangeSets)?;
|
||||
if first_block > 0 {
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
}
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some((first_block, Address::ZERO).into()))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Some(Ok((key, _))) = walker.peek() {
|
||||
if key.block_number() != block {
|
||||
break;
|
||||
}
|
||||
let (key, entry) = walker.next().expect("peeked")?;
|
||||
entries.push(StorageBeforeTx {
|
||||
address: key.address(),
|
||||
key: entry.key,
|
||||
value: entry.value,
|
||||
});
|
||||
}
|
||||
|
||||
count += entries.len() as u64;
|
||||
writer.append_storage_changeset(entries, block)?;
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
|
||||
info!(target: "reth::cli", count, "StorageChangeSets migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_receipts<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N::Primitives: reth_primitives_traits::NodePrimitives<
|
||||
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
|
||||
>,
|
||||
{
|
||||
let provider = factory.provider()?;
|
||||
if !provider.prune_modes_ref().receipts_log_filter.is_empty() {
|
||||
info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX");
|
||||
return Ok(());
|
||||
}
|
||||
drop(provider);
|
||||
|
||||
let sf_provider = factory.static_file_provider();
|
||||
let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts);
|
||||
|
||||
if existing.is_some_and(|b| b >= tip) {
|
||||
info!(target: "reth::cli", "Receipts already in static files, skipping");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Migrating Receipts → static files");
|
||||
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let prune_start = provider
|
||||
.get_prune_checkpoint(PruneSegment::Receipts)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
if first_block > 0 {
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::Receipts)?;
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
writer.commit()?;
|
||||
}
|
||||
|
||||
let before = sf_provider
|
||||
.get_highest_static_file_tx(StaticFileSegment::Receipts)
|
||||
.map_or(0, |tx| tx + 1);
|
||||
|
||||
let block_range = first_block..=tip;
|
||||
|
||||
let segment = reth_static_file::segments::Receipts;
|
||||
reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?;
|
||||
|
||||
sf_provider.commit()?;
|
||||
|
||||
let after = sf_provider
|
||||
.get_highest_static_file_tx(StaticFileSegment::Receipts)
|
||||
.map_or(0, |tx| tx + 1);
|
||||
let count = after - before;
|
||||
info!(target: "reth::cli", count, "Receipts migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clears tables that can be recomputed by the pipeline and resets their
|
||||
/// stage checkpoints.
|
||||
fn clear_recomputable_tables<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Clearing recomputable MDBX tables");
|
||||
let db = factory.db_ref();
|
||||
|
||||
macro_rules! clear_table {
|
||||
($table:ty) => {{
|
||||
let tx = db.tx_mut()?;
|
||||
tx.clear::<$table>()?;
|
||||
tx.commit()?;
|
||||
info!(target: "reth::cli", table = <$table as Table>::NAME, "Cleared");
|
||||
}};
|
||||
}
|
||||
|
||||
// Migrated changeset tables (now in static files)
|
||||
clear_table!(tables::AccountChangeSets);
|
||||
clear_table!(tables::StorageChangeSets);
|
||||
|
||||
// Senders — rebuilt by SenderRecovery
|
||||
clear_table!(tables::TransactionSenders);
|
||||
|
||||
// Indices — rebuilt by TransactionLookup / IndexAccountHistory / IndexStorageHistory
|
||||
clear_table!(tables::TransactionHashNumbers);
|
||||
clear_table!(tables::AccountsHistory);
|
||||
clear_table!(tables::StoragesHistory);
|
||||
|
||||
// Plain state — superseded by hashed state in v2
|
||||
clear_table!(tables::PlainAccountState);
|
||||
clear_table!(tables::PlainStorageState);
|
||||
|
||||
// Trie — rebuilt by MerkleExecute
|
||||
clear_table!(tables::AccountsTrie);
|
||||
clear_table!(tables::StoragesTrie);
|
||||
|
||||
// Reset stage checkpoints so the pipeline rebuilds everything
|
||||
info!(target: "reth::cli", "Resetting stage checkpoints");
|
||||
let provider_rw = factory.database_provider_rw()?;
|
||||
for stage in [
|
||||
StageId::SenderRecovery,
|
||||
StageId::TransactionLookup,
|
||||
StageId::IndexAccountHistory,
|
||||
StageId::IndexStorageHistory,
|
||||
StageId::MerkleExecute,
|
||||
StageId::MerkleUnwind,
|
||||
] {
|
||||
provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(0))?;
|
||||
info!(target: "reth::cli", %stage, "Checkpoint reset to 0");
|
||||
}
|
||||
provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
info!(target: "reth::cli", "Recomputable tables cleared");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a compacted copy of the MDBX database.
|
||||
fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
|
||||
let db_path = db.path();
|
||||
let compact_path = db_path.with_file_name("db_compact");
|
||||
|
||||
reth_fs_util::create_dir_all(&compact_path)?;
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?compact_path, "Compacting MDBX database");
|
||||
|
||||
let compact_dest = compact_path.join("mdbx.dat");
|
||||
let dest_cstr = std::ffi::CString::new(
|
||||
compact_dest.to_str().ok_or_else(|| eyre::eyre!("compact path must be valid UTF-8"))?,
|
||||
)?;
|
||||
|
||||
let flags = ffi::MDBX_CP_COMPACT | ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
|
||||
|
||||
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
|
||||
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
|
||||
});
|
||||
|
||||
if rc != 0 {
|
||||
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
|
||||
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
|
||||
});
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "MDBX compaction complete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Swaps the original MDBX database with a compacted copy.
|
||||
fn swap_compacted_db(
|
||||
db_path: &std::path::Path,
|
||||
compact_path: &std::path::Path,
|
||||
) -> eyre::Result<()> {
|
||||
let backup_path = db_path.with_file_name("db_pre_compact");
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database");
|
||||
|
||||
std::fs::rename(db_path, &backup_path)?;
|
||||
|
||||
if let Err(e) = std::fs::rename(compact_path, db_path) {
|
||||
let _ = std::fs::rename(&backup_path, db_path);
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
std::fs::remove_dir_all(&backup_path)?;
|
||||
|
||||
info!(target: "reth::cli", "Database compaction swap complete");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ mod copy;
|
||||
mod diff;
|
||||
mod get;
|
||||
mod list;
|
||||
mod migrate_v2;
|
||||
mod prune_checkpoints;
|
||||
mod repair_trie;
|
||||
mod settings;
|
||||
@@ -77,6 +78,9 @@ pub enum Subcommands {
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
/// Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
|
||||
#[command(name = "migrate-v2")]
|
||||
MigrateV2(migrate_v2::Command),
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
|
||||
@@ -231,6 +235,13 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::MigrateV2(command) => {
|
||||
let Environment { provider_factory, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, ctx.task_executor.clone())?;
|
||||
|
||||
// Migrate changesets+receipts, clear tables, compact MDBX
|
||||
command.execute::<N>(provider_factory).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
368
crates/cli/commands/src/download/archive.rs
Normal file
368
crates/cli/commands/src/download/archive.rs
Normal file
@@ -0,0 +1,368 @@
|
||||
use super::{
|
||||
extract::{extract_archive_raw, streaming_download_and_extract, CompressionFormat},
|
||||
fetch::ArchiveFetcher,
|
||||
manifest::SnapshotArchive,
|
||||
planning::{PlannedArchive, PlannedDownloads},
|
||||
progress::{
|
||||
spawn_progress_display, ArchiveDownloadProgress, ArchiveExtractionProgress,
|
||||
ArchiveVerificationProgress, DownloadRequestLimiter, SharedProgress,
|
||||
},
|
||||
session::{ArchiveProcessContext, DownloadSession},
|
||||
verify::OutputVerifier,
|
||||
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
|
||||
};
|
||||
use eyre::Result;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::task;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const DOWNLOAD_CACHE_DIR: &str = ".download-cache";
|
||||
|
||||
/// Runs all planned modular archive downloads for one command invocation.
|
||||
pub(crate) async fn run_modular_downloads(
|
||||
planned_downloads: PlannedDownloads,
|
||||
target_dir: &Path,
|
||||
download_concurrency: usize,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let download_cache_dir = target_dir.join(DOWNLOAD_CACHE_DIR);
|
||||
fs::create_dir_all(&download_cache_dir)?;
|
||||
|
||||
let shared = SharedProgress::new(
|
||||
planned_downloads.total_download_size,
|
||||
planned_downloads.total_output_size,
|
||||
planned_downloads.total_archives() as u64,
|
||||
cancel_token.clone(),
|
||||
);
|
||||
let session = DownloadSession::new(
|
||||
Some(Arc::clone(&shared)),
|
||||
Some(DownloadRequestLimiter::new(download_concurrency)),
|
||||
cancel_token,
|
||||
);
|
||||
let ctx =
|
||||
ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session);
|
||||
|
||||
ModularDownloadJob::new(ctx, download_concurrency).run(planned_downloads).await
|
||||
}
|
||||
|
||||
/// Schedules modular archive work for one run of `reth download`.
|
||||
struct ModularDownloadJob {
|
||||
/// Shared paths and session state for each archive in this job.
|
||||
ctx: ArchiveProcessContext,
|
||||
/// Maximum number of archives processed at once.
|
||||
archive_concurrency: usize,
|
||||
}
|
||||
|
||||
impl ModularDownloadJob {
|
||||
/// Creates the modular download job for one command run.
|
||||
const fn new(ctx: ArchiveProcessContext, archive_concurrency: usize) -> Self {
|
||||
Self { ctx, archive_concurrency }
|
||||
}
|
||||
|
||||
/// Runs all planned archives and waits for the shared progress task to finish.
|
||||
async fn run(self, planned_downloads: PlannedDownloads) -> Result<()> {
|
||||
let shared = Arc::clone(
|
||||
self.ctx.session().progress().expect("modular downloads always use shared progress"),
|
||||
);
|
||||
let progress_handle = spawn_progress_display(Arc::clone(&shared));
|
||||
let ctx = self.ctx.clone();
|
||||
let results: Vec<Result<()>> = stream::iter(planned_downloads.archives)
|
||||
.map(move |archive| {
|
||||
let ctx = ctx.clone();
|
||||
async move { Self::process_archive(ctx, archive).await }
|
||||
})
|
||||
.buffer_unordered(self.archive_concurrency)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
shared.done.store(true, Ordering::Relaxed);
|
||||
let _ = progress_handle.await;
|
||||
|
||||
for result in results {
|
||||
result?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs one archive on the blocking pool so fetch and extraction stay off the async executor.
|
||||
async fn process_archive(ctx: ArchiveProcessContext, archive: PlannedArchive) -> Result<()> {
|
||||
task::spawn_blocking(move || ArchiveProcessor::new(archive, ctx).run()).await??;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit retry states for one modular archive.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ArchiveAttemptState {
|
||||
/// Start or restart one full archive attempt.
|
||||
RunAttempt,
|
||||
/// Check whether the extracted outputs verify.
|
||||
VerifyOutputs,
|
||||
/// Wait and decide whether another full attempt should run.
|
||||
RetryAttempt,
|
||||
/// Finish successfully.
|
||||
Complete,
|
||||
/// Stop with an error after retries are exhausted.
|
||||
Fail,
|
||||
}
|
||||
|
||||
/// Processes one modular archive from reuse check through extraction and verification.
|
||||
struct ArchiveProcessor {
|
||||
/// The concrete archive and component being processed.
|
||||
archive: PlannedArchive,
|
||||
/// Shared paths and session state for this archive attempt.
|
||||
ctx: ArchiveProcessContext,
|
||||
}
|
||||
|
||||
impl ArchiveProcessor {
|
||||
/// Creates a processor for one archive and the shared download context.
|
||||
fn new(archive: PlannedArchive, ctx: ArchiveProcessContext) -> Self {
|
||||
Self { archive, ctx }
|
||||
}
|
||||
|
||||
/// Runs the archive retry state machine until outputs are verified or retries are exhausted.
|
||||
fn run(self) -> Result<()> {
|
||||
let archive = self.archive();
|
||||
if self.try_reuse_outputs()? {
|
||||
info!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, "Skipping already verified plain files");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mode = ArchiveMode::new(&self.ctx)?;
|
||||
let format = CompressionFormat::from_url(&archive.file_name)?;
|
||||
let mut attempt = 1;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
let mut state = ArchiveAttemptState::RunAttempt;
|
||||
|
||||
loop {
|
||||
match state {
|
||||
ArchiveAttemptState::RunAttempt => {
|
||||
self.cleanup_outputs();
|
||||
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
file = %archive.file_name,
|
||||
component = %self.archive.component,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
"Retrying archive from scratch"
|
||||
);
|
||||
}
|
||||
|
||||
match self.run_attempt(mode, format) {
|
||||
Ok(()) => state = ArchiveAttemptState::VerifyOutputs,
|
||||
Err(error) if mode.retries_fetch_errors() => {
|
||||
warn!(target: "reth::cli",
|
||||
file = %archive.file_name,
|
||||
component = %self.archive.component,
|
||||
attempt,
|
||||
err = %format_args!("{error:#}"),
|
||||
"Archive attempt failed, retrying from scratch"
|
||||
);
|
||||
last_error = Some(error);
|
||||
state = ArchiveAttemptState::RetryAttempt;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::VerifyOutputs => {
|
||||
if self.verify_outputs_with_progress()? {
|
||||
state = ArchiveAttemptState::Complete;
|
||||
} else {
|
||||
warn!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, attempt, "Archive extracted, but output verification failed, retrying");
|
||||
state = ArchiveAttemptState::RetryAttempt;
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::RetryAttempt => {
|
||||
if attempt >= MAX_DOWNLOAD_RETRIES {
|
||||
state = ArchiveAttemptState::Fail;
|
||||
} else {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
attempt += 1;
|
||||
state = ArchiveAttemptState::RunAttempt;
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::Complete => return Ok(()),
|
||||
ArchiveAttemptState::Fail => {
|
||||
if let Some(error) = last_error {
|
||||
return Err(error.wrap_err(format!(
|
||||
"Failed after {} attempts for {}",
|
||||
MAX_DOWNLOAD_RETRIES, archive.file_name
|
||||
)));
|
||||
}
|
||||
|
||||
eyre::bail!(
|
||||
"Failed integrity validation after {} attempts for {}",
|
||||
MAX_DOWNLOAD_RETRIES,
|
||||
archive.file_name
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the concrete archive being fetched or verified.
|
||||
fn archive(&self) -> &SnapshotArchive {
|
||||
&self.archive.archive
|
||||
}
|
||||
|
||||
/// Returns the verifier for this archive's output files.
|
||||
fn output_verifier(&self) -> OutputVerifier<'_> {
|
||||
OutputVerifier::new(self.ctx.target_dir())
|
||||
}
|
||||
|
||||
/// Returns `true` if this archive can be reused from existing verified outputs.
|
||||
/// Returns `false` if a fresh archive attempt is still needed.
|
||||
fn try_reuse_outputs(&self) -> Result<bool> {
|
||||
if self.verify_outputs()? {
|
||||
self.mark_complete();
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Removes any partial outputs before a fresh archive attempt.
|
||||
fn cleanup_outputs(&self) {
|
||||
self.output_verifier().cleanup(&self.archive().output_files);
|
||||
}
|
||||
|
||||
/// Returns `true` if all declared plain outputs verify.
|
||||
/// Returns `false` if any output is missing or does not match.
|
||||
fn verify_outputs(&self) -> Result<bool> {
|
||||
self.output_verifier().verify(&self.archive().output_files)
|
||||
}
|
||||
|
||||
/// Records archive completion in shared progress once outputs verify.
|
||||
fn mark_complete(&self) {
|
||||
self.ctx.session().record_reused_archive(self.archive().size, self.archive().output_size());
|
||||
}
|
||||
|
||||
/// Executes one archive attempt according to the selected cache-vs-stream mode.
|
||||
fn run_attempt(&self, mode: ArchiveMode, format: CompressionFormat) -> Result<()> {
|
||||
mode.execute(self, format)
|
||||
}
|
||||
|
||||
/// Downloads the archive into the cache, then extracts from the cached file.
|
||||
fn run_cached_attempt(&self, format: CompressionFormat) -> Result<()> {
|
||||
let cache_dir =
|
||||
self.ctx.cache_dir().ok_or_else(|| eyre::eyre!("Missing download cache directory"))?;
|
||||
let fetcher =
|
||||
ArchiveFetcher::new(self.archive().url.clone(), cache_dir, self.ctx.session().clone());
|
||||
|
||||
if self.archive.ty == super::manifest::SnapshotComponentType::State {
|
||||
debug!(target: "reth::cli", url = %self.archive().url, "Downloading state snapshot archive");
|
||||
}
|
||||
|
||||
let download_result = {
|
||||
let mut download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
|
||||
let result = fetcher.download(Some(&mut download_progress));
|
||||
if let Ok(ref downloaded) = result &&
|
||||
download_progress.has_tracked_bytes()
|
||||
{
|
||||
download_progress.complete(downloaded.size);
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
let downloaded = match download_result {
|
||||
Ok(downloaded) => downloaded,
|
||||
Err(error) => {
|
||||
fetcher.cleanup_downloaded_files();
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
info!(target: "reth::cli",
|
||||
file = %self.archive().file_name,
|
||||
component = %self.archive.component,
|
||||
size = %super::progress::DownloadProgress::format_size(downloaded.size),
|
||||
"Archive download complete"
|
||||
);
|
||||
|
||||
let extract_result = self.extract_cached_archive(&downloaded.path, format);
|
||||
fetcher.cleanup_downloaded_files();
|
||||
extract_result
|
||||
}
|
||||
|
||||
/// Streams the archive directly into extraction without keeping a cached copy.
|
||||
fn run_streaming_attempt(&self, format: CompressionFormat) -> Result<()> {
|
||||
let _download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
|
||||
streaming_download_and_extract(
|
||||
&self.archive().url,
|
||||
format,
|
||||
self.ctx.target_dir(),
|
||||
self.ctx.session(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Extracts a cached archive file while updating shared extraction activity.
|
||||
fn extract_cached_archive(&self, archive_path: &Path, format: CompressionFormat) -> Result<()> {
|
||||
let mut extraction_progress = ArchiveExtractionProgress::new(self.ctx.session().progress());
|
||||
let file = fs::open(archive_path)?;
|
||||
let result = extract_archive_raw(
|
||||
file,
|
||||
format,
|
||||
self.ctx.target_dir(),
|
||||
Some(&mut extraction_progress),
|
||||
);
|
||||
extraction_progress.finish();
|
||||
result
|
||||
}
|
||||
|
||||
/// Returns `true` if all declared plain outputs verify while updating shared verification
|
||||
/// progress.
|
||||
fn verify_outputs_with_progress(&self) -> Result<bool> {
|
||||
let mut verification_progress =
|
||||
ArchiveVerificationProgress::new(self.ctx.session().progress());
|
||||
let verified = self
|
||||
.output_verifier()
|
||||
.verify_with_progress(&self.archive().output_files, Some(&mut verification_progress))?;
|
||||
if verified {
|
||||
verification_progress.complete(self.archive().output_size());
|
||||
}
|
||||
Ok(verified)
|
||||
}
|
||||
}
|
||||
|
||||
/// Chooses whether an archive attempt uses the cache or streams directly.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ArchiveMode {
|
||||
/// Download the archive to the cache, then extract it.
|
||||
Cached,
|
||||
/// Stream the archive directly into extraction.
|
||||
Streaming,
|
||||
}
|
||||
|
||||
impl ArchiveMode {
|
||||
/// Picks the archive mode from the process context.
|
||||
fn new(ctx: &ArchiveProcessContext) -> Result<Self> {
|
||||
if ctx.cache_dir().is_some() {
|
||||
ctx.session().require_request_limiter()?;
|
||||
return Ok(Self::Cached)
|
||||
}
|
||||
|
||||
Ok(Self::Streaming)
|
||||
}
|
||||
|
||||
/// Returns `true` when fetch failures should retry the whole archive attempt.
|
||||
const fn retries_fetch_errors(&self) -> bool {
|
||||
matches!(self, Self::Cached)
|
||||
}
|
||||
|
||||
/// Runs the selected archive mode for a single attempt.
|
||||
fn execute(&self, processor: &ArchiveProcessor, format: CompressionFormat) -> Result<()> {
|
||||
match self {
|
||||
Self::Cached => processor.run_cached_attempt(format),
|
||||
Self::Streaming => processor.run_streaming_attempt(format),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -248,6 +248,7 @@ fn selection_to_prune_mode(
|
||||
ComponentSelection::Distance(d) => {
|
||||
Some(PruneMode::Distance(min_distance.map_or(d, |min| d.max(min))))
|
||||
}
|
||||
ComponentSelection::Since(block) => Some(PruneMode::Before(block)),
|
||||
ComponentSelection::None => Some(min_distance.map_or(PruneMode::Full, PruneMode::Distance)),
|
||||
}
|
||||
}
|
||||
@@ -269,6 +270,7 @@ pub(crate) fn describe_prune_config(config: &Config) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Formats one prune mode for the generated config summary.
|
||||
fn format_mode(mode: &PruneMode) -> String {
|
||||
match mode {
|
||||
PruneMode::Full => "\"full\"".to_string(),
|
||||
@@ -453,6 +455,36 @@ mod tests {
|
||||
assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Distance(10_064)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn selections_since_maps_to_before_prune_mode() {
|
||||
let mut selections = BTreeMap::new();
|
||||
selections.insert(SnapshotComponentType::State, ComponentSelection::All);
|
||||
selections.insert(SnapshotComponentType::Headers, ComponentSelection::All);
|
||||
selections
|
||||
.insert(SnapshotComponentType::Transactions, ComponentSelection::Since(15_537_394));
|
||||
selections.insert(SnapshotComponentType::Receipts, ComponentSelection::Since(15_537_394));
|
||||
selections.insert(
|
||||
SnapshotComponentType::AccountChangesets,
|
||||
ComponentSelection::Since(15_537_394),
|
||||
);
|
||||
selections.insert(
|
||||
SnapshotComponentType::StorageChangesets,
|
||||
ComponentSelection::Since(15_537_394),
|
||||
);
|
||||
|
||||
let config = config_for_selections(
|
||||
&selections,
|
||||
&empty_manifest(),
|
||||
None,
|
||||
None::<&reth_chainspec::ChainSpec>,
|
||||
);
|
||||
|
||||
assert_eq!(config.prune.segments.bodies_history, Some(PruneMode::Before(15_537_394)));
|
||||
assert_eq!(config.prune.segments.receipts, Some(PruneMode::Before(15_537_394)));
|
||||
assert_eq!(config.prune.segments.account_history, Some(PruneMode::Before(15_537_394)));
|
||||
assert_eq!(config.prune.segments.storage_history, Some(PruneMode::Before(15_537_394)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn full_preset_matches_default_full_prune_config() {
|
||||
let mut selections = BTreeMap::new();
|
||||
|
||||
490
crates/cli/commands/src/download/extract.rs
Normal file
490
crates/cli/commands/src/download/extract.rs
Normal file
@@ -0,0 +1,490 @@
|
||||
use super::{
|
||||
fetch::{ArchiveFetcher, DownloadedArchive},
|
||||
progress::{
|
||||
ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress,
|
||||
DownloadRequestLimiter, ProgressReader, SharedProgress, SharedProgressReader,
|
||||
},
|
||||
session::DownloadSession,
|
||||
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
|
||||
};
|
||||
use eyre::{Result, WrapErr};
|
||||
use lz4::Decoder;
|
||||
use reqwest::blocking::Client as BlockingClient;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
io::Read,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tar::Archive;
|
||||
use tokio::task;
|
||||
use tracing::{info, warn};
|
||||
use url::Url;
|
||||
use zstd::stream::read::Decoder as ZstdDecoder;
|
||||
|
||||
const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
|
||||
const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
|
||||
const STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE: u64 = 64 * 1024 * 1024;
|
||||
const EXTRACTION_PROGRESS_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Supported compression formats for snapshots
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) enum CompressionFormat {
|
||||
/// LZ4-compressed tar archive.
|
||||
Lz4,
|
||||
/// Zstandard-compressed tar archive.
|
||||
Zstd,
|
||||
}
|
||||
|
||||
impl CompressionFormat {
|
||||
/// Detect compression format from file extension
|
||||
pub(crate) fn from_url(url: &str) -> Result<Self> {
|
||||
let path =
|
||||
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
|
||||
|
||||
if path.ends_with(EXTENSION_TAR_LZ4) {
|
||||
Ok(Self::Lz4)
|
||||
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
Ok(Self::Zstd)
|
||||
} else {
|
||||
Err(eyre::eyre!(
|
||||
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
|
||||
path
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a compressed tar archive to the target directory with progress tracking.
|
||||
fn extract_archive<R: Read>(
|
||||
reader: R,
|
||||
total_size: u64,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let progress_reader = ProgressReader::new(reader, total_size, cancel_token);
|
||||
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
let decoder = Decoder::new(progress_reader)?;
|
||||
Archive::new(decoder).unpack(target_dir)?;
|
||||
}
|
||||
CompressionFormat::Zstd => {
|
||||
let decoder = ZstdDecoder::new(progress_reader)?;
|
||||
Archive::new(decoder).unpack(target_dir)?;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extracts a compressed tar archive without progress tracking.
|
||||
pub(crate) fn extract_archive_raw<R: Read>(
|
||||
reader: R,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
unpack_archive(Archive::new(Decoder::new(reader)?), target_dir, progress)?;
|
||||
}
|
||||
CompressionFormat::Zstd => {
|
||||
unpack_archive(Archive::new(ZstdDecoder::new(reader)?), target_dir, progress)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unpack_archive<R: Read>(
|
||||
mut archive: Archive<R>,
|
||||
target_dir: &Path,
|
||||
mut progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
let entries = archive.entries().wrap_err_with(|| {
|
||||
format!("failed to read archive entries for `{}`", target_dir.display())
|
||||
})?;
|
||||
|
||||
for entry in entries {
|
||||
let mut entry = entry.wrap_err_with(|| {
|
||||
format!("failed to read archive entry for `{}`", target_dir.display())
|
||||
})?;
|
||||
extract_entry_with_progress(&mut entry, target_dir, progress.as_deref_mut())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_entry_with_progress<R: Read>(
|
||||
entry: &mut tar::Entry<'_, R>,
|
||||
target_dir: &Path,
|
||||
progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
let size = entry.header().entry_size().unwrap_or(0);
|
||||
let entry_type = entry.header().entry_type();
|
||||
|
||||
if !entry_type.is_file() || size == 0 {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
if let Some(progress) = progress {
|
||||
progress.record_extracted(size);
|
||||
}
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let Some(entry_path) = entry_destination_path(entry, target_dir)? else {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop));
|
||||
let unpack_result = entry
|
||||
.unpack_in(target_dir)
|
||||
.wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display()));
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
|
||||
let monitor_result = monitor.join();
|
||||
unpack_result?;
|
||||
|
||||
monitor_result.map_err(|_| eyre::eyre!("extraction progress monitor panicked"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn entry_destination_path<R: Read>(
|
||||
entry: &tar::Entry<'_, R>,
|
||||
target_dir: &Path,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
let mut file_dst = target_dir.to_path_buf();
|
||||
let path = entry.path().wrap_err("invalid path in archive entry")?;
|
||||
|
||||
for part in path.components() {
|
||||
match part {
|
||||
Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
|
||||
Component::ParentDir => return Ok(None),
|
||||
Component::Normal(part) => file_dst.push(part),
|
||||
}
|
||||
}
|
||||
|
||||
if file_dst == target_dir {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
Ok(Some(file_dst))
|
||||
}
|
||||
|
||||
fn spawn_extraction_progress_monitor(
|
||||
entry_path: PathBuf,
|
||||
progress: ArchiveExtractionProgressHandle,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> thread::JoinHandle<()> {
|
||||
thread::spawn(move || {
|
||||
let mut extracted = 0_u64;
|
||||
|
||||
loop {
|
||||
record_extracted_file_bytes(&entry_path, &progress, &mut extracted);
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
thread::sleep(EXTRACTION_PROGRESS_POLL_INTERVAL);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn record_extracted_file_bytes(
|
||||
entry_path: &Path,
|
||||
progress: &ArchiveExtractionProgressHandle,
|
||||
extracted: &mut u64,
|
||||
) {
|
||||
let Ok(meta) = fs::metadata(entry_path) else { return };
|
||||
let len = meta.len();
|
||||
if len > *extracted {
|
||||
progress.record_extracted(len - *extracted);
|
||||
*extracted = len;
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a snapshot from a local file.
|
||||
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let total_size = file.metadata()?.len();
|
||||
info!(target: "reth::cli",
|
||||
file = %path.display(),
|
||||
size = %DownloadProgress::format_size(total_size),
|
||||
"Extracting local archive"
|
||||
);
|
||||
let start = Instant::now();
|
||||
extract_archive(file, total_size, format, target_dir, CancellationToken::new())?;
|
||||
info!(target: "reth::cli",
|
||||
file = %path.display(),
|
||||
elapsed = %DownloadProgress::format_duration(start.elapsed()),
|
||||
"Local extraction complete"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Streams a remote archive directly into the extractor without writing to disk.
|
||||
///
|
||||
/// On failure, retries from scratch up to [`MAX_DOWNLOAD_RETRIES`] times.
|
||||
pub(crate) fn streaming_download_and_extract(
|
||||
url: &str,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
session: &DownloadSession,
|
||||
) -> Result<()> {
|
||||
let shared = session.progress();
|
||||
let quiet = session.progress().is_some();
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
"Retrying streaming download from scratch"
|
||||
);
|
||||
}
|
||||
|
||||
let client = BlockingClient::builder().connect_timeout(Duration::from_secs(30)).build()?;
|
||||
let _request_permit = session
|
||||
.request_limiter()
|
||||
.map(|limiter| limiter.acquire(session.progress(), session.cancel_token()))
|
||||
.transpose()?;
|
||||
|
||||
let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
|
||||
Ok(r) => r,
|
||||
Err(error) => {
|
||||
let err = eyre::Error::from(error);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
warn!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
err = %err,
|
||||
"Streaming request failed, retrying"
|
||||
);
|
||||
}
|
||||
last_error = Some(err);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !quiet && let Some(size) = response.content_length() {
|
||||
info!(target: "reth::cli",
|
||||
url = %url,
|
||||
size = %DownloadProgress::format_size(size),
|
||||
"Streaming archive"
|
||||
);
|
||||
}
|
||||
|
||||
let result = if let Some(progress) = shared {
|
||||
let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) };
|
||||
extract_archive_raw(reader, format, target_dir, None)
|
||||
} else {
|
||||
let total_size = response.content_length().unwrap_or(0);
|
||||
extract_archive(
|
||||
response,
|
||||
total_size,
|
||||
format,
|
||||
target_dir,
|
||||
session.cancel_token().clone(),
|
||||
)
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(error) => {
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
warn!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
err = %error,
|
||||
"Streaming extraction failed, retrying"
|
||||
);
|
||||
}
|
||||
last_error = Some(error);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or_else(|| {
|
||||
eyre::eyre!("Streaming download failed after {MAX_DOWNLOAD_RETRIES} attempts")
|
||||
}))
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
|
||||
fn download_and_extract(
|
||||
url: &str,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
session: DownloadSession,
|
||||
) -> Result<()> {
|
||||
let quiet = session.progress().is_some();
|
||||
let fetcher = ArchiveFetcher::new(url.to_string(), target_dir, session.clone());
|
||||
let DownloadedArchive { path: downloaded_path, size: total_size } = fetcher.download(None)?;
|
||||
|
||||
let file_name =
|
||||
downloaded_path.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
|
||||
|
||||
if !quiet {
|
||||
info!(target: "reth::cli",
|
||||
file = %file_name,
|
||||
size = %DownloadProgress::format_size(total_size),
|
||||
"Extracting archive"
|
||||
);
|
||||
}
|
||||
let file = fs::open(&downloaded_path)?;
|
||||
|
||||
if quiet {
|
||||
extract_archive_raw(file, format, target_dir, None)?;
|
||||
} else {
|
||||
extract_archive(file, total_size, format, target_dir, session.cancel_token().clone())?;
|
||||
info!(target: "reth::cli",
|
||||
file = %file_name,
|
||||
"Extraction complete"
|
||||
);
|
||||
}
|
||||
|
||||
fetcher.cleanup_downloaded_files();
|
||||
session.record_archive_output_complete(total_size);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
///
|
||||
/// Supports `file://` URLs for local files and HTTP(S) URLs for remote downloads.
|
||||
/// When `resumable` is true, downloads to a `.part` file first with HTTP Range resume
|
||||
/// support. Otherwise streams directly into the extractor.
|
||||
fn blocking_download_and_extract(
|
||||
url: &str,
|
||||
target_dir: &Path,
|
||||
shared: Option<Arc<SharedProgress>>,
|
||||
resumable: bool,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
|
||||
if let Ok(parsed_url) = Url::parse(url) &&
|
||||
parsed_url.scheme() == "file"
|
||||
{
|
||||
let session = DownloadSession::new(shared, request_limiter, cancel_token);
|
||||
let file_path = parsed_url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
|
||||
let result = extract_from_file(&file_path, format, target_dir);
|
||||
if result.is_ok() {
|
||||
session.record_archive_output_complete(file_path.metadata()?.len());
|
||||
}
|
||||
result
|
||||
} else if let Some(request_limiter) = request_limiter {
|
||||
download_and_extract(
|
||||
url,
|
||||
format,
|
||||
target_dir,
|
||||
DownloadSession::new(shared, Some(request_limiter), cancel_token),
|
||||
)
|
||||
} else if resumable {
|
||||
let session =
|
||||
DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token);
|
||||
download_and_extract(url, format, target_dir, session)
|
||||
} else {
|
||||
let session = DownloadSession::new(shared, None, cancel_token);
|
||||
let result = streaming_download_and_extract(url, format, target_dir, &session);
|
||||
if result.is_ok() {
|
||||
session.record_archive_output_complete(0);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot archive asynchronously.
|
||||
///
|
||||
/// When `shared` is provided, download progress is reported to the shared
|
||||
/// counter for aggregated display. Otherwise uses a local progress bar.
|
||||
/// When `resumable` is true, uses two-phase download with `.part` files.
|
||||
pub(crate) async fn stream_and_extract(
|
||||
url: &str,
|
||||
target_dir: &Path,
|
||||
shared: Option<Arc<SharedProgress>>,
|
||||
resumable: bool,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let target_dir = target_dir.to_path_buf();
|
||||
let url = url.to_string();
|
||||
task::spawn_blocking(move || {
|
||||
blocking_download_and_extract(
|
||||
&url,
|
||||
&target_dir,
|
||||
shared,
|
||||
resumable,
|
||||
request_limiter,
|
||||
cancel_token,
|
||||
)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_detection() {
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
|
||||
}
|
||||
}
|
||||
1018
crates/cli/commands/src/download/fetch.rs
Normal file
1018
crates/cli/commands/src/download/fetch.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,10 @@ use std::{
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
fn is_zero(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
/// A snapshot manifest describes available components for a snapshot at a given block height.
|
||||
///
|
||||
/// Each component is either a single archive (state) or a set of chunked archives (static file
|
||||
@@ -62,6 +66,12 @@ pub struct SingleArchive {
|
||||
pub file: String,
|
||||
/// Compressed archive size in bytes.
|
||||
pub size: u64,
|
||||
/// Total extracted plain-output size in bytes.
|
||||
///
|
||||
/// Older manifests may omit this, in which case downloaders should derive it from
|
||||
/// `output_files`.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
pub decompressed_size: u64,
|
||||
/// Optional BLAKE3 checksum of the compressed archive.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub blake3: Option<String>,
|
||||
@@ -83,6 +93,12 @@ pub struct ChunkedArchive {
|
||||
/// Computed during manifest generation. Older manifests may omit this.
|
||||
#[serde(default)]
|
||||
pub chunk_sizes: Vec<u64>,
|
||||
/// Extracted plain-output size of each chunk in bytes, ordered from first to last.
|
||||
///
|
||||
/// Older manifests may omit this, in which case downloaders should derive it from
|
||||
/// `chunk_output_files`.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub chunk_decompressed_sizes: Vec<u64>,
|
||||
/// Expected extracted plain files per chunk, ordered from first to last.
|
||||
///
|
||||
/// This is the authoritative integrity source for the modular download path.
|
||||
@@ -101,9 +117,9 @@ pub struct OutputFileChecksum {
|
||||
pub blake3: String,
|
||||
}
|
||||
|
||||
/// A single archive with concrete URL and optional integrity metadata.
|
||||
/// A concrete snapshot archive with its download and verification metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ArchiveDescriptor {
|
||||
pub struct SnapshotArchive {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub size: u64,
|
||||
@@ -111,6 +127,13 @@ pub struct ArchiveDescriptor {
|
||||
pub output_files: Vec<OutputFileChecksum>,
|
||||
}
|
||||
|
||||
impl SnapshotArchive {
|
||||
/// Returns the total extracted plain-output size for this archive.
|
||||
pub fn output_size(&self) -> u64 {
|
||||
self.output_files.iter().map(|file| file.size).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// How much of a component to download.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ComponentSelection {
|
||||
@@ -119,6 +142,9 @@ pub enum ComponentSelection {
|
||||
/// Download only the most recent chunks covering at least `distance` blocks.
|
||||
/// Maps to `PruneMode::Distance(distance)` in the generated config.
|
||||
Distance(u64),
|
||||
/// Download chunks starting at the specified block number.
|
||||
/// Maps to `PruneMode::Before(block)` in the generated config.
|
||||
Since(u64),
|
||||
/// Don't download this component at all.
|
||||
/// Maps to `PruneMode::Full` for tx-based segments, or a minimal distance for others.
|
||||
None,
|
||||
@@ -129,6 +155,7 @@ impl std::fmt::Display for ComponentSelection {
|
||||
match self {
|
||||
Self::All => write!(f, "All"),
|
||||
Self::Distance(d) => write!(f, "Last {d} blocks"),
|
||||
Self::Since(block) => write!(f, "Since block {block}"),
|
||||
Self::None => write!(f, "None"),
|
||||
}
|
||||
}
|
||||
@@ -311,19 +338,19 @@ impl SnapshotManifest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns concrete archive descriptors for a component, optionally limited to distance.
|
||||
pub fn archive_descriptors_for_distance(
|
||||
/// Returns concrete snapshot archives for a component, optionally limited to distance.
|
||||
pub fn snapshot_archives_for_distance(
|
||||
&self,
|
||||
ty: SnapshotComponentType,
|
||||
distance: Option<u64>,
|
||||
) -> Vec<ArchiveDescriptor> {
|
||||
) -> Vec<SnapshotArchive> {
|
||||
let Some(component) = self.component(ty) else {
|
||||
return vec![];
|
||||
};
|
||||
|
||||
match component {
|
||||
ComponentManifest::Single(single) => {
|
||||
vec![ArchiveDescriptor {
|
||||
vec![SnapshotArchive {
|
||||
url: format!("{}/{}", self.base_url_or_empty(), single.file),
|
||||
file_name: single.file.clone(),
|
||||
size: single.size,
|
||||
@@ -353,7 +380,7 @@ impl SnapshotManifest {
|
||||
let output_files =
|
||||
chunked.chunk_output_files.get(i as usize).cloned().unwrap_or_default();
|
||||
|
||||
ArchiveDescriptor {
|
||||
SnapshotArchive {
|
||||
url: format!("{}/{}", self.base_url_or_empty(), file_name),
|
||||
file_name,
|
||||
size,
|
||||
@@ -394,6 +421,36 @@ impl SnapshotManifest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the exact extracted plain-output size for a component given a distance selection.
|
||||
pub fn output_size_for_distance(
|
||||
&self,
|
||||
ty: SnapshotComponentType,
|
||||
distance: Option<u64>,
|
||||
) -> u64 {
|
||||
let Some(component) = self.component(ty) else {
|
||||
return 0;
|
||||
};
|
||||
|
||||
match component {
|
||||
ComponentManifest::Single(single) => single.output_size(),
|
||||
ComponentManifest::Chunked(chunked) => {
|
||||
let num_chunks = chunked.num_chunks();
|
||||
let start_chunk = match distance {
|
||||
Some(dist) => {
|
||||
let needed = dist.min(chunked.total_blocks);
|
||||
let needed_chunks = needed.div_ceil(chunked.blocks_per_file);
|
||||
num_chunks.saturating_sub(needed_chunks)
|
||||
}
|
||||
None => 0,
|
||||
};
|
||||
|
||||
(start_chunk..num_chunks)
|
||||
.map(|index| chunked.chunk_output_size(index as usize))
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of chunks that would be downloaded for a given distance.
|
||||
pub fn chunks_for_distance(&self, ty: SnapshotComponentType, distance: Option<u64>) -> u64 {
|
||||
let Some(ComponentManifest::Chunked(chunked)) = self.component(ty) else {
|
||||
@@ -417,6 +474,14 @@ impl ComponentManifest {
|
||||
Self::Chunked(c) => c.chunk_sizes.iter().sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total extracted plain-output size for this component.
|
||||
pub fn total_output_size(&self) -> u64 {
|
||||
match self {
|
||||
Self::Single(single) => single.output_size(),
|
||||
Self::Chunked(chunked) => chunked.total_output_size(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkedArchive {
|
||||
@@ -424,6 +489,39 @@ impl ChunkedArchive {
|
||||
pub fn num_chunks(&self) -> u64 {
|
||||
self.total_blocks.div_ceil(self.blocks_per_file)
|
||||
}
|
||||
|
||||
/// Returns the extracted plain-output size for one chunk.
|
||||
pub fn chunk_output_size(&self, index: usize) -> u64 {
|
||||
self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| {
|
||||
self.chunk_output_files
|
||||
.get(index)
|
||||
.map(|files| files.iter().map(|file| file.size).sum())
|
||||
.unwrap_or(0)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the total extracted plain-output size across all chunks.
|
||||
pub fn total_output_size(&self) -> u64 {
|
||||
if !self.chunk_decompressed_sizes.is_empty() {
|
||||
self.chunk_decompressed_sizes.iter().sum()
|
||||
} else {
|
||||
self.chunk_output_files
|
||||
.iter()
|
||||
.map(|files| files.iter().map(|file| file.size).sum::<u64>())
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SingleArchive {
|
||||
/// Returns the total extracted plain-output size for this archive.
|
||||
pub fn output_size(&self) -> u64 {
|
||||
if self.decompressed_size != 0 {
|
||||
self.decompressed_size
|
||||
} else {
|
||||
self.output_files.iter().map(|file| file.size).sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a snapshot manifest from a URL.
|
||||
@@ -512,6 +610,10 @@ pub fn generate_manifest(
|
||||
blocks_per_file,
|
||||
total_blocks: block,
|
||||
chunk_sizes,
|
||||
chunk_decompressed_sizes: chunk_output_files
|
||||
.iter()
|
||||
.map(|files| files.iter().map(|file| file.size).sum())
|
||||
.collect(),
|
||||
chunk_output_files,
|
||||
}),
|
||||
);
|
||||
@@ -528,6 +630,7 @@ pub fn generate_manifest(
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: state_size,
|
||||
decompressed_size: state_output_files.iter().map(|file| file.size).sum(),
|
||||
blake3: None,
|
||||
output_files: state_output_files,
|
||||
}),
|
||||
@@ -542,6 +645,7 @@ pub fn generate_manifest(
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: rocksdb_size,
|
||||
decompressed_size: rocksdb_output_files.iter().map(|file| file.size).sum(),
|
||||
blake3: None,
|
||||
output_files: rocksdb_output_files,
|
||||
}),
|
||||
@@ -810,6 +914,7 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 0,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
}),
|
||||
@@ -820,6 +925,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_500_000,
|
||||
chunk_sizes: vec![80_000, 100_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![], vec![], vec![]],
|
||||
}),
|
||||
);
|
||||
@@ -829,6 +935,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_500_000,
|
||||
chunk_sizes: vec![40_000, 50_000, 60_000],
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![], vec![], vec![]],
|
||||
}),
|
||||
);
|
||||
@@ -879,6 +986,7 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: 777,
|
||||
decompressed_size: 0,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
}),
|
||||
@@ -936,6 +1044,7 @@ mod tests {
|
||||
fn component_selection_display() {
|
||||
assert_eq!(ComponentSelection::All.to_string(), "All");
|
||||
assert_eq!(ComponentSelection::Distance(10_064).to_string(), "Last 10064 blocks");
|
||||
assert_eq!(ComponentSelection::Since(15_537_394).to_string(), "Since block 15537394");
|
||||
assert_eq!(ComponentSelection::None.to_string(), "None");
|
||||
}
|
||||
|
||||
@@ -950,6 +1059,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 24_396_822,
|
||||
chunk_sizes: vec![100; 49], // 49 chunks
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![]; 49],
|
||||
}),
|
||||
);
|
||||
@@ -992,6 +1102,68 @@ mod tests {
|
||||
assert_eq!(m.size_for_distance(SnapshotComponentType::Receipts, None), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_size_for_distance_uses_manifest_or_output_files() {
|
||||
let m = test_manifest();
|
||||
assert_eq!(m.output_size_for_distance(SnapshotComponentType::Transactions, None), 0);
|
||||
|
||||
let mut components = BTreeMap::new();
|
||||
components.insert(
|
||||
"state".to_string(),
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 1_000,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
size: 1_000,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
components.insert(
|
||||
"transactions".to_string(),
|
||||
ComponentManifest::Chunked(ChunkedArchive {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![80_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![111, 222],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
|
||||
size: 111,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_500000_999999.bin".to_string(),
|
||||
size: 222,
|
||||
blake3: "h1".to_string(),
|
||||
}],
|
||||
],
|
||||
}),
|
||||
);
|
||||
let manifest = SnapshotManifest {
|
||||
block: 1_000_000,
|
||||
chain_id: 1,
|
||||
storage_version: 2,
|
||||
timestamp: 0,
|
||||
base_url: Some("https://example.com".to_string()),
|
||||
reth_version: None,
|
||||
components,
|
||||
};
|
||||
|
||||
assert_eq!(manifest.output_size_for_distance(SnapshotComponentType::State, None), 1_000);
|
||||
assert_eq!(
|
||||
manifest.output_size_for_distance(SnapshotComponentType::Transactions, None),
|
||||
333
|
||||
);
|
||||
assert_eq!(
|
||||
manifest.output_size_for_distance(SnapshotComponentType::Transactions, Some(500_000)),
|
||||
222
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_descriptors_include_checksum_metadata() {
|
||||
let mut components = BTreeMap::new();
|
||||
@@ -1000,6 +1172,7 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 1_000,
|
||||
blake3: Some("abc123".to_string()),
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
@@ -1014,6 +1187,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![80_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![111, 222],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
|
||||
@@ -1039,13 +1213,13 @@ mod tests {
|
||||
components,
|
||||
};
|
||||
|
||||
let state = m.archive_descriptors_for_distance(SnapshotComponentType::State, None);
|
||||
let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None);
|
||||
assert_eq!(state.len(), 1);
|
||||
assert_eq!(state[0].file_name, "state.tar.zst");
|
||||
assert_eq!(state[0].blake3.as_deref(), Some("abc123"));
|
||||
assert_eq!(state[0].output_files.len(), 1);
|
||||
|
||||
let tx = m.archive_descriptors_for_distance(SnapshotComponentType::Transactions, None);
|
||||
let tx = m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, None);
|
||||
assert_eq!(tx.len(), 2);
|
||||
assert_eq!(tx[0].blake3, None);
|
||||
assert_eq!(tx[1].blake3, None);
|
||||
@@ -1068,6 +1242,7 @@ mod tests {
|
||||
panic!("state should be a single archive")
|
||||
};
|
||||
assert_eq!(state.file, "state.tar.zst");
|
||||
assert!(state.decompressed_size > 0);
|
||||
assert!(!state.output_files.is_empty());
|
||||
assert_eq!(state.output_files[0].path, "db/mdbx.dat");
|
||||
assert!(output.path().join("state.tar.zst").exists());
|
||||
@@ -1092,6 +1267,7 @@ mod tests {
|
||||
panic!("rocksdb indices should be a single archive")
|
||||
};
|
||||
assert_eq!(rocksdb.file, "rocksdb_indices.tar.zst");
|
||||
assert!(rocksdb.decompressed_size > 0);
|
||||
assert!(!rocksdb.output_files.is_empty());
|
||||
assert_eq!(rocksdb.output_files[0].path, "rocksdb/CURRENT");
|
||||
assert!(output.path().join("rocksdb_indices.tar.zst").exists());
|
||||
|
||||
@@ -45,6 +45,7 @@ pub struct SnapshotManifestCommand {
|
||||
}
|
||||
|
||||
impl SnapshotManifestCommand {
|
||||
/// Packages snapshot archives and writes the manifest file.
|
||||
pub fn execute(self) -> Result<()> {
|
||||
let block = match self.block {
|
||||
Some(block) => block,
|
||||
@@ -88,6 +89,7 @@ impl SnapshotManifestCommand {
|
||||
}
|
||||
}
|
||||
|
||||
/// Infers the snapshot block from the source datadir.
|
||||
fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
if let Ok(block) = infer_snapshot_block_from_db(source_datadir) {
|
||||
return Ok(block);
|
||||
@@ -102,6 +104,7 @@ fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
Ok(block)
|
||||
}
|
||||
|
||||
/// Reads the snapshot block from the source database Finish stage checkpoint.
|
||||
fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let candidates = [source_datadir.join("db"), source_datadir.to_path_buf()];
|
||||
|
||||
@@ -126,6 +129,7 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64>
|
||||
)
|
||||
}
|
||||
|
||||
/// Infers the snapshot block from the highest header static-file range.
|
||||
fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let max_end = header_ranges(source_datadir)?
|
||||
.into_iter()
|
||||
@@ -135,6 +139,7 @@ fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result
|
||||
Ok(max_end)
|
||||
}
|
||||
|
||||
/// Infers the static-file block span from header file ranges.
|
||||
fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let mut inferred = None;
|
||||
for (start, end) in header_ranges(source_datadir)? {
|
||||
@@ -161,6 +166,7 @@ fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Collects header static-file ranges from the source datadir.
|
||||
fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
|
||||
let static_files_dir = source_datadir.join("static_files");
|
||||
let static_files_dir =
|
||||
@@ -183,6 +189,7 @@ fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
/// Parses the block range from a header static-file name.
|
||||
fn parse_headers_range(file_name: &str) -> Option<(u64, u64)> {
|
||||
let remainder = file_name.strip_prefix("static_file_headers_")?;
|
||||
let (start, end_with_suffix) = remainder.split_once('_')?;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
322
crates/cli/commands/src/download/planning.rs
Normal file
322
crates/cli/commands/src/download/planning.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
use super::{manifest::*, verify::OutputVerifier};
|
||||
use eyre::Result;
|
||||
use std::{collections::BTreeMap, path::Path};
|
||||
use tracing::info;
|
||||
|
||||
/// One archive selected from the manifest, along with its component name.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PlannedArchive {
|
||||
/// Snapshot component type this archive belongs to.
|
||||
pub(crate) ty: SnapshotComponentType,
|
||||
/// User-facing component name used in logs.
|
||||
pub(crate) component: String,
|
||||
/// Concrete snapshot archive metadata resolved from the manifest.
|
||||
pub(crate) archive: SnapshotArchive,
|
||||
}
|
||||
|
||||
/// The archive list for a modular snapshot download.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct PlannedDownloads {
|
||||
/// Concrete archives that still need reuse checks or processing.
|
||||
pub(crate) archives: Vec<PlannedArchive>,
|
||||
/// Total compressed download size of all planned archives.
|
||||
pub(crate) total_download_size: u64,
|
||||
/// Total extracted plain-output size of all planned archives.
|
||||
pub(crate) total_output_size: u64,
|
||||
}
|
||||
|
||||
impl PlannedDownloads {
|
||||
/// Returns the number of concrete archives queued for this snapshot selection.
|
||||
pub(crate) const fn total_archives(&self) -> usize {
|
||||
self.archives.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the sort priority used to schedule archives.
|
||||
pub(crate) const fn archive_priority_rank(ty: SnapshotComponentType) -> u8 {
|
||||
match ty {
|
||||
SnapshotComponentType::State => 0,
|
||||
SnapshotComponentType::RocksdbIndices => 1,
|
||||
_ => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Startup summary showing how much of the selected work can be reused.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub(crate) struct DownloadStartupSummary {
|
||||
/// Archives whose declared outputs already verify on disk.
|
||||
pub(crate) reusable: usize,
|
||||
/// Archives that still need to be downloaded or retried.
|
||||
pub(crate) needs_download: usize,
|
||||
}
|
||||
|
||||
/// Checks selected archives against existing output files before work begins.
|
||||
pub(crate) fn summarize_download_startup(
|
||||
all_downloads: &[PlannedArchive],
|
||||
target_dir: &Path,
|
||||
) -> Result<DownloadStartupSummary> {
|
||||
let mut summary = DownloadStartupSummary::default();
|
||||
let verifier = OutputVerifier::new(target_dir);
|
||||
|
||||
for planned in all_downloads {
|
||||
if verifier.verify(&planned.archive.output_files)? {
|
||||
summary.reusable += 1;
|
||||
} else {
|
||||
summary.needs_download += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Converts a selection into the manifest distance form used for archive lookup.
|
||||
fn selection_archive_distance(
|
||||
selection: &ComponentSelection,
|
||||
snapshot_block: u64,
|
||||
) -> Option<Option<u64>> {
|
||||
match selection {
|
||||
ComponentSelection::All => Some(None),
|
||||
ComponentSelection::Distance(distance) => Some(Some(*distance)),
|
||||
ComponentSelection::Since(block) => Some(Some(snapshot_block.saturating_sub(*block) + 1)),
|
||||
ComponentSelection::None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sorts planned archives into a stable processing order.
|
||||
fn sort_planned_archives(all_downloads: &mut [PlannedArchive]) {
|
||||
all_downloads.sort_by(|a, b| {
|
||||
archive_priority_rank(a.ty)
|
||||
.cmp(&archive_priority_rank(b.ty))
|
||||
.then_with(|| a.component.cmp(&b.component))
|
||||
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
|
||||
});
|
||||
}
|
||||
|
||||
/// Expands component selections into the archives that need to be processed.
|
||||
pub(crate) fn collect_planned_archives(
|
||||
manifest: &SnapshotManifest,
|
||||
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
|
||||
) -> Result<PlannedDownloads> {
|
||||
let mut archives = Vec::new();
|
||||
let mut total_download_size = 0;
|
||||
let mut total_output_size = 0;
|
||||
|
||||
for (ty, selection) in selections {
|
||||
let Some(distance) = selection_archive_distance(selection, manifest.block) else {
|
||||
continue;
|
||||
};
|
||||
total_download_size += manifest.size_for_distance(*ty, distance);
|
||||
total_output_size += manifest.output_size_for_distance(*ty, distance);
|
||||
|
||||
let snapshot_archives = manifest.snapshot_archives_for_distance(*ty, distance);
|
||||
let component = ty.display_name().to_string();
|
||||
if !snapshot_archives.is_empty() {
|
||||
info!(target: "reth::cli",
|
||||
component = %component,
|
||||
archives = snapshot_archives.len(),
|
||||
selection = %selection,
|
||||
"Queued component for download"
|
||||
);
|
||||
}
|
||||
|
||||
for archive in snapshot_archives {
|
||||
if archive.output_files.is_empty() {
|
||||
eyre::bail!(
|
||||
"Invalid modular manifest: {} is missing plain output checksum metadata",
|
||||
archive.file_name
|
||||
);
|
||||
}
|
||||
|
||||
archives.push(PlannedArchive { ty: *ty, component: component.clone(), archive });
|
||||
}
|
||||
}
|
||||
|
||||
sort_planned_archives(&mut archives);
|
||||
Ok(PlannedDownloads { archives, total_download_size, total_output_size })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn summarize_download_startup_counts_reusable_and_needs_download() {
|
||||
let dir = tempdir().unwrap();
|
||||
let target_dir = dir.path();
|
||||
let ok_file = target_dir.join("ok.bin");
|
||||
std::fs::write(&ok_file, vec![1_u8; 4]).unwrap();
|
||||
let ok_hash = blake3::hash(&[1_u8; 4]).to_hex().to_string();
|
||||
|
||||
let planned = vec![
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::State,
|
||||
component: "State".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/ok.tar.zst".to_string(),
|
||||
file_name: "ok.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "ok.bin".to_string(),
|
||||
size: 4,
|
||||
blake3: ok_hash,
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Headers,
|
||||
component: "Headers".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/missing.tar.zst".to_string(),
|
||||
file_name: "missing.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "missing.bin".to_string(),
|
||||
size: 1,
|
||||
blake3: "deadbeef".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Transactions,
|
||||
component: "Transactions".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/bad-size.tar.zst".to_string(),
|
||||
file_name: "bad-size.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let summary = summarize_download_startup(&planned, target_dir).unwrap();
|
||||
assert_eq!(summary.reusable, 1);
|
||||
assert_eq!(summary.needs_download, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_priority_prefers_state_then_rocksdb() {
|
||||
let mut planned = [
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Transactions,
|
||||
component: "Transactions".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u3".to_string(),
|
||||
file_name: "t.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "a".to_string(),
|
||||
size: 1,
|
||||
blake3: "x".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::RocksdbIndices,
|
||||
component: "RocksDB Indices".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u2".to_string(),
|
||||
file_name: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "b".to_string(),
|
||||
size: 1,
|
||||
blake3: "y".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::State,
|
||||
component: "State (mdbx)".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u1".to_string(),
|
||||
file_name: "state.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "c".to_string(),
|
||||
size: 1,
|
||||
blake3: "z".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
planned.sort_by(|a, b| {
|
||||
archive_priority_rank(a.ty)
|
||||
.cmp(&archive_priority_rank(b.ty))
|
||||
.then_with(|| a.component.cmp(&b.component))
|
||||
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
|
||||
});
|
||||
|
||||
assert_eq!(planned[0].ty, SnapshotComponentType::State);
|
||||
assert_eq!(planned[1].ty, SnapshotComponentType::RocksdbIndices);
|
||||
assert_eq!(planned[2].ty, SnapshotComponentType::Transactions);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_planned_archives_tracks_download_and_output_totals() {
|
||||
let mut components = BTreeMap::new();
|
||||
components.insert(
|
||||
"state".to_string(),
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 10,
|
||||
decompressed_size: 100,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
size: 100,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
components.insert(
|
||||
"transactions".to_string(),
|
||||
ComponentManifest::Chunked(ChunkedArchive {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![20, 30],
|
||||
chunk_decompressed_sizes: vec![200, 300],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/tx-0".to_string(),
|
||||
size: 200,
|
||||
blake3: "h1".to_string(),
|
||||
}],
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/tx-1".to_string(),
|
||||
size: 300,
|
||||
blake3: "h2".to_string(),
|
||||
}],
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
let manifest = SnapshotManifest {
|
||||
block: 1_000_000,
|
||||
chain_id: 1,
|
||||
storage_version: 2,
|
||||
timestamp: 0,
|
||||
base_url: Some("https://example.com".to_string()),
|
||||
reth_version: None,
|
||||
components,
|
||||
};
|
||||
|
||||
let selections = BTreeMap::from([
|
||||
(SnapshotComponentType::State, ComponentSelection::All),
|
||||
(SnapshotComponentType::Transactions, ComponentSelection::Distance(500_000)),
|
||||
]);
|
||||
|
||||
let planned = collect_planned_archives(&manifest, &selections).unwrap();
|
||||
|
||||
assert_eq!(planned.total_download_size, 40);
|
||||
assert_eq!(planned.total_output_size, 400);
|
||||
assert_eq!(planned.archives.len(), 2);
|
||||
}
|
||||
}
|
||||
844
crates/cli/commands/src/download/progress.rs
Normal file
844
crates/cli/commands/src/download/progress.rs
Normal file
@@ -0,0 +1,844 @@
|
||||
use eyre::Result;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use std::{
|
||||
io::{self, Read, Write},
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc, Condvar, Mutex,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
|
||||
|
||||
/// Tracks download progress and throttles display updates to every 100ms.
|
||||
pub(crate) struct DownloadProgress {
|
||||
/// Bytes copied so far for this single download.
|
||||
pub(crate) downloaded: u64,
|
||||
/// Total bytes expected for this single download.
|
||||
total_size: u64,
|
||||
/// Time when the progress line was last printed.
|
||||
last_displayed: Instant,
|
||||
/// Time when this progress tracker started.
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
impl DownloadProgress {
|
||||
/// Creates new progress tracker with given total size
|
||||
pub(crate) fn new(total_size: u64) -> Self {
|
||||
let now = Instant::now();
|
||||
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
|
||||
}
|
||||
|
||||
/// Converts bytes to human readable format (B, KB, MB, GB)
|
||||
pub(crate) fn format_size(size: u64) -> String {
|
||||
let mut size = size as f64;
|
||||
let mut unit_index = 0;
|
||||
|
||||
while size >= 1024.0 && unit_index < BYTE_UNITS.len() - 1 {
|
||||
size /= 1024.0;
|
||||
unit_index += 1;
|
||||
}
|
||||
|
||||
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
|
||||
}
|
||||
|
||||
/// Format duration as human readable string
|
||||
pub(crate) fn format_duration(duration: Duration) -> String {
|
||||
let secs = duration.as_secs();
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m {}s", secs / 60, secs % 60)
|
||||
} else {
|
||||
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates progress bar (for single-archive legacy downloads)
|
||||
pub(crate) fn update(&mut self, chunk_size: u64) -> Result<()> {
|
||||
self.downloaded += chunk_size;
|
||||
|
||||
if self.last_displayed.elapsed() >= Duration::from_millis(100) {
|
||||
let formatted_downloaded = Self::format_size(self.downloaded);
|
||||
let formatted_total = Self::format_size(self.total_size);
|
||||
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
|
||||
|
||||
let elapsed = self.started_at.elapsed();
|
||||
let eta = if self.downloaded > 0 {
|
||||
let remaining = self.total_size.saturating_sub(self.downloaded);
|
||||
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
|
||||
if speed > 0.0 {
|
||||
Duration::from_secs_f64(remaining as f64 / speed)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
}
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let eta_str = Self::format_duration(eta);
|
||||
|
||||
print!(
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
|
||||
);
|
||||
io::stdout().flush()?;
|
||||
self.last_displayed = Instant::now();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct PhaseStart {
|
||||
started_at: Instant,
|
||||
baseline_bytes: u64,
|
||||
}
|
||||
|
||||
/// Shared progress counters for parallel downloads.
|
||||
pub(crate) struct SharedProgress {
|
||||
/// Raw HTTP bytes fetched during this session, including retries.
|
||||
pub(crate) session_fetched_bytes: AtomicU64,
|
||||
/// Compressed bytes from archives that have fully downloaded.
|
||||
pub(crate) completed_download_bytes: AtomicU64,
|
||||
/// Compressed bytes written for currently active archive download attempts.
|
||||
pub(crate) active_download_bytes: AtomicU64,
|
||||
/// Total compressed bytes expected across all planned archives.
|
||||
pub(crate) total_download_bytes: u64,
|
||||
/// Plain-output bytes from archives that have fully verified.
|
||||
pub(crate) completed_output_bytes: AtomicU64,
|
||||
/// Plain-output bytes unpacked by currently active extractions.
|
||||
pub(crate) active_extracted_output_bytes: AtomicU64,
|
||||
/// Plain-output bytes hashed by currently active verifications.
|
||||
pub(crate) active_verified_output_bytes: AtomicU64,
|
||||
/// Total plain-output bytes expected across all planned archives.
|
||||
pub(crate) total_output_bytes: u64,
|
||||
/// Total number of planned archives.
|
||||
pub(crate) total_archives: u64,
|
||||
/// Time when the modular download job started.
|
||||
pub(crate) started_at: Instant,
|
||||
/// Time and baseline when the current extraction phase started.
|
||||
extraction_phase: Mutex<Option<PhaseStart>>,
|
||||
/// Time and baseline when the current verification phase started.
|
||||
verification_phase: Mutex<Option<PhaseStart>>,
|
||||
/// Number of archives that have fully finished.
|
||||
pub(crate) archives_done: AtomicU64,
|
||||
/// Number of archives currently in the fetch phase.
|
||||
pub(crate) active_downloads: AtomicU64,
|
||||
/// Number of in-flight HTTP requests.
|
||||
pub(crate) active_download_requests: AtomicU64,
|
||||
/// Number of archives currently extracting.
|
||||
pub(crate) active_extractions: AtomicU64,
|
||||
/// Number of archives currently verifying extracted outputs.
|
||||
pub(crate) active_verifications: AtomicU64,
|
||||
/// Signals the background progress task to exit.
|
||||
pub(crate) done: AtomicBool,
|
||||
/// Cancellation token shared by the whole command.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl SharedProgress {
|
||||
/// Creates the shared progress state for a modular download job.
|
||||
pub(crate) fn new(
|
||||
total_download_bytes: u64,
|
||||
total_output_bytes: u64,
|
||||
total_archives: u64,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
session_fetched_bytes: AtomicU64::new(0),
|
||||
completed_download_bytes: AtomicU64::new(0),
|
||||
active_download_bytes: AtomicU64::new(0),
|
||||
total_download_bytes,
|
||||
completed_output_bytes: AtomicU64::new(0),
|
||||
active_extracted_output_bytes: AtomicU64::new(0),
|
||||
active_verified_output_bytes: AtomicU64::new(0),
|
||||
total_output_bytes,
|
||||
total_archives,
|
||||
started_at: Instant::now(),
|
||||
extraction_phase: Mutex::new(None),
|
||||
verification_phase: Mutex::new(None),
|
||||
archives_done: AtomicU64::new(0),
|
||||
active_downloads: AtomicU64::new(0),
|
||||
active_download_requests: AtomicU64::new(0),
|
||||
active_extractions: AtomicU64::new(0),
|
||||
active_verifications: AtomicU64::new(0),
|
||||
done: AtomicBool::new(false),
|
||||
cancel_token,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether the whole command has been cancelled.
|
||||
pub(crate) fn is_cancelled(&self) -> bool {
|
||||
self.cancel_token.is_cancelled()
|
||||
}
|
||||
|
||||
/// Adds raw session traffic bytes without affecting logical progress.
|
||||
pub(crate) fn record_session_fetched_bytes(&self, bytes: u64) {
|
||||
self.session_fetched_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn add_active_download_bytes(&self, bytes: u64) {
|
||||
self.active_download_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn sub_active_download_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_download_bytes, bytes);
|
||||
}
|
||||
|
||||
fn add_active_extracted_output_bytes(&self, bytes: u64) {
|
||||
self.active_extracted_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn sub_active_extracted_output_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_extracted_output_bytes, bytes);
|
||||
}
|
||||
|
||||
fn add_active_verified_output_bytes(&self, bytes: u64) {
|
||||
self.active_verified_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn sub_active_verified_output_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_verified_output_bytes, bytes);
|
||||
}
|
||||
|
||||
/// Records an archive whose outputs were already present locally.
|
||||
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
|
||||
self.completed_download_bytes.fetch_add(download_bytes, Ordering::Relaxed);
|
||||
self.completed_output_bytes.fetch_add(output_bytes, Ordering::Relaxed);
|
||||
self.archives_done.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records an archive whose compressed download completed successfully.
|
||||
pub(crate) fn record_archive_download_complete(&self, bytes: u64) {
|
||||
self.completed_download_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records an archive whose extracted outputs have fully verified.
|
||||
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
|
||||
self.completed_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.archives_done.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Returns logical compressed download progress.
|
||||
pub(crate) fn logical_downloaded_bytes(&self) -> u64 {
|
||||
(self.completed_download_bytes.load(Ordering::Relaxed) +
|
||||
self.active_download_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_download_bytes)
|
||||
}
|
||||
|
||||
/// Returns verified plain-output bytes.
|
||||
pub(crate) fn verified_output_bytes(&self) -> u64 {
|
||||
self.completed_output_bytes.load(Ordering::Relaxed).min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Returns plain-output bytes currently represented by extraction progress.
|
||||
pub(crate) fn extracting_output_bytes(&self) -> u64 {
|
||||
(self.completed_output_bytes.load(Ordering::Relaxed) +
|
||||
self.active_extracted_output_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Returns plain-output bytes currently represented by verification progress.
|
||||
pub(crate) fn verifying_output_bytes(&self) -> u64 {
|
||||
(self.completed_output_bytes.load(Ordering::Relaxed) +
|
||||
self.active_verified_output_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
fn restart_phase(slot: &Mutex<Option<PhaseStart>>, baseline_bytes: u64) {
|
||||
*slot.lock().unwrap() = Some(PhaseStart { started_at: Instant::now(), baseline_bytes });
|
||||
}
|
||||
|
||||
fn phase_eta(
|
||||
slot: &Mutex<Option<PhaseStart>>,
|
||||
current_bytes: u64,
|
||||
total_bytes: u64,
|
||||
) -> Option<Duration> {
|
||||
let phase = *slot.lock().unwrap();
|
||||
let phase = phase?;
|
||||
let done = current_bytes.saturating_sub(phase.baseline_bytes);
|
||||
let total = total_bytes.saturating_sub(phase.baseline_bytes);
|
||||
eta_from_progress(phase.started_at.elapsed(), done, total)
|
||||
}
|
||||
|
||||
fn extraction_eta(&self, current_bytes: u64) -> Option<Duration> {
|
||||
Self::phase_eta(&self.extraction_phase, current_bytes, self.total_output_bytes)
|
||||
}
|
||||
|
||||
fn verification_eta(&self, current_bytes: u64) -> Option<Duration> {
|
||||
Self::phase_eta(&self.verification_phase, current_bytes, self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Marks one archive as actively downloading.
|
||||
pub(crate) fn download_started(&self) {
|
||||
self.active_downloads.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Marks one archive download as finished.
|
||||
pub(crate) fn download_finished(&self) {
|
||||
sub_bytes(&self.active_downloads, 1);
|
||||
}
|
||||
|
||||
/// Marks one HTTP request as in flight.
|
||||
pub(crate) fn request_started(&self) {
|
||||
self.active_download_requests.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Marks one HTTP request as finished.
|
||||
pub(crate) fn request_finished(&self) {
|
||||
sub_bytes(&self.active_download_requests, 1);
|
||||
}
|
||||
|
||||
/// Marks one archive as actively extracting.
|
||||
pub(crate) fn extraction_started(&self) {
|
||||
if self.active_extractions.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
Self::restart_phase(
|
||||
&self.extraction_phase,
|
||||
self.completed_output_bytes.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one archive extraction as finished.
|
||||
pub(crate) fn extraction_finished(&self) {
|
||||
sub_bytes(&self.active_extractions, 1);
|
||||
}
|
||||
|
||||
/// Marks one archive as actively verifying outputs.
|
||||
pub(crate) fn verification_started(&self) {
|
||||
if self.active_verifications.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
Self::restart_phase(
|
||||
&self.verification_phase,
|
||||
self.completed_output_bytes.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one archive verification as finished.
|
||||
pub(crate) fn verification_finished(&self) {
|
||||
sub_bytes(&self.active_verifications, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn sub_bytes(counter: &AtomicU64, bytes: u64) {
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_sub(bytes))
|
||||
});
|
||||
}
|
||||
|
||||
fn eta_from_progress(elapsed: Duration, done: u64, total: u64) -> Option<Duration> {
|
||||
if done == 0 || done >= total {
|
||||
return None;
|
||||
}
|
||||
|
||||
let secs = elapsed.as_secs_f64();
|
||||
if secs <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let speed = done as f64 / secs;
|
||||
if speed <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(Duration::from_secs_f64((total - done) as f64 / speed))
|
||||
}
|
||||
|
||||
fn format_percent(done: u64, total: u64) -> String {
|
||||
if total == 0 {
|
||||
return "100.0%".to_string();
|
||||
}
|
||||
|
||||
format!("{:.1}%", (done as f64 / total as f64) * 100.0)
|
||||
}
|
||||
|
||||
fn format_eta(eta: Option<Duration>) -> String {
|
||||
eta.map(DownloadProgress::format_duration).unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
|
||||
/// Global request limit for the blocking downloader.
|
||||
///
|
||||
/// This uses `Mutex + Condvar` because the segmented path runs blocking reqwest
|
||||
/// clients on OS threads.
|
||||
pub(crate) struct DownloadRequestLimiter {
|
||||
/// Maximum number of in-flight HTTP requests.
|
||||
limit: usize,
|
||||
/// Current number of acquired request slots.
|
||||
active: Mutex<usize>,
|
||||
/// Wakes blocked threads when a slot is released.
|
||||
notify: Condvar,
|
||||
}
|
||||
|
||||
impl DownloadRequestLimiter {
|
||||
/// Creates the shared request limiter.
|
||||
pub(crate) fn new(limit: usize) -> Arc<Self> {
|
||||
Arc::new(Self { limit: limit.max(1), active: Mutex::new(0), notify: Condvar::new() })
|
||||
}
|
||||
|
||||
/// Returns the configured request limit.
|
||||
pub(crate) fn max_concurrency(&self) -> usize {
|
||||
self.limit
|
||||
}
|
||||
|
||||
pub(crate) fn acquire<'a>(
|
||||
&'a self,
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
cancel_token: &CancellationToken,
|
||||
) -> Result<DownloadRequestPermit<'a>> {
|
||||
let mut active = self.active.lock().unwrap();
|
||||
loop {
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(eyre::eyre!("Download cancelled"));
|
||||
}
|
||||
|
||||
if *active < self.limit {
|
||||
*active += 1;
|
||||
if let Some(progress) = progress {
|
||||
progress.request_started();
|
||||
}
|
||||
return Ok(DownloadRequestPermit { limiter: self, progress });
|
||||
}
|
||||
|
||||
// Wake periodically so cancellation can interrupt waiters even if
|
||||
// no request finishes.
|
||||
let (next_active, _) =
|
||||
self.notify.wait_timeout(active, Duration::from_millis(100)).unwrap();
|
||||
active = next_active;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII permit for one in-flight HTTP request.
|
||||
///
|
||||
/// Dropping the permit releases a slot in the shared request limit and updates
|
||||
/// the live progress counters.
|
||||
pub(crate) struct DownloadRequestPermit<'a> {
|
||||
/// Limiter that owns the request slot.
|
||||
limiter: &'a DownloadRequestLimiter,
|
||||
/// Shared progress counters updated when the permit drops.
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
}
|
||||
|
||||
impl Drop for DownloadRequestPermit<'_> {
|
||||
/// Releases the request slot and updates shared progress counters.
|
||||
fn drop(&mut self) {
|
||||
let mut active = self.limiter.active.lock().unwrap();
|
||||
*active = active.saturating_sub(1);
|
||||
drop(active);
|
||||
self.limiter.notify.notify_one();
|
||||
|
||||
if let Some(progress) = self.progress {
|
||||
progress.request_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive download attempt.
|
||||
pub(crate) struct ArchiveDownloadProgress<'a> {
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
downloaded: u64,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveDownloadProgress<'a> {
|
||||
/// Starts tracking one archive download attempt.
|
||||
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.download_started();
|
||||
}
|
||||
Self { progress, downloaded: 0, completed: false }
|
||||
}
|
||||
|
||||
/// Adds logical compressed bytes written by this attempt.
|
||||
pub(crate) fn record_downloaded(&mut self, bytes: u64) {
|
||||
self.downloaded += bytes;
|
||||
if let Some(progress) = self.progress {
|
||||
progress.add_active_download_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this tracker has recorded any logical bytes itself.
|
||||
pub(crate) fn has_tracked_bytes(&self) -> bool {
|
||||
self.downloaded > 0
|
||||
}
|
||||
|
||||
/// Moves this archive from active download bytes into completed download bytes.
|
||||
pub(crate) fn complete(&mut self, total_bytes: u64) {
|
||||
if self.completed {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_download_bytes(self.downloaded);
|
||||
progress.record_archive_download_complete(total_bytes);
|
||||
}
|
||||
self.downloaded = 0;
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveDownloadProgress<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_download_bytes(self.downloaded);
|
||||
progress.download_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive extraction attempt.
|
||||
pub(crate) struct ArchiveExtractionProgress {
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
extracted: Arc<AtomicU64>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
/// Cloneable handle for reporting extracted bytes from background monitoring.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ArchiveExtractionProgressHandle {
|
||||
progress: Arc<SharedProgress>,
|
||||
extracted: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl ArchiveExtractionProgress {
|
||||
/// Starts tracking one archive extraction attempt.
|
||||
pub(crate) fn new(progress: Option<&Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.extraction_started();
|
||||
}
|
||||
Self {
|
||||
progress: progress.cloned(),
|
||||
extracted: Arc::new(AtomicU64::new(0)),
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a cloneable handle that can report extraction progress from another thread.
|
||||
pub(crate) fn handle(&self) -> Option<ArchiveExtractionProgressHandle> {
|
||||
Some(ArchiveExtractionProgressHandle {
|
||||
progress: Arc::clone(self.progress.as_ref()?),
|
||||
extracted: Arc::clone(&self.extracted),
|
||||
})
|
||||
}
|
||||
|
||||
/// Adds plain-output bytes extracted by this attempt.
|
||||
pub(crate) fn record_extracted(&mut self, bytes: u64) {
|
||||
if let Some(handle) = self.handle() {
|
||||
handle.record_extracted(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ends extraction tracking before verification begins.
|
||||
pub(crate) fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
|
||||
}
|
||||
self.finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveExtractionProgress {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
|
||||
progress.extraction_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ArchiveExtractionProgressHandle {
|
||||
/// Adds plain-output bytes extracted by this attempt.
|
||||
pub(crate) fn record_extracted(&self, bytes: u64) {
|
||||
self.extracted.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.progress.add_active_extracted_output_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive verification attempt.
|
||||
pub(crate) struct ArchiveVerificationProgress<'a> {
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
verified: u64,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveVerificationProgress<'a> {
|
||||
/// Starts tracking one archive verification attempt.
|
||||
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.verification_started();
|
||||
}
|
||||
Self { progress, verified: 0, completed: false }
|
||||
}
|
||||
|
||||
/// Adds plain-output bytes hashed by this verification attempt.
|
||||
pub(crate) fn record_verified(&mut self, bytes: u64) {
|
||||
self.verified += bytes;
|
||||
if let Some(progress) = self.progress {
|
||||
progress.add_active_verified_output_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Moves this archive from active verification bytes into completed output bytes.
|
||||
pub(crate) fn complete(&mut self, total_bytes: u64) {
|
||||
if self.completed {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_verified_output_bytes(self.verified);
|
||||
progress.record_archive_output_complete(total_bytes);
|
||||
}
|
||||
self.verified = 0;
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveVerificationProgress<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_verified_output_bytes(self.verified);
|
||||
progress.verification_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Adapter to track progress while reading (used for extraction in legacy path)
|
||||
pub(crate) struct ProgressReader<R> {
|
||||
/// Wrapped reader that provides archive bytes.
|
||||
reader: R,
|
||||
/// Per-download progress tracker for legacy paths.
|
||||
progress: DownloadProgress,
|
||||
/// Cancellation token checked between reads.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl<R: Read> ProgressReader<R> {
|
||||
/// Wraps a reader with per-download progress tracking.
|
||||
pub(crate) fn new(reader: R, total_size: u64, cancel_token: CancellationToken) -> Self {
|
||||
Self { reader, progress: DownloadProgress::new(total_size), cancel_token }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for ProgressReader<R> {
|
||||
/// Reads bytes, checks cancellation, and updates the local progress bar.
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let bytes = self.reader.read(buf)?;
|
||||
if bytes > 0 &&
|
||||
let Err(error) = self.progress.update(bytes as u64)
|
||||
{
|
||||
return Err(io::Error::other(error));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that bumps a shared atomic counter while writing data.
|
||||
/// Used for parallel downloads where a single display task shows aggregated progress.
|
||||
pub(crate) struct SharedProgressWriter<'a, W> {
|
||||
/// Wrapped writer receiving downloaded bytes.
|
||||
pub(crate) inner: W,
|
||||
/// Shared counters updated as bytes are written.
|
||||
pub(crate) progress: Arc<SharedProgress>,
|
||||
/// Optional callback for logical bytes written by the current archive attempt.
|
||||
pub(crate) on_written: Option<&'a mut dyn FnMut(u64)>,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for SharedProgressWriter<'_, W> {
|
||||
/// Writes bytes and records them in shared progress.
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
if self.progress.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let n = self.inner.write(buf)?;
|
||||
self.progress.record_session_fetched_bytes(n as u64);
|
||||
if let Some(on_written) = self.on_written.as_deref_mut() {
|
||||
on_written(n as u64);
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Flushes the wrapped writer.
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that bumps a shared atomic counter while reading data.
|
||||
/// Used for streaming downloads where a single display task shows aggregated progress.
|
||||
pub(crate) struct SharedProgressReader<R> {
|
||||
/// Wrapped reader producing streamed bytes.
|
||||
pub(crate) inner: R,
|
||||
/// Shared counters updated as bytes are read.
|
||||
pub(crate) progress: Arc<SharedProgress>,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for SharedProgressReader<R> {
|
||||
/// Reads bytes and records them in shared progress.
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
if self.progress.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let n = self.inner.read(buf)?;
|
||||
self.progress.record_session_fetched_bytes(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns a background task that prints aggregated download progress.
|
||||
/// Returns a handle; drop it (or call `.abort()`) to stop.
|
||||
pub(crate) fn spawn_progress_display(progress: Arc<SharedProgress>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(3));
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
if progress.done.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
|
||||
let download_total = progress.total_download_bytes;
|
||||
let output_total = progress.total_output_bytes;
|
||||
if download_total == 0 && output_total == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let done = progress.archives_done.load(Ordering::Relaxed);
|
||||
let all = progress.total_archives;
|
||||
let active_downloads = progress.active_downloads.load(Ordering::Relaxed);
|
||||
let active_requests = progress.active_download_requests.load(Ordering::Relaxed);
|
||||
let active_extractions = progress.active_extractions.load(Ordering::Relaxed);
|
||||
let active_verifications = progress.active_verifications.load(Ordering::Relaxed);
|
||||
let downloaded = progress.logical_downloaded_bytes();
|
||||
let extracted = progress.extracting_output_bytes();
|
||||
let verified = progress.verifying_output_bytes();
|
||||
let elapsed = DownloadProgress::format_duration(progress.started_at.elapsed());
|
||||
let download_total_display = DownloadProgress::format_size(download_total);
|
||||
let output_total_display = DownloadProgress::format_size(output_total);
|
||||
let downloaded_display = DownloadProgress::format_size(downloaded);
|
||||
let extracted_display = DownloadProgress::format_size(extracted);
|
||||
let active_download_phase = active_downloads > 0 || active_requests > 0;
|
||||
|
||||
if active_download_phase {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(downloaded, download_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(eta_from_progress(progress.started_at.elapsed(), downloaded, download_total)),
|
||||
bytes = format_args!("{downloaded_display}/{download_total_display}"),
|
||||
"Downloading snapshot archives"
|
||||
);
|
||||
} else if active_extractions > 0 {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(extracted, output_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(progress.extraction_eta(extracted)),
|
||||
bytes = format_args!("{extracted_display}/{output_total_display}"),
|
||||
"Extracting snapshot archives"
|
||||
);
|
||||
} else if active_verifications > 0 {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(verified, output_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(progress.verification_eta(verified)),
|
||||
bytes = format_args!("{}/{output_total_display}", DownloadProgress::format_size(verified)),
|
||||
"Verifying snapshot archives"
|
||||
);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let completed = progress.verified_output_bytes();
|
||||
let completed_display = DownloadProgress::format_size(completed);
|
||||
let output_total = DownloadProgress::format_size(progress.total_output_bytes);
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{}/{}", progress.total_archives, progress.total_archives),
|
||||
progress = "100.0%",
|
||||
elapsed = %DownloadProgress::format_duration(progress.started_at.elapsed()),
|
||||
eta = "0s",
|
||||
bytes = format_args!("{completed_display}/{output_total}"),
|
||||
"Snapshot archive processing complete"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[test]
|
||||
fn shared_progress_separates_session_fetch_from_logical_progress() {
|
||||
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
|
||||
|
||||
progress.record_session_fetched_bytes(10);
|
||||
progress.record_session_fetched_bytes(10);
|
||||
progress.record_archive_download_complete(10);
|
||||
progress.record_archive_output_complete(20);
|
||||
|
||||
assert_eq!(progress.session_fetched_bytes.load(Ordering::Relaxed), 20);
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 10);
|
||||
assert_eq!(progress.verified_output_bytes(), 20);
|
||||
assert_eq!(progress.archives_done.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_download_progress_rolls_back_unfinished_attempts() {
|
||||
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
|
||||
|
||||
{
|
||||
let mut download = ArchiveDownloadProgress::new(Some(&progress));
|
||||
download.record_downloaded(4);
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 4);
|
||||
}
|
||||
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 0);
|
||||
assert_eq!(progress.active_downloads.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_phase_baseline_restarts_after_idle() {
|
||||
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
|
||||
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.completed_output_bytes.store(25, Ordering::Relaxed);
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.extraction_finished();
|
||||
progress.extraction_finished();
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_phase_baseline_restarts_after_idle() {
|
||||
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
|
||||
|
||||
progress.verification_started();
|
||||
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.completed_output_bytes.store(40, Ordering::Relaxed);
|
||||
progress.verification_started();
|
||||
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.verification_finished();
|
||||
progress.verification_finished();
|
||||
progress.verification_started();
|
||||
assert_eq!(
|
||||
progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes,
|
||||
40
|
||||
);
|
||||
}
|
||||
}
|
||||
100
crates/cli/commands/src/download/session.rs
Normal file
100
crates/cli/commands/src/download/session.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
use super::progress::{DownloadRequestLimiter, SharedProgress};
|
||||
use eyre::Result;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
/// Shared state for one run of `reth download`.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DownloadSession {
|
||||
/// Shared progress counters for this command, when enabled.
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
/// Shared limit for concurrent HTTP requests, when enabled.
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
/// Cancellation token shared by the whole command.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl DownloadSession {
|
||||
/// Stores the shared progress, request limiter, and cancellation token.
|
||||
pub(crate) fn new(
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Self {
|
||||
Self { progress, request_limiter, cancel_token }
|
||||
}
|
||||
|
||||
/// Returns the shared progress tracker, if this flow uses one.
|
||||
pub(crate) fn progress(&self) -> Option<&Arc<SharedProgress>> {
|
||||
self.progress.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the shared HTTP request limiter, if this flow uses one.
|
||||
pub(crate) fn request_limiter(&self) -> Option<&Arc<DownloadRequestLimiter>> {
|
||||
self.request_limiter.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the request limiter or errors if the caller needs one.
|
||||
pub(crate) fn require_request_limiter(&self) -> Result<&Arc<DownloadRequestLimiter>> {
|
||||
self.request_limiter().ok_or_else(|| eyre::eyre!("Missing download request limiter"))
|
||||
}
|
||||
|
||||
/// Returns the cancellation token for this command.
|
||||
pub(crate) fn cancel_token(&self) -> &CancellationToken {
|
||||
&self.cancel_token
|
||||
}
|
||||
|
||||
/// Records one archive whose outputs were already reusable on disk.
|
||||
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
|
||||
if let Some(progress) = self.progress() {
|
||||
progress.record_reused_archive(download_bytes, output_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Records one archive whose extracted outputs fully verified.
|
||||
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
|
||||
if let Some(progress) = self.progress() {
|
||||
progress.record_archive_output_complete(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Paths used while processing one archive, plus the shared download session.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ArchiveProcessContext {
|
||||
/// Directory where extracted output files are written.
|
||||
target_dir: PathBuf,
|
||||
/// Directory used for cached archive downloads, when enabled.
|
||||
cache_dir: Option<PathBuf>,
|
||||
/// Shared command-scoped download state.
|
||||
session: DownloadSession,
|
||||
}
|
||||
|
||||
impl ArchiveProcessContext {
|
||||
/// Creates the context used while processing modular archives.
|
||||
pub(crate) fn new(
|
||||
target_dir: PathBuf,
|
||||
cache_dir: Option<PathBuf>,
|
||||
session: DownloadSession,
|
||||
) -> Self {
|
||||
Self { target_dir, cache_dir, session }
|
||||
}
|
||||
|
||||
/// Returns the directory where extracted outputs should be written.
|
||||
pub(crate) fn target_dir(&self) -> &Path {
|
||||
&self.target_dir
|
||||
}
|
||||
|
||||
/// Returns the cache directory for two-phase downloads, if enabled.
|
||||
pub(crate) fn cache_dir(&self) -> Option<&Path> {
|
||||
self.cache_dir.as_deref()
|
||||
}
|
||||
|
||||
/// Returns the shared download session.
|
||||
pub(crate) fn session(&self) -> &DownloadSession {
|
||||
&self.session
|
||||
}
|
||||
}
|
||||
230
crates/cli/commands/src/download/source.rs
Normal file
230
crates/cli/commands/src/download/source.rs
Normal file
@@ -0,0 +1,230 @@
|
||||
use super::{manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults};
|
||||
use eyre::{Result, WrapErr};
|
||||
use reqwest::Client;
|
||||
use reth_fs_util as fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
|
||||
/// An entry from the snapshot discovery API listing.
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct SnapshotApiEntry {
|
||||
#[serde(deserialize_with = "deserialize_string_or_u64")]
|
||||
chain_id: u64,
|
||||
#[serde(deserialize_with = "deserialize_string_or_u64")]
|
||||
block: u64,
|
||||
#[serde(default)]
|
||||
date: Option<String>,
|
||||
#[serde(default)]
|
||||
profile: Option<String>,
|
||||
metadata_url: String,
|
||||
#[serde(default)]
|
||||
size: u64,
|
||||
}
|
||||
|
||||
impl SnapshotApiEntry {
|
||||
/// Returns whether this discovery entry points to a modular manifest.
|
||||
fn is_modular(&self) -> bool {
|
||||
self.metadata_url.ends_with("manifest.json")
|
||||
}
|
||||
}
|
||||
|
||||
/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
|
||||
///
|
||||
/// Queries the configured snapshot API and returns the manifest URL for the most
|
||||
/// recent modular snapshot matching the requested chain.
|
||||
pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
|
||||
let defaults = DownloadDefaults::get_global();
|
||||
let api_url = &*defaults.snapshot_api_url;
|
||||
|
||||
info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
|
||||
|
||||
let entries = fetch_snapshot_api_entries(chain_id).await?;
|
||||
let entry =
|
||||
entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"No modular snapshot manifest found for chain \
|
||||
{chain_id} at {api_url}\n\n\
|
||||
You can provide a manifest URL directly with --manifest-url, or\n\
|
||||
use a direct snapshot URL with -u from:\n\
|
||||
\t- {}\n\n\
|
||||
Use --list to see all available snapshots.",
|
||||
api_url.trim_end_matches("/api/snapshots"),
|
||||
)
|
||||
})?;
|
||||
|
||||
info!(target: "reth::cli",
|
||||
block = entry.block,
|
||||
url = %entry.metadata_url,
|
||||
"Found latest snapshot manifest"
|
||||
);
|
||||
|
||||
Ok(entry.metadata_url.clone())
|
||||
}
|
||||
|
||||
/// Deserializes a JSON value that may be either a number or a string-encoded number.
|
||||
fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
use serde::Deserialize;
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
match &value {
|
||||
serde_json::Value::Number(n) => {
|
||||
n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
|
||||
}
|
||||
serde_json::Value::String(s) => {
|
||||
s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
|
||||
}
|
||||
_ => Err(serde::de::Error::custom("expected number or string")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
|
||||
pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
|
||||
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
|
||||
|
||||
let entries: Vec<SnapshotApiEntry> = Client::new()
|
||||
.get(api_url)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
Ok(entries.into_iter().filter(|entry| entry.chain_id == chain_id).collect())
|
||||
}
|
||||
|
||||
/// Prints a formatted table of available modular snapshots.
|
||||
pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
|
||||
let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
|
||||
|
||||
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
|
||||
println!(
|
||||
"Available snapshots for chain {chain_id} ({}):\n",
|
||||
api_url.trim_end_matches("/api/snapshots"),
|
||||
);
|
||||
println!("{:<12} {:>10} {:<10} {:>10} MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
|
||||
println!("{}", "-".repeat(100));
|
||||
|
||||
for entry in &modular {
|
||||
let date = entry.date.as_deref().unwrap_or("-");
|
||||
let profile = entry.profile.as_deref().unwrap_or("-");
|
||||
let size = if entry.size > 0 {
|
||||
DownloadProgress::format_size(entry.size)
|
||||
} else {
|
||||
"-".to_string()
|
||||
};
|
||||
|
||||
println!(
|
||||
"{date:<12} {:>10} {profile:<10} {size:>10} {}",
|
||||
entry.block, entry.metadata_url
|
||||
);
|
||||
}
|
||||
|
||||
if modular.is_empty() {
|
||||
println!(" (no modular snapshots found)");
|
||||
}
|
||||
|
||||
println!(
|
||||
"\nTo download a specific snapshot, copy its manifest URL and run:\n \
|
||||
reth download --manifest-url <URL>"
|
||||
);
|
||||
}
|
||||
|
||||
/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path.
|
||||
pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
|
||||
if let Ok(parsed) = Url::parse(source) {
|
||||
return match parsed.scheme() {
|
||||
"http" | "https" => {
|
||||
let response = Client::new()
|
||||
.get(source)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.wrap_err_with(|| {
|
||||
let sources = DownloadDefaults::get_global()
|
||||
.available_snapshots
|
||||
.iter()
|
||||
.map(|snapshot| format!("\t- {snapshot}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
format!(
|
||||
"Failed to fetch snapshot manifest from {source}\n\n\
|
||||
The manifest endpoint may not be available for this snapshot source.\n\
|
||||
You can use a direct snapshot URL instead:\n\n\
|
||||
\treth download -u <snapshot-url>\n\n\
|
||||
Available snapshot sources:\n{sources}"
|
||||
)
|
||||
})?;
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
"file" => {
|
||||
let path = parsed
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
|
||||
let content = fs::read_to_string(path)?;
|
||||
Ok(serde_json::from_str(&content)?)
|
||||
}
|
||||
_ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
|
||||
};
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(source)?;
|
||||
Ok(serde_json::from_str(&content)?)
|
||||
}
|
||||
|
||||
/// Resolves the base URL used to join relative archive paths in a manifest.
|
||||
pub(crate) fn resolve_manifest_base_url(
|
||||
manifest: &SnapshotManifest,
|
||||
source: &str,
|
||||
) -> Result<String> {
|
||||
if let Some(base_url) = manifest.base_url.as_deref() &&
|
||||
!base_url.is_empty()
|
||||
{
|
||||
return Ok(base_url.trim_end_matches('/').to_string());
|
||||
}
|
||||
|
||||
if let Ok(mut url) = Url::parse(source) {
|
||||
if url.scheme() == "file" {
|
||||
let mut path = url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
|
||||
path.pop();
|
||||
let mut base = Url::from_directory_path(path)
|
||||
.map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
|
||||
.to_string();
|
||||
if base.ends_with('/') {
|
||||
base.pop();
|
||||
}
|
||||
return Ok(base);
|
||||
}
|
||||
|
||||
{
|
||||
let mut segments = url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
|
||||
segments.pop_if_empty();
|
||||
segments.pop();
|
||||
}
|
||||
return Ok(url.as_str().trim_end_matches('/').to_string());
|
||||
}
|
||||
|
||||
let path = Path::new(source);
|
||||
let manifest_dir = if path.is_absolute() {
|
||||
path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
|
||||
} else {
|
||||
let joined = std::env::current_dir()?.join(path);
|
||||
joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
|
||||
};
|
||||
|
||||
let mut base = Url::from_directory_path(&manifest_dir)
|
||||
.map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
|
||||
.to_string();
|
||||
if base.ends_with('/') {
|
||||
base.pop();
|
||||
}
|
||||
Ok(base)
|
||||
}
|
||||
@@ -262,6 +262,7 @@ impl SelectorApp {
|
||||
ComponentSelection::None => return 0,
|
||||
ComponentSelection::All => None,
|
||||
ComponentSelection::Distance(d) => Some(d),
|
||||
ComponentSelection::Since(block) => Some(self.manifest.block - block + 1),
|
||||
};
|
||||
self.groups[group_idx]
|
||||
.types
|
||||
@@ -344,6 +345,7 @@ fn format_selection(sel: &ComponentSelection) -> String {
|
||||
match sel {
|
||||
ComponentSelection::All => "All".to_string(),
|
||||
ComponentSelection::Distance(d) => format!("Last {d} blocks"),
|
||||
ComponentSelection::Since(block) => format!("Since block {block}"),
|
||||
ComponentSelection::None => "None".to_string(),
|
||||
}
|
||||
}
|
||||
|
||||
84
crates/cli/commands/src/download/verify.rs
Normal file
84
crates/cli/commands/src/download/verify.rs
Normal file
@@ -0,0 +1,84 @@
|
||||
use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress};
|
||||
use blake3::Hasher;
|
||||
use eyre::Result;
|
||||
use reth_fs_util as fs;
|
||||
use std::{io::Read, path::Path};
|
||||
|
||||
/// Verifies and cleans up extracted output files in one target directory.
|
||||
pub(crate) struct OutputVerifier<'a> {
|
||||
/// Directory containing the output files declared by the manifest.
|
||||
target_dir: &'a Path,
|
||||
}
|
||||
|
||||
impl<'a> OutputVerifier<'a> {
|
||||
/// Creates a verifier for one extraction target directory.
|
||||
pub(crate) const fn new(target_dir: &'a Path) -> Self {
|
||||
Self { target_dir }
|
||||
}
|
||||
|
||||
/// Returns `true` only when every declared output file exists and matches size and BLAKE3.
|
||||
/// Returns `false` if any file is missing, mismatched, or no outputs were declared.
|
||||
pub(crate) fn verify(&self, output_files: &[OutputFileChecksum]) -> Result<bool> {
|
||||
self.verify_with_progress(output_files, None)
|
||||
}
|
||||
|
||||
/// Returns `true` only when every declared output file exists and matches size and BLAKE3,
|
||||
/// updating the optional verification progress as file bytes are hashed.
|
||||
pub(crate) fn verify_with_progress(
|
||||
&self,
|
||||
output_files: &[OutputFileChecksum],
|
||||
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
|
||||
) -> Result<bool> {
|
||||
if output_files.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
for expected in output_files {
|
||||
let output_path = self.target_dir.join(&expected.path);
|
||||
let meta = match fs::metadata(&output_path) {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
if meta.len() != expected.size {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let actual = Self::file_blake3_hex(&output_path, progress.as_deref_mut())?;
|
||||
if !actual.eq_ignore_ascii_case(&expected.blake3) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Removes any declared output files so a fresh archive attempt can restart cleanly.
|
||||
pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) {
|
||||
for output in output_files {
|
||||
let _ = fs::remove_file(self.target_dir.join(&output.path));
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the hex-encoded BLAKE3 checksum for one plain output file.
|
||||
fn file_blake3_hex(
|
||||
path: &Path,
|
||||
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
|
||||
) -> Result<String> {
|
||||
let mut file = fs::open(path)?;
|
||||
let mut hasher = Hasher::new();
|
||||
let mut buf = [0_u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let n = file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
if let Some(progress) = progress.as_deref_mut() {
|
||||
progress.record_verified(n as u64);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
}
|
||||
@@ -78,9 +78,10 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
|
||||
self.env.init::<N>(AccessRights::RW, runtime)?;
|
||||
|
||||
let static_file_provider = provider_factory.static_file_provider();
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
|
||||
if self.without_evm {
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
|
||||
// ensure header, total difficulty and header hash are provided
|
||||
let header = self.header.ok_or_else(|| eyre::eyre!("Header file must be provided"))?;
|
||||
let header = without_evm::read_header_from_file::<
|
||||
@@ -106,23 +107,22 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> InitStateC
|
||||
// SAFETY: it's safe to commit static files, since in the event of a crash, they
|
||||
// will be unwound according to database checkpoints.
|
||||
//
|
||||
// Necessary to commit, so the header is accessible to provider_rw and
|
||||
// init_state_dump
|
||||
// Necessary to commit, so the header is accessible to init_from_state_dump
|
||||
static_file_provider.commit()?;
|
||||
} else if last_block_number > 0 && last_block_number < header.number() {
|
||||
return Err(eyre::eyre!(
|
||||
"Data directory should be empty when calling init-state with --without-evm."
|
||||
));
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Initiating state dump");
|
||||
|
||||
let reader = BufReader::new(reth_fs_util::open(self.state)?);
|
||||
|
||||
let hash = init_from_state_dump(reader, &provider_rw, config.stages.etl)?;
|
||||
|
||||
provider_rw.commit()?;
|
||||
let hash = init_from_state_dump(reader, &provider_factory, config.stages.etl)?;
|
||||
|
||||
info!(target: "reth::cli", hash = ?hash, "Genesis block written");
|
||||
Ok(())
|
||||
|
||||
@@ -50,8 +50,13 @@ where
|
||||
info!(target: "reth::cli", new_tip = ?header.num_hash(), "Setting up dummy EVM chain before importing state.");
|
||||
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
// Write EVM dummy data up to `header - 1` block
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
// Write EVM dummy data up to `header - 1` block. Skip when the supplied
|
||||
// header is at block 0: `header.number() - 1` would underflow in u64 to
|
||||
// `u64::MAX`, sending `append_dummy_chain` into a 1..=u64::MAX loop that
|
||||
// exhausts memory before failing.
|
||||
if header.number() > 0 {
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Appending first valid block.");
|
||||
|
||||
@@ -191,7 +196,13 @@ mod tests {
|
||||
use alloy_primitives::{address, b256};
|
||||
use reth_db_common::init::init_genesis;
|
||||
use reth_provider::{test_utils::create_test_provider_factory, DatabaseProviderFactory};
|
||||
use std::io::Write;
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
@@ -264,4 +275,45 @@ mod tests {
|
||||
|
||||
assert_eq!(actual_next_height, expected_next_height);
|
||||
}
|
||||
|
||||
/// Regression: a header at block 0 used to send `append_dummy_chain` into
|
||||
/// a `1..=u64::MAX` loop because `header.number() - 1` underflowed in
|
||||
/// u64. The guard `if header.number() > 0` skips the dummy-chain step
|
||||
/// when there is no pre-genesis range to backfill, so `header_factory`
|
||||
/// is never invoked.
|
||||
#[test]
|
||||
fn test_setup_without_evm_skips_dummy_chain_for_genesis_header() {
|
||||
let header = Header { number: 0, ..Default::default() };
|
||||
let header_hash = header.hash_slow();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
init_genesis(&provider_factory).unwrap();
|
||||
let provider_rw = provider_factory.database_provider_rw().unwrap();
|
||||
|
||||
let factory_calls = Arc::new(AtomicU64::new(0));
|
||||
let factory_calls_inner = Arc::clone(&factory_calls);
|
||||
|
||||
// The Result of `setup_without_evm` itself is not asserted: with
|
||||
// `number == 0` plus a genesis already written by `init_genesis`,
|
||||
// the subsequent `append_first_block` may legitimately fail. The
|
||||
// bug under test is the OOM in the dummy-chain loop, observable
|
||||
// through the factory-call counter below.
|
||||
let _ = setup_without_evm(
|
||||
&provider_rw,
|
||||
SealedHeader::new(header, header_hash),
|
||||
move |number| {
|
||||
// Bound calls so a regression cannot exhaust the test
|
||||
// runner's memory; the only correct value here is 0.
|
||||
let n = factory_calls_inner.fetch_add(1, Ordering::Relaxed);
|
||||
assert!(n < 8, "header_factory must not be invoked for a genesis-block header");
|
||||
Header { number, ..Default::default() }
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
factory_calls.load(Ordering::Relaxed),
|
||||
0,
|
||||
"append_dummy_chain must be skipped when header.number() == 0"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
|
||||
)
|
||||
}
|
||||
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
|
||||
let default_secret_key_path = data_dir.p2p_secret();
|
||||
let p2p_secret_key = self.network.secret_key(default_secret_key_path)?;
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::common::{
|
||||
EnvironmentArgs,
|
||||
};
|
||||
use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt};
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use clap::Parser;
|
||||
use eyre::WrapErr;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
|
||||
@@ -12,14 +13,22 @@ use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_consensus::FullConsensus;
|
||||
use reth_evm::{execute::Executor, ConfigureEvm};
|
||||
use reth_primitives_traits::{format_gas_throughput, BlockBody, GotExpected};
|
||||
use reth_primitives_traits::{format_gas_throughput, Account, BlockBody, GotExpected};
|
||||
use reth_provider::{
|
||||
BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider,
|
||||
StaticFileProviderFactory, TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_revm::{
|
||||
database::StateProviderDatabase,
|
||||
db::{
|
||||
states::reverts::{AccountInfoRevert, RevertToSlot},
|
||||
BundleState,
|
||||
},
|
||||
};
|
||||
use reth_stages::stages::calculate_gas_used_from_headers;
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
@@ -68,13 +77,18 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>> Command<C> {
|
||||
/// Execute `re-execute` command
|
||||
pub async fn execute<N>(
|
||||
self,
|
||||
mut self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
{
|
||||
// Default to 4GB RocksDB block cache for re-execute unless explicitly set.
|
||||
if self.env.db.rocksdb_block_cache_size.is_none() {
|
||||
self.env.db.rocksdb_block_cache_size = Some(4 << 30);
|
||||
}
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO, runtime)?;
|
||||
|
||||
let components = components(provider_factory.chain_spec());
|
||||
@@ -108,15 +122,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
min_block..=max_block,
|
||||
)?;
|
||||
|
||||
let db_at = {
|
||||
let provider_factory = provider_factory.clone();
|
||||
move |block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider_factory.history_by_block_number(block_number).unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let skip_invalid_blocks = self.skip_invalid_blocks;
|
||||
let blocks_per_chunk = self.blocks_per_chunk;
|
||||
let (stats_tx, mut stats_rx) = mpsc::unbounded_channel();
|
||||
@@ -132,13 +137,23 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let provider_factory = provider_factory.clone();
|
||||
let evm_config = components.evm_config().clone();
|
||||
let consensus = components.consensus().clone();
|
||||
let db_at = db_at.clone();
|
||||
let stats_tx = stats_tx.clone();
|
||||
let info_tx = info_tx.clone();
|
||||
let cancellation = cancellation.clone();
|
||||
let next_block = Arc::clone(&next_block);
|
||||
tasks.spawn_blocking(move || {
|
||||
let executor_lifetime = Duration::from_secs(120);
|
||||
let executor_lifetime = Duration::from_secs(600);
|
||||
let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety();
|
||||
|
||||
let db_at = {
|
||||
|block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider
|
||||
.history_by_block_number(block_number)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
@@ -245,14 +260,31 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let _ = stats_tx.send(block.gas_used());
|
||||
|
||||
// Reset DB once in a while to avoid OOM or read tx timeouts
|
||||
if executor.size_hint() > 1_000_000 ||
|
||||
if executor.size_hint() > 5_000_000 ||
|
||||
executor_created.elapsed() > executor_lifetime
|
||||
{
|
||||
executor =
|
||||
evm_config.batch_executor(db_at(block.number()));
|
||||
let last_block = block.number();
|
||||
let old_executor = std::mem::replace(
|
||||
&mut executor,
|
||||
evm_config.batch_executor(db_at(last_block)),
|
||||
);
|
||||
let bundle = old_executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
last_block,
|
||||
)?;
|
||||
executor_created = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Full verification at chunk end for remaining unverified blocks
|
||||
let bundle = executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
chunk_end - 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
eyre::Ok(())
|
||||
@@ -333,3 +365,103 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies reverts against database changesets.
|
||||
///
|
||||
/// For each block, reverts must match changeset entries exactly. No extra slots/accounts
|
||||
/// in reverts for non-destroyed accounts. Destroyed accounts may have extra changeset slots
|
||||
/// (from DB storage wipe) absent from reverts.
|
||||
fn verify_bundle_against_changesets<P>(
|
||||
provider: &P,
|
||||
bundle: &BundleState,
|
||||
last_block: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
P: ChangeSetReader + StorageChangeSetReader,
|
||||
{
|
||||
// Verify reverts against changesets per block
|
||||
for (i, block_reverts) in bundle.reverts.iter().rev().enumerate() {
|
||||
let block_number = last_block - i as u64;
|
||||
|
||||
let mut cs_accounts: HashMap<Address, Option<Account>> = provider
|
||||
.account_block_changeset(block_number)?
|
||||
.into_iter()
|
||||
.map(|cs| (cs.address, cs.info))
|
||||
.collect();
|
||||
|
||||
let mut cs_storage: HashMap<Address, HashMap<B256, U256>> = HashMap::new();
|
||||
for (bna, entry) in provider.storage_changeset(block_number)? {
|
||||
cs_storage.entry(bna.address()).or_default().insert(entry.key, entry.value);
|
||||
}
|
||||
|
||||
for (addr, revert) in block_reverts {
|
||||
// Verify account info
|
||||
match &revert.account {
|
||||
AccountInfoRevert::DoNothing => {
|
||||
eyre::ensure!(
|
||||
!cs_accounts.contains_key(addr),
|
||||
"Block {block_number}: account {addr} in changeset but revert is DoNothing",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::DeleteIt => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is DeleteIt but not in changeset")
|
||||
})?;
|
||||
eyre::ensure!(
|
||||
cs_info.is_none(),
|
||||
"Block {block_number}: account {addr} revert is DeleteIt but changeset has {cs_info:?}",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::RevertTo(info) => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is RevertTo but not in changeset")
|
||||
})?;
|
||||
let revert_acct = Some(Account::from(info));
|
||||
eyre::ensure!(
|
||||
revert_acct == cs_info,
|
||||
"Block {block_number}: account {addr} info mismatch: revert={revert_acct:?} cs={cs_info:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify storage slots — remove matched changeset entries as we go
|
||||
let mut cs_slots = cs_storage.get_mut(addr);
|
||||
for (slot_key, revert_slot) in &revert.storage {
|
||||
let b256_key = B256::from(*slot_key);
|
||||
let cs_value = cs_slots.as_mut().and_then(|s| s.remove(&b256_key));
|
||||
match (revert_slot, cs_value) {
|
||||
// When a contract is selfdestructed and re-created at the same address
|
||||
// within the same block, revm marks slots touched by the new contract
|
||||
// as `Destroyed` and never reads the original DB value, so
|
||||
// `to_previous_value()` would resolve to zero, which might be wrong.
|
||||
(RevertToSlot::Destroyed, _) => {}
|
||||
(RevertToSlot::Some(prev), Some(cs_value)) => eyre::ensure!(
|
||||
*prev == cs_value,
|
||||
"Block {block_number}: {addr} slot {b256_key} mismatch: \
|
||||
revert={prev} cs={cs_value}",
|
||||
),
|
||||
(RevertToSlot::Some(_), None) => eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} slot {b256_key} in reverts but not in changeset",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_storage slots for this address must be from a destroyed account
|
||||
if let Some(remaining) = cs_slots.filter(|s| !s.is_empty()) {
|
||||
eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} has {} unmatched storage slots in changeset",
|
||||
remaining.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_accounts entries had no corresponding revert
|
||||
if let Some(addr) = cs_accounts.keys().next() {
|
||||
eyre::bail!("Block {block_number}: account {addr} in changeset but not in reverts");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user