mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
90 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
fcfa8287f6 | ||
|
|
d25de30050 | ||
|
|
4ffde69d94 | ||
|
|
077e5eecfe | ||
|
|
709485dcb7 | ||
|
|
88505c7fcb | ||
|
|
c14bc59236 | ||
|
|
347c1325cc | ||
|
|
5f85eb7ac8 | ||
|
|
a12454d2e6 | ||
|
|
c194c17a27 | ||
|
|
43a7452b0e | ||
|
|
73ec2c9d56 | ||
|
|
76e886578b | ||
|
|
ad08829288 | ||
|
|
b89288582b | ||
|
|
87d878a979 | ||
|
|
473f85c558 | ||
|
|
a8eee6028f | ||
|
|
671da55884 | ||
|
|
a8fc13dc25 | ||
|
|
4c1f6b6507 | ||
|
|
b850f2a81d | ||
|
|
79144cb430 | ||
|
|
b04346ffe5 | ||
|
|
674623f14e | ||
|
|
97b1b56b2d | ||
|
|
af6d20b5ea | ||
|
|
64cf412aaf | ||
|
|
5b10e03c5c | ||
|
|
91d248e6fa | ||
|
|
344037d04e | ||
|
|
aca6261107 | ||
|
|
d4ca2e2687 | ||
|
|
e5e0abb47e | ||
|
|
225e3ae238 | ||
|
|
345fbbbfdb | ||
|
|
db17c899c3 | ||
|
|
2c86c0b876 | ||
|
|
bd4cd28a8d | ||
|
|
6fa48a497a | ||
|
|
6886cd7742 | ||
|
|
eeb223f0b8 | ||
|
|
f344f5abfb | ||
|
|
68845d1114 | ||
|
|
ecfb6cc089 | ||
|
|
b271694301 | ||
|
|
41c68729ab | ||
|
|
79578e35b8 | ||
|
|
e4f14b2ae1 | ||
|
|
05e6da66e1 | ||
|
|
6be5520e34 | ||
|
|
d29db3b765 | ||
|
|
40c30dbc73 | ||
|
|
5c383818a6 | ||
|
|
cf6ffb1599 | ||
|
|
ba3cd2872a | ||
|
|
4f9af7c16a | ||
|
|
13c5504aa2 | ||
|
|
fa6b44b038 | ||
|
|
6377a957c1 | ||
|
|
378d4052ee | ||
|
|
62d99888d2 | ||
|
|
73f5d77b51 | ||
|
|
b62f71977a | ||
|
|
ad27be67be | ||
|
|
63f80907cc | ||
|
|
a57930481c | ||
|
|
bbcfe354a1 | ||
|
|
7839f3d876 | ||
|
|
e89b4611e4 | ||
|
|
2b7d4b54d4 | ||
|
|
fe7a4c80b6 | ||
|
|
122c5b322b | ||
|
|
f1ed5f0ade | ||
|
|
6364fb87d0 | ||
|
|
d55458479d | ||
|
|
42f49132b7 | ||
|
|
f39c47bd11 | ||
|
|
b1ac264107 | ||
|
|
0195da5b84 | ||
|
|
b964195ef8 | ||
|
|
252fe42c54 | ||
|
|
3edb271183 | ||
|
|
165a80441b | ||
|
|
981e32d4d9 | ||
|
|
d7522904a0 | ||
|
|
e92af360ae | ||
|
|
408ef4657d | ||
|
|
3574ecaaa0 |
4
.github/dependabot.yml
vendored
4
.github/dependabot.yml
vendored
@@ -4,10 +4,14 @@ updates:
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
- package-ecosystem: "cargo"
|
||||
directory: "/"
|
||||
schedule:
|
||||
interval: "weekly"
|
||||
cooldown:
|
||||
default-days: 7
|
||||
labels:
|
||||
- "A-dependencies"
|
||||
commit-message:
|
||||
|
||||
116
.github/scripts/bench-reth-build.sh
vendored
116
.github/scripts/bench-reth-build.sh
vendored
@@ -1,24 +1,21 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Builds (or fetches from cache) reth binaries for benchmarking.
|
||||
# Builds reth binaries for benchmarking from local source only.
|
||||
#
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit> [branch-sha]
|
||||
# Usage: bench-reth-build.sh <baseline|feature> <source-dir> <commit>
|
||||
#
|
||||
# baseline — build/fetch the baseline binary at <commit> (merge-base)
|
||||
# baseline — build the baseline binary at <commit> (merge-base)
|
||||
# source-dir must be checked out at <commit>
|
||||
# feature — build/fetch the candidate binary + reth-bench at <commit>
|
||||
# feature — build the candidate binary + reth-bench at <commit>
|
||||
# source-dir must be checked out at <commit>
|
||||
# optional branch-sha is the PR head commit for cache key
|
||||
#
|
||||
# Outputs:
|
||||
# baseline: <source-dir>/target/profiling/reth (or reth-bb if BENCH_BIG_BLOCKS=true)
|
||||
# feature: <source-dir>/target/profiling/reth (or reth-bb), reth-bench installed to cargo bin
|
||||
#
|
||||
# Required: mc (MinIO client) with a configured alias
|
||||
# Optional env: BENCH_BIG_BLOCKS (true/false) — build reth-bb instead of reth
|
||||
set -euxo pipefail
|
||||
|
||||
MC="mc"
|
||||
MODE="$1"
|
||||
SOURCE_DIR="$2"
|
||||
COMMIT="$3"
|
||||
@@ -42,103 +39,38 @@ if [ "${BENCH_TRACY:-off}" != "off" ]; then
|
||||
EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
|
||||
fi
|
||||
|
||||
# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
|
||||
if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
|
||||
BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
|
||||
else
|
||||
BUILD_SUFFIX=""
|
||||
fi
|
||||
# Build the requested node binary with the benchmark profile.
|
||||
build_node_binary() {
|
||||
local features_arg=""
|
||||
local workspace_arg=""
|
||||
|
||||
# Verify a cached reth binary was built from the expected commit.
|
||||
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
|
||||
verify_binary() {
|
||||
local binary="$1" expected_commit="$2"
|
||||
local version binary_sha
|
||||
version=$("$binary" --version 2>/dev/null) || return 1
|
||||
binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
|
||||
if [ -z "$binary_sha" ]; then
|
||||
echo "Warning: could not extract commit SHA from version output"
|
||||
return 1
|
||||
cd "$SOURCE_DIR"
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
features_arg="--features ${EXTRA_FEATURES}"
|
||||
workspace_arg="--workspace"
|
||||
fi
|
||||
if [ "$binary_sha" = "$expected_commit" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
|
||||
return 1
|
||||
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --locked --profile profiling $NODE_PKG $workspace_arg $features_arg
|
||||
}
|
||||
|
||||
case "$MODE" in
|
||||
baseline|main)
|
||||
BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null; then
|
||||
echo "Cache hit for baseline (${COMMIT}), downloading ${NODE_BIN}..."
|
||||
if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
|
||||
verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached baseline binary is stale or download failed, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
FEATURES_ARG=""
|
||||
WORKSPACE_ARG=""
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
FEATURES_ARG="--features ${EXTRA_FEATURES}"
|
||||
WORKSPACE_ARG="--workspace"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling $NODE_PKG $WORKSPACE_ARG $FEATURES_ARG
|
||||
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
|
||||
fi
|
||||
echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
|
||||
build_node_binary
|
||||
;;
|
||||
|
||||
feature|branch)
|
||||
BRANCH_SHA="${4:-$COMMIT}"
|
||||
BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
|
||||
|
||||
CACHE_VALID=false
|
||||
if $MC stat --no-list "${BUCKET}/${NODE_BIN}" &>/dev/null && $MC stat --no-list "${BUCKET}/reth-bench" &>/dev/null; then
|
||||
echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
|
||||
mkdir -p "${SOURCE_DIR}/target/profiling"
|
||||
if $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}" && \
|
||||
$MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench && \
|
||||
chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" /home/ubuntu/.cargo/bin/reth-bench && \
|
||||
verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
|
||||
CACHE_VALID=true
|
||||
else
|
||||
echo "Cached feature binary is stale or download failed, rebuilding..."
|
||||
fi
|
||||
fi
|
||||
if [ "$CACHE_VALID" = false ]; then
|
||||
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
|
||||
cd "${SOURCE_DIR}"
|
||||
rustup show active-toolchain || rustup default stable
|
||||
if [ -n "$EXTRA_FEATURES" ]; then
|
||||
# Can't use `make profiling` when adding features; build explicitly
|
||||
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling --workspace $NODE_PKG --features "${EXTRA_FEATURES}"
|
||||
else
|
||||
# shellcheck disable=SC2086
|
||||
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
|
||||
cargo build --profile profiling $NODE_PKG
|
||||
fi
|
||||
make install-reth-bench
|
||||
$MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
|
||||
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
|
||||
fi
|
||||
echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
|
||||
rustup show active-toolchain || rustup default stable
|
||||
build_node_binary
|
||||
make -C "$SOURCE_DIR" install-reth-bench
|
||||
;;
|
||||
|
||||
*)
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit> [branch-sha]"
|
||||
echo "Usage: $0 <baseline|feature> <source-dir> <commit>"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
37
.github/scripts/bench-reth-local.sh
vendored
37
.github/scripts/bench-reth-local.sh
vendored
@@ -2,7 +2,7 @@
|
||||
#
|
||||
# local-reth-bench.sh — Run the reth Engine API benchmark locally.
|
||||
#
|
||||
# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
|
||||
# Replicates the CI bench.yml workflow (build, local snapshot validation, system tuning,
|
||||
# interleaved B-F-F-B execution, summary, charts) without any GitHub
|
||||
# Actions glue (no PR comments, no artifact upload, no Slack).
|
||||
#
|
||||
@@ -21,15 +21,17 @@
|
||||
# Requires: the reth repo at RETH_REPO (default: ~/reth)
|
||||
#
|
||||
# Dependencies (install before first run):
|
||||
# mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
|
||||
# make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
|
||||
# schelk, cpupower, taskset, stdbuf, python3, curl,
|
||||
# make, uv, jq, Rust toolchain (cargo/rustup)
|
||||
# Optional:
|
||||
# mc for Tracy profile upload
|
||||
#
|
||||
# The script delegates to the existing bench-reth-*.sh scripts in the reth
|
||||
# repo for the actual build, snapshot, and run steps.
|
||||
set -euxo pipefail
|
||||
|
||||
# ── PATH ──────────────────────────────────────────────────────────────
|
||||
# Ensure cargo and user-local bins (mc, uv) are visible
|
||||
# Ensure cargo and user-local bins (uv) are visible
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
|
||||
# ── Defaults ──────────────────────────────────────────────────────────
|
||||
@@ -106,7 +108,7 @@ fi
|
||||
|
||||
# ── Check dependencies ───────────────────────────────────────────────
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq cargo; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -238,19 +240,14 @@ echo " Baseline src : $BASELINE_SRC"
|
||||
echo " Feature src : $FEATURE_SRC"
|
||||
echo
|
||||
|
||||
# ── Step 3: Check / download snapshot ────────────────────────────────
|
||||
echo "▸ Checking snapshot..."
|
||||
# ── Step 3: Validate local snapshot ──────────────────────────────────
|
||||
echo "▸ Validating local snapshot..."
|
||||
cd "$RETH_REPO"
|
||||
SNAPSHOT_NEEDED=false
|
||||
if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
|
||||
SNAPSHOT_NEEDED=true
|
||||
echo " Snapshot needs update."
|
||||
else
|
||||
echo " Snapshot is up-to-date."
|
||||
fi
|
||||
"${SCRIPTS_DIR}/bench-reth-snapshot.sh"
|
||||
echo " Snapshot is ready."
|
||||
echo
|
||||
|
||||
# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
|
||||
# ── Step 4: Build binaries in parallel ───────────────────────────────
|
||||
echo "▸ Building binaries (parallel)..."
|
||||
cd "$RETH_REPO"
|
||||
|
||||
@@ -262,19 +259,11 @@ PID_BASELINE=$!
|
||||
"${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
|
||||
PID_FEATURE=$!
|
||||
|
||||
PID_SNAPSHOT=
|
||||
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
|
||||
echo " Also downloading snapshot in parallel..."
|
||||
"${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
|
||||
PID_SNAPSHOT=$!
|
||||
fi
|
||||
|
||||
wait $PID_BASELINE || FAIL=1
|
||||
wait $PID_FEATURE || FAIL=1
|
||||
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
|
||||
|
||||
if [ $FAIL -ne 0 ]; then
|
||||
echo "Error: one or more parallel tasks failed (builds / snapshot)"
|
||||
echo "Error: one or more build tasks failed"
|
||||
exit 1
|
||||
fi
|
||||
echo " Binaries built successfully."
|
||||
|
||||
29
.github/scripts/bench-reth-run.sh
vendored
29
.github/scripts/bench-reth-run.sh
vendored
@@ -88,10 +88,16 @@ trap cleanup EXIT
|
||||
# Stop any leftover reth process in the scope, then recover schelk state.
|
||||
sudo systemctl stop "$RETH_SCOPE" 2>/dev/null || true
|
||||
sudo systemctl reset-failed "$RETH_SCOPE" 2>/dev/null || true
|
||||
sudo schelk recover -y --kill || true
|
||||
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
|
||||
|
||||
# Mount
|
||||
sudo schelk mount -y
|
||||
sudo schelk mount -y || true
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Failed to mount benchmark datadir at ${DATADIR}"
|
||||
ls -la "$SCHELK_MOUNT" || true
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
sync
|
||||
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
|
||||
echo "=== Cache state after drop ==="
|
||||
@@ -317,13 +323,18 @@ if [ "$BIG_BLOCKS" = "true" ]; then
|
||||
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
|
||||
else
|
||||
# Standard mode: warmup + new-payload-fcu
|
||||
# Warmup
|
||||
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
|
||||
"${EXTRA_BENCH_ARGS[@]}" 2>&1 | sed -u "s/^/[bench] /"
|
||||
WARMUP="${BENCH_WARMUP_BLOCKS:-50}"
|
||||
if [ "$WARMUP" -gt 0 ] 2>/dev/null; then
|
||||
# Warm up the node before measuring the benchmark window.
|
||||
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
|
||||
--rpc-url "$BENCH_RPC_URL" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
--advance "$WARMUP" \
|
||||
"${EXTRA_BENCH_ARGS[@]}" 2>&1 | sed -u "s/^/[bench] /"
|
||||
else
|
||||
echo "Skipping warmup (0 blocks)..."
|
||||
fi
|
||||
|
||||
# Start tracy-capture after warmup so profile only covers the benchmark
|
||||
if [ "${BENCH_TRACY:-off}" != "off" ]; then
|
||||
|
||||
137
.github/scripts/bench-reth-snapshot.sh
vendored
137
.github/scripts/bench-reth-snapshot.sh
vendored
@@ -1,129 +1,56 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Downloads the latest snapshot into the schelk volume using
|
||||
# `reth download` with progress reporting to the GitHub PR comment.
|
||||
#
|
||||
# Skips the download if the manifest content hasn't changed since
|
||||
# the last successful download (checked via SHA-256 of the manifest).
|
||||
# Validates that the benchmark snapshot has already been populated into the
|
||||
# local schelk volume.
|
||||
#
|
||||
# Usage: bench-reth-snapshot.sh [--check]
|
||||
# --check Only check if a download is needed; exits 0 if up-to-date, 10 if not.
|
||||
# --check Exit 0 if the local snapshot is ready, 10 if it is missing.
|
||||
#
|
||||
# Required env:
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# BENCH_RETH_BINARY – path to the reth binary
|
||||
# GITHUB_TOKEN – token for GitHub API calls (only for download)
|
||||
# BENCH_COMMENT_ID – PR comment ID to update (optional)
|
||||
# BENCH_REPO – owner/repo (e.g. paradigmxyz/reth)
|
||||
# BENCH_JOB_URL – link to the Actions job
|
||||
# BENCH_ACTOR – user who triggered the benchmark
|
||||
# BENCH_CONFIG – config summary line
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# Optional env:
|
||||
# BENCH_BIG_BLOCKS – true when validating the big-blocks snapshot datadir
|
||||
# BENCH_SNAPSHOT_NAME – expected snapshot label for log/error output
|
||||
set -euxo pipefail
|
||||
|
||||
MC="mc"
|
||||
BUCKET="minio/reth-snapshots"
|
||||
# Allow overriding the snapshot name (e.g. for big-blocks mode where the
|
||||
# big-blocks manifest specifies which base snapshot to use).
|
||||
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable-previous}"
|
||||
MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
|
||||
: "${SCHELK_MOUNT:?SCHELK_MOUNT must be set}"
|
||||
|
||||
DATADIR_NAME="datadir"
|
||||
HASH_MODE_SUFFIX=""
|
||||
if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
DATADIR_NAME="datadir-big-blocks"
|
||||
HASH_MODE_SUFFIX="-big-blocks"
|
||||
fi
|
||||
DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
|
||||
HASH_FILE="$HOME/.reth-bench-snapshot-hash${HASH_MODE_SUFFIX}"
|
||||
|
||||
# Fetch manifest and compute content hash for reliable freshness check
|
||||
MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
|
||||
exit 2
|
||||
describe_snapshot() {
|
||||
if [ -n "${BENCH_SNAPSHOT_NAME:-}" ]; then
|
||||
printf '%s' "${BENCH_SNAPSHOT_NAME}"
|
||||
elif [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
printf '%s' 'big-block weekly snapshot'
|
||||
else
|
||||
printf '%s' 'benchmark snapshot'
|
||||
fi
|
||||
}
|
||||
REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
|
||||
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
snapshot_ready() {
|
||||
[ -d "$DATADIR/db" ] && [ -d "$DATADIR/static_files" ]
|
||||
}
|
||||
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
EXPECTED_SNAPSHOT="$(describe_snapshot)"
|
||||
|
||||
sudo schelk recover -y --kill || sudo schelk full-recover -y || true
|
||||
sudo schelk mount -y || true
|
||||
|
||||
if snapshot_ready; then
|
||||
echo "Found local ${EXPECTED_SNAPSHOT} at ${DATADIR}"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
echo "::error::Missing local ${EXPECTED_SNAPSHOT} at ${DATADIR}. Benchmarks no longer download snapshots; pre-populate the local schelk data first."
|
||||
ls -la "$SCHELK_MOUNT" || true
|
||||
ls -la "$DATADIR" || true
|
||||
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 10
|
||||
fi
|
||||
|
||||
RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
|
||||
if [ ! -x "$RETH" ]; then
|
||||
echo "::error::reth binary not found or not executable at $RETH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve the MinIO HTTP endpoint from the mc alias so reth can
|
||||
# fetch archives over HTTP (the manifest's embedded base_url points
|
||||
# to the cluster-internal address which is unreachable from runners).
|
||||
MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
|
||||
if [ -z "$MINIO_ENDPOINT" ]; then
|
||||
echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
|
||||
exit 1
|
||||
fi
|
||||
BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/${SNAPSHOT_NAME}"
|
||||
|
||||
# Rewrite manifest's base_url with the runner-reachable endpoint
|
||||
MANIFEST_TMP=$(mktemp --suffix=.json)
|
||||
trap 'rm -f -- "$MANIFEST_TMP"' EXIT
|
||||
echo "$MANIFEST_CONTENT" \
|
||||
| jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
|
||||
|
||||
# Prepare mount. If a previous run left the volume mounted, recover first.
|
||||
sudo schelk recover -y --kill || true
|
||||
sudo schelk mount -y
|
||||
sudo rm -rf "$DATADIR"
|
||||
sudo mkdir -p "$DATADIR"
|
||||
# reth download runs as current user (not root), needs write access
|
||||
sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
|
||||
|
||||
update_comment() {
|
||||
local status="$1"
|
||||
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
|
||||
local body
|
||||
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
|
||||
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
|
||||
curl -sf -X PATCH \
|
||||
-H "Authorization: token ${GITHUB_TOKEN}" \
|
||||
-H "Accept: application/vnd.github.v3+json" \
|
||||
"https://api.github.com/repos/${BENCH_REPO}/issues/comments/${BENCH_COMMENT_ID}" \
|
||||
-d "$(jq -nc --arg body "$body" '{body: $body}')" \
|
||||
> /dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
update_comment "Downloading snapshot…"
|
||||
|
||||
# Download using reth download (manifest-path with rewritten base_url)
|
||||
"$RETH" download \
|
||||
--manifest-path "$MANIFEST_TMP" \
|
||||
-y \
|
||||
--minimal \
|
||||
--datadir "$DATADIR"
|
||||
|
||||
update_comment "Downloading snapshot… done"
|
||||
echo "Snapshot download complete"
|
||||
|
||||
# Sanity check: verify expected directories exist
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Promote the new snapshot to become the schelk baseline (virgin volume).
|
||||
# This copies changed blocks from scratch → virgin so that future
|
||||
# `schelk recover` calls restore to this new state.
|
||||
sync
|
||||
sudo schelk promote -y
|
||||
|
||||
# Save manifest hash
|
||||
echo "$REMOTE_HASH" > "$HASH_FILE"
|
||||
echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
exit 1
|
||||
|
||||
25
.github/scripts/hive/build_simulators.sh
vendored
25
.github/scripts/hive/build_simulators.sh
vendored
@@ -1,6 +1,23 @@
|
||||
#!/usr/bin/env bash
|
||||
set -eo pipefail
|
||||
|
||||
fixture_variant="${1:-osaka}"
|
||||
|
||||
case "${fixture_variant}" in
|
||||
amsterdam)
|
||||
eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/snobal-devnet-5@v8037.0.0/fixtures_snobal-devnet-5.tar.gz"
|
||||
eels_branch="devnets/snobal/5"
|
||||
;;
|
||||
osaka)
|
||||
eels_fixtures="https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz"
|
||||
eels_branch="forks/osaka"
|
||||
;;
|
||||
*)
|
||||
echo "unknown hive fixture variant: ${fixture_variant}"
|
||||
exit 1
|
||||
;;
|
||||
esac
|
||||
|
||||
# Create the hive_assets directory
|
||||
mkdir hive_assets/
|
||||
|
||||
@@ -12,12 +29,12 @@ go build .
|
||||
# Run each hive command in the background for each simulator and wait
|
||||
echo "Building images"
|
||||
./hive -client reth --sim "ethereum/eels/consume-engine" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
|
||||
--sim.buildarg branch=forks/osaka \
|
||||
--sim.buildarg fixtures="${eels_fixtures}" \
|
||||
--sim.buildarg branch="${eels_branch}" \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/eels/consume-rlp" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
|
||||
--sim.buildarg branch=forks/osaka \
|
||||
--sim.buildarg fixtures="${eels_fixtures}" \
|
||||
--sim.buildarg branch="${eels_branch}" \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
|
||||
|
||||
78
.github/scripts/hive/expected_failures.yaml
vendored
78
.github/scripts/hive/expected_failures.yaml
vendored
@@ -13,15 +13,15 @@ rpc-compat:
|
||||
# syncing mode, the test expects syncing to be false on start
|
||||
- eth_syncing/check-syncing (reth)
|
||||
|
||||
engine-withdrawals: [ ]
|
||||
engine-withdrawals: []
|
||||
|
||||
engine-api: [ ]
|
||||
engine-api: []
|
||||
|
||||
engine-cancun: [ ]
|
||||
engine-cancun: []
|
||||
|
||||
sync: [ ]
|
||||
sync: []
|
||||
|
||||
engine-auth: [ ]
|
||||
engine-auth: []
|
||||
|
||||
# EIP-7610 related tests (Revert creation in case of non-empty storage):
|
||||
#
|
||||
@@ -99,6 +99,40 @@ eels/consume-engine:
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Prague-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Shanghai-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Paris-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Cancun-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Osaka-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
|
||||
# Blob limit tests:
|
||||
#
|
||||
@@ -193,3 +227,37 @@ eels/consume-rlp:
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Prague-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Shanghai-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Paris-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Cancun-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Shanghai-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Osaka-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Prague-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Cancun-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Paris-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Osaka-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
|
||||
6
.github/scripts/hive/run_simulator.sh
vendored
6
.github/scripts/hive/run_simulator.sh
vendored
@@ -5,6 +5,12 @@ cd hivetests/
|
||||
|
||||
sim="${1}"
|
||||
limit="${2}"
|
||||
fixture_variant="${3:-}"
|
||||
|
||||
if [[ "${fixture_variant}" == "osaka" && "${sim}" == *"eels"* && "${limit}" == *"tests/amsterdam"* ]]; then
|
||||
echo "osaka fixtures do not support amsterdam tests"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Use lower parallelism for eels tests to avoid OOM-killing the runner
|
||||
parallelism=16
|
||||
|
||||
121
.github/workflows/bench-scheduled.yml
vendored
121
.github/workflows/bench-scheduled.yml
vendored
@@ -54,9 +54,7 @@ env:
|
||||
|
||||
name: bench-scheduled
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -65,6 +63,9 @@ jobs:
|
||||
resolve-refs:
|
||||
name: resolve-refs
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
outputs:
|
||||
mode: ${{ steps.mode.outputs.mode }}
|
||||
baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
|
||||
@@ -76,21 +77,26 @@ jobs:
|
||||
long-running: ${{ steps.refs.outputs.long-running }}
|
||||
release-tag: ${{ steps.refs.outputs.release-tag }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
sparse-checkout: .github/scripts
|
||||
sparse-checkout-cone-mode: true
|
||||
fetch-depth: 2
|
||||
|
||||
- name: Detect mode
|
||||
id: mode
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
INPUT_MODE: ${{ inputs.mode }}
|
||||
SCHEDULE: ${{ github.event.schedule }}
|
||||
run: |
|
||||
# Maps cron schedules to modes (must match the schedule entries above)
|
||||
if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
|
||||
MODE="${{ inputs.mode || 'nightly' }}"
|
||||
elif [ "${{ github.event.schedule }}" = "30 5 * * *" ]; then
|
||||
if [ "$EVENT_NAME" = "workflow_dispatch" ]; then
|
||||
MODE="${INPUT_MODE:-nightly}"
|
||||
elif [ "$SCHEDULE" = "30 5 * * *" ]; then
|
||||
MODE="nightly"
|
||||
elif [ "${{ github.event.schedule }}" = "0 9 * * *" ]; then
|
||||
elif [ "$SCHEDULE" = "0 9 * * *" ]; then
|
||||
MODE="release"
|
||||
else
|
||||
MODE="hourly"
|
||||
@@ -105,14 +111,15 @@ jobs:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
GITHUB_RUN_ID: ${{ github.run_id }}
|
||||
INPUT_FORCE: ${{ inputs.force || 'false' }}
|
||||
run: |
|
||||
FORCE="${{ inputs.force || 'false' }}"
|
||||
FORCE="${INPUT_FORCE:-false}"
|
||||
MODE="${{ steps.mode.outputs.mode }}"
|
||||
.github/scripts/bench-scheduled-refs.sh "$FORCE" "$MODE"
|
||||
|
||||
- name: Alert on long-running hourly
|
||||
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -154,7 +161,7 @@ jobs:
|
||||
|
||||
- name: Alert on stale nightly
|
||||
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -242,6 +249,9 @@ jobs:
|
||||
needs.resolve-refs.outputs.is-stale != 'true'
|
||||
name: bench-scheduled
|
||||
runs-on: [self-hosted, Linux, X64, available]
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
@@ -270,15 +280,16 @@ jobs:
|
||||
- name: Clean up previous bench-work
|
||||
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
|
||||
- name: Resolve job URL
|
||||
id: job-url
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
@@ -291,8 +302,9 @@ jobs:
|
||||
core.exportVariable('BENCH_JOB_URL', jobUrl);
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
continue-on-error: true
|
||||
- uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
@@ -307,12 +319,6 @@ jobs:
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
@@ -340,7 +346,7 @@ jobs:
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -366,35 +372,30 @@ jobs:
|
||||
echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
- name: Validate local snapshot
|
||||
id: snapshot-check
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
@@ -418,15 +419,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -648,7 +640,7 @@ jobs:
|
||||
|
||||
- name: Upload results
|
||||
if: "!cancelled()"
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: bench-scheduled-results
|
||||
path: ${{ env.BENCH_WORK_DIR }}
|
||||
@@ -656,10 +648,12 @@ jobs:
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success() && env.BENCH_MODE != 'hourly'
|
||||
env:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="${BENCH_MODE}/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
|
||||
CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
|
||||
@@ -680,7 +674,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
@@ -759,7 +753,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -914,7 +908,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -925,8 +919,8 @@ jobs:
|
||||
if (!token || !channel) return;
|
||||
|
||||
const steps_status = [
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
|
||||
@@ -975,6 +969,11 @@ jobs:
|
||||
}),
|
||||
});
|
||||
|
||||
- name: Clean build outputs
|
||||
if: always()
|
||||
run: |
|
||||
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
|
||||
296
.github/workflows/bench.yml
vendored
296
.github/workflows/bench.yml
vendored
@@ -82,7 +82,7 @@ on:
|
||||
- on-error
|
||||
- never
|
||||
abba:
|
||||
description: "Run ABBA (BFFB) interleaved order; false = single AB pass"
|
||||
description: "Run ABBA (FBBF) interleaved order; false = single FB pass"
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
@@ -99,9 +99,7 @@ env:
|
||||
|
||||
name: bench
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
reth-bench-ack:
|
||||
@@ -110,6 +108,9 @@ jobs:
|
||||
github.event_name == 'workflow_dispatch'
|
||||
name: reth-bench-ack
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
outputs:
|
||||
pr: ${{ steps.args.outputs.pr }}
|
||||
actor: ${{ steps.args.outputs.actor }}
|
||||
@@ -133,7 +134,7 @@ jobs:
|
||||
steps:
|
||||
- name: Check org membership
|
||||
if: github.event_name == 'issue_comment'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
@@ -152,7 +153,7 @@ jobs:
|
||||
|
||||
- name: Parse arguments
|
||||
id: args
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -359,7 +360,7 @@ jobs:
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -445,7 +446,7 @@ jobs:
|
||||
|
||||
- name: Poll queue position
|
||||
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -529,6 +530,9 @@ jobs:
|
||||
needs: reth-bench-ack
|
||||
name: reth-bench
|
||||
runs-on: [self-hosted, Linux, X64, available]
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
@@ -560,7 +564,7 @@ jobs:
|
||||
|
||||
- name: Resolve checkout ref
|
||||
id: checkout-ref
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
if (!process.env.BENCH_PR) {
|
||||
@@ -578,15 +582,16 @@ jobs:
|
||||
core.info(`PR #${process.env.BENCH_PR} (${pr.state}), using head SHA ${pr.head.sha}`);
|
||||
core.setOutput('ref', pr.head.sha);
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ steps.checkout-ref.outputs.ref }}
|
||||
|
||||
- name: Resolve job URL and update status
|
||||
if: env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -634,8 +639,9 @@ jobs:
|
||||
});
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
continue-on-error: true
|
||||
- uses: astral-sh/setup-uv@bd01e18f51369d5a26f1651c3cb451d3417e3bba # v6.3.1
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
@@ -650,12 +656,6 @@ jobs:
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
@@ -690,7 +690,7 @@ jobs:
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
for cmd in schelk cpupower taskset stdbuf python3 curl make uv jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
@@ -702,7 +702,7 @@ jobs:
|
||||
# Build binaries
|
||||
- name: Resolve PR head branch
|
||||
id: pr-info
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
if (process.env.BENCH_PR) {
|
||||
@@ -720,7 +720,7 @@ jobs:
|
||||
|
||||
- name: Resolve refs
|
||||
id: refs
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const { execSync } = require('child_process');
|
||||
@@ -766,79 +766,68 @@ jobs:
|
||||
core.setOutput('feature-ref', featureRef);
|
||||
core.setOutput('feature-name', featureName);
|
||||
|
||||
- name: Check big-blocks freshness
|
||||
- name: Validate local big blocks
|
||||
if: env.BENCH_BIG_BLOCKS == 'true'
|
||||
id: big-blocks-check
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
|
||||
HASH_FILE="$HOME/.reth-bench-big-blocks-hash"
|
||||
echo "Fetching big-blocks manifest from $MANIFEST..."
|
||||
BB_MANIFEST=$($MC cat "$MANIFEST" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch big-blocks manifest from $MANIFEST"
|
||||
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
|
||||
PAYLOAD_DIR="$BIG_BLOCKS_DIR/payloads"
|
||||
MANIFEST="$BIG_BLOCKS_DIR/manifest.json"
|
||||
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
|
||||
|
||||
if [ ! -f "$MANIFEST" ]; then
|
||||
echo "::error::Missing local big-blocks manifest at $MANIFEST"
|
||||
exit 1
|
||||
}
|
||||
BASE_SNAPSHOT=$(echo "$BB_MANIFEST" | jq -r '.base_snapshot // empty')
|
||||
fi
|
||||
|
||||
BASE_SNAPSHOT=$(jq -r '.base_snapshot // empty' "$MANIFEST")
|
||||
if [ -z "$BASE_SNAPSHOT" ]; then
|
||||
echo "::error::Big-blocks manifest missing base_snapshot field"
|
||||
exit 1
|
||||
fi
|
||||
echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
|
||||
echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
|
||||
|
||||
REMOTE_HASH=$(echo "$BB_MANIFEST" | sha256sum | awk '{print $1}')
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Big blocks up-to-date (hash: ${REMOTE_HASH:0:16}…)"
|
||||
echo "needed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "Big blocks need update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
echo "needed=true" >> "$GITHUB_OUTPUT"
|
||||
echo "remote-hash=${REMOTE_HASH}" >> "$GITHUB_OUTPUT"
|
||||
if [ ! -d "$PAYLOAD_DIR" ]; then
|
||||
echo "::error::Missing local big-block payload directory at $PAYLOAD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
PAYLOAD_COUNT=$(find "$PAYLOAD_DIR" -name '*.json' | wc -l)
|
||||
if [ "$PAYLOAD_COUNT" -eq 0 ]; then
|
||||
echo "::error::No payload files found in $PAYLOAD_DIR"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Update status (snapshot needed)
|
||||
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const s = require('./.github/scripts/bench-update-status.js');
|
||||
await s({github, context, status: 'Building binaries (snapshot update pending)...'});
|
||||
echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
|
||||
echo "Payload files: $PAYLOAD_COUNT"
|
||||
echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
|
||||
|
||||
- name: Validate local snapshot
|
||||
id: snapshot-check
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
prepare_source_dir() {
|
||||
local dir="$1"
|
||||
local ref="$2"
|
||||
|
||||
if [ -d "$dir" ]; then
|
||||
git -C "$dir" reset --hard HEAD
|
||||
git -C "$dir" clean -fdx
|
||||
git -C "$dir" fetch origin "$ref"
|
||||
else
|
||||
git clone . "$dir"
|
||||
fi
|
||||
|
||||
git -C "$dir" checkout --force "$ref"
|
||||
}
|
||||
|
||||
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
prepare_source_dir ../reth-baseline "$BASELINE_REF"
|
||||
|
||||
FEATURE_REF="${{ steps.refs.outputs.feature-ref }}"
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
prepare_source_dir ../reth-feature "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
@@ -862,15 +851,6 @@ jobs:
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -941,45 +921,6 @@ jobs:
|
||||
rm -rf "$BENCH_WORK_DIR"
|
||||
mkdir -p "$BENCH_WORK_DIR"
|
||||
|
||||
- name: Download big blocks
|
||||
if: env.BENCH_BIG_BLOCKS == 'true'
|
||||
run: |
|
||||
set -euo pipefail
|
||||
BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
|
||||
echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
|
||||
if [ "${{ steps.big-blocks-check.outputs.needed }}" = "false" ]; then
|
||||
echo "Big blocks cached at $BIG_BLOCKS_DIR, skipping download"
|
||||
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
|
||||
exit 0
|
||||
fi
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
|
||||
rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
|
||||
|
||||
# Download and parse manifest
|
||||
echo "Downloading manifest from $MANIFEST..."
|
||||
$MC cat "$MANIFEST" > "$BIG_BLOCKS_DIR/manifest.json"
|
||||
UPLOAD_PATH=$(jq -r '.upload_path' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
COUNT=$(jq -r '.count' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
TARGET_GAS=$(jq -r '.target_gas' "$BIG_BLOCKS_DIR/manifest.json")
|
||||
echo "Manifest: count=$COUNT, target_gas=$TARGET_GAS, archive=$UPLOAD_PATH"
|
||||
|
||||
# Download and extract archive
|
||||
ARCHIVE="minio/$UPLOAD_PATH"
|
||||
echo "Downloading big blocks from $ARCHIVE..."
|
||||
$MC cat "$ARCHIVE" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
|
||||
echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
|
||||
|
||||
# Verify expected directory structure
|
||||
if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
|
||||
echo "::error::Big blocks archive missing expected payloads/ directory"
|
||||
ls -laR "$BIG_BLOCKS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
|
||||
# Save manifest hash for freshness check on next run
|
||||
echo "${{ steps.big-blocks-check.outputs.remote-hash }}" > "$HOME/.reth-bench-big-blocks-hash"
|
||||
|
||||
- name: Start metrics proxy
|
||||
run: |
|
||||
BENCH_ID="ci-${{ github.run_id }}"
|
||||
@@ -1002,28 +943,15 @@ jobs:
|
||||
|
||||
- name: Update status (running benchmarks)
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const s = require('./.github/scripts/bench-update-status.js');
|
||||
await s({github, context, status: 'Running benchmarks...'});
|
||||
|
||||
# Interleaved run order (B-F-F-B) to reduce systematic bias from
|
||||
# Interleaved run order (F-B-B-F) to reduce systematic bias from
|
||||
# thermal drift and cache warming.
|
||||
- name: "Run benchmark: baseline (1/2)"
|
||||
id: run-baseline-1
|
||||
env:
|
||||
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=baseline-1,run_type=baseline,git_ref=${{ steps.refs.outputs.baseline-ref }}"
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-1"
|
||||
|
||||
- name: "Run benchmark: feature (1/2)"
|
||||
id: run-feature-1
|
||||
env:
|
||||
@@ -1037,19 +965,18 @@ jobs:
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-1"
|
||||
|
||||
- name: "Run benchmark: feature (2/2)"
|
||||
if: env.BENCH_ABBA != 'false'
|
||||
id: run-feature-2
|
||||
- name: "Run benchmark: baseline (1/2)"
|
||||
id: run-baseline-1
|
||||
env:
|
||||
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=feature-2,run_type=feature,git_ref=${{ steps.refs.outputs.feature-ref }}"
|
||||
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=baseline-1,run_type=baseline,git_ref=${{ steps.refs.outputs.baseline-ref }}"
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-2"
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-1"
|
||||
|
||||
- name: "Run benchmark: baseline (2/2)"
|
||||
if: env.BENCH_ABBA != 'false'
|
||||
@@ -1065,6 +992,20 @@ jobs:
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-2"
|
||||
|
||||
- name: "Run benchmark: feature (2/2)"
|
||||
if: env.BENCH_ABBA != 'false'
|
||||
id: run-feature-2
|
||||
env:
|
||||
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
|
||||
OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=feature-2,run_type=feature,git_ref=${{ steps.refs.outputs.feature-ref }}"
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-2"
|
||||
|
||||
- name: Stop metrics proxy & generate Grafana URL
|
||||
id: metrics
|
||||
if: "!cancelled()"
|
||||
@@ -1231,7 +1172,7 @@ jobs:
|
||||
|
||||
- name: Upload results
|
||||
if: "!cancelled()"
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: bench-reth-results
|
||||
path: ${{ env.BENCH_WORK_DIR }}
|
||||
@@ -1239,11 +1180,13 @@ jobs:
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success()
|
||||
env:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
RUN_ID: ${{ github.run_id }}
|
||||
run: |
|
||||
PR_NUMBER="${BENCH_PR:-0}"
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
|
||||
CHARTS_REPO="https://x-access-token:${DEREK_TOKEN}@github.com/decofe/reth-bench-charts.git"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
|
||||
@@ -1256,15 +1199,35 @@ jobs:
|
||||
mkdir -p "${TMP_DIR}/${CHART_DIR}"
|
||||
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
|
||||
git -C "${TMP_DIR}" add "${CHART_DIR}"
|
||||
if git -C "${TMP_DIR}" diff --cached --quiet; then
|
||||
echo "Charts for ${CHART_DIR} are already present, skipping push"
|
||||
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
rm -rf "${TMP_DIR}"
|
||||
exit 0
|
||||
fi
|
||||
git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
||||
commit -m "bench charts for PR #${PR_NUMBER} run ${RUN_ID}"
|
||||
git -C "${TMP_DIR}" push origin HEAD:main
|
||||
|
||||
for attempt in 1 2 3 4 5; do
|
||||
if git -C "${TMP_DIR}" push origin HEAD:main; then
|
||||
break
|
||||
fi
|
||||
if [ "$attempt" -eq 5 ]; then
|
||||
echo "::error::Failed to push charts after ${attempt} attempts"
|
||||
rm -rf "${TMP_DIR}"
|
||||
exit 1
|
||||
fi
|
||||
sleep "$attempt"
|
||||
git -C "${TMP_DIR}" fetch origin main
|
||||
git -C "${TMP_DIR}" rebase origin/main
|
||||
done
|
||||
|
||||
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
rm -rf "${TMP_DIR}"
|
||||
|
||||
- name: Compare & comment
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1300,7 +1263,7 @@ jobs:
|
||||
// Samply profile links (URLs point directly to Firefox Profiler)
|
||||
if (process.env.BENCH_SAMPLY === 'true') {
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const runs = abba ? ['baseline-1', 'feature-1', 'feature-2', 'baseline-2'] : ['baseline-1', 'feature-1'];
|
||||
const runs = abba ? ['feature-1', 'baseline-1', 'baseline-2', 'feature-2'] : ['feature-1', 'baseline-1'];
|
||||
const links = [];
|
||||
for (const run of runs) {
|
||||
try {
|
||||
@@ -1344,7 +1307,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const jobSummary = require('./.github/scripts/bench-job-summary.js');
|
||||
@@ -1358,7 +1321,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -1369,18 +1332,20 @@ jobs:
|
||||
|
||||
- name: Update status (failed)
|
||||
if: failure() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const steps_status = [
|
||||
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
...(abba ? [['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}']] : []),
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
];
|
||||
const failed = steps_status.find(([, o]) => o === 'failure');
|
||||
const failedStep = failed ? failed[0] : 'unknown step';
|
||||
@@ -1403,20 +1368,22 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const bigBlocks = process.env.BENCH_BIG_BLOCKS === 'true';
|
||||
const steps_status = [
|
||||
...(bigBlocks ? [['validating local big-block data', '${{ steps.big-blocks-check.outcome }}']] : []),
|
||||
['validating local snapshot', '${{ steps.snapshot-check.outcome }}'],
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
...(abba ? [['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}']] : []),
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
];
|
||||
const failed = steps_status.find(([, o]) => o === 'failure');
|
||||
const failedStep = failed ? failed[0] : 'unknown step';
|
||||
@@ -1425,7 +1392,7 @@ jobs:
|
||||
|
||||
- name: Update status (cancelled)
|
||||
if: cancelled() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1436,6 +1403,11 @@ jobs:
|
||||
body: `cc @${process.env.BENCH_ACTOR}\n\n⚠️ Benchmark cancelled. [View logs](${jobUrl})`,
|
||||
});
|
||||
|
||||
- name: Clean build outputs
|
||||
if: always()
|
||||
run: |
|
||||
sudo rm -rf ../reth-baseline/target ../reth-feature/target "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
|
||||
19
.github/workflows/book.yml
vendored
19
.github/workflows/book.yml
vendored
@@ -10,19 +10,22 @@ on:
|
||||
types: [opened, reopened, synchronize, closed]
|
||||
merge_group:
|
||||
|
||||
env:
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- name: Checkout
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Install bun
|
||||
uses: oven-sh/setup-bun@v2
|
||||
uses: oven-sh/setup-bun@0c5077e51419868618aeaa5fe8019c62421857d6 # v2.2.0
|
||||
with:
|
||||
bun-version: v1.2.23
|
||||
|
||||
@@ -36,8 +39,6 @@ jobs:
|
||||
- name: Install Rust nightly
|
||||
uses: dtolnay/rust-toolchain@nightly
|
||||
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
|
||||
- name: Build docs
|
||||
run: cd docs/vocs && bash scripts/build-cargo-docs.sh
|
||||
|
||||
@@ -47,10 +48,10 @@ jobs:
|
||||
echo "Vocs Build Complete"
|
||||
|
||||
- name: Setup Pages
|
||||
uses: actions/configure-pages@v6
|
||||
uses: actions/configure-pages@45bfe0192ca1faeb007ade9deae92b16b8254a0d # v6.0.0
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v5
|
||||
uses: actions/upload-pages-artifact@fc324d3547104276b827a68afc52ff2a11cc49c9 # v5.0.0
|
||||
with:
|
||||
path: "./docs/vocs/docs/dist"
|
||||
|
||||
@@ -74,4 +75,4 @@ jobs:
|
||||
steps:
|
||||
- name: Deploy to GitHub Pages
|
||||
id: deployment
|
||||
uses: actions/deploy-pages@v5
|
||||
uses: actions/deploy-pages@cd2ce8fcbc39b97be8ca5fce6e763baed58fa128 # v5.0.0
|
||||
|
||||
26
.github/workflows/check-alloy.yml
vendored
26
.github/workflows/check-alloy.yml
vendored
@@ -22,31 +22,41 @@ on:
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check:
|
||||
name: Check compilation with patched alloy
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
|
||||
- name: Apply alloy patches
|
||||
env:
|
||||
ALLOY_BRANCH: ${{ inputs.alloy_branch }}
|
||||
ALLOY_EVM_BRANCH: ${{ inputs.alloy_evm_branch }}
|
||||
OP_ALLOY_BRANCH: ${{ inputs.op_alloy_branch }}
|
||||
run: |
|
||||
ARGS=""
|
||||
if [ -n "${{ inputs.alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --alloy ${{ inputs.alloy_branch }}"
|
||||
if [ -n "$ALLOY_BRANCH" ]; then
|
||||
ARGS="$ARGS --alloy $ALLOY_BRANCH"
|
||||
fi
|
||||
if [ -n "${{ inputs.alloy_evm_branch }}" ]; then
|
||||
ARGS="$ARGS --evm ${{ inputs.alloy_evm_branch }}"
|
||||
if [ -n "$ALLOY_EVM_BRANCH" ]; then
|
||||
ARGS="$ARGS --evm $ALLOY_EVM_BRANCH"
|
||||
fi
|
||||
if [ -n "${{ inputs.op_alloy_branch }}" ]; then
|
||||
ARGS="$ARGS --op ${{ inputs.op_alloy_branch }}"
|
||||
if [ -n "$OP_ALLOY_BRANCH" ]; then
|
||||
ARGS="$ARGS --op $OP_ALLOY_BRANCH"
|
||||
fi
|
||||
|
||||
if [ -z "$ARGS" ]; then
|
||||
|
||||
16
.github/workflows/compact.yml
vendored
16
.github/workflows/compact.yml
vendored
@@ -16,32 +16,38 @@ env:
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
name: compact-codec
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
compact-codec:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
bin:
|
||||
- cargo run --bin reth --features "dev"
|
||||
steps:
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Checkout base
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
ref: ${{ github.base_ref || 'main' }}
|
||||
persist-credentials: false
|
||||
# On `main` branch, generates test vectors and serializes them to disk using `Compact`.
|
||||
- name: Generate compact vectors
|
||||
run: |
|
||||
${{ matrix.bin }} -- test-vectors compact --write
|
||||
- name: Checkout PR
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
clean: false
|
||||
persist-credentials: false
|
||||
# On incoming merge try to read and decode previously generated vectors with `Compact`
|
||||
- name: Read vectors
|
||||
run: ${{ matrix.bin }} -- test-vectors compact --read
|
||||
|
||||
9
.github/workflows/dependencies.yml
vendored
9
.github/workflows/dependencies.yml
vendored
@@ -9,13 +9,14 @@ on:
|
||||
workflow_dispatch:
|
||||
# Needed so we can run it manually
|
||||
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
update:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
|
||||
secrets:
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
token: ${{ secrets.GITHUB_TOKEN }}
|
||||
|
||||
14
.github/workflows/docker-tag-latest.yml
vendored
14
.github/workflows/docker-tag-latest.yml
vendored
@@ -17,6 +17,8 @@ on:
|
||||
env:
|
||||
DOCKER_USERNAME: ${{ github.actor }}
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
tag-reth-latest:
|
||||
name: Tag reth as latest
|
||||
@@ -27,16 +29,22 @@ jobs:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Log in to Docker
|
||||
env:
|
||||
DOCKER_PASSWORD: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
|
||||
echo "$DOCKER_PASSWORD" | docker login ghcr.io --username "${DOCKER_USERNAME}" --password-stdin
|
||||
|
||||
- name: Pull reth release image
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/reth:${{ inputs.version }}
|
||||
docker pull ghcr.io/${{ github.repository_owner }}/reth:${VERSION}
|
||||
|
||||
- name: Tag reth as latest
|
||||
env:
|
||||
VERSION: ${{ inputs.version }}
|
||||
run: |
|
||||
docker tag ghcr.io/${{ github.repository_owner }}/reth:${{ inputs.version }} ghcr.io/${{ github.repository_owner }}/reth:latest
|
||||
docker tag ghcr.io/${{ github.repository_owner }}/reth:${VERSION} ghcr.io/${{ github.repository_owner }}/reth:latest
|
||||
|
||||
- name: Push reth latest tag
|
||||
run: |
|
||||
|
||||
22
.github/workflows/docker-test.yml
vendored
22
.github/workflows/docker-test.yml
vendored
@@ -13,6 +13,8 @@ on:
|
||||
default: "artifacts"
|
||||
description: "Name for the uploaded artifact"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
timeout-minutes: 45
|
||||
@@ -21,7 +23,9 @@ jobs:
|
||||
id-token: write
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- run: mkdir -p artifacts
|
||||
|
||||
- name: Get git info
|
||||
@@ -32,8 +36,12 @@ jobs:
|
||||
|
||||
- name: Detect fork
|
||||
id: fork
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name }}
|
||||
REPO: ${{ github.repository }}
|
||||
run: |
|
||||
if [ "${{ github.event_name }}" = "pull_request" ] && [ "${{ github.event.pull_request.head.repo.full_name }}" != "${{ github.repository }}" ]; then
|
||||
if [ "$EVENT_NAME" = "pull_request" ] && [ "$HEAD_REPO" != "$REPO" ]; then
|
||||
echo "is_fork=true" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "is_fork=false" >> "$GITHUB_OUTPUT"
|
||||
@@ -42,11 +50,11 @@ jobs:
|
||||
# Depot build (upstream only)
|
||||
- name: Set up Depot CLI
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/setup-action@v1
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
|
||||
|
||||
- name: Build reth image (Depot)
|
||||
if: steps.fork.outputs.is_fork == 'false'
|
||||
uses: depot/bake-action@v1
|
||||
uses: depot/bake-action@1d58c2668346981089b088b7ef36755b206b20e9 # v1.13.0
|
||||
env:
|
||||
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
@@ -60,11 +68,11 @@ jobs:
|
||||
# Docker build (forks)
|
||||
- name: Set up Docker Buildx
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build reth image (Docker)
|
||||
if: steps.fork.outputs.is_fork == 'true'
|
||||
uses: docker/bake-action@v6
|
||||
uses: docker/bake-action@82490499d2e5613fcead7e128237ef0b0ea210f7 # v7.0.0
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
@@ -76,7 +84,7 @@ jobs:
|
||||
*.dockerfile=Dockerfile
|
||||
|
||||
- name: Upload reth image
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: ${{ inputs.artifact_name }}
|
||||
path: ./artifacts
|
||||
|
||||
23
.github/workflows/docker.yml
vendored
23
.github/workflows/docker.yml
vendored
@@ -29,6 +29,8 @@ on:
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
@@ -39,13 +41,15 @@ jobs:
|
||||
contents: read
|
||||
id-token: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- name: Set up Depot CLI
|
||||
uses: depot/setup-action@v1
|
||||
uses: depot/setup-action@15c09a5f77a0840ad4bce955686522a257853461 # v1.7.1
|
||||
|
||||
- name: Log in to GHCR
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -60,10 +64,13 @@ jobs:
|
||||
|
||||
- name: Determine build parameters
|
||||
id: params
|
||||
env:
|
||||
EVENT_NAME: ${{ github.event_name }}
|
||||
BUILD_TYPE: ${{ inputs.build_type }}
|
||||
run: |
|
||||
REGISTRY="ghcr.io/${{ github.repository_owner }}"
|
||||
|
||||
if [[ "${{ github.event_name }}" == "push" ]]; then
|
||||
if [[ "${EVENT_NAME}" == "push" ]]; then
|
||||
VERSION="${GITHUB_REF#refs/tags/}"
|
||||
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
|
||||
|
||||
@@ -81,7 +88,7 @@ jobs:
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
|
||||
elif [[ "${EVENT_NAME}" == "schedule" ]] || [[ "${BUILD_TYPE}" == "nightly" ]]; then
|
||||
echo "targets=nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
echo "ethereum_set=ethereum.tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
|
||||
@@ -94,7 +101,7 @@ jobs:
|
||||
fi
|
||||
|
||||
- name: Build and push images
|
||||
uses: depot/bake-action@v1
|
||||
uses: depot/bake-action@1d58c2668346981089b088b7ef36755b206b20e9 # v1.13.0
|
||||
env:
|
||||
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
|
||||
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
|
||||
@@ -105,6 +112,8 @@ jobs:
|
||||
files: docker-bake.hcl
|
||||
targets: ${{ steps.params.outputs.targets }}
|
||||
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
|
||||
save: false
|
||||
load: false
|
||||
set: |
|
||||
${{ steps.params.outputs.ethereum_set }}
|
||||
|
||||
@@ -124,7 +133,7 @@ jobs:
|
||||
if: failure() && github.event_name == 'schedule'
|
||||
steps:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3
|
||||
env:
|
||||
SLACK_COLOR: danger
|
||||
SLACK_ICON_EMOJI: ":rotating_light:"
|
||||
|
||||
30
.github/workflows/e2e.yml
vendored
30
.github/workflows/e2e.yml
vendored
@@ -17,19 +17,27 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: e2e-testsuite
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 90
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run e2e tests
|
||||
@@ -48,15 +56,21 @@ jobs:
|
||||
rocksdb:
|
||||
name: e2e-rocksdb
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run RocksDB e2e tests
|
||||
|
||||
24
.github/workflows/fetch-grafana-dashboard.yml
vendored
24
.github/workflows/fetch-grafana-dashboard.yml
vendored
@@ -12,6 +12,8 @@ on:
|
||||
required: true
|
||||
default: "etc/grafana/dashboards/overview.json"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
fetch:
|
||||
runs-on: ubuntu-latest
|
||||
@@ -19,9 +21,11 @@ jobs:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
@@ -29,14 +33,18 @@ jobs:
|
||||
env:
|
||||
FETCH_GRAFANA_DASHBOARD_URL: ${{ secrets.FETCH_GRAFANA_DASHBOARD_URL }}
|
||||
FETCH_GRAFANA_DASHBOARD_TOKEN: ${{ secrets.FETCH_GRAFANA_DASHBOARD_TOKEN }}
|
||||
DASHBOARD_UID: ${{ inputs.dashboard_uid }}
|
||||
TARGET_PATH: ${{ inputs.target_path }}
|
||||
run: |
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py "${{ inputs.dashboard_uid }}" \
|
||||
> "${{ inputs.target_path }}"
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py "${DASHBOARD_UID}" \
|
||||
> "${TARGET_PATH}"
|
||||
|
||||
- name: Check for changes
|
||||
id: diff
|
||||
env:
|
||||
TARGET_PATH: ${{ inputs.target_path }}
|
||||
run: |
|
||||
if git diff --quiet "${{ inputs.target_path }}"; then
|
||||
if git diff --quiet "${TARGET_PATH}"; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No changes detected."
|
||||
else
|
||||
@@ -47,8 +55,10 @@ jobs:
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
DASHBOARD_UID: ${{ inputs.dashboard_uid }}
|
||||
TARGET_PATH: ${{ inputs.target_path }}
|
||||
run: |
|
||||
TARGET="${{ inputs.target_path }}"
|
||||
TARGET="${TARGET_PATH}"
|
||||
FILENAME="$(basename "$TARGET")"
|
||||
BRANCH="chore/sync-grafana-${FILENAME%.*}-$(date +%Y%m%d-%H%M%S)"
|
||||
git config user.name "github-actions[bot]"
|
||||
@@ -59,4 +69,4 @@ jobs:
|
||||
git push origin "$BRANCH"
|
||||
gh pr create \
|
||||
--title "chore: update Grafana dashboard ${FILENAME}" \
|
||||
--body "Automated export from Grafana (dashboard UID: \`${{ inputs.dashboard_uid }}\`, target: \`${TARGET}\`)."
|
||||
--body "Automated export from Grafana (dashboard UID: \`${DASHBOARD_UID}\`, target: \`${TARGET}\`)."
|
||||
|
||||
8
.github/workflows/grafana.yml
vendored
8
.github/workflows/grafana.yml
vendored
@@ -6,11 +6,17 @@ on:
|
||||
push:
|
||||
branches: [main]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
check-dashboard:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Validate dashboard format
|
||||
run: |
|
||||
python3 -c "
|
||||
|
||||
252
.github/workflows/hive.yml
vendored
252
.github/workflows/hive.yml
vendored
@@ -14,8 +14,13 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build-reth:
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/docker-test.yml
|
||||
with:
|
||||
hive_target: hive
|
||||
@@ -25,34 +30,46 @@ jobs:
|
||||
prepare-hive:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-16' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
variant:
|
||||
- amsterdam
|
||||
- osaka
|
||||
name: Prepare Hive - ${{ matrix.variant == 'amsterdam' && 'Amsterdam' || 'Osaka' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
path: hivetests
|
||||
persist-credentials: false
|
||||
|
||||
- name: Get hive commit hash
|
||||
id: hive-commit
|
||||
run: echo "hash=$(cd hivetests && git rev-parse HEAD)" >> $GITHUB_OUTPUT
|
||||
|
||||
- uses: actions/setup-go@v6
|
||||
- uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0
|
||||
with:
|
||||
go-version: "^1.13.1"
|
||||
- run: go version
|
||||
|
||||
- name: Restore hive assets cache
|
||||
id: cache-hive
|
||||
uses: actions/cache@v5
|
||||
uses: actions/cache@668228422ae6a00e4ad889ee87cd7109ec5666a7 # v5.0.4
|
||||
with:
|
||||
path: ./hive_assets
|
||||
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
key: hive-assets-${{ matrix.variant }}-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
|
||||
|
||||
- name: Build hive assets
|
||||
if: steps.cache-hive.outputs.cache-hit != 'true'
|
||||
run: .github/scripts/hive/build_simulators.sh
|
||||
run: .github/scripts/hive/build_simulators.sh ${{ matrix.variant }}
|
||||
|
||||
- name: Load cached Docker images
|
||||
if: steps.cache-hive.outputs.cache-hit == 'true'
|
||||
@@ -68,11 +85,195 @@ jobs:
|
||||
chmod +x hive
|
||||
|
||||
- name: Upload hive assets
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: hive_assets
|
||||
name: hive_assets_${{ matrix.variant }}
|
||||
path: ./hive_assets
|
||||
test:
|
||||
test-amsterdam:
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
# ethereum/rpc to be deprecated:
|
||||
# https://github.com/ethereum/hive/pull/1117
|
||||
scenario:
|
||||
- sim: smoke/genesis
|
||||
- sim: smoke/network
|
||||
- sim: ethereum/sync
|
||||
- sim: devp2p
|
||||
limit: discv4
|
||||
# started failing after https://github.com/ethereum/go-ethereum/pull/31843, no
|
||||
# action on our side, remove from here when we get unexpected passes on these tests
|
||||
# - sim: devp2p
|
||||
# limit: eth
|
||||
# include:
|
||||
# - MaliciousHandshake
|
||||
# # failures tracked in https://github.com/paradigmxyz/reth/issues/14825
|
||||
# - Status
|
||||
# - GetBlockHeaders
|
||||
# - ZeroRequestID
|
||||
# - GetBlockBodies
|
||||
# - Transaction
|
||||
# - NewPooledTxs
|
||||
- sim: devp2p
|
||||
limit: discv5
|
||||
include:
|
||||
# failures tracked at https://github.com/paradigmxyz/reth/issues/14825
|
||||
- PingLargeRequestID
|
||||
- sim: ethereum/engine
|
||||
limit: engine-exchange-capabilities
|
||||
- sim: ethereum/engine
|
||||
limit: engine-withdrawals
|
||||
- sim: ethereum/engine
|
||||
limit: engine-auth
|
||||
- sim: ethereum/engine
|
||||
limit: engine-api
|
||||
- sim: ethereum/engine
|
||||
limit: cancun
|
||||
# eth_ rpc methods
|
||||
- sim: ethereum/rpc-compat
|
||||
include:
|
||||
- eth_blockNumber
|
||||
- eth_call
|
||||
- eth_chainId
|
||||
- eth_createAccessList
|
||||
- eth_estimateGas
|
||||
- eth_feeHistory
|
||||
- eth_getBalance
|
||||
- eth_getBlockBy
|
||||
- eth_getBlockTransactionCountBy
|
||||
- eth_getCode
|
||||
- eth_getProof
|
||||
- eth_getStorage
|
||||
- eth_getTransactionBy
|
||||
- eth_getTransactionCount
|
||||
- eth_getTransactionReceipt
|
||||
- eth_sendRawTransaction
|
||||
- eth_syncing
|
||||
# debug_ rpc methods
|
||||
- debug_
|
||||
|
||||
# consume-engine
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/prague.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/cancun.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/shanghai.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/berlin.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/istanbul.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/homestead.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/frontier.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/paris.*
|
||||
|
||||
# consume-rlp
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/prague.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/cancun.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/shanghai.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/berlin.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/istanbul.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/homestead.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/frontier.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/paris.*
|
||||
needs:
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: Hive-Amsterdam / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download hive assets
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: hive_assets_amsterdam
|
||||
path: /tmp
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reth
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker images
|
||||
run: .github/scripts/hive/load_images.sh
|
||||
|
||||
- name: Move hive binary
|
||||
run: |
|
||||
mv /tmp/hive /usr/local/bin
|
||||
chmod +x /usr/local/bin/hive
|
||||
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
ref: master
|
||||
path: hivetests
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run simulator
|
||||
env:
|
||||
SCENARIO_SIM: ${{ matrix.scenario.sim }}
|
||||
SCENARIO_LIMIT: ${{ matrix.scenario.limit }}
|
||||
SCENARIO_TESTS: ${{ join(matrix.scenario.include, '|') }}
|
||||
run: |
|
||||
LIMIT="$SCENARIO_LIMIT"
|
||||
TESTS="$SCENARIO_TESTS"
|
||||
if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then
|
||||
FILTER="$LIMIT/$TESTS"
|
||||
elif [ -n "$LIMIT" ]; then
|
||||
FILTER="$LIMIT"
|
||||
elif [ -n "$TESTS" ]; then
|
||||
FILTER="/$TESTS"
|
||||
else
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "$SCENARIO_SIM" "$FILTER" "amsterdam"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
|
||||
|
||||
- name: Print simulator output
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
cat hivetests/workspace/logs/*simulator*.log
|
||||
|
||||
- name: Print reth client logs
|
||||
if: ${{ failure() }}
|
||||
run: |
|
||||
cat hivetests/workspace/logs/reth/client-*.log
|
||||
|
||||
test-osaka:
|
||||
timeout-minutes: 120
|
||||
strategy:
|
||||
fail-fast: false
|
||||
@@ -178,24 +379,26 @@ jobs:
|
||||
needs:
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
name: Hive-Osaka / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download hive assets
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: hive_assets
|
||||
name: hive_assets_osaka
|
||||
path: /tmp
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: reth
|
||||
path: /tmp
|
||||
@@ -209,16 +412,21 @@ jobs:
|
||||
chmod +x /usr/local/bin/hive
|
||||
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/hive
|
||||
ref: master
|
||||
path: hivetests
|
||||
persist-credentials: false
|
||||
|
||||
- name: Run simulator
|
||||
env:
|
||||
SCENARIO_SIM: ${{ matrix.scenario.sim }}
|
||||
SCENARIO_LIMIT: ${{ matrix.scenario.limit }}
|
||||
SCENARIO_TESTS: ${{ join(matrix.scenario.include, '|') }}
|
||||
run: |
|
||||
LIMIT="${{ matrix.scenario.limit }}"
|
||||
TESTS="${{ join(matrix.scenario.include, '|') }}"
|
||||
LIMIT="$SCENARIO_LIMIT"
|
||||
TESTS="$SCENARIO_TESTS"
|
||||
if [ -n "$LIMIT" ] && [ -n "$TESTS" ]; then
|
||||
FILTER="$LIMIT/$TESTS"
|
||||
elif [ -n "$LIMIT" ]; then
|
||||
@@ -229,7 +437,7 @@ jobs:
|
||||
FILTER="/"
|
||||
fi
|
||||
echo "filter: $FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
|
||||
.github/scripts/hive/run_simulator.sh "$SCENARIO_SIM" "$FILTER" "osaka"
|
||||
|
||||
- name: Parse hive output
|
||||
run: |
|
||||
@@ -245,12 +453,14 @@ jobs:
|
||||
run: |
|
||||
cat hivetests/workspace/logs/reth/client-*.log
|
||||
notify-on-error:
|
||||
needs: test
|
||||
needs:
|
||||
- test-amsterdam
|
||||
- test-osaka
|
||||
if: failure()
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3
|
||||
env:
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
|
||||
36
.github/workflows/integration.yml
vendored
36
.github/workflows/integration.yml
vendored
@@ -20,11 +20,15 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.network }}
|
||||
if: github.event_name != 'schedule'
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
strategy:
|
||||
@@ -32,14 +36,18 @@ jobs:
|
||||
network: ["ethereum"]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Install Geth
|
||||
run: .github/scripts/install_geth.sh
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run tests
|
||||
@@ -58,7 +66,7 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Decide whether the needed jobs succeeded or failed
|
||||
uses: re-actors/alls-green@release/v1
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
@@ -66,13 +74,19 @@ jobs:
|
||||
name: era1 file integration tests once a day
|
||||
if: github.event_name == 'schedule' && github.repository == 'paradigmxyz/reth'
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: run era1 files integration tests
|
||||
|
||||
16
.github/workflows/kurtosis.yml
vendored
16
.github/workflows/kurtosis.yml
vendored
@@ -18,9 +18,14 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build-reth:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
permissions:
|
||||
contents: read
|
||||
id-token: write
|
||||
uses: ./.github/workflows/docker-test.yml
|
||||
with:
|
||||
hive_target: kurtosis
|
||||
@@ -32,15 +37,18 @@ jobs:
|
||||
fail-fast: false
|
||||
name: run kurtosis
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
needs:
|
||||
- build-reth
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: artifacts
|
||||
path: /tmp
|
||||
@@ -52,7 +60,7 @@ jobs:
|
||||
docker image ls -a
|
||||
|
||||
- name: Run kurtosis
|
||||
uses: ethpandaops/kurtosis-assertoor-github-action@v1
|
||||
uses: ethpandaops/kurtosis-assertoor-github-action@f64942cbc780df731a731ea9f45765b161d2c8df # v1.0.1
|
||||
with:
|
||||
ethereum_package_args: ".github/assets/kurtosis_network_params.yaml"
|
||||
|
||||
@@ -62,7 +70,7 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Slack Webhook Action
|
||||
uses: rtCamp/action-slack-notify@v2
|
||||
uses: rtCamp/action-slack-notify@e31e87e03dd19038e411e38ae27cbad084a90661 # v2.3.3
|
||||
env:
|
||||
SLACK_COLOR: ${{ job.status }}
|
||||
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
|
||||
|
||||
8
.github/workflows/label-pr.yml
vendored
8
.github/workflows/label-pr.yml
vendored
@@ -4,19 +4,23 @@ on:
|
||||
pull_request:
|
||||
types: [opened]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
label_prs:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Label PRs
|
||||
uses: actions/github-script@v9
|
||||
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
|
||||
with:
|
||||
script: |
|
||||
const label_pr = require('./.github/scripts/label_pr.js')
|
||||
|
||||
8
.github/workflows/lint-actions.yml
vendored
8
.github/workflows/lint-actions.yml
vendored
@@ -8,11 +8,17 @@ on:
|
||||
paths:
|
||||
- '.github/**'
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
actionlint:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Download actionlint
|
||||
id: get_actionlint
|
||||
run: bash <(curl https://raw.githubusercontent.com/rhysd/actionlint/main/scripts/download-actionlint.bash)
|
||||
|
||||
190
.github/workflows/lint.yml
vendored
190
.github/workflows/lint.yml
vendored
@@ -10,10 +10,14 @@ env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
clippy-binaries:
|
||||
name: clippy binaries / ${{ matrix.type }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
strategy:
|
||||
matrix:
|
||||
@@ -22,17 +26,19 @@ jobs:
|
||||
args: --workspace --lib --examples --tests --benches --locked
|
||||
features: "ethereum asm-keccak jemalloc jemalloc-prof min-error-logs min-warn-logs min-info-logs min-debug-logs min-trace-logs"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@clippy
|
||||
with:
|
||||
components: clippy
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- if: "${{ matrix.type == 'book' }}"
|
||||
uses: arduino/setup-protoc@v3
|
||||
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Run clippy on binaries
|
||||
@@ -43,15 +49,19 @@ jobs:
|
||||
clippy:
|
||||
name: clippy
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: clippy
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
|
||||
@@ -60,19 +70,25 @@ jobs:
|
||||
|
||||
wasm:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: wasm32-wasip1
|
||||
- uses: taiki-e/install-action@cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: dcarbone/install-jq-action@v3
|
||||
- uses: dcarbone/install-jq-action@b7ef57d46ece78760b4019dbc4080a1ba2a40b45 # v3.2.0
|
||||
- name: Run Wasm checks
|
||||
run: |
|
||||
sudo apt update && sudo apt install gcc-multilib
|
||||
@@ -80,37 +96,49 @@ jobs:
|
||||
|
||||
riscv:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: riscv32imac-unknown-none-elf
|
||||
- uses: taiki-e/install-action@cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: dcarbone/install-jq-action@v3
|
||||
- uses: dcarbone/install-jq-action@b7ef57d46ece78760b4019dbc4080a1ba2a40b45 # v3.2.0
|
||||
- name: Run RISC-V checks
|
||||
run: .github/scripts/check_rv32imac.sh
|
||||
|
||||
crate-checks:
|
||||
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
partition: [1, 2, 3]
|
||||
total_partitions: [3]
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-hack
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo hack check --workspace --partition ${{ matrix.partition }}/${{ matrix.total_partitions }}
|
||||
@@ -118,15 +146,19 @@ jobs:
|
||||
msrv:
|
||||
name: MSRV
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@master
|
||||
with:
|
||||
toolchain: "1.93" # MSRV
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo build --bin reth --workspace
|
||||
@@ -136,13 +168,17 @@ jobs:
|
||||
docs:
|
||||
name: docs
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo docs --document-private-items
|
||||
@@ -154,42 +190,56 @@ jobs:
|
||||
fmt:
|
||||
name: fmt
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
with:
|
||||
components: rustfmt
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- name: Run fmt
|
||||
run: cargo fmt --all --check
|
||||
|
||||
udeps:
|
||||
name: udeps
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: taiki-e/install-action@cargo-udeps
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: cargo-udeps
|
||||
- run: cargo udeps --workspace --lib --examples --tests --benches --all-features --locked
|
||||
|
||||
book:
|
||||
name: book
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@nightly
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo build --bin reth --workspace
|
||||
@@ -201,38 +251,54 @@ jobs:
|
||||
|
||||
typos:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: crate-ci/typos@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: crate-ci/typos@02ea592e44b3a53c302f697cddca7641cd051c3d # v1.45.0
|
||||
|
||||
check-toml:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Run dprint
|
||||
uses: dprint/check@v2.3
|
||||
uses: dprint/check@9cb3a2b17a8e606d37aae341e49df3654933fc23 # v2.3
|
||||
with:
|
||||
config-path: dprint.json
|
||||
|
||||
grafana:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Check dashboard JSON with jq
|
||||
uses: sergeysova/jq-action@v2
|
||||
uses: sergeysova/jq-action@a3f0d4ff59cc1dddf023fc0b325dd75b10deec58 # v2.3.0
|
||||
with:
|
||||
cmd: jq empty etc/grafana/dashboards/overview.json
|
||||
|
||||
no-test-deps:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- name: Ensure no arbitrary or proptest dependency on default build
|
||||
run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0
|
||||
@@ -240,13 +306,17 @@ jobs:
|
||||
# Check crates correctly propagate features
|
||||
feature-propagation:
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: taiki-e/cache-cargo-install-action@v3
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: taiki-e/cache-cargo-install-action@a8b9ecf8e0c0ea09d7481cfc583a5203ecd585b5 # v3.0.5
|
||||
with:
|
||||
tool: zepter
|
||||
- name: Eagerly pull dependencies
|
||||
@@ -254,6 +324,8 @@ jobs:
|
||||
- run: zepter run check
|
||||
|
||||
deny:
|
||||
permissions:
|
||||
contents: read
|
||||
uses: tempoxyz/ci/.github/workflows/deny.yml@main
|
||||
|
||||
lint-success:
|
||||
@@ -277,6 +349,6 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Decide whether the needed jobs succeeded or failed
|
||||
uses: re-actors/alls-green@release/v1
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
23
.github/workflows/pr-audit.yml
vendored
23
.github/workflows/pr-audit.yml
vendored
@@ -4,27 +4,36 @@ on:
|
||||
pull_request:
|
||||
types: [labeled]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
publish:
|
||||
runs-on: ubuntu-latest
|
||||
if: github.event.label.name == 'cyclops'
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Publish event
|
||||
env:
|
||||
EVENTS_KEY: ${{ secrets.EVENTS_KEY }}
|
||||
EVENTS_CERT: ${{ secrets.EVENTS_CERT }}
|
||||
EVENTS_ARGS: ${{ secrets.EVENTS_ARGS }}
|
||||
PR_NUMBER: ${{ github.event.pull_request.number }}
|
||||
PR_SHA: ${{ github.event.pull_request.head.sha }}
|
||||
run: |
|
||||
set -euo pipefail
|
||||
|
||||
echo "${{ secrets.EVENTS_KEY }}" > ${{ runner.temp }}/key
|
||||
echo "${{ secrets.EVENTS_CERT }}" > ${{ runner.temp }}/cert
|
||||
echo "$EVENTS_KEY" > "${{ runner.temp }}/key"
|
||||
echo "$EVENTS_CERT" > "${{ runner.temp }}/cert"
|
||||
|
||||
curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \
|
||||
curl -sf -o /dev/null -X POST $EVENTS_ARGS \
|
||||
-H "Content-Type: application/json" \
|
||||
--key ${{ runner.temp }}/key \
|
||||
--cert ${{ runner.temp }}/cert \
|
||||
--key "${{ runner.temp }}/key" \
|
||||
--cert "${{ runner.temp }}/cert" \
|
||||
-d '{
|
||||
"repository": "${{ github.repository }}",
|
||||
"event": "pr_audit",
|
||||
"data": {
|
||||
"pr_number": ${{ github.event.pull_request.number }},
|
||||
"sha": "${{ github.event.pull_request.head.sha }}"
|
||||
"pr_number": '"$PR_NUMBER"',
|
||||
"sha": "'"$PR_SHA"'"
|
||||
}
|
||||
}'
|
||||
|
||||
11
.github/workflows/pr-title.yml
vendored
11
.github/workflows/pr-title.yml
vendored
@@ -8,20 +8,19 @@ on:
|
||||
- edited
|
||||
- synchronize
|
||||
|
||||
permissions:
|
||||
pull-requests: read
|
||||
contents: read
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
conventional-title:
|
||||
name: Validate PR title is Conventional Commit
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
pull-requests: write
|
||||
steps:
|
||||
- name: Check title
|
||||
id: lint_pr_title
|
||||
uses: amannn/action-semantic-pull-request@v6
|
||||
uses: amannn/action-semantic-pull-request@48f256284bd46cdaab1048c3721360e808335d50 # v6.1.1
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
with:
|
||||
@@ -40,7 +39,7 @@ jobs:
|
||||
continue-on-error: true
|
||||
- name: Add PR Comment for Invalid Title
|
||||
if: steps.lint_pr_title.outcome == 'failure'
|
||||
uses: marocchino/sticky-pull-request-comment@v2
|
||||
uses: marocchino/sticky-pull-request-comment@d4d6b0936434b21bc8345ad45a440c5f7d2c40ff # v3.0.3
|
||||
with:
|
||||
header: pr-title-lint-error
|
||||
message: |
|
||||
@@ -76,7 +75,7 @@ jobs:
|
||||
|
||||
- name: Remove Comment for Valid Title
|
||||
if: steps.lint_pr_title.outcome == 'success'
|
||||
uses: marocchino/sticky-pull-request-comment@v2
|
||||
uses: marocchino/sticky-pull-request-comment@d4d6b0936434b21bc8345ad45a440c5f7d2c40ff # v3.0.3
|
||||
with:
|
||||
header: pr-title-lint-error
|
||||
delete: true
|
||||
|
||||
5
.github/workflows/release-dist.yml
vendored
5
.github/workflows/release-dist.yml
vendored
@@ -7,12 +7,15 @@ on:
|
||||
release:
|
||||
types: [published]
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
release-homebrew:
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Update Homebrew formula
|
||||
uses: dawidd6/action-homebrew-bump-formula@v7
|
||||
uses: dawidd6/action-homebrew-bump-formula@1446dca236b0440c6f02723a3f14f13be2c04ab0 # v7
|
||||
with:
|
||||
token: ${{ secrets.HOMEBREW }}
|
||||
no_fork: true
|
||||
|
||||
24
.github/workflows/release-reproducible.yml
vendored
24
.github/workflows/release-reproducible.yml
vendored
@@ -2,6 +2,8 @@
|
||||
|
||||
name: release-reproducible
|
||||
|
||||
permissions: {}
|
||||
|
||||
on:
|
||||
workflow_run:
|
||||
workflows: [release]
|
||||
@@ -15,20 +17,23 @@ jobs:
|
||||
name: extract version
|
||||
if: ${{ github.event.workflow_run.conclusion == 'success' }}
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
steps:
|
||||
- name: Extract version from triggering tag
|
||||
id: extract_version
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
HEAD_SHA: ${{ github.event.workflow_run.head_sha }}
|
||||
run: |
|
||||
# Get the tag that points to the head SHA of the triggering workflow
|
||||
TAG=$(gh api /repos/${{ github.repository }}/git/refs/tags \
|
||||
--jq '.[] | select(.object.sha == "${{ github.event.workflow_run.head_sha }}") | .ref' \
|
||||
--jq ".[] | select(.object.sha == \"${HEAD_SHA}\") | .ref" \
|
||||
| head -1 \
|
||||
| sed 's|refs/tags/||')
|
||||
|
||||
if [ -z "$TAG" ]; then
|
||||
echo "No tag found for SHA ${{ github.event.workflow_run.head_sha }}"
|
||||
echo "No tag found for SHA ${HEAD_SHA}"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
@@ -44,15 +49,16 @@ jobs:
|
||||
packages: write
|
||||
contents: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
ref: ${{ needs.extract-version.outputs.VERSION }}
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Log in to GitHub Container Registry
|
||||
uses: docker/login-action@v4
|
||||
uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
|
||||
with:
|
||||
registry: ghcr.io
|
||||
username: ${{ github.actor }}
|
||||
@@ -65,7 +71,7 @@ jobs:
|
||||
echo "RUST_TOOLCHAIN=$RUST_TOOLCHAIN" >> $GITHUB_OUTPUT
|
||||
|
||||
- name: Build reproducible artifacts
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
id: docker_build
|
||||
with:
|
||||
context: .
|
||||
@@ -75,13 +81,11 @@ jobs:
|
||||
VERSION=${{ needs.extract-version.outputs.VERSION }}
|
||||
target: artifacts
|
||||
outputs: type=local,dest=./docker-artifacts
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
env:
|
||||
DOCKER_BUILD_RECORD_UPLOAD: false
|
||||
|
||||
- name: Build and push final image
|
||||
uses: docker/build-push-action@v6
|
||||
uses: docker/build-push-action@d08e5c354a6adb9ed34480a06d141179aa583294 # v7.0.0
|
||||
with:
|
||||
context: .
|
||||
file: ./Dockerfile.reproducible
|
||||
@@ -92,8 +96,6 @@ jobs:
|
||||
tags: |
|
||||
${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:${{ needs.extract-version.outputs.VERSION }}
|
||||
${{ env.DOCKER_REPRODUCIBLE_IMAGE_NAME }}:latest
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
provenance: false
|
||||
env:
|
||||
DOCKER_BUILD_RECORD_UPLOAD: false
|
||||
|
||||
48
.github/workflows/release.yml
vendored
48
.github/workflows/release.yml
vendored
@@ -3,6 +3,8 @@
|
||||
|
||||
name: release
|
||||
|
||||
permissions: {}
|
||||
|
||||
on:
|
||||
push:
|
||||
tags:
|
||||
@@ -20,21 +22,24 @@ env:
|
||||
REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/reth-reproducible
|
||||
CARGO_TERM_COLOR: always
|
||||
DOCKER_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/reth
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
jobs:
|
||||
dry-run:
|
||||
name: check dry run
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- run: |
|
||||
echo "Dry run value: ${{ github.event.inputs.dry_run }}"
|
||||
echo "Dry run enabled: ${{ github.event.inputs.dry_run == 'true'}}"
|
||||
echo "Dry run disabled: ${{ github.event.inputs.dry_run != 'true'}}"
|
||||
- env:
|
||||
DRY_RUN: ${{ github.event.inputs.dry_run }}
|
||||
run: |
|
||||
echo "Dry run value: ${DRY_RUN}"
|
||||
echo "Dry run enabled: $( [ "${DRY_RUN}" = 'true' ] && echo true || echo false )"
|
||||
echo "Dry run disabled: $( [ "${DRY_RUN}" != 'true' ] && echo true || echo false )"
|
||||
|
||||
extract-version:
|
||||
name: extract version
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
steps:
|
||||
- name: Extract version
|
||||
run: echo "VERSION=${GITHUB_REF_NAME//\//-}" >> $GITHUB_OUTPUT
|
||||
@@ -45,12 +50,15 @@ jobs:
|
||||
check-version:
|
||||
name: check version
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: read
|
||||
needs: extract-version
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- name: Verify crate version matches tag
|
||||
# Check that the Cargo version starts with the tag,
|
||||
# so that Cargo version 1.4.8 can be matched against both v1.4.8 and v1.4.8-rc.1
|
||||
@@ -63,6 +71,8 @@ jobs:
|
||||
build:
|
||||
name: build release
|
||||
runs-on: ${{ matrix.configs.os }}
|
||||
permissions:
|
||||
contents: read
|
||||
needs: extract-version
|
||||
continue-on-error: ${{ matrix.configs.allow_fail }}
|
||||
strategy:
|
||||
@@ -95,20 +105,20 @@ jobs:
|
||||
- command: build
|
||||
binary: reth
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: ${{ matrix.configs.target }}
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- name: Install cross main
|
||||
if: ${{ !matrix.configs.native }}
|
||||
id: cross_main
|
||||
run: |
|
||||
cargo install cross --locked --git https://github.com/cross-rs/cross
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
with:
|
||||
cache-on-failure: true
|
||||
cargo install cross --locked \
|
||||
--git https://github.com/cross-rs/cross \
|
||||
--rev 65fe72b0cdb1e7e0cc0652517498d4389cc8f5cf
|
||||
|
||||
- name: Apple M1 setup
|
||||
if: matrix.configs.target == 'aarch64-apple-darwin'
|
||||
@@ -145,14 +155,14 @@ jobs:
|
||||
|
||||
- name: Upload artifact
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
|
||||
path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
|
||||
|
||||
- name: Upload signature
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
|
||||
path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
|
||||
@@ -171,11 +181,12 @@ jobs:
|
||||
steps:
|
||||
# This is necessary for generating the changelog.
|
||||
# It has to come before "Download Artifacts" or else it deletes the artifacts.
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
fetch-depth: 0
|
||||
- name: Download artifacts
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
- name: Generate full changelog
|
||||
id: changelog
|
||||
run: |
|
||||
@@ -261,6 +272,7 @@ jobs:
|
||||
dry-run-summary:
|
||||
name: dry run summary
|
||||
runs-on: ubuntu-latest
|
||||
permissions: {}
|
||||
needs: [build, extract-version]
|
||||
if: ${{ github.event.inputs.dry_run == 'true' }}
|
||||
env:
|
||||
|
||||
16
.github/workflows/reproducible-build.yml
vendored
16
.github/workflows/reproducible-build.yml
vendored
@@ -5,11 +5,15 @@ on:
|
||||
schedule:
|
||||
- cron: "0 1 */2 * *"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
build:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: build reproducible binaries
|
||||
runs-on: ${{ matrix.runner }}
|
||||
permissions:
|
||||
contents: read
|
||||
strategy:
|
||||
matrix:
|
||||
include:
|
||||
@@ -18,14 +22,16 @@ jobs:
|
||||
- runner: ubuntu-22.04
|
||||
machine: machine-2
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
with:
|
||||
target: x86_64-unknown-linux-gnu
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
uses: docker/setup-buildx-action@4d04d5d9486b7bd6fa91e7baf45bbb4f8b9deedd # v4.0.0
|
||||
|
||||
- name: Build reproducible binary with Docker
|
||||
run: |
|
||||
@@ -43,7 +49,7 @@ jobs:
|
||||
echo "Binaries SHA256 on ${{ matrix.machine }}: $(cat checksum.sha256)"
|
||||
|
||||
- name: Upload the hash
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@bbbca2ddaa5d8feaa63e36b76fdaad77386f024f # v7.0.0
|
||||
with:
|
||||
name: checksum-${{ matrix.machine }}
|
||||
path: |
|
||||
@@ -56,12 +62,12 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Download artifacts from machine-1
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: checksum-machine-1
|
||||
path: machine-1/
|
||||
- name: Download artifacts from machine-2
|
||||
uses: actions/download-artifact@v8
|
||||
uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
|
||||
with:
|
||||
name: checksum-machine-2
|
||||
path: machine-2/
|
||||
|
||||
14
.github/workflows/stage.yml
vendored
14
.github/workflows/stage.yml
vendored
@@ -18,22 +18,28 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
stage:
|
||||
name: stage-run-test
|
||||
# Only run stage commands test in merge groups
|
||||
if: github.event_name == 'merge_group'
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 60
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Build reth
|
||||
|
||||
4
.github/workflows/stale.yml
vendored
4
.github/workflows/stale.yml
vendored
@@ -7,6 +7,8 @@ on:
|
||||
schedule:
|
||||
- cron: "30 1 * * *"
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
close-issues:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
@@ -15,7 +17,7 @@ jobs:
|
||||
issues: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/stale@v10
|
||||
- uses: actions/stale@b5d41d4e1d5dceea10e7104786b73624c18a190f # v10.2.0
|
||||
with:
|
||||
days-before-stale: 21
|
||||
days-before-close: 7
|
||||
|
||||
14
.github/workflows/sync-era.yml
vendored
14
.github/workflows/sync-era.yml
vendored
@@ -15,11 +15,15 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -34,11 +38,13 @@ jobs:
|
||||
block: 100000
|
||||
unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Build ${{ matrix.chain.bin }}
|
||||
|
||||
14
.github/workflows/sync.yml
vendored
14
.github/workflows/sync.yml
vendored
@@ -15,11 +15,15 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
sync:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
name: sync (${{ matrix.chain.bin }})
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
@@ -34,11 +38,13 @@ jobs:
|
||||
block: 100000
|
||||
unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a"
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Build ${{ matrix.chain.bin }}
|
||||
|
||||
53
.github/workflows/unit.yml
vendored
53
.github/workflows/unit.yml
vendored
@@ -17,10 +17,14 @@ concurrency:
|
||||
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
||||
cancel-in-progress: true
|
||||
|
||||
permissions: {}
|
||||
|
||||
jobs:
|
||||
test:
|
||||
name: test / ${{ matrix.type }}
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
strategy:
|
||||
@@ -32,16 +36,20 @@ jobs:
|
||||
exclude_args: ""
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- if: "${{ matrix.type == 'book' }}"
|
||||
uses: arduino/setup-protoc@v3
|
||||
uses: arduino/setup-protoc@c65c819552d16ad3c9b72d9dfd5ba5237b9c906b # v3.0.0
|
||||
with:
|
||||
repo-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
- name: Run tests
|
||||
@@ -56,20 +64,25 @@ jobs:
|
||||
state:
|
||||
name: Ethereum state tests
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_LOG: info,sync=error
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- name: Checkout ethereum/tests
|
||||
uses: actions/checkout@v6
|
||||
uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
repository: ethereum/tests
|
||||
ref: 81862e4848585a438d64f911a19b3825f0f4cd95
|
||||
path: testing/ef-tests/ethereum-tests
|
||||
submodules: recursive
|
||||
fetch-depth: 1
|
||||
persist-credentials: false
|
||||
- name: Download & extract EEST fixtures (public)
|
||||
shell: bash
|
||||
env:
|
||||
@@ -79,11 +92,13 @@ jobs:
|
||||
mkdir -p testing/ef-tests/execution-spec-tests
|
||||
URL="https://github.com/ethereum/execution-spec-tests/releases/download/${EEST_TESTS_TAG}/fixtures_stable.tar.gz"
|
||||
curl -L "$URL" | tar -xz --strip-components=1 -C testing/ef-tests/execution-spec-tests
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: taiki-e/install-action@nextest
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: taiki-e/install-action@1f2425cdb59f8fffb99ee16a5968edf6f57a2b93 # v2.75.24
|
||||
with:
|
||||
tool: nextest
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- run: cargo nextest run --no-fail-fast --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
|
||||
@@ -91,15 +106,19 @@ jobs:
|
||||
doc:
|
||||
name: doc tests
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
contents: read
|
||||
env:
|
||||
RUST_BACKTRACE: 1
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- uses: rui314/setup-mold@v1
|
||||
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
|
||||
with:
|
||||
persist-credentials: false
|
||||
- uses: rui314/setup-mold@725a8794d15fc7563f59595bd9556495c0564878 # v1
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
- uses: Swatinem/rust-cache@v2
|
||||
- uses: mozilla-actions/sccache-action@7d986dd989559c6ecdb630a3fd2557667be217ad # v0.0.9
|
||||
- uses: Swatinem/rust-cache@c19371144df3bb44fab255c43d04cbc2ab54d1c4 # v2.9.1
|
||||
with:
|
||||
cache-on-failure: true
|
||||
- name: Run doctests
|
||||
@@ -113,6 +132,6 @@ jobs:
|
||||
timeout-minutes: 30
|
||||
steps:
|
||||
- name: Decide whether the needed jobs succeeded or failed
|
||||
uses: re-actors/alls-green@release/v1
|
||||
uses: re-actors/alls-green@05ac9388f0aebcb5727afa17fcccfecd6f8ec5fe # release/v1
|
||||
with:
|
||||
jobs: ${{ toJSON(needs) }}
|
||||
|
||||
605
Cargo.lock
generated
605
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
72
Cargo.toml
72
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "2.1.0"
|
||||
version = "2.2.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.93"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -326,8 +326,8 @@ reth-cli = { path = "crates/cli/cli" }
|
||||
reth-cli-commands = { path = "crates/cli/commands" }
|
||||
reth-cli-runner = { path = "crates/cli/runner" }
|
||||
reth-cli-util = { path = "crates/cli/util" }
|
||||
reth-codecs = { version = "0.3.0", default-features = false }
|
||||
reth-codecs-derive = "0.3.0"
|
||||
reth-codecs = { version = "0.3.1", default-features = false }
|
||||
reth-codecs-derive = "0.3.1"
|
||||
reth-config = { path = "crates/config", default-features = false }
|
||||
reth-consensus = { path = "crates/consensus/consensus", default-features = false }
|
||||
reth-consensus-common = { path = "crates/consensus/common", default-features = false }
|
||||
@@ -395,7 +395,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
|
||||
reth-payload-primitives = { path = "crates/payload/primitives" }
|
||||
reth-payload-validator = { path = "crates/payload/validator" }
|
||||
reth-payload-util = { path = "crates/payload/util" }
|
||||
reth-primitives-traits = { version = "0.3.0", default-features = false }
|
||||
reth-primitives-traits = { version = "0.3.1", default-features = false }
|
||||
reth-provider = { path = "crates/storage/provider" }
|
||||
reth-prune = { path = "crates/prune/prune" }
|
||||
reth-prune-types = { path = "crates/prune/types", default-features = false }
|
||||
@@ -411,7 +411,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal
|
||||
reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
|
||||
reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
|
||||
reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
|
||||
reth-rpc-traits = { version = "0.3.0", default-features = false }
|
||||
reth-rpc-traits = { version = "0.3.1", default-features = false }
|
||||
reth-stages = { path = "crates/stages/stages" }
|
||||
reth-stages-api = { path = "crates/stages/api" }
|
||||
reth-stages-types = { path = "crates/stages/types", default-features = false }
|
||||
@@ -430,7 +430,7 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
|
||||
reth-trie-db = { path = "crates/trie/db" }
|
||||
reth-trie-parallel = { path = "crates/trie/parallel" }
|
||||
reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.0", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.1", default-features = false }
|
||||
|
||||
# revm
|
||||
revm = { version = "38.0.0", default-features = false }
|
||||
@@ -449,40 +449,40 @@ alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.33", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.33.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.4", default-features = false }
|
||||
alloy-evm = { version = "0.34.0", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.7"
|
||||
|
||||
alloy-consensus = { version = "2.0.0", default-features = false }
|
||||
alloy-contract = { version = "2.0.0", default-features = false }
|
||||
alloy-eips = { version = "2.0.0", default-features = false }
|
||||
alloy-genesis = { version = "2.0.0", default-features = false }
|
||||
alloy-json-rpc = { version = "2.0.0", default-features = false }
|
||||
alloy-network = { version = "2.0.0", default-features = false }
|
||||
alloy-network-primitives = { version = "2.0.0", default-features = false }
|
||||
alloy-provider = { version = "2.0.0", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-client = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types = { version = "2.0.0", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "2.0.0", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "2.0.0", default-features = false }
|
||||
alloy-serde = { version = "2.0.0", default-features = false }
|
||||
alloy-signer = { version = "2.0.0", default-features = false }
|
||||
alloy-signer-local = { version = "2.0.0", default-features = false }
|
||||
alloy-transport = { version = "2.0.0" }
|
||||
alloy-transport-http = { version = "2.0.0", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "2.0.0", default-features = false }
|
||||
alloy-transport-ws = { version = "2.0.0", default-features = false }
|
||||
alloy-consensus = { version = "2.0.4", default-features = false }
|
||||
alloy-contract = { version = "2.0.4", default-features = false }
|
||||
alloy-eips = { version = "2.0.4", default-features = false }
|
||||
alloy-genesis = { version = "2.0.4", default-features = false }
|
||||
alloy-json-rpc = { version = "2.0.4", default-features = false }
|
||||
alloy-network = { version = "2.0.4", default-features = false }
|
||||
alloy-network-primitives = { version = "2.0.4", default-features = false }
|
||||
alloy-provider = { version = "2.0.4", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-client = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types = { version = "2.0.4", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-debug = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-engine = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-eth = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-mev = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-trace = { version = "2.0.4", default-features = false }
|
||||
alloy-rpc-types-txpool = { version = "2.0.4", default-features = false }
|
||||
alloy-serde = { version = "2.0.4", default-features = false }
|
||||
alloy-signer = { version = "2.0.4", default-features = false }
|
||||
alloy-signer-local = { version = "2.0.4", default-features = false }
|
||||
alloy-transport = { version = "2.0.4" }
|
||||
alloy-transport-http = { version = "2.0.4", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "2.0.4", default-features = false }
|
||||
alloy-transport-ws = { version = "2.0.4", default-features = false }
|
||||
|
||||
# misc
|
||||
either = { version = "1.15.0", default-features = false }
|
||||
@@ -581,7 +581,7 @@ tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
|
||||
# p2p
|
||||
discv5 = "0.10"
|
||||
discv5 = { git = "https://github.com/sigp/discv5", rev = "7663c00" }
|
||||
if-addrs = "0.14"
|
||||
|
||||
# rpc
|
||||
|
||||
13
Dockerfile
13
Dockerfile
@@ -33,15 +33,24 @@ ENV FEATURES=$FEATURES
|
||||
RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json
|
||||
|
||||
# Build application
|
||||
# Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
|
||||
#
|
||||
# TARGETPLATFORM is set by BuildKit: https://docs.docker.com/reference/dockerfile#automatic-platform-args-in-the-global-scope
|
||||
ARG TARGETPLATFORM
|
||||
COPY --exclude=dist . .
|
||||
RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
|
||||
RUN if [ -n "$RUSTFLAGS" ]; then \
|
||||
export RUSTFLAGS="$RUSTFLAGS"; \
|
||||
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
|
||||
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
|
||||
fi && \
|
||||
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin reth
|
||||
|
||||
# ARG is not resolved in COPY so we have to hack around it by copying the
|
||||
# binary to a temporary location
|
||||
RUN cp /app/target/$BUILD_PROFILE/reth /app/reth
|
||||
|
||||
# Use Ubuntu as the release image
|
||||
FROM ubuntu AS runtime
|
||||
FROM ubuntu:24.04 AS runtime
|
||||
WORKDIR /app
|
||||
|
||||
# Copy reth over from the build stage
|
||||
|
||||
@@ -69,6 +69,7 @@ default = [
|
||||
"jemalloc",
|
||||
"reth-cli-util/jemalloc",
|
||||
"asm-keccak",
|
||||
"keccak-cache-global",
|
||||
"min-debug-logs",
|
||||
]
|
||||
|
||||
@@ -89,6 +90,12 @@ asm-keccak = [
|
||||
"revm-primitives/asm-keccak",
|
||||
]
|
||||
|
||||
keccak-cache-global = [
|
||||
"reth-node-core/keccak-cache-global",
|
||||
"reth-node-ethereum/keccak-cache-global",
|
||||
"alloy-primitives/keccak-cache-global",
|
||||
]
|
||||
|
||||
min-debug-logs = [
|
||||
"tracing/release_max_level_debug",
|
||||
"reth-ethereum-cli/min-debug-logs",
|
||||
|
||||
@@ -7,15 +7,16 @@
|
||||
//! `execute_transaction` to apply segment-boundary changes.
|
||||
|
||||
use crate::evm_config::BigBlockSegment;
|
||||
use alloy_consensus::TransactionEnvelope;
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_evm::{
|
||||
block::{
|
||||
BlockExecutionError, BlockExecutionResult, BlockExecutor, BlockExecutorFactory,
|
||||
BlockExecutorFor, ExecutableTx, GasOutput, OnStateHook, StateChangeSource, StateDB,
|
||||
ExecutableTx, GasOutput, OnStateHook, StateChangeSource, StateDB,
|
||||
},
|
||||
eth::{EthBlockExecutionCtx, EthBlockExecutor, EthEvmContext, EthTxResult},
|
||||
precompiles::PrecompilesMap,
|
||||
Database, EthEvm, EthEvmFactory, Evm, FromRecoveredTx, FromTxWithEncoded,
|
||||
Database, EthEvm, EthEvmFactory, Evm, EvmFactory, FromRecoveredTx, FromTxWithEncoded,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use reth_ethereum_primitives::{Receipt, TransactionSigned};
|
||||
@@ -36,6 +37,7 @@ use tracing::{debug, trace};
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/// Runtime state for segment boundary tracking.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct BbEvmPlan {
|
||||
/// The segment boundaries and environments.
|
||||
pub(crate) segments: Vec<BigBlockSegment>,
|
||||
@@ -73,6 +75,10 @@ impl BbEvmPlan {
|
||||
.filter(|(n, _)| *n >= min && *n < block_number)
|
||||
.collect()
|
||||
}
|
||||
|
||||
pub(crate) fn segment_index_for_tx(&self, tx_index: usize) -> usize {
|
||||
self.segments.partition_point(|segment| segment.start_tx <= tx_index).saturating_sub(1)
|
||||
}
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for BbEvmPlan {
|
||||
@@ -97,6 +103,9 @@ impl std::fmt::Debug for BbEvmPlan {
|
||||
/// segment boundaries without requiring additional trait bounds on `DB`.
|
||||
pub(crate) type BlockHashSeeder<DB> = fn(&mut DB, &[(u64, B256)]);
|
||||
|
||||
/// Function pointer that reads the BAL index from the DB.
|
||||
pub(crate) type BalIndexReader<DB> = fn(&DB) -> u64;
|
||||
|
||||
/// Block executor that wraps [`EthBlockExecutor`] and handles segment-boundary
|
||||
/// changes for big-block execution.
|
||||
///
|
||||
@@ -108,7 +117,8 @@ pub(crate) type BlockHashSeeder<DB> = fn(&mut DB, &[(u64, B256)]);
|
||||
/// Gas counters reset at each boundary so that each segment's real gas limit
|
||||
/// is used (preserving correct GASLIMIT opcode behavior). Accumulated offsets
|
||||
/// are applied to receipts and totals in `finish()`.
|
||||
pub(crate) struct BbBlockExecutor<'a, DB, I, P, Spec>
|
||||
#[expect(missing_debug_implementations)]
|
||||
pub struct BbBlockExecutor<'a, DB, I, P, Spec>
|
||||
where
|
||||
DB: Database,
|
||||
{
|
||||
@@ -131,6 +141,10 @@ where
|
||||
/// Callback to reseed block hashes into the DB's cache at segment
|
||||
/// boundaries. See [`BlockHashSeeder`].
|
||||
block_hash_seeder: Option<BlockHashSeeder<DB>>,
|
||||
/// Callback to read the BAL index from the DB.
|
||||
bal_index_reader: Option<BalIndexReader<DB>>,
|
||||
/// Whether the executor has selected its starting segment.
|
||||
initialized: bool,
|
||||
}
|
||||
|
||||
impl<'a, DB, I, P, Spec> BbBlockExecutor<'a, DB, I, P, Spec>
|
||||
@@ -156,6 +170,7 @@ where
|
||||
receipt_builder: RethReceiptBuilder,
|
||||
plan: Option<BbEvmPlan>,
|
||||
block_hash_seeder: Option<BlockHashSeeder<DB>>,
|
||||
bal_index_reader: Option<BalIndexReader<DB>>,
|
||||
) -> Self {
|
||||
let inner = EthBlockExecutor::new(evm, ctx, spec, receipt_builder);
|
||||
Self {
|
||||
@@ -166,9 +181,63 @@ where
|
||||
blob_gas_used_offset: 0,
|
||||
shared_hook: Arc::new(Mutex::new(None)),
|
||||
block_hash_seeder,
|
||||
bal_index_reader,
|
||||
initialized: false,
|
||||
}
|
||||
}
|
||||
|
||||
fn initialize(&mut self) -> Result<(), BlockExecutionError> {
|
||||
if self.initialized {
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let plan = match &self.plan {
|
||||
Some(plan) => plan,
|
||||
None => return Ok(()),
|
||||
};
|
||||
|
||||
self.initialized = true;
|
||||
|
||||
let bal_index =
|
||||
self.bal_index_reader.map(|reader| reader(self.inner().evm().db())).unwrap_or(0);
|
||||
let segment_idx =
|
||||
if bal_index == 0 { 0 } else { plan.segment_index_for_tx((bal_index - 1) as usize) };
|
||||
let segment = &plan.segments[segment_idx];
|
||||
|
||||
// Swap the EVM's block_env and executor ctx to the selected segment's
|
||||
// values so that EIP-2935/EIP-4788 system calls use the correct block
|
||||
// number and parent hash. Without this, the outer big block header's
|
||||
// block_number (which is synthetic) would be used, writing to wrong
|
||||
// EIP-2935 slots and corrupting state.
|
||||
let block_env = segment.evm_env.block_env.clone();
|
||||
let block_number = block_env.number.saturating_to::<u64>();
|
||||
let mut cfg_env = segment.evm_env.cfg_env.clone();
|
||||
cfg_env.disable_base_fee = true;
|
||||
let ctx = EthBlockExecutionCtx {
|
||||
parent_hash: segment.ctx.parent_hash,
|
||||
parent_beacon_block_root: segment.ctx.parent_beacon_block_root,
|
||||
ommers: segment.ctx.ommers,
|
||||
withdrawals: segment.ctx.withdrawals.clone(),
|
||||
extra_data: segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: segment.ctx.tx_count_hint,
|
||||
slot_number: segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
let inner = self.inner_mut();
|
||||
let evm_ctx = inner.evm.ctx_mut();
|
||||
evm_ctx.block = block_env;
|
||||
evm_ctx.cfg = cfg_env;
|
||||
inner.ctx = ctx;
|
||||
|
||||
self.reseed_block_hashes_for(block_number);
|
||||
|
||||
if bal_index > 0 {
|
||||
self.plan = None;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a forwarding `OnStateHook` that delegates to the shared hook.
|
||||
fn forwarding_hook(&self) -> Option<Box<dyn OnStateHook>> {
|
||||
let shared = self.shared_hook.clone();
|
||||
@@ -349,35 +418,9 @@ where
|
||||
type Result = EthTxResult<HaltReason, alloy_consensus::TxType>;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
// Swap the EVM's block_env and executor ctx to the first segment's
|
||||
// values so that the initial EIP-2935/EIP-4788 system calls use the
|
||||
// correct block number and parent hash. Without this, the outer big
|
||||
// block header's block_number (which is synthetic) would be used,
|
||||
// writing to wrong EIP-2935 slots and corrupting state.
|
||||
if let Some(seg0) = self.plan.as_ref().map(|p| &p.segments[0]) {
|
||||
let block_env = seg0.evm_env.block_env.clone();
|
||||
let block_number = block_env.number.saturating_to::<u64>();
|
||||
let mut cfg_env = seg0.evm_env.cfg_env.clone();
|
||||
cfg_env.disable_base_fee = true;
|
||||
let seg0_ctx = EthBlockExecutionCtx {
|
||||
parent_hash: seg0.ctx.parent_hash,
|
||||
parent_beacon_block_root: seg0.ctx.parent_beacon_block_root,
|
||||
ommers: seg0.ctx.ommers,
|
||||
withdrawals: seg0.ctx.withdrawals.clone(),
|
||||
extra_data: seg0.ctx.extra_data.clone(),
|
||||
tx_count_hint: seg0.ctx.tx_count_hint,
|
||||
slot_number: seg0.ctx.slot_number,
|
||||
};
|
||||
|
||||
let inner = self.inner_mut();
|
||||
let evm_ctx = inner.evm.ctx_mut();
|
||||
evm_ctx.block = block_env;
|
||||
evm_ctx.cfg = cfg_env;
|
||||
inner.ctx = seg0_ctx;
|
||||
|
||||
self.reseed_block_hashes_for(block_number);
|
||||
}
|
||||
|
||||
// The outer big-block header uses a synthetic block number, so start
|
||||
// system calls must run against the selected real segment env.
|
||||
self.initialize()?;
|
||||
self.inner_mut().apply_pre_execution_changes()
|
||||
}
|
||||
|
||||
@@ -385,15 +428,16 @@ where
|
||||
&mut self,
|
||||
tx: impl ExecutableTx<Self>,
|
||||
) -> Result<Self::Result, BlockExecutionError> {
|
||||
self.initialize()?;
|
||||
self.maybe_apply_boundary()?;
|
||||
self.inner_mut().execute_transaction_without_commit(tx)
|
||||
}
|
||||
|
||||
fn commit_transaction(
|
||||
&mut self,
|
||||
output: Self::Result,
|
||||
) -> Result<GasOutput, BlockExecutionError> {
|
||||
let gas_used = self.inner_mut().commit_transaction(output)?;
|
||||
fn commit_transaction(&mut self, output: Self::Result) -> GasOutput {
|
||||
self.maybe_apply_boundary()
|
||||
.expect("segment boundary application must succeed before committing transaction");
|
||||
|
||||
let gas_used = self.inner_mut().commit_transaction(output);
|
||||
|
||||
// Fix up cumulative_gas_used on the just-committed receipt so that
|
||||
// the receipt root task (which reads receipts incrementally) sees
|
||||
@@ -408,7 +452,7 @@ where
|
||||
if let Some(plan) = &mut self.plan {
|
||||
plan.tx_counter += 1;
|
||||
}
|
||||
Ok(gas_used)
|
||||
gas_used
|
||||
}
|
||||
|
||||
fn finish(
|
||||
@@ -499,7 +543,7 @@ pub struct BbBlockExecutorFactory<Spec> {
|
||||
receipt_builder: RethReceiptBuilder,
|
||||
spec: Spec,
|
||||
evm_factory: EthEvmFactory,
|
||||
/// Staged plan consumed by the next [`BbBlockExecutor`].
|
||||
/// Staged plan cloned into each [`BbBlockExecutor`].
|
||||
pub(crate) staged_plan: Arc<Mutex<Option<BbEvmPlan>>>,
|
||||
}
|
||||
|
||||
@@ -528,8 +572,12 @@ impl<Spec> BbBlockExecutorFactory<Spec> {
|
||||
*self.staged_plan.lock().unwrap() = Some(plan);
|
||||
}
|
||||
|
||||
fn take_plan(&self) -> Option<BbEvmPlan> {
|
||||
self.staged_plan.lock().unwrap().take()
|
||||
pub(crate) fn clear_staged_plan(&self) {
|
||||
*self.staged_plan.lock().unwrap() = None;
|
||||
}
|
||||
|
||||
fn peek_plan(&self) -> Option<BbEvmPlan> {
|
||||
self.staged_plan.lock().unwrap().clone()
|
||||
}
|
||||
|
||||
pub(crate) fn create_executor_with_seeder<'a, DB, I>(
|
||||
@@ -537,14 +585,23 @@ impl<Spec> BbBlockExecutorFactory<Spec> {
|
||||
evm: EthEvm<DB, I, PrecompilesMap>,
|
||||
ctx: EthBlockExecutionCtx<'a>,
|
||||
block_hash_seeder: Option<BlockHashSeeder<DB>>,
|
||||
bal_index_reader: Option<BalIndexReader<DB>>,
|
||||
) -> BbBlockExecutor<'a, DB, I, PrecompilesMap, &'a Spec>
|
||||
where
|
||||
Spec: alloy_evm::eth::spec::EthExecutorSpec,
|
||||
DB: StateDB + 'a,
|
||||
I: Inspector<EthEvmContext<DB>> + 'a,
|
||||
{
|
||||
let plan = self.take_plan();
|
||||
BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, block_hash_seeder)
|
||||
let plan = self.peek_plan();
|
||||
BbBlockExecutor::new(
|
||||
evm,
|
||||
ctx,
|
||||
&self.spec,
|
||||
self.receipt_builder,
|
||||
plan,
|
||||
block_hash_seeder,
|
||||
bal_index_reader,
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -557,6 +614,12 @@ where
|
||||
type ExecutionCtx<'a> = EthBlockExecutionCtx<'a>;
|
||||
type Transaction = TransactionSigned;
|
||||
type Receipt = Receipt;
|
||||
type TxExecutionResult = EthTxResult<
|
||||
<EthEvmFactory as EvmFactory>::HaltReason,
|
||||
<TransactionSigned as TransactionEnvelope>::TxType,
|
||||
>;
|
||||
type Executor<'a, DB: StateDB, I: Inspector<EthEvmContext<DB>>> =
|
||||
BbBlockExecutor<'a, DB, I, PrecompilesMap, &'a Spec>;
|
||||
|
||||
fn evm_factory(&self) -> &Self::EvmFactory {
|
||||
&self.evm_factory
|
||||
@@ -566,12 +629,12 @@ where
|
||||
&'a self,
|
||||
evm: EthEvm<DB, I, PrecompilesMap>,
|
||||
ctx: EthBlockExecutionCtx<'a>,
|
||||
) -> impl BlockExecutorFor<'a, Self, DB, I>
|
||||
) -> Self::Executor<'a, DB, I>
|
||||
where
|
||||
DB: StateDB + 'a,
|
||||
I: Inspector<EthEvmContext<DB>> + 'a,
|
||||
DB: StateDB,
|
||||
I: Inspector<EthEvmContext<DB>>,
|
||||
{
|
||||
let plan = self.take_plan();
|
||||
BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, None)
|
||||
let plan = self.peek_plan();
|
||||
BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, None, None)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,11 +8,14 @@
|
||||
pub(crate) use reth_engine_primitives::BigBlockData;
|
||||
|
||||
use crate::{
|
||||
evm::{BbBlockExecutorFactory, BbEvmPlan},
|
||||
evm::{BalIndexReader, BbBlockExecutorFactory, BbEvmPlan},
|
||||
BigBlockMap,
|
||||
};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_evm::eth::EthBlockExecutionCtx;
|
||||
use alloy_evm::{
|
||||
eth::{spec::EthExecutorSpec, EthBlockExecutionCtx},
|
||||
EthEvmFactory,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types::engine::ExecutionData;
|
||||
use core::convert::Infallible;
|
||||
@@ -20,8 +23,8 @@ use reth_chainspec::{ChainSpec, EthChainSpec};
|
||||
use reth_ethereum_forks::Hardforks;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
use reth_evm::{
|
||||
ConfigureEngineEvm, ConfigureEvm, Database, EvmEnv, ExecutableTxIterator,
|
||||
NextBlockEnvAttributes,
|
||||
ConfigureEngineEvm, ConfigureEvm, Database, EvmEnv, EvmEnvFor, ExecutableTxIterator,
|
||||
ExecutionCtxFor, NextBlockEnvAttributes,
|
||||
};
|
||||
use reth_evm_ethereum::{EthBlockAssembler, EthEvmConfig, RethReceiptBuilder};
|
||||
use reth_primitives_traits::{SealedBlock, SealedHeader};
|
||||
@@ -29,9 +32,6 @@ use revm::primitives::hardfork::SpecId;
|
||||
use std::sync::Arc;
|
||||
use tracing::debug;
|
||||
|
||||
use alloy_evm::{eth::spec::EthExecutorSpec, EthEvmFactory};
|
||||
use reth_evm::{EvmEnvFor, ExecutionCtxFor};
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Execution plan types
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -55,7 +55,7 @@ pub(crate) struct BigBlockSegment {
|
||||
///
|
||||
/// Wraps [`EthEvmConfig`] and a shared [`BigBlockMap`]. When a big-block
|
||||
/// payload is received, the plan is staged on the [`BbBlockExecutorFactory`]
|
||||
/// and consumed when the executor is created. Block hashes for inter-segment
|
||||
/// and cloned when executors are created. Block hashes for inter-segment
|
||||
/// BLOCKHASH resolution are reseeded into `State::block_hashes` at each
|
||||
/// segment boundary via a [`BlockHashSeeder`](crate::evm::BlockHashSeeder)
|
||||
/// callback injected in [`ConfigureEvm::create_executor`].
|
||||
@@ -106,6 +106,10 @@ fn seed_state_block_hashes<DB>(state: &mut &mut revm::database::State<DB>, hashe
|
||||
}
|
||||
}
|
||||
|
||||
fn read_bal_index<DB>(state: &&mut revm::database::State<DB>) -> u64 {
|
||||
state.bal_state.bal_index()
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// ConfigureEvm
|
||||
// ---------------------------------------------------------------------------
|
||||
@@ -144,6 +148,12 @@ where
|
||||
&self,
|
||||
block: &'a SealedBlock<reth_ethereum_primitives::Block>,
|
||||
) -> Result<EthBlockExecutionCtx<'a>, Self::Error> {
|
||||
if let Some(plan) = self.plan_for_payload_hash(&block.hash()) {
|
||||
self.executor_factory.stage_plan(plan);
|
||||
} else {
|
||||
self.executor_factory.clear_staged_plan();
|
||||
}
|
||||
|
||||
self.inner.context_for_block(block)
|
||||
}
|
||||
|
||||
@@ -159,7 +169,7 @@ where
|
||||
&'a self,
|
||||
evm: reth_evm::EvmFor<Self, &'a mut revm::database::State<DB>, I>,
|
||||
ctx: EthBlockExecutionCtx<'a>,
|
||||
) -> impl alloy_evm::block::BlockExecutorFor<
|
||||
) -> alloy_evm::block::BlockExecutorFor<
|
||||
'a,
|
||||
Self::BlockExecutorFactory,
|
||||
&'a mut revm::database::State<DB>,
|
||||
@@ -169,15 +179,16 @@ where
|
||||
DB: Database,
|
||||
I: reth_evm::InspectorFor<Self, &'a mut revm::database::State<DB>> + 'a,
|
||||
{
|
||||
// Use create_executor_with_seeder to inject a concrete seeder that
|
||||
// can reseed State::block_hashes at segment boundaries. The seeder
|
||||
// is a function pointer that knows the concrete State<DB> type,
|
||||
// allowing the generic BbBlockExecutor to reseed without additional
|
||||
// trait bounds on DB.
|
||||
let bal_index_reader: Option<BalIndexReader<&'a mut revm::database::State<DB>>> =
|
||||
Some(read_bal_index::<DB>);
|
||||
|
||||
// Inject concrete function pointers that know the `State<DB>` type so
|
||||
// the generic executor can reseed block hashes and read `bal_index`.
|
||||
self.executor_factory.create_executor_with_seeder(
|
||||
evm,
|
||||
ctx,
|
||||
Some(seed_state_block_hashes::<DB>),
|
||||
bal_index_reader,
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -214,6 +225,7 @@ where
|
||||
|
||||
Ok(env)
|
||||
} else {
|
||||
self.executor_factory.clear_staged_plan();
|
||||
self.inner.evm_env_for_payload(payload)
|
||||
}
|
||||
}
|
||||
@@ -248,10 +260,12 @@ where
|
||||
/// In practice, this is called from `evm_env_for_payload` in the
|
||||
/// engine pipeline.
|
||||
pub fn stage_plan_for_payload(&self, payload_hash: &B256) {
|
||||
let bb = match self.pending.lock().unwrap().remove(payload_hash) {
|
||||
Some(bb) => bb,
|
||||
None => return,
|
||||
};
|
||||
let Some(plan) = self.plan_for_payload_hash(payload_hash) else { return };
|
||||
self.executor_factory.stage_plan(plan);
|
||||
}
|
||||
|
||||
fn plan_for_payload_hash(&self, payload_hash: &B256) -> Option<BbEvmPlan> {
|
||||
let bb = self.pending.lock().unwrap().remove(payload_hash)?;
|
||||
|
||||
let segments: Vec<_> = bb
|
||||
.env_switches
|
||||
@@ -287,6 +301,6 @@ where
|
||||
|
||||
plan.block_hashes_to_seed.sort_unstable_by_key(|(n, _)| *n);
|
||||
|
||||
self.executor_factory.stage_plan(plan);
|
||||
Some(plan)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -21,6 +21,8 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) auth_provider: RootProvider<AnyNetwork>,
|
||||
/// The block provider is used for block queries.
|
||||
pub(crate) block_provider: RootProvider<AnyNetwork>,
|
||||
/// The local regular RPC provider is used for non-authenticated node RPCs like `testing_*`.
|
||||
pub(crate) local_rpc_provider: RootProvider<AnyNetwork>,
|
||||
/// The benchmark mode, which defines whether the benchmark should run for a closed or open
|
||||
/// range of blocks.
|
||||
pub(crate) benchmark_mode: BenchMode,
|
||||
@@ -83,6 +85,11 @@ impl BenchContext {
|
||||
let client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let auth_provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
let local_rpc_url = Url::parse(&bench_args.local_rpc_url)?;
|
||||
info!(target: "reth-bench", "Connecting to local regular RPC at {} for testing namespace calls", local_rpc_url);
|
||||
let local_rpc_provider =
|
||||
RootProvider::<AnyNetwork>::new(ClientBuilder::default().http(local_rpc_url));
|
||||
|
||||
// Computes the block range for the benchmark.
|
||||
//
|
||||
// - If `--advance` is provided, fetches the latest block from the engine and sets:
|
||||
@@ -159,6 +166,7 @@ impl BenchContext {
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
local_rpc_provider,
|
||||
benchmark_mode,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
|
||||
@@ -21,6 +21,7 @@ use alloy_rpc_types_engine::{
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use futures::{stream, StreamExt};
|
||||
use reth_chainspec::EthChainSpec;
|
||||
use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_runner::CliContext;
|
||||
@@ -29,7 +30,10 @@ use reth_ethereum_cli::chainspec::EthereumChainSpecParser;
|
||||
use reth_ethereum_primitives::Receipt;
|
||||
use reth_primitives_traits::proofs;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, future::Future};
|
||||
use std::{
|
||||
collections::{HashMap, HashSet},
|
||||
future::Future,
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
@@ -267,6 +271,15 @@ pub struct Command {
|
||||
/// the flattened BAL on the stored payload.
|
||||
#[arg(long, default_value_t = false)]
|
||||
bal: bool,
|
||||
|
||||
/// Maximum number of in-flight RPC fetches to keep buffered ahead of the merger.
|
||||
///
|
||||
/// Each entry is one full per-block fetch (block + receipts, plus BAL when `--bal` is
|
||||
/// set). Larger values absorb RPC latency at the cost of more concurrent connections
|
||||
/// and memory; the buffer persists across `--num-big-blocks` so prefetching continues
|
||||
/// across big-block boundaries.
|
||||
#[arg(long, value_name = "PREFETCH_BUFFER", default_value_t = 32)]
|
||||
prefetch_buffer: usize,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -319,13 +332,27 @@ impl Command {
|
||||
}
|
||||
let mut prev_big_block_header: Option<PrevBigBlockHeader> = None;
|
||||
|
||||
// Track the next block to fetch across big blocks so they don't overlap.
|
||||
// Persistent prefetch stream: keeps `prefetch_buffer` per-block fetches in flight
|
||||
// ahead of the merger across all big blocks. Each item is a fully materialized
|
||||
// `FetchedBlock` (or `None` once the chain tip is reached on this fetch).
|
||||
let prefetch_buffer = self.prefetch_buffer.max(1);
|
||||
let bal_enabled = self.bal;
|
||||
let block_stream = stream::iter(self.from_block..)
|
||||
.map(|block_number| {
|
||||
let provider = provider.clone();
|
||||
async move { fetch_one_block(provider, block_number, bal_enabled).await }
|
||||
})
|
||||
.buffered(prefetch_buffer);
|
||||
let mut block_stream = Box::pin(block_stream);
|
||||
|
||||
// Track the next block number we expect from the stream (purely for logging /
|
||||
// big-block range bookkeeping; the stream produces blocks in `from_block..` order).
|
||||
let mut next_block = self.from_block;
|
||||
|
||||
for big_block_idx in 0..self.num_big_blocks {
|
||||
let range_start = next_block;
|
||||
|
||||
// Fetch consecutive blocks until the gas target is reached.
|
||||
// Drain the prefetch stream until the gas target is reached for this big block.
|
||||
let mut blocks = Vec::new();
|
||||
let mut block_receipts: Vec<Vec<Receipt>> = Vec::new();
|
||||
let mut block_access_lists: Vec<Option<BlockAccessList>> = Vec::new();
|
||||
@@ -334,16 +361,11 @@ impl Command {
|
||||
let mut reached_chain_tip = false;
|
||||
while accumulated_block_gas < self.target_gas {
|
||||
let block_number = next_block;
|
||||
info!(target: "reth-bench", block_number, big_block = big_block_idx, "Fetching block");
|
||||
info!(target: "reth-bench", block_number, big_block = big_block_idx, "Awaiting prefetched block");
|
||||
|
||||
let fetch_result = tokio::try_join!(
|
||||
provider.get_block_by_number(block_number.into()).full(),
|
||||
provider.get_block_receipts(block_number.into()),
|
||||
);
|
||||
|
||||
let (rpc_block, receipts) = match fetch_result {
|
||||
Ok((Some(block), Some(receipts))) => (block, receipts),
|
||||
Ok((None, _) | (_, None)) => {
|
||||
let fetched = match block_stream.next().await {
|
||||
Some(Ok(Some(fetched))) => fetched,
|
||||
Some(Ok(None)) => {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
@@ -352,52 +374,16 @@ impl Command {
|
||||
reached_chain_tip = true;
|
||||
break;
|
||||
}
|
||||
Err(e) => return Err(e.into()),
|
||||
Some(Err(e)) => return Err(e),
|
||||
// The block-number stream is open-ended; this only fires if the
|
||||
// upstream `iter(from..)` is somehow exhausted.
|
||||
None => {
|
||||
reached_chain_tip = true;
|
||||
break;
|
||||
}
|
||||
};
|
||||
|
||||
let block_access_list = if self.bal {
|
||||
Some(fetch_block_access_list(&provider, block_number).await.wrap_err_with(
|
||||
|| format!("Failed to fetch BAL for block {block_number}"),
|
||||
)?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
// Convert RPC receipts to consensus receipts
|
||||
let consensus_receipts: Vec<Receipt> = receipts
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let inner = &r.inner.inner.inner;
|
||||
let tx_type = r.inner.inner.r#type.try_into().unwrap_or_default();
|
||||
Receipt {
|
||||
tx_type,
|
||||
success: inner.receipt.status.coerce_status(),
|
||||
cumulative_gas_used: inner.receipt.cumulative_gas_used,
|
||||
logs: inner
|
||||
.receipt
|
||||
.logs
|
||||
.iter()
|
||||
.map(|log| alloy_primitives::Log {
|
||||
address: log.inner.address,
|
||||
data: log.inner.data.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Convert to consensus block
|
||||
let block = rpc_block
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
.try_map_transactions(|tx| -> eyre::Result<TxEnvelope> {
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
// Convert to ExecutionData
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
let execution_data = ExecutionData { payload, sidecar };
|
||||
let FetchedBlock { execution_data, consensus_receipts, block_access_list } =
|
||||
fetched;
|
||||
|
||||
let block_gas = execution_data.payload.as_v1().gas_used;
|
||||
let block_blob_gas =
|
||||
@@ -671,6 +657,79 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
/// One fully-materialized block fetched by the prefetcher.
|
||||
struct FetchedBlock {
|
||||
/// Execution payload with sidecar derived from the RPC block.
|
||||
execution_data: ExecutionData,
|
||||
/// Consensus-format receipts (`cumulative_gas_used` is still per-block, callers offset
|
||||
/// it when merging).
|
||||
consensus_receipts: Vec<Receipt>,
|
||||
/// `eth_getBlockAccessListByBlockNumber` result when `--bal` is enabled.
|
||||
block_access_list: Option<BlockAccessList>,
|
||||
}
|
||||
|
||||
/// Fetches one block + receipts (and optionally its BAL) from the RPC. Returns `Ok(None)`
|
||||
/// when the block doesn't exist yet (chain-tip reached).
|
||||
async fn fetch_one_block(
|
||||
provider: RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
bal_enabled: bool,
|
||||
) -> eyre::Result<Option<FetchedBlock>> {
|
||||
let (rpc_block, receipts) = tokio::try_join!(
|
||||
provider.get_block_by_number(block_number.into()).full(),
|
||||
provider.get_block_receipts(block_number.into()),
|
||||
)?;
|
||||
let (rpc_block, receipts) = match (rpc_block, receipts) {
|
||||
(Some(b), Some(r)) => (b, r),
|
||||
_ => return Ok(None),
|
||||
};
|
||||
|
||||
let block_access_list = if bal_enabled {
|
||||
Some(
|
||||
fetch_block_access_list(&provider, block_number)
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch BAL for block {block_number}"))?,
|
||||
)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let consensus_receipts: Vec<Receipt> = receipts
|
||||
.iter()
|
||||
.map(|r| {
|
||||
let inner = &r.inner.inner.inner;
|
||||
let tx_type = r.inner.inner.r#type.try_into().unwrap_or_default();
|
||||
Receipt {
|
||||
tx_type,
|
||||
success: inner.receipt.status.coerce_status(),
|
||||
cumulative_gas_used: inner.receipt.cumulative_gas_used,
|
||||
logs: inner
|
||||
.receipt
|
||||
.logs
|
||||
.iter()
|
||||
.map(|log| alloy_primitives::Log {
|
||||
address: log.inner.address,
|
||||
data: log.inner.data.clone(),
|
||||
})
|
||||
.collect(),
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
|
||||
let block = rpc_block
|
||||
.into_inner()
|
||||
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
|
||||
.try_map_transactions(|tx| -> eyre::Result<TxEnvelope> {
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
let execution_data = ExecutionData { payload, sidecar };
|
||||
|
||||
Ok(Some(FetchedBlock { execution_data, consensus_receipts, block_access_list }))
|
||||
}
|
||||
|
||||
fn merge_block_access_list(
|
||||
merged: &mut BlockAccessList,
|
||||
incoming: BlockAccessList,
|
||||
@@ -717,6 +776,17 @@ fn merge_account_changes(existing: &mut AccountChanges, incoming: AccountChanges
|
||||
existing.balance_changes.extend(incoming.balance_changes);
|
||||
existing.nonce_changes.extend(incoming.nonce_changes);
|
||||
existing.code_changes.extend(incoming.code_changes);
|
||||
|
||||
// EIP-7928 invariant: a slot must appear in either storage_changes or storage_reads,
|
||||
// not both. Per-block BALs respect this, but merging blocks can produce a slot
|
||||
// that is read in one block and changed in another. Without this normalization,
|
||||
// an empty read entry can shadow the real writes during BAL deserialization,
|
||||
// making reads of that slot fall through to stale snapshot state.
|
||||
let written: HashSet<_> =
|
||||
existing.storage_changes.iter().map(|slot_changes| slot_changes.slot).collect();
|
||||
existing.storage_reads.retain(|slot| !written.contains(slot));
|
||||
let mut seen = HashSet::with_capacity(existing.storage_reads.len());
|
||||
existing.storage_reads.retain(|slot| seen.insert(*slot));
|
||||
}
|
||||
|
||||
fn merge_slot_changes(existing: &mut Vec<SlotChanges>, incoming: Vec<SlotChanges>) {
|
||||
@@ -836,4 +906,54 @@ mod tests {
|
||||
assert_eq!(other.address, Address::repeat_byte(0x22));
|
||||
assert_eq!(other.storage_changes[0].changes[0].block_access_index, 3);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn merge_account_changes_normalizes_storage_reads_after_cross_block_merge() {
|
||||
let address = Address::repeat_byte(0x33);
|
||||
const A: U256 = U256::from_limbs([1, 0, 0, 0]);
|
||||
const B: U256 = U256::from_limbs([2, 0, 0, 0]);
|
||||
const C: U256 = U256::from_limbs([3, 0, 0, 0]);
|
||||
const D: U256 = U256::from_limbs([4, 0, 0, 0]);
|
||||
|
||||
// Each AccountChanges value is valid on its own: storage slots only appear in
|
||||
// either reads or changes. The invalid read/change overlap is introduced when
|
||||
// these per-block BAL entries are merged for a standalone big block.
|
||||
let mut existing = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![SlotChanges::new(A, vec![StorageChange::new(0, U256::from(10))])],
|
||||
storage_reads: vec![B, C],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
// B is read before it is written by the incoming block, and A is written before
|
||||
// it appears as a read in the incoming block. C is read in both blocks, so the
|
||||
// merge should also dedupe it. D remains read-only.
|
||||
let incoming = AccountChanges {
|
||||
address,
|
||||
storage_changes: vec![SlotChanges::new(B, vec![StorageChange::new(1, U256::from(20))])],
|
||||
storage_reads: vec![A, C, D],
|
||||
balance_changes: vec![],
|
||||
nonce_changes: vec![],
|
||||
code_changes: vec![],
|
||||
};
|
||||
|
||||
merge_account_changes(&mut existing, incoming);
|
||||
|
||||
// Written slots remain represented by storage_changes, while storage_reads only
|
||||
// keeps unique read-only slots in first-seen order.
|
||||
assert_eq!(
|
||||
existing
|
||||
.storage_changes
|
||||
.iter()
|
||||
.map(|slot_changes| slot_changes.slot)
|
||||
.collect::<Vec<_>>(),
|
||||
vec![A, B]
|
||||
);
|
||||
assert_eq!(existing.storage_reads, vec![C, D]);
|
||||
assert!(existing.storage_reads.iter().all(|read_slot| {
|
||||
!existing.storage_changes.iter().any(|slot_changes| slot_changes.slot == *read_slot)
|
||||
}));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -14,14 +14,24 @@ use crate::{
|
||||
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
|
||||
},
|
||||
};
|
||||
use alloy_provider::{ext::DebugApi, Provider};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::Encodable2718;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_provider::{
|
||||
ext::DebugApi,
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
Provider, RootProvider,
|
||||
};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionData, ExecutionPayloadEnvelopeV5, ForkchoiceState, PayloadAttributes,
|
||||
};
|
||||
use clap::Parser;
|
||||
use eyre::{Context, OptionExt};
|
||||
use eyre::{bail, ensure, Context, OptionExt};
|
||||
use futures::{stream, StreamExt, TryStreamExt};
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
use reth_rpc_api::{RethNewPayloadInput, TestingBuildBlockRequestV1};
|
||||
use std::time::{Duration, Instant};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
@@ -32,6 +42,22 @@ pub struct Command {
|
||||
#[arg(long, value_name = "RPC_URL", verbatim_doc_comment)]
|
||||
rpc_url: String,
|
||||
|
||||
/// Build a separate fork with `testing_buildBlockV1` and alternate forkchoice updates between
|
||||
/// the canonical chain and that fork on every block while the fork grows up to the configured
|
||||
/// depth.
|
||||
///
|
||||
/// This requires enabling the hidden `testing` RPC module on the target node,
|
||||
/// for example with `reth node --http --http.api eth,testing`.
|
||||
#[arg(
|
||||
long,
|
||||
value_name = "DEPTH",
|
||||
num_args = 0..=1,
|
||||
default_missing_value = "8",
|
||||
value_parser = parse_reorg_depth,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
reorg: Option<usize>,
|
||||
|
||||
/// How long to wait after a forkchoice update before sending the next payload.
|
||||
///
|
||||
/// Accepts a duration string (e.g. `100ms`, `2s`) or a bare integer treated as
|
||||
@@ -75,22 +101,87 @@ pub struct Command {
|
||||
)]
|
||||
rpc_block_buffer_size: usize,
|
||||
|
||||
/// Weather to enable bal by default or not.
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
enable_bal: bool,
|
||||
|
||||
#[command(flatten)]
|
||||
benchmark: BenchmarkArgs,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct PreparedBuiltBlock {
|
||||
block_hash: B256,
|
||||
params: serde_json::Value,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct QueuedForkBlock {
|
||||
block_number: u64,
|
||||
prepared: PreparedBuiltBlock,
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct ReorgState {
|
||||
depth: usize,
|
||||
fork_length: usize,
|
||||
branch_point_hash: Option<B256>,
|
||||
fork_parent_hash: Option<B256>,
|
||||
}
|
||||
|
||||
impl ReorgState {
|
||||
const fn new(depth: usize) -> Self {
|
||||
Self { depth, fork_length: 0, branch_point_hash: None, fork_parent_hash: None }
|
||||
}
|
||||
|
||||
const fn push_fork_head(&mut self, canonical_parent_hash: B256, fork_head_hash: B256) {
|
||||
if self.fork_length == 0 {
|
||||
self.branch_point_hash = Some(canonical_parent_hash);
|
||||
}
|
||||
self.fork_length += 1;
|
||||
self.fork_parent_hash = Some(fork_head_hash);
|
||||
}
|
||||
|
||||
fn forkchoice_state(&self, fork_head_hash: B256) -> eyre::Result<ForkchoiceState> {
|
||||
let branch_point_hash = self.branch_point_hash.ok_or_eyre("missing reorg branch point")?;
|
||||
|
||||
Ok(ForkchoiceState {
|
||||
head_block_hash: fork_head_hash,
|
||||
safe_block_hash: branch_point_hash,
|
||||
finalized_block_hash: branch_point_hash,
|
||||
})
|
||||
}
|
||||
|
||||
const fn reset(&mut self) {
|
||||
self.fork_length = 0;
|
||||
self.branch_point_hash = None;
|
||||
self.fork_parent_hash = None;
|
||||
}
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark new-payload-fcu` command
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
if self.reorg.is_some() && self.benchmark.rlp_blocks {
|
||||
bail!("--reorg cannot be combined with --rlp-blocks")
|
||||
}
|
||||
if self.reorg.is_some() && self.enable_bal {
|
||||
bail!("--reorg cannot be combined with --enable-bal")
|
||||
}
|
||||
|
||||
// Log mode configuration
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!(target: "reth-bench", "Using wait-time mode with {}ms minimum interval between blocks", duration.as_millis());
|
||||
}
|
||||
if let Some(depth) = self.reorg {
|
||||
info!(target: "reth-bench", depth, "Using testing_buildBlockV1 reorg mode");
|
||||
}
|
||||
|
||||
let BenchContext {
|
||||
benchmark_mode,
|
||||
block_provider,
|
||||
auth_provider,
|
||||
local_rpc_provider,
|
||||
next_block,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
@@ -178,7 +269,8 @@ impl Command {
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
let mut total_wait_time = Duration::ZERO;
|
||||
|
||||
let mut reorg_state = self.reorg.map(ReorgState::new);
|
||||
let mut queued_fork_block = None;
|
||||
while let Some((block, head, safe, finalized, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = blocks.try_next().await?;
|
||||
@@ -188,17 +280,21 @@ impl Command {
|
||||
let gas_used = block.header.gas_used;
|
||||
let gas_limit = block.header.gas_limit;
|
||||
let block_number = block.header.number;
|
||||
let canonical_parent_hash = block.header.parent_hash;
|
||||
let transaction_count = block.transactions.len() as u64;
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
let deferred_branch_start_block = reorg_state
|
||||
.as_ref()
|
||||
.filter(|state| state.fork_length == 0 && queued_fork_block.is_none())
|
||||
.map(|_| block.clone());
|
||||
let canonical_forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: head,
|
||||
safe_block_hash: safe,
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
|
||||
let bal = if rlp.is_none() &&
|
||||
(block.header.block_access_list_hash.is_some() || self.enable_bal)
|
||||
{
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
@@ -212,10 +308,11 @@ impl Command {
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
debug!(target: "reth-bench", ?block_number, "Sending payload");
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
|
||||
let np_latency =
|
||||
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
|
||||
let new_payload_result = NewPayloadResult {
|
||||
@@ -236,17 +333,12 @@ impl Command {
|
||||
};
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
|
||||
call_forkchoice_updated_with_reth(&auth_provider, version, canonical_forkchoice_state)
|
||||
.await?;
|
||||
let fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let total_latency = if server_timings.is_some() {
|
||||
// When using server-side latency for newPayload, derive total from the
|
||||
// independently measured components to avoid mixing server-side and
|
||||
// client-side (network-inclusive) timings.
|
||||
np_latency + fcu_latency
|
||||
} else {
|
||||
start.elapsed()
|
||||
};
|
||||
let total_latency =
|
||||
if server_timings.is_some() { np_latency + fcu_latency } else { start.elapsed() };
|
||||
let combined_result = CombinedResult {
|
||||
block_number,
|
||||
gas_limit,
|
||||
@@ -256,6 +348,88 @@ impl Command {
|
||||
total_latency,
|
||||
};
|
||||
|
||||
if let Some(reorg_state) = reorg_state.as_mut() {
|
||||
if queued_fork_block.is_none() && reorg_state.fork_length == 0 {
|
||||
// A branch start uses a canonical parent, so it can be built lazily here
|
||||
// instead of being queued ahead of time.
|
||||
let block = deferred_branch_start_block
|
||||
.as_ref()
|
||||
.ok_or_eyre("missing deferred fork block for reorg branch start")?;
|
||||
queued_fork_block = Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
&local_rpc_provider,
|
||||
block,
|
||||
canonical_parent_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
});
|
||||
}
|
||||
|
||||
let queued = queued_fork_block
|
||||
.take()
|
||||
.ok_or_eyre("missing queued fork block for reorg replay")?;
|
||||
ensure!(
|
||||
queued.block_number == block_number,
|
||||
"queued fork block {} does not match source block {}",
|
||||
queued.block_number,
|
||||
block_number
|
||||
);
|
||||
let prepared = queued.prepared;
|
||||
|
||||
call_new_payload_with_reth(&auth_provider, None, prepared.params).await?;
|
||||
|
||||
reorg_state.push_fork_head(canonical_parent_hash, prepared.block_hash);
|
||||
let forkchoice_state = reorg_state.forkchoice_state(prepared.block_hash)?;
|
||||
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
branch_point = %forkchoice_state.safe_block_hash,
|
||||
fork_head = %prepared.block_hash,
|
||||
fork_depth = reorg_state.fork_length,
|
||||
max_reorg_depth = reorg_state.depth,
|
||||
"Switching forkchoice to reorg branch"
|
||||
);
|
||||
|
||||
let fcu_start = Instant::now();
|
||||
call_forkchoice_updated_with_reth(&auth_provider, None, forkchoice_state).await?;
|
||||
let _fork_fcu_latency = fcu_start.elapsed();
|
||||
|
||||
let next_fork_block_number = block_number + 1;
|
||||
if reorg_state.fork_length < reorg_state.depth {
|
||||
queued_fork_block = queue_fork_block(
|
||||
&block_provider,
|
||||
&local_rpc_provider,
|
||||
&benchmark_mode,
|
||||
next_fork_block_number,
|
||||
Some(prepared.block_hash),
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?;
|
||||
} else {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
block_number,
|
||||
reorg_depth = reorg_state.depth,
|
||||
"Resetting reorg branch after reaching max depth"
|
||||
);
|
||||
|
||||
// `testing_buildBlockV1` resolves the parent from canonical state, so switch
|
||||
// back to the source chain before reseeding the next queued fork block.
|
||||
call_forkchoice_updated_with_reth(
|
||||
&auth_provider,
|
||||
version,
|
||||
canonical_forkchoice_state,
|
||||
)
|
||||
.await?;
|
||||
|
||||
reorg_state.reset();
|
||||
queued_fork_block = None;
|
||||
}
|
||||
}
|
||||
|
||||
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
|
||||
// We want to measure engine throughput, not RPC fetch latency.
|
||||
blocks_processed += 1;
|
||||
@@ -312,3 +486,155 @@ impl Command {
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
async fn prepare_built_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<PreparedBuiltBlock> {
|
||||
const MAX_BUILD_ATTEMPTS: usize = 10;
|
||||
const BUILD_RETRY_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
let request = build_block_request(block, parent_block_hash)?;
|
||||
let built_payload: ExecutionPayloadEnvelopeV5 = {
|
||||
let mut attempts_remaining = MAX_BUILD_ATTEMPTS;
|
||||
|
||||
loop {
|
||||
match block_provider.client().request("testing_buildBlockV1", [request.clone()]).await {
|
||||
Ok(payload) => break payload,
|
||||
Err(err) if attempts_remaining > 1 && is_retryable_build_block_error(&err) => {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
block_number = block.header.number,
|
||||
%parent_block_hash,
|
||||
attempts_remaining,
|
||||
error = %err,
|
||||
"Retrying testing_buildBlockV1 after transient fork build failure"
|
||||
);
|
||||
attempts_remaining -= 1;
|
||||
tokio::time::sleep(BUILD_RETRY_INTERVAL).await;
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(err).wrap_err_with(|| {
|
||||
format!(
|
||||
"Failed to build block {} via testing_buildBlockV1",
|
||||
block.header.number
|
||||
)
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
let payload = &built_payload.execution_payload.payload_inner.payload_inner;
|
||||
let block_hash = payload.block_hash;
|
||||
let (payload, sidecar) = built_payload
|
||||
.into_payload_and_sidecar(block.header.parent_beacon_block_root.unwrap_or_default());
|
||||
// Fork payloads are built immediately before the next `testing_buildBlockV1` call. Leaving
|
||||
// reth's default persistence wait enabled here gives the regular RPC side a consistent base
|
||||
// state for the next synthetic fork block build.
|
||||
let params = serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(ExecutionData { payload, sidecar }),
|
||||
None::<bool>,
|
||||
no_wait_for_caches.then_some(false),
|
||||
))?;
|
||||
|
||||
Ok(PreparedBuiltBlock { block_hash, params })
|
||||
}
|
||||
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
async fn queue_fork_block(
|
||||
block_provider: &RootProvider<AnyNetwork>,
|
||||
local_rpc_provider: &RootProvider<AnyNetwork>,
|
||||
benchmark_mode: &crate::bench_mode::BenchMode,
|
||||
block_number: u64,
|
||||
parent_block_hash: Option<B256>,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<Option<QueuedForkBlock>> {
|
||||
if !benchmark_mode.contains(block_number) {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
let future_block = block_provider
|
||||
.get_block_by_number(alloy_eips::BlockNumberOrTag::Number(block_number))
|
||||
.full()
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch block by number {block_number}"))?
|
||||
.ok_or_eyre("Block not found")?;
|
||||
let parent_block_hash = parent_block_hash.unwrap_or(future_block.header.parent_hash);
|
||||
|
||||
Ok(Some(QueuedForkBlock {
|
||||
block_number,
|
||||
prepared: prepare_built_block(
|
||||
local_rpc_provider,
|
||||
&future_block,
|
||||
parent_block_hash,
|
||||
no_wait_for_caches,
|
||||
)
|
||||
.await?,
|
||||
}))
|
||||
}
|
||||
|
||||
fn is_retryable_build_block_error(err: &alloy_transport::TransportError) -> bool {
|
||||
let message = err.to_string();
|
||||
message.contains("block not found: hash") ||
|
||||
message.contains("block hash not found for block number")
|
||||
}
|
||||
|
||||
fn build_block_request(
|
||||
block: &AnyRpcBlock,
|
||||
parent_block_hash: B256,
|
||||
) -> eyre::Result<TestingBuildBlockRequestV1> {
|
||||
let mut transactions = block
|
||||
.clone()
|
||||
.try_into_transactions()
|
||||
.map_err(|_| eyre::eyre!("Block transactions must be fetched in full for --reorg"))?
|
||||
.into_iter()
|
||||
.map(|tx| {
|
||||
let tx: TxEnvelope =
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type in RPC block"))?;
|
||||
if tx.is_eip4844() {
|
||||
return Ok(None)
|
||||
}
|
||||
Ok(Some(tx.encoded_2718().into()))
|
||||
})
|
||||
.filter_map(|tx| tx.transpose())
|
||||
.collect::<eyre::Result<Vec<_>>>()?;
|
||||
|
||||
// `testing_buildBlockV1` only takes raw transaction bytes, so we exclude blob transactions
|
||||
// from the synthetic fork blocks rather than trying to reconstruct their sidecars.
|
||||
// Keep only 90% of the remaining transactions so the alternate branch produces a materially
|
||||
// different post-state instead of only differing by header data.
|
||||
let keep = transactions.len().saturating_mul(9) / 10;
|
||||
transactions.truncate(keep);
|
||||
|
||||
let rpc_block = block.clone().into_inner();
|
||||
|
||||
Ok(TestingBuildBlockRequestV1 {
|
||||
parent_block_hash,
|
||||
payload_attributes: PayloadAttributes {
|
||||
timestamp: block.header.timestamp,
|
||||
prev_randao: block.header.mix_hash.unwrap_or_default(),
|
||||
suggested_fee_recipient: block.header.beneficiary,
|
||||
withdrawals: rpc_block.withdrawals.map(|withdrawals| withdrawals.into_inner()),
|
||||
parent_beacon_block_root: block.header.parent_beacon_block_root,
|
||||
slot_number: block.header.slot_number,
|
||||
},
|
||||
transactions,
|
||||
extra_data: Some(block.header.extra_data.clone()),
|
||||
})
|
||||
}
|
||||
|
||||
fn parse_reorg_depth(value: &str) -> Result<usize, String> {
|
||||
let depth = value
|
||||
.trim()
|
||||
.parse::<usize>()
|
||||
.map_err(|_| format!("invalid reorg depth {value:?}, expected a positive integer"))?;
|
||||
|
||||
if depth == 0 {
|
||||
return Err("reorg depth must be greater than 0".to_string())
|
||||
}
|
||||
|
||||
Ok(depth)
|
||||
}
|
||||
|
||||
@@ -54,6 +54,7 @@ impl Command {
|
||||
rlp_blocks,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
..
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
|
||||
@@ -18,7 +18,7 @@ reth-errors.workspace = true
|
||||
reth-execution-types.workspace = true
|
||||
reth-metrics.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-primitives-traits.workspace = true
|
||||
reth-primitives-traits = { workspace = true, features = ["dashmap"] }
|
||||
reth-storage-api.workspace = true
|
||||
reth-trie.workspace = true
|
||||
|
||||
|
||||
@@ -4,26 +4,32 @@
|
||||
//! lazily on first access. This allows execution to start before the trie overlay
|
||||
//! is fully computed.
|
||||
|
||||
use crate::DeferredTrieData;
|
||||
use crate::{EthPrimitives, ExecutedBlock};
|
||||
use alloy_primitives::B256;
|
||||
use reth_primitives_traits::{
|
||||
dashmap::{self, DashMap},
|
||||
AlloyBlockHeader, NodePrimitives,
|
||||
};
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostStateSorted, TrieInputSorted};
|
||||
use std::sync::{Arc, OnceLock};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace};
|
||||
|
||||
/// Inputs captured for lazy overlay computation.
|
||||
#[derive(Clone)]
|
||||
struct LazyOverlayInputs {
|
||||
/// The persisted ancestor hash (anchor) this overlay should be built on.
|
||||
anchor_hash: B256,
|
||||
/// Deferred trie data handles for all in-memory blocks (newest to oldest).
|
||||
blocks: Vec<DeferredTrieData>,
|
||||
struct LazyOverlayInputs<N: NodePrimitives = EthPrimitives> {
|
||||
/// In-memory blocks from tip to anchor child.
|
||||
///
|
||||
/// Blocks must be provided in reverse chain order (newest to oldest).
|
||||
blocks: Vec<ExecutedBlock<N>>,
|
||||
}
|
||||
|
||||
/// Lazily computed trie overlay.
|
||||
///
|
||||
/// Captures the inputs needed to compute a [`TrieInputSorted`] and defers the actual
|
||||
/// computation until first access. This is conceptually similar to [`DeferredTrieData`]
|
||||
/// but for overlay computation.
|
||||
/// computation until first access.
|
||||
///
|
||||
/// Blocks must be provided in reverse chain order (newest to oldest), so the first block is the
|
||||
/// chain tip and the last block is the oldest in-memory block in the chain segment.
|
||||
///
|
||||
/// # Fast Path vs Slow Path
|
||||
///
|
||||
@@ -31,37 +37,41 @@ struct LazyOverlayInputs {
|
||||
/// matches our expected anchor, we can reuse it directly (O(1)).
|
||||
/// - **Slow path**: Otherwise, we merge all ancestor blocks' trie data into a new overlay.
|
||||
#[derive(Clone)]
|
||||
pub struct LazyOverlay {
|
||||
/// Computed result, cached after first access.
|
||||
inner: Arc<OnceLock<TrieInputSorted>>,
|
||||
pub struct LazyOverlay<N: NodePrimitives = EthPrimitives> {
|
||||
/// Computed results, cached by requested anchor hash.
|
||||
inner: Arc<DashMap<B256, Arc<TrieInputSorted>>>,
|
||||
/// Inputs for lazy computation.
|
||||
inputs: LazyOverlayInputs,
|
||||
inputs: LazyOverlayInputs<N>,
|
||||
}
|
||||
|
||||
impl std::fmt::Debug for LazyOverlay {
|
||||
impl<N: NodePrimitives> std::fmt::Debug for LazyOverlay<N> {
|
||||
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
|
||||
f.debug_struct("LazyOverlay")
|
||||
.field("anchor_hash", &self.inputs.anchor_hash)
|
||||
.field(
|
||||
"oldest_block_parent_hash",
|
||||
&self.inputs.blocks.last().map(|block| block.recovered_block().parent_hash()),
|
||||
)
|
||||
.field("num_blocks", &self.inputs.blocks.len())
|
||||
.field("computed", &self.inner.get().is_some())
|
||||
.field("cached_anchors", &self.inner.len())
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
|
||||
impl LazyOverlay {
|
||||
/// Create a new lazy overlay with the given anchor hash and block handles.
|
||||
impl<N: NodePrimitives> LazyOverlay<N> {
|
||||
/// Create a new lazy overlay from in-memory blocks.
|
||||
///
|
||||
/// # Arguments
|
||||
///
|
||||
/// * `anchor_hash` - The persisted ancestor hash this overlay is built on top of
|
||||
/// * `blocks` - Deferred trie data handles for in-memory blocks (newest to oldest)
|
||||
pub fn new(anchor_hash: B256, blocks: Vec<DeferredTrieData>) -> Self {
|
||||
Self { inner: Arc::new(OnceLock::new()), inputs: LazyOverlayInputs { anchor_hash, blocks } }
|
||||
}
|
||||
/// * `blocks` - Executed blocks in reverse chain order (newest to oldest)
|
||||
pub fn new(blocks: Vec<ExecutedBlock<N>>) -> Self {
|
||||
debug_assert!(
|
||||
blocks.windows(2).all(|window| {
|
||||
window[0].recovered_block().parent_hash() == window[1].recovered_block().hash()
|
||||
}),
|
||||
"LazyOverlay blocks must be ordered newest to oldest along a single chain"
|
||||
);
|
||||
|
||||
/// Returns the anchor hash this overlay is built on.
|
||||
pub const fn anchor_hash(&self) -> B256 {
|
||||
self.inputs.anchor_hash
|
||||
Self { inner: Default::default(), inputs: LazyOverlayInputs { blocks } }
|
||||
}
|
||||
|
||||
/// Returns the number of in-memory blocks this overlay covers.
|
||||
@@ -69,43 +79,75 @@ impl LazyOverlay {
|
||||
self.inputs.blocks.len()
|
||||
}
|
||||
|
||||
/// Returns true if the overlay has already been computed.
|
||||
pub fn is_computed(&self) -> bool {
|
||||
self.inner.get().is_some()
|
||||
/// Returns the oldest anchor hash this overlay can serve.
|
||||
///
|
||||
/// This is the parent hash of the oldest block in the stored newest-to-oldest chain segment.
|
||||
pub fn anchor_hash(&self) -> Option<B256> {
|
||||
self.inputs.blocks.last().map(|block| block.recovered_block().parent_hash())
|
||||
}
|
||||
|
||||
/// Returns the computed trie input, computing it if necessary.
|
||||
/// Returns true if there are no blocks in the overlay, or if one of the blocks has the given
|
||||
/// hash as a parent hash.
|
||||
pub fn has_anchor_hash(&self, hash: B256) -> bool {
|
||||
self.inputs.blocks.is_empty() ||
|
||||
self.inputs.blocks.iter().any(|b| b.recovered_block().parent_hash() == hash)
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
/// Returns true if the overlay has already been computed for the requested anchor.
|
||||
pub fn is_computed(&self, anchor_hash: B256) -> bool {
|
||||
self.inner.contains_key(&anchor_hash)
|
||||
}
|
||||
|
||||
/// Returns the computed trie input for the requested anchor, computing it if necessary.
|
||||
///
|
||||
/// The first call triggers computation (which may block waiting for deferred data).
|
||||
/// Subsequent calls return the cached result immediately.
|
||||
pub fn get(&self) -> &TrieInputSorted {
|
||||
self.inner.get_or_init(|| self.compute())
|
||||
/// Subsequent calls for the same anchor return the cached result immediately.
|
||||
pub fn get(&self, anchor_hash: B256) -> Arc<TrieInputSorted> {
|
||||
match self.inner.entry(anchor_hash) {
|
||||
dashmap::Entry::Occupied(entry) => Arc::clone(entry.get()),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
let input = self.compute(anchor_hash);
|
||||
entry.insert(Arc::clone(&input));
|
||||
input
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the overlay as (nodes, state) tuple for use with `OverlayStateProviderFactory`.
|
||||
pub fn as_overlay(&self) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get();
|
||||
pub fn as_overlay(
|
||||
&self,
|
||||
anchor_hash: B256,
|
||||
) -> (Arc<TrieUpdatesSorted>, Arc<HashedPostStateSorted>) {
|
||||
let input = self.get(anchor_hash);
|
||||
(Arc::clone(&input.nodes), Arc::clone(&input.state))
|
||||
}
|
||||
|
||||
/// Compute the trie input overlay.
|
||||
fn compute(&self) -> TrieInputSorted {
|
||||
let anchor_hash = self.inputs.anchor_hash;
|
||||
fn compute(&self, anchor_hash: B256) -> Arc<TrieInputSorted> {
|
||||
let blocks = &self.inputs.blocks;
|
||||
|
||||
if blocks.is_empty() {
|
||||
debug!(target: "chain_state::lazy_overlay", "No in-memory blocks, returning empty overlay");
|
||||
return TrieInputSorted::default();
|
||||
return Default::default()
|
||||
}
|
||||
|
||||
let Some(last_index) =
|
||||
blocks.iter().position(|block| block.recovered_block().parent_hash() == anchor_hash)
|
||||
else {
|
||||
panic!(
|
||||
"LazyOverlay does not contain a block whose parent hash matches requested anchor {anchor_hash}"
|
||||
);
|
||||
};
|
||||
let blocks = &blocks[..=last_index];
|
||||
|
||||
// Fast path: Check if tip block's overlay is ready and anchor matches.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors.
|
||||
// The tip block (first in list) has the cumulative overlay from all ancestors up to the
|
||||
// requested anchor.
|
||||
if let Some(tip) = blocks.first() {
|
||||
let data = tip.wait_cloned();
|
||||
let data = tip.trie_data();
|
||||
if let Some(anchored) = &data.anchored_trie_input {
|
||||
if anchored.anchor_hash == anchor_hash {
|
||||
trace!(target: "chain_state::lazy_overlay", %anchor_hash, "Reusing tip block's cached overlay (fast path)");
|
||||
return (*anchored.trie_input).clone();
|
||||
return Arc::clone(&anchored.trie_input);
|
||||
}
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
@@ -116,23 +158,30 @@ impl LazyOverlay {
|
||||
}
|
||||
}
|
||||
|
||||
// Slow path: Merge all blocks' trie data into a new overlay.
|
||||
debug!(target: "chain_state::lazy_overlay", num_blocks = blocks.len(), "Merging blocks (slow path)");
|
||||
Self::merge_blocks(blocks)
|
||||
// Slow path: Merge the prefix of blocks from the tip back to the requested anchor.
|
||||
debug!(
|
||||
target: "chain_state::lazy_overlay",
|
||||
%anchor_hash,
|
||||
num_blocks = blocks.len(),
|
||||
"Merging blocks (slow path)"
|
||||
);
|
||||
Arc::new(Self::merge_blocks(blocks))
|
||||
}
|
||||
|
||||
/// Merge all blocks' trie data into a single [`TrieInputSorted`].
|
||||
///
|
||||
/// Blocks are ordered newest to oldest.
|
||||
fn merge_blocks(blocks: &[DeferredTrieData]) -> TrieInputSorted {
|
||||
fn merge_blocks(blocks: &[ExecutedBlock<N>]) -> TrieInputSorted {
|
||||
if blocks.is_empty() {
|
||||
return TrieInputSorted::default();
|
||||
}
|
||||
|
||||
let state =
|
||||
HashedPostStateSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().hashed_state));
|
||||
let nodes =
|
||||
TrieUpdatesSorted::merge_batch(blocks.iter().map(|b| b.wait_cloned().trie_updates));
|
||||
let state = HashedPostStateSorted::merge_batch(
|
||||
blocks.iter().map(|block| block.trie_data().hashed_state),
|
||||
);
|
||||
let nodes = TrieUpdatesSorted::merge_batch(
|
||||
blocks.iter().map(|block| block.trie_data().trie_updates),
|
||||
);
|
||||
|
||||
TrieInputSorted { state, nodes, prefix_sets: Default::default() }
|
||||
}
|
||||
@@ -141,46 +190,138 @@ impl LazyOverlay {
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use reth_trie::{updates::TrieUpdates, HashedPostState};
|
||||
use crate::{test_utils::TestBlockBuilder, ComputedTrieData, EthPrimitives, ExecutedBlock};
|
||||
use alloy_primitives::U256;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_trie::{updates::TrieUpdatesSorted, HashedPostState, HashedStorage};
|
||||
use std::sync::Arc;
|
||||
|
||||
fn empty_deferred(anchor: B256) -> DeferredTrieData {
|
||||
DeferredTrieData::pending(
|
||||
Arc::new(HashedPostState::default()),
|
||||
Arc::new(TrieUpdates::default()),
|
||||
anchor,
|
||||
Vec::new(),
|
||||
fn with_unique_state(
|
||||
block: &ExecutedBlock<EthPrimitives>,
|
||||
id: u8,
|
||||
) -> ExecutedBlock<EthPrimitives> {
|
||||
let hashed_address = B256::with_last_byte(id);
|
||||
let hashed_slot = B256::with_last_byte(id.saturating_add(32));
|
||||
let hashed_state = HashedPostState::default()
|
||||
.with_accounts([(hashed_address, Some(Account::default()))])
|
||||
.with_storages([(
|
||||
hashed_address,
|
||||
HashedStorage::from_iter(false, [(hashed_slot, U256::from(id))]),
|
||||
)])
|
||||
.into_sorted();
|
||||
|
||||
ExecutedBlock::new(
|
||||
Arc::clone(&block.recovered_block),
|
||||
Arc::clone(&block.execution_output),
|
||||
ComputedTrieData::without_trie_input(
|
||||
Arc::new(hashed_state),
|
||||
Arc::new(TrieUpdatesSorted::default()),
|
||||
),
|
||||
)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn empty_blocks_returns_default() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
let result = overlay.get();
|
||||
assert!(result.state.is_empty());
|
||||
assert!(result.nodes.is_empty());
|
||||
fn test_blocks() -> Vec<ExecutedBlock<EthPrimitives>> {
|
||||
TestBlockBuilder::eth()
|
||||
.get_executed_blocks(1..4)
|
||||
.collect::<Vec<_>>()
|
||||
.into_iter()
|
||||
.rev()
|
||||
.enumerate()
|
||||
.map(|(index, block)| with_unique_state(&block, index as u8 + 1))
|
||||
.collect()
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn single_block_uses_data_directly() {
|
||||
let anchor = B256::random();
|
||||
let deferred = empty_deferred(anchor);
|
||||
let overlay = LazyOverlay::new(anchor, vec![deferred]);
|
||||
let block = TestBlockBuilder::eth().get_executed_block_with_number(1, B256::random());
|
||||
let anchor_hash = block.recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(vec![block]);
|
||||
|
||||
assert!(!overlay.is_computed());
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
assert!(!overlay.is_computed(anchor_hash));
|
||||
let _ = overlay.get(anchor_hash);
|
||||
assert!(overlay.is_computed(anchor_hash));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn cached_after_first_access() {
|
||||
let overlay = LazyOverlay::new(B256::ZERO, vec![]);
|
||||
fn caches_results_per_anchor() {
|
||||
let blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let full_anchor = blocks[2].recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
// First access computes
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
let prefix = overlay.get(prefix_anchor);
|
||||
let full = overlay.get(full_anchor);
|
||||
|
||||
// Second access uses cache
|
||||
let _ = overlay.get();
|
||||
assert!(overlay.is_computed());
|
||||
assert!(overlay.is_computed(prefix_anchor));
|
||||
assert!(overlay.is_computed(full_anchor));
|
||||
assert!(!Arc::ptr_eq(&prefix, &full));
|
||||
assert!(Arc::ptr_eq(&prefix, &overlay.get(prefix_anchor)));
|
||||
assert!(Arc::ptr_eq(&full, &overlay.get(full_anchor)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn requested_anchor_limits_the_merged_prefix() {
|
||||
let blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let expected = LazyOverlay::merge_blocks(&blocks[..2]);
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
let actual = overlay.get(prefix_anchor);
|
||||
|
||||
assert_eq!(actual.nodes.as_ref(), expected.nodes.as_ref());
|
||||
assert_eq!(actual.state.as_ref(), expected.state.as_ref());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn anchor_hash_returns_oldest_served_anchor() {
|
||||
let blocks = test_blocks();
|
||||
let expected_anchor = blocks.last().unwrap().recovered_block().parent_hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
assert_eq!(overlay.anchor_hash(), Some(expected_anchor));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn reuses_tip_overlay_when_anchor_matches() {
|
||||
let mut blocks = test_blocks();
|
||||
let prefix_anchor = blocks[2].recovered_block().hash();
|
||||
let tip_overlay = Arc::new(LazyOverlay::merge_blocks(&blocks[..2]));
|
||||
let tip_data = blocks[0].trie_data();
|
||||
|
||||
blocks[0] = ExecutedBlock::new(
|
||||
Arc::clone(&blocks[0].recovered_block),
|
||||
Arc::clone(&blocks[0].execution_output),
|
||||
ComputedTrieData::with_trie_input(
|
||||
tip_data.hashed_state,
|
||||
tip_data.trie_updates,
|
||||
prefix_anchor,
|
||||
Arc::clone(&tip_overlay),
|
||||
),
|
||||
);
|
||||
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
let actual = overlay.get(prefix_anchor);
|
||||
|
||||
assert!(Arc::ptr_eq(&actual, &tip_overlay));
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "LazyOverlay does not contain a block whose parent hash matches requested anchor"
|
||||
)]
|
||||
fn missing_anchor_panics() {
|
||||
let blocks = test_blocks();
|
||||
let missing_anchor = blocks[0].recovered_block().hash();
|
||||
let overlay = LazyOverlay::new(blocks);
|
||||
|
||||
let _ = overlay.get(missing_anchor);
|
||||
}
|
||||
|
||||
#[test]
|
||||
#[should_panic(
|
||||
expected = "LazyOverlay blocks must be ordered newest to oldest along a single chain"
|
||||
)]
|
||||
fn misordered_blocks_panic() {
|
||||
let blocks: Vec<_> = TestBlockBuilder::eth().get_executed_blocks(1..3).collect();
|
||||
let _ = LazyOverlay::new(blocks);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -150,16 +150,22 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
// commands can proceed.
|
||||
debug!(target: "reth::cli", ?rocksdb_path, "RocksDB not found, initializing empty database");
|
||||
reth_fs_util::create_dir_all(&rocksdb_path)?;
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.build()?
|
||||
.with_database_log_level(self.db.log_level);
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
} else {
|
||||
RocksDBProvider::builder(data_dir.rocksdb())
|
||||
let mut builder = RocksDBProvider::builder(data_dir.rocksdb())
|
||||
.with_default_tables()
|
||||
.with_database_log_level(self.db.log_level)
|
||||
.with_read_only(!access.is_read_write())
|
||||
.build()?
|
||||
.with_read_only(!access.is_read_write());
|
||||
if let Some(cache_size) = self.db.rocksdb_block_cache_size {
|
||||
builder = builder.with_block_cache_size(cache_size);
|
||||
}
|
||||
builder.build()?
|
||||
};
|
||||
|
||||
let provider_factory =
|
||||
|
||||
@@ -14,7 +14,7 @@ use reth_db_api::{
|
||||
table::{Compress, Decompress, DupSort, Table},
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
RawKey, RawTable, Receipts, TableViewer, Transactions,
|
||||
RawKey, RawTable, TableViewer,
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
@@ -264,15 +264,12 @@ impl Command {
|
||||
);
|
||||
}
|
||||
StaticFileSegment::Transactions => {
|
||||
let transaction = <<Transactions as Table>::Value>::decompress(
|
||||
content[0].as_slice(),
|
||||
)?;
|
||||
let transaction = TxTy::<N>::decompress(content[0].as_slice())?;
|
||||
println!("{}", serde_json::to_string_pretty(&transaction)?);
|
||||
}
|
||||
StaticFileSegment::Receipts => {
|
||||
let receipt = <<Receipts as Table>::Value>::decompress(
|
||||
content[0].as_slice(),
|
||||
)?;
|
||||
let receipt =
|
||||
ReceiptTy::<N>::decompress(content[0].as_slice())?;
|
||||
println!("{}", serde_json::to_string_pretty(&receipt)?);
|
||||
}
|
||||
StaticFileSegment::TransactionSenders => {
|
||||
|
||||
@@ -5,6 +5,7 @@
|
||||
//! state), compacts MDBX, then runs the pipeline to rebuild them.
|
||||
|
||||
use crate::common::CliNodeTypes;
|
||||
use alloy_primitives::Address;
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
mdbx::{self, ffi},
|
||||
@@ -132,8 +133,12 @@ impl Command {
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?;
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::AccountChangeSets)?;
|
||||
if first_block > 0 {
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
}
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(first_block))?.peekable();
|
||||
@@ -174,11 +179,15 @@ impl Command {
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?;
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::StorageChangeSets)?;
|
||||
if first_block > 0 {
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
}
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(Default::default()))?.peekable();
|
||||
let mut walker = cursor.walk(Some((first_block, Address::ZERO).into()))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
@@ -238,6 +247,18 @@ impl Command {
|
||||
.map_or(0, |b| b + 1);
|
||||
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
|
||||
|
||||
// The writer always starts at the fixed range boundary (e.g. 2500000) which may be
|
||||
// earlier than first_block (e.g. 2603897 from prune checkpoint).
|
||||
if first_block > 0 {
|
||||
let mut writer = sf_provider.latest_writer(StaticFileSegment::Receipts)?;
|
||||
writer.ensure_at_block(first_block - 1)?;
|
||||
writer.commit()?;
|
||||
}
|
||||
|
||||
let before = sf_provider
|
||||
.get_highest_static_file_tx(StaticFileSegment::Receipts)
|
||||
.map_or(0, |tx| tx + 1);
|
||||
|
||||
let block_range = first_block..=tip;
|
||||
|
||||
let segment = reth_static_file::segments::Receipts;
|
||||
@@ -245,7 +266,11 @@ impl Command {
|
||||
|
||||
sf_provider.commit()?;
|
||||
|
||||
info!(target: "reth::cli", "Receipts migrated");
|
||||
let after = sf_provider
|
||||
.get_highest_static_file_tx(StaticFileSegment::Receipts)
|
||||
.map_or(0, |tx| tx + 1);
|
||||
let count = after - before;
|
||||
info!(target: "reth::cli", count, "Receipts migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
|
||||
@@ -50,8 +50,13 @@ where
|
||||
info!(target: "reth::cli", new_tip = ?header.num_hash(), "Setting up dummy EVM chain before importing state.");
|
||||
|
||||
let static_file_provider = provider_rw.static_file_provider();
|
||||
// Write EVM dummy data up to `header - 1` block
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
// Write EVM dummy data up to `header - 1` block. Skip when the supplied
|
||||
// header is at block 0: `header.number() - 1` would underflow in u64 to
|
||||
// `u64::MAX`, sending `append_dummy_chain` into a 1..=u64::MAX loop that
|
||||
// exhausts memory before failing.
|
||||
if header.number() > 0 {
|
||||
append_dummy_chain(&static_file_provider, header.number() - 1, header_factory)?;
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Appending first valid block.");
|
||||
|
||||
@@ -191,7 +196,13 @@ mod tests {
|
||||
use alloy_primitives::{address, b256};
|
||||
use reth_db_common::init::init_genesis;
|
||||
use reth_provider::{test_utils::create_test_provider_factory, DatabaseProviderFactory};
|
||||
use std::io::Write;
|
||||
use std::{
|
||||
io::Write,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
},
|
||||
};
|
||||
use tempfile::NamedTempFile;
|
||||
|
||||
#[test]
|
||||
@@ -264,4 +275,45 @@ mod tests {
|
||||
|
||||
assert_eq!(actual_next_height, expected_next_height);
|
||||
}
|
||||
|
||||
/// Regression: a header at block 0 used to send `append_dummy_chain` into
|
||||
/// a `1..=u64::MAX` loop because `header.number() - 1` underflowed in
|
||||
/// u64. The guard `if header.number() > 0` skips the dummy-chain step
|
||||
/// when there is no pre-genesis range to backfill, so `header_factory`
|
||||
/// is never invoked.
|
||||
#[test]
|
||||
fn test_setup_without_evm_skips_dummy_chain_for_genesis_header() {
|
||||
let header = Header { number: 0, ..Default::default() };
|
||||
let header_hash = header.hash_slow();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
init_genesis(&provider_factory).unwrap();
|
||||
let provider_rw = provider_factory.database_provider_rw().unwrap();
|
||||
|
||||
let factory_calls = Arc::new(AtomicU64::new(0));
|
||||
let factory_calls_inner = Arc::clone(&factory_calls);
|
||||
|
||||
// The Result of `setup_without_evm` itself is not asserted: with
|
||||
// `number == 0` plus a genesis already written by `init_genesis`,
|
||||
// the subsequent `append_first_block` may legitimately fail. The
|
||||
// bug under test is the OOM in the dummy-chain loop, observable
|
||||
// through the factory-call counter below.
|
||||
let _ = setup_without_evm(
|
||||
&provider_rw,
|
||||
SealedHeader::new(header, header_hash),
|
||||
move |number| {
|
||||
// Bound calls so a regression cannot exhaust the test
|
||||
// runner's memory; the only correct value here is 0.
|
||||
let n = factory_calls_inner.fetch_add(1, Ordering::Relaxed);
|
||||
assert!(n < 8, "header_factory must not be invoked for a genesis-block header");
|
||||
Header { number, ..Default::default() }
|
||||
},
|
||||
);
|
||||
|
||||
assert_eq!(
|
||||
factory_calls.load(Ordering::Relaxed),
|
||||
0,
|
||||
"append_dummy_chain must be skipped when header.number() == 0"
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -188,7 +188,7 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
|
||||
)
|
||||
}
|
||||
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
|
||||
let default_secret_key_path = data_dir.p2p_secret();
|
||||
let p2p_secret_key = self.network.secret_key(default_secret_key_path)?;
|
||||
|
||||
@@ -5,6 +5,7 @@ use crate::common::{
|
||||
EnvironmentArgs,
|
||||
};
|
||||
use alloy_consensus::{transaction::TxHashRef, BlockHeader, TxReceipt};
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use clap::Parser;
|
||||
use eyre::WrapErr;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks, Hardforks};
|
||||
@@ -12,14 +13,22 @@ use reth_cli::chainspec::ChainSpecParser;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_consensus::FullConsensus;
|
||||
use reth_evm::{execute::Executor, ConfigureEvm};
|
||||
use reth_primitives_traits::{format_gas_throughput, BlockBody, GotExpected};
|
||||
use reth_primitives_traits::{format_gas_throughput, Account, BlockBody, GotExpected};
|
||||
use reth_provider::{
|
||||
BlockNumReader, BlockReader, ChainSpecProvider, DatabaseProviderFactory, ReceiptProvider,
|
||||
StaticFileProviderFactory, TransactionVariant,
|
||||
};
|
||||
use reth_revm::database::StateProviderDatabase;
|
||||
use reth_revm::{
|
||||
database::StateProviderDatabase,
|
||||
db::{
|
||||
states::reverts::{AccountInfoRevert, RevertToSlot},
|
||||
BundleState,
|
||||
},
|
||||
};
|
||||
use reth_stages::stages::calculate_gas_used_from_headers;
|
||||
use reth_storage_api::{ChangeSetReader, DBProvider, StorageChangeSetReader};
|
||||
use std::{
|
||||
collections::HashMap,
|
||||
sync::{
|
||||
atomic::{AtomicU64, Ordering},
|
||||
Arc,
|
||||
@@ -68,13 +77,18 @@ impl<C: ChainSpecParser> Command<C> {
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>> Command<C> {
|
||||
/// Execute `re-execute` command
|
||||
pub async fn execute<N>(
|
||||
self,
|
||||
mut self,
|
||||
components: impl CliComponentsBuilder<N>,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N: CliNodeTypes<ChainSpec = C::ChainSpec>,
|
||||
{
|
||||
// Default to 4GB RocksDB block cache for re-execute unless explicitly set.
|
||||
if self.env.db.rocksdb_block_cache_size.is_none() {
|
||||
self.env.db.rocksdb_block_cache_size = Some(4 << 30);
|
||||
}
|
||||
|
||||
let Environment { provider_factory, .. } = self.env.init::<N>(AccessRights::RO, runtime)?;
|
||||
|
||||
let components = components(provider_factory.chain_spec());
|
||||
@@ -108,15 +122,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
min_block..=max_block,
|
||||
)?;
|
||||
|
||||
let db_at = {
|
||||
let provider_factory = provider_factory.clone();
|
||||
move |block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider_factory.history_by_block_number(block_number).unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
let skip_invalid_blocks = self.skip_invalid_blocks;
|
||||
let blocks_per_chunk = self.blocks_per_chunk;
|
||||
let (stats_tx, mut stats_rx) = mpsc::unbounded_channel();
|
||||
@@ -132,13 +137,23 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let provider_factory = provider_factory.clone();
|
||||
let evm_config = components.evm_config().clone();
|
||||
let consensus = components.consensus().clone();
|
||||
let db_at = db_at.clone();
|
||||
let stats_tx = stats_tx.clone();
|
||||
let info_tx = info_tx.clone();
|
||||
let cancellation = cancellation.clone();
|
||||
let next_block = Arc::clone(&next_block);
|
||||
tasks.spawn_blocking(move || {
|
||||
let executor_lifetime = Duration::from_secs(600);
|
||||
let provider = provider_factory.database_provider_ro()?.disable_long_read_transaction_safety();
|
||||
|
||||
let db_at = {
|
||||
|block_number: u64| {
|
||||
StateProviderDatabase(
|
||||
provider
|
||||
.history_by_block_number(block_number)
|
||||
.unwrap(),
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
@@ -248,11 +263,28 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
if executor.size_hint() > 5_000_000 ||
|
||||
executor_created.elapsed() > executor_lifetime
|
||||
{
|
||||
executor =
|
||||
evm_config.batch_executor(db_at(block.number()));
|
||||
let last_block = block.number();
|
||||
let old_executor = std::mem::replace(
|
||||
&mut executor,
|
||||
evm_config.batch_executor(db_at(last_block)),
|
||||
);
|
||||
let bundle = old_executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
last_block,
|
||||
)?;
|
||||
executor_created = Instant::now();
|
||||
}
|
||||
}
|
||||
|
||||
// Full verification at chunk end for remaining unverified blocks
|
||||
let bundle = executor.into_state().take_bundle();
|
||||
verify_bundle_against_changesets(
|
||||
&provider,
|
||||
&bundle,
|
||||
chunk_end - 1,
|
||||
)?;
|
||||
}
|
||||
|
||||
eyre::Ok(())
|
||||
@@ -333,3 +365,103 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies reverts against database changesets.
|
||||
///
|
||||
/// For each block, reverts must match changeset entries exactly. No extra slots/accounts
|
||||
/// in reverts for non-destroyed accounts. Destroyed accounts may have extra changeset slots
|
||||
/// (from DB storage wipe) absent from reverts.
|
||||
fn verify_bundle_against_changesets<P>(
|
||||
provider: &P,
|
||||
bundle: &BundleState,
|
||||
last_block: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
P: ChangeSetReader + StorageChangeSetReader,
|
||||
{
|
||||
// Verify reverts against changesets per block
|
||||
for (i, block_reverts) in bundle.reverts.iter().rev().enumerate() {
|
||||
let block_number = last_block - i as u64;
|
||||
|
||||
let mut cs_accounts: HashMap<Address, Option<Account>> = provider
|
||||
.account_block_changeset(block_number)?
|
||||
.into_iter()
|
||||
.map(|cs| (cs.address, cs.info))
|
||||
.collect();
|
||||
|
||||
let mut cs_storage: HashMap<Address, HashMap<B256, U256>> = HashMap::new();
|
||||
for (bna, entry) in provider.storage_changeset(block_number)? {
|
||||
cs_storage.entry(bna.address()).or_default().insert(entry.key, entry.value);
|
||||
}
|
||||
|
||||
for (addr, revert) in block_reverts {
|
||||
// Verify account info
|
||||
match &revert.account {
|
||||
AccountInfoRevert::DoNothing => {
|
||||
eyre::ensure!(
|
||||
!cs_accounts.contains_key(addr),
|
||||
"Block {block_number}: account {addr} in changeset but revert is DoNothing",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::DeleteIt => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is DeleteIt but not in changeset")
|
||||
})?;
|
||||
eyre::ensure!(
|
||||
cs_info.is_none(),
|
||||
"Block {block_number}: account {addr} revert is DeleteIt but changeset has {cs_info:?}",
|
||||
);
|
||||
}
|
||||
AccountInfoRevert::RevertTo(info) => {
|
||||
let cs_info = cs_accounts.remove(addr).ok_or_else(|| {
|
||||
eyre::eyre!("Block {block_number}: account {addr} revert is RevertTo but not in changeset")
|
||||
})?;
|
||||
let revert_acct = Some(Account::from(info));
|
||||
eyre::ensure!(
|
||||
revert_acct == cs_info,
|
||||
"Block {block_number}: account {addr} info mismatch: revert={revert_acct:?} cs={cs_info:?}",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Verify storage slots — remove matched changeset entries as we go
|
||||
let mut cs_slots = cs_storage.get_mut(addr);
|
||||
for (slot_key, revert_slot) in &revert.storage {
|
||||
let b256_key = B256::from(*slot_key);
|
||||
let cs_value = cs_slots.as_mut().and_then(|s| s.remove(&b256_key));
|
||||
match (revert_slot, cs_value) {
|
||||
// When a contract is selfdestructed and re-created at the same address
|
||||
// within the same block, revm marks slots touched by the new contract
|
||||
// as `Destroyed` and never reads the original DB value, so
|
||||
// `to_previous_value()` would resolve to zero, which might be wrong.
|
||||
(RevertToSlot::Destroyed, _) => {}
|
||||
(RevertToSlot::Some(prev), Some(cs_value)) => eyre::ensure!(
|
||||
*prev == cs_value,
|
||||
"Block {block_number}: {addr} slot {b256_key} mismatch: \
|
||||
revert={prev} cs={cs_value}",
|
||||
),
|
||||
(RevertToSlot::Some(_), None) => eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} slot {b256_key} in reverts but not in changeset",
|
||||
),
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_storage slots for this address must be from a destroyed account
|
||||
if let Some(remaining) = cs_slots.filter(|s| !s.is_empty()) {
|
||||
eyre::ensure!(
|
||||
revert.wipe_storage,
|
||||
"Block {block_number}: {addr} has {} unmatched storage slots in changeset",
|
||||
remaining.len(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Any remaining cs_accounts entries had no corresponding revert
|
||||
if let Some(addr) = cs_accounts.keys().next() {
|
||||
eyre::bail!("Block {block_number}: account {addr} in changeset but not in reverts");
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use reth_db_api::{
|
||||
};
|
||||
use reth_db_common::DbTool;
|
||||
use reth_evm::ConfigureEvm;
|
||||
use reth_node_api::HeaderTy;
|
||||
use reth_node_api::{HeaderTy, TxTy};
|
||||
use reth_node_core::dirs::{ChainPath, DataDirPath};
|
||||
use reth_provider::{
|
||||
providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
|
||||
@@ -88,7 +88,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
|
||||
)
|
||||
})??;
|
||||
output_db.update(|tx| {
|
||||
tx.import_table_with_range::<tables::BlockOmmers, _>(
|
||||
tx.import_table_with_range::<tables::BlockOmmers<HeaderTy<N>>, _>(
|
||||
&db_tool.provider_factory.db_ref().tx()?,
|
||||
Some(from),
|
||||
to,
|
||||
@@ -110,7 +110,7 @@ fn import_tables_with_range<N: ProviderNodeTypes>(
|
||||
})??;
|
||||
|
||||
output_db.update(|tx| {
|
||||
tx.import_table_with_range::<tables::Transactions, _>(
|
||||
tx.import_table_with_range::<tables::Transactions<TxTy<N>>, _>(
|
||||
&db_tool.provider_factory.db_ref().tx()?,
|
||||
Some(from_tx),
|
||||
to_tx,
|
||||
|
||||
@@ -210,7 +210,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let consensus = Arc::new(components.consensus().clone());
|
||||
|
||||
let mut config = config;
|
||||
config.peers.trusted_nodes_only = self.network.trusted_only;
|
||||
config.peers.trusted_nodes_only |= self.network.trusted_only;
|
||||
config.peers.trusted_nodes.extend(self.network.trusted_peers.clone());
|
||||
|
||||
let network_secret_path = self
|
||||
|
||||
@@ -30,10 +30,16 @@
|
||||
|
||||
extern crate alloc;
|
||||
|
||||
use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
|
||||
use alloc::{
|
||||
boxed::Box,
|
||||
fmt::Debug,
|
||||
string::{String, ToString},
|
||||
sync::Arc,
|
||||
vec::Vec,
|
||||
};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
|
||||
use core::error::Error;
|
||||
use core::{error::Error, fmt::Display};
|
||||
|
||||
/// Pre-computed receipt root and logs bloom.
|
||||
///
|
||||
@@ -104,6 +110,18 @@ pub trait Consensus<B: Block>: HeaderValidator<B::Header> {
|
||||
/// Note: validating blocks does not include other validations of the Consensus
|
||||
fn validate_block_pre_execution(&self, block: &SealedBlock<B>) -> Result<(), ConsensusError>;
|
||||
|
||||
/// Returns `true` if the given consensus error is transient and may resolve on its own.
|
||||
///
|
||||
/// On fast chains, clock skew between nodes can cause a valid block's timestamp to
|
||||
/// appear briefly in the future. Caching such blocks as permanently invalid would
|
||||
/// prevent them from being re-validated once the local clock catches up.
|
||||
///
|
||||
/// Transient errors will not cause the block hash to be cached as permanently invalid,
|
||||
/// allowing the block to be re-validated later.
|
||||
fn is_transient_error(&self, _error: &ConsensusError) -> bool {
|
||||
false
|
||||
}
|
||||
|
||||
/// Validate a block disregarding world state using an optional pre-computed transaction root.
|
||||
///
|
||||
/// If `transaction_root` is provided, the implementation should use the pre-computed
|
||||
@@ -456,19 +474,49 @@ pub enum ConsensusError {
|
||||
/// EIP-7825: Transaction gas limit exceeds maximum allowed
|
||||
#[error(transparent)]
|
||||
TransactionGasLimitTooHigh(Box<TxGasLimitTooHighErr>),
|
||||
/// Other, likely an injected L2 error.
|
||||
#[error("{0}")]
|
||||
Other(String),
|
||||
/// Other unspecified error.
|
||||
/// Any additional consensus error, for example L2-specific errors.
|
||||
#[error(transparent)]
|
||||
Custom(#[from] Arc<dyn Error + Send + Sync>),
|
||||
Other(#[from] Arc<dyn Error + Send + Sync>),
|
||||
}
|
||||
|
||||
impl ConsensusError {
|
||||
/// Returns a new [`ConsensusError::Other`] instance with the given error.
|
||||
pub fn other<E>(error: E) -> Self
|
||||
where
|
||||
E: Error + Send + Sync + 'static,
|
||||
{
|
||||
Self::Other(Arc::new(error))
|
||||
}
|
||||
|
||||
/// Returns a new [`ConsensusError::Other`] instance with the given message.
|
||||
pub fn msg(msg: impl Display) -> Self {
|
||||
Self::other(MessageError(msg.to_string()))
|
||||
}
|
||||
|
||||
/// Returns `true` if the error is a state root error.
|
||||
pub const fn is_state_root_error(&self) -> bool {
|
||||
matches!(self, Self::BodyStateRootDiff(_))
|
||||
}
|
||||
|
||||
/// Returns the arbitrary error if it is [`ConsensusError::Other`].
|
||||
pub fn as_other(&self) -> Option<&(dyn Error + Send + Sync + 'static)> {
|
||||
match self {
|
||||
Self::Other(err) => Some(err.as_ref()),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a reference to the [`ConsensusError::Other`] value if it is of that type.
|
||||
/// Returns `None` otherwise.
|
||||
pub fn downcast_other_ref<T: Error + 'static>(&self) -> Option<&T> {
|
||||
let other = self.as_other()?;
|
||||
other.downcast_ref()
|
||||
}
|
||||
|
||||
/// Returns `true` if this type is a [`ConsensusError::Other`] of that error type.
|
||||
pub fn is_other<T: Error + 'static>(&self) -> bool {
|
||||
self.as_other().map(|err| err.is::<T>()).unwrap_or(false)
|
||||
}
|
||||
}
|
||||
|
||||
impl From<InvalidTransactionError> for ConsensusError {
|
||||
@@ -500,6 +548,10 @@ pub struct TxGasLimitTooHighErr {
|
||||
pub max_allowed: u64,
|
||||
}
|
||||
|
||||
#[derive(Debug, thiserror::Error)]
|
||||
#[error("{0}")]
|
||||
struct MessageError(String);
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
@@ -509,24 +561,31 @@ mod tests {
|
||||
struct CustomL2Error;
|
||||
|
||||
#[test]
|
||||
fn test_custom_error_conversion() {
|
||||
// Test conversion from custom error to ConsensusError
|
||||
let custom_err = CustomL2Error;
|
||||
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
|
||||
let consensus_err: ConsensusError = arc_err.into();
|
||||
|
||||
// Verify it's the Custom variant
|
||||
assert!(matches!(consensus_err, ConsensusError::Custom(_)));
|
||||
fn test_other_error_conversion() {
|
||||
let consensus_err = ConsensusError::other(CustomL2Error);
|
||||
assert!(matches!(consensus_err, ConsensusError::Other(_)));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_custom_error_display() {
|
||||
let custom_err = CustomL2Error;
|
||||
let arc_err: Arc<dyn Error + Send + Sync> = Arc::new(custom_err);
|
||||
let consensus_err: ConsensusError = arc_err.into();
|
||||
|
||||
// Verify the error message is preserved through transparent attribute
|
||||
fn test_other_error_display() {
|
||||
let consensus_err = ConsensusError::other(CustomL2Error);
|
||||
let error_message = format!("{}", consensus_err);
|
||||
assert_eq!(error_message, "Custom L2 consensus error");
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_other_error_downcast() {
|
||||
let consensus_err = ConsensusError::other(CustomL2Error);
|
||||
|
||||
assert!(consensus_err.is_other::<CustomL2Error>());
|
||||
assert!(consensus_err.downcast_other_ref::<CustomL2Error>().is_some());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_other_msg() {
|
||||
let consensus_err = ConsensusError::msg("consensus message");
|
||||
|
||||
assert_eq!(consensus_err.to_string(), "consensus message");
|
||||
assert!(consensus_err.downcast_other_ref::<MessageError>().is_some());
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
use futures_util::StreamExt;
|
||||
use reth_node_api::{BlockBody, PayloadAttributes, PayloadKind};
|
||||
use reth_node_api::{PayloadAttributes, PayloadKind};
|
||||
use reth_payload_builder::{PayloadBuilderHandle, PayloadId};
|
||||
use reth_payload_builder_primitives::Events;
|
||||
use reth_payload_primitives::{BuiltPayload, PayloadTypes};
|
||||
use reth_payload_primitives::PayloadTypes;
|
||||
use tokio_stream::wrappers::BroadcastStream;
|
||||
|
||||
/// Helper for payload operations
|
||||
@@ -53,27 +53,11 @@ impl<T: PayloadTypes> PayloadTestContext<T> {
|
||||
///
|
||||
/// Panics if the payload builder does not produce a non-empty payload within 30 seconds.
|
||||
pub async fn wait_for_built_payload(&self, payload_id: PayloadId) {
|
||||
let start = std::time::Instant::now();
|
||||
loop {
|
||||
let payload =
|
||||
self.payload_builder.best_payload(payload_id).await.transpose().ok().flatten();
|
||||
if payload.is_none_or(|p| p.block().body().transactions().is_empty()) {
|
||||
assert!(
|
||||
start.elapsed() < std::time::Duration::from_secs(30),
|
||||
"timed out waiting for a non-empty payload for {payload_id} — \
|
||||
check that the chain spec supports all generated tx types"
|
||||
);
|
||||
tokio::time::sleep(std::time::Duration::from_millis(20)).await;
|
||||
continue
|
||||
}
|
||||
// Resolve payload once its built
|
||||
self.payload_builder
|
||||
.resolve_kind(payload_id, PayloadKind::Earliest)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
break;
|
||||
}
|
||||
self.payload_builder
|
||||
.resolve_kind(payload_id, PayloadKind::WaitForPending)
|
||||
.await
|
||||
.unwrap()
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// Expects the next event to be a built payload event or panics
|
||||
|
||||
@@ -15,6 +15,9 @@ pub const DEFAULT_MEMORY_BLOCK_BUFFER_TARGET: u64 = 0;
|
||||
/// The size of proof targets chunk to spawn in one multiproof calculation.
|
||||
pub const DEFAULT_MULTIPROOF_TASK_CHUNK_SIZE: usize = 5;
|
||||
|
||||
/// Default number of cache hits before an invalid header entry is evicted and reprocessed.
|
||||
pub const DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
|
||||
|
||||
/// Gas threshold below which the small block chunk size is used.
|
||||
pub const SMALL_BLOCK_GAS_THRESHOLD: u64 = 20_000_000;
|
||||
|
||||
@@ -102,6 +105,11 @@ pub struct TreeConfig {
|
||||
block_buffer_limit: u32,
|
||||
/// Number of invalid headers to keep in cache.
|
||||
max_invalid_header_cache_length: u32,
|
||||
/// Number of cache hits before an invalid header entry is evicted and reprocessed.
|
||||
///
|
||||
/// Setting this to `0` effectively disables the cache because entries are evicted on the
|
||||
/// first lookup.
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
/// Maximum number of blocks to execute sequentially in a batch.
|
||||
///
|
||||
/// This is used as a cutoff to prevent long-running sequential block execution when we receive
|
||||
@@ -170,6 +178,23 @@ pub struct TreeConfig {
|
||||
share_execution_cache_with_payload_builder: bool,
|
||||
/// Whether to share sparse trie with the payload builder.
|
||||
share_sparse_trie_with_payload_builder: bool,
|
||||
/// Whether to suppress persistence cycles while building a payload.
|
||||
///
|
||||
/// When enabled, persistence is deferred from the moment an FCU with payload attributes
|
||||
/// arrives until the next FCU without attributes. This avoids persistence I/O competing
|
||||
/// with block building on latency-sensitive chains.
|
||||
suppress_persistence_during_build: bool,
|
||||
/// Whether to disable BAL (Block Access List, EIP-7928) based parallel execution.
|
||||
/// When disabled, falls back to transaction-based prewarming even when a BAL is available.
|
||||
disable_bal_parallel_execution: bool,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
/// When disabled, the BAL hashed post state is not sent to the multiproof task for
|
||||
/// early parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
/// Whether to disable BAL (Block Access List) storage prefetch IO during prewarming.
|
||||
/// When set, BAL storage slots are not read into the execution cache. BAL hashed-state
|
||||
/// streaming for parallel state-root computation is controlled separately.
|
||||
disable_bal_batch_io: bool,
|
||||
/// Maximum random jitter applied before each proof computation (trie-debug only).
|
||||
/// When set, each proof worker sleeps for a random duration up to this value
|
||||
/// before starting a proof calculation.
|
||||
@@ -189,6 +214,7 @@ impl Default for TreeConfig {
|
||||
persistence_backpressure_threshold: DEFAULT_PERSISTENCE_BACKPRESSURE_THRESHOLD,
|
||||
block_buffer_limit: DEFAULT_BLOCK_BUFFER_LIMIT,
|
||||
max_invalid_header_cache_length: DEFAULT_MAX_INVALID_HEADER_CACHE_LENGTH,
|
||||
invalid_header_hit_eviction_threshold: DEFAULT_INVALID_HEADER_HIT_EVICTION_THRESHOLD,
|
||||
max_execute_block_batch_size: DEFAULT_MAX_EXECUTE_BLOCK_BATCH_SIZE,
|
||||
legacy_state_root: false,
|
||||
always_compare_trie_updates: false,
|
||||
@@ -212,6 +238,10 @@ impl Default for TreeConfig {
|
||||
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
|
||||
share_execution_cache_with_payload_builder: false,
|
||||
share_sparse_trie_with_payload_builder: false,
|
||||
suppress_persistence_during_build: false,
|
||||
disable_bal_parallel_execution: true,
|
||||
disable_bal_parallel_state_root: false,
|
||||
disable_bal_batch_io: false,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
proof_jitter: None,
|
||||
}
|
||||
@@ -227,6 +257,7 @@ impl TreeConfig {
|
||||
persistence_backpressure_threshold: u64,
|
||||
block_buffer_limit: u32,
|
||||
max_invalid_header_cache_length: u32,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
max_execute_block_batch_size: usize,
|
||||
legacy_state_root: bool,
|
||||
always_compare_trie_updates: bool,
|
||||
@@ -260,6 +291,7 @@ impl TreeConfig {
|
||||
persistence_backpressure_threshold,
|
||||
block_buffer_limit,
|
||||
max_invalid_header_cache_length,
|
||||
invalid_header_hit_eviction_threshold,
|
||||
max_execute_block_batch_size,
|
||||
legacy_state_root,
|
||||
always_compare_trie_updates,
|
||||
@@ -283,6 +315,10 @@ impl TreeConfig {
|
||||
state_root_task_timeout,
|
||||
share_execution_cache_with_payload_builder,
|
||||
share_sparse_trie_with_payload_builder,
|
||||
suppress_persistence_during_build: false,
|
||||
disable_bal_parallel_execution: true,
|
||||
disable_bal_parallel_state_root: false,
|
||||
disable_bal_batch_io: false,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
proof_jitter: None,
|
||||
}
|
||||
@@ -313,6 +349,14 @@ impl TreeConfig {
|
||||
self.max_invalid_header_cache_length
|
||||
}
|
||||
|
||||
/// Return the invalid header cache hit eviction threshold.
|
||||
///
|
||||
/// Setting this to `0` effectively disables the cache because entries are evicted on the
|
||||
/// first lookup.
|
||||
pub const fn invalid_header_hit_eviction_threshold(&self) -> u8 {
|
||||
self.invalid_header_hit_eviction_threshold
|
||||
}
|
||||
|
||||
/// Return the maximum execute block batch size.
|
||||
pub const fn max_execute_block_batch_size(&self) -> usize {
|
||||
self.max_execute_block_batch_size
|
||||
@@ -443,6 +487,15 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for the invalid header cache hit eviction threshold.
|
||||
pub const fn with_invalid_header_hit_eviction_threshold(
|
||||
mut self,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
) -> Self {
|
||||
self.invalid_header_hit_eviction_threshold = invalid_header_hit_eviction_threshold;
|
||||
self
|
||||
}
|
||||
|
||||
/// Setter for maximum execute block batch size.
|
||||
pub const fn with_max_execute_block_batch_size(
|
||||
mut self,
|
||||
@@ -646,6 +699,56 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether persistence is suppressed during payload building.
|
||||
pub const fn suppress_persistence_during_build(&self) -> bool {
|
||||
self.suppress_persistence_during_build
|
||||
}
|
||||
|
||||
/// Setter for whether to suppress persistence during payload building.
|
||||
pub const fn with_suppress_persistence_during_build(mut self, value: bool) -> Self {
|
||||
self.suppress_persistence_during_build = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL-based parallel execution is disabled.
|
||||
pub const fn disable_bal_parallel_execution(&self) -> bool {
|
||||
self.disable_bal_parallel_execution
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL-based parallel execution.
|
||||
pub const fn without_bal_parallel_execution(
|
||||
mut self,
|
||||
disable_bal_parallel_execution: bool,
|
||||
) -> Self {
|
||||
self.disable_bal_parallel_execution = disable_bal_parallel_execution;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL-driven parallel state root computation is disabled.
|
||||
pub const fn disable_bal_parallel_state_root(&self) -> bool {
|
||||
self.disable_bal_parallel_state_root
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL-driven parallel state root computation.
|
||||
pub const fn without_bal_parallel_state_root(
|
||||
mut self,
|
||||
disable_bal_parallel_state_root: bool,
|
||||
) -> Self {
|
||||
self.disable_bal_parallel_state_root = disable_bal_parallel_state_root;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL batched IO is disabled.
|
||||
pub const fn disable_bal_batch_io(&self) -> bool {
|
||||
self.disable_bal_batch_io
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL batched IO.
|
||||
pub const fn without_bal_batch_io(mut self, disable_bal_batch_io: bool) -> Self {
|
||||
self.disable_bal_batch_io = disable_bal_batch_io;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the proof jitter duration, if configured (trie-debug only).
|
||||
#[cfg(feature = "trie-debug")]
|
||||
pub const fn proof_jitter(&self) -> Option<Duration> {
|
||||
|
||||
@@ -21,7 +21,8 @@ impl ForkchoiceStateTracker {
|
||||
/// `sync_target` to `None`, since we're now fully synced.
|
||||
pub const fn set_latest(&mut self, state: ForkchoiceState, status: ForkchoiceStatus) {
|
||||
if status.is_valid() {
|
||||
self.set_valid(state);
|
||||
self.last_syncing = None;
|
||||
self.last_valid = Some(state);
|
||||
} else if status.is_syncing() {
|
||||
self.last_syncing = Some(state);
|
||||
}
|
||||
@@ -30,11 +31,24 @@ impl ForkchoiceStateTracker {
|
||||
self.latest = Some(received);
|
||||
}
|
||||
|
||||
const fn set_valid(&mut self, state: ForkchoiceState) {
|
||||
// we no longer need to sync to this state.
|
||||
/// Promotes a previously tracked syncing forkchoice state to valid, without overwriting a
|
||||
/// newer `latest` state.
|
||||
///
|
||||
/// This is used when a `Syncing` FCU's head finally becomes canonical via the downloaded-block
|
||||
/// flow, so the safe/finalized anchors of that FCU can be applied. Unlike
|
||||
/// [`Self::set_latest`], this preserves a newer `latest` (e.g. an `Invalid` FCU received
|
||||
/// after the syncing one) and only flips `latest` to `Valid` when it still refers to the same
|
||||
/// syncing FCU being promoted.
|
||||
pub fn promote_sync_target_to_valid(&mut self, state: ForkchoiceState) {
|
||||
self.last_syncing = None;
|
||||
|
||||
self.last_valid = Some(state);
|
||||
|
||||
if let Some(received) = self.latest.as_mut() &&
|
||||
received.state == state &&
|
||||
received.status.is_syncing()
|
||||
{
|
||||
received.status = ForkchoiceStatus::Valid;
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the [`ForkchoiceStatus`] of the latest received FCU.
|
||||
|
||||
@@ -8,25 +8,28 @@ use schnellru::{ByLength, LruMap};
|
||||
use std::fmt::Debug;
|
||||
use tracing::warn;
|
||||
|
||||
/// The max hit counter for invalid headers in the cache before it is forcefully evicted.
|
||||
///
|
||||
/// In other words, if a header is referenced more than this number of times, it will be evicted to
|
||||
/// allow for reprocessing.
|
||||
const INVALID_HEADER_HIT_EVICTION_THRESHOLD: u8 = 128;
|
||||
|
||||
/// Keeps track of invalid headers.
|
||||
#[derive(Debug)]
|
||||
pub struct InvalidHeaderCache {
|
||||
/// This maps a header hash to a reference to its invalid ancestor.
|
||||
headers: LruMap<B256, HeaderEntry>,
|
||||
/// Number of cache hits before an invalid header entry is evicted and reprocessed.
|
||||
hit_eviction_threshold: u8,
|
||||
/// Metrics for the cache.
|
||||
metrics: InvalidHeaderCacheMetrics,
|
||||
}
|
||||
|
||||
impl InvalidHeaderCache {
|
||||
/// Invalid header cache constructor.
|
||||
pub fn new(max_length: u32) -> Self {
|
||||
Self { headers: LruMap::new(ByLength::new(max_length)), metrics: Default::default() }
|
||||
///
|
||||
/// Setting `hit_eviction_threshold` to `0` effectively disables the cache because entries are
|
||||
/// evicted on the first lookup.
|
||||
pub fn new(max_length: u32, hit_eviction_threshold: u8) -> Self {
|
||||
Self {
|
||||
headers: LruMap::new(ByLength::new(max_length)),
|
||||
hit_eviction_threshold,
|
||||
metrics: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
fn insert_entry(&mut self, hash: B256, header: BlockWithParent) {
|
||||
@@ -41,7 +44,7 @@ impl InvalidHeaderCache {
|
||||
{
|
||||
let entry = self.headers.get(hash)?;
|
||||
entry.hit_count += 1;
|
||||
if entry.hit_count < INVALID_HEADER_HIT_EVICTION_THRESHOLD {
|
||||
if entry.hit_count < self.hit_eviction_threshold {
|
||||
return Some(entry.header)
|
||||
}
|
||||
}
|
||||
@@ -110,17 +113,28 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_hit_eviction() {
|
||||
let mut cache = InvalidHeaderCache::new(10);
|
||||
let hit_eviction_threshold = 3;
|
||||
let mut cache = InvalidHeaderCache::new(10, hit_eviction_threshold);
|
||||
let header = Header::default();
|
||||
let header = SealedHeader::seal_slow(header);
|
||||
cache.insert(header.block_with_parent());
|
||||
assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, 0);
|
||||
|
||||
for hit in 1..INVALID_HEADER_HIT_EVICTION_THRESHOLD {
|
||||
for hit in 1..hit_eviction_threshold {
|
||||
assert!(cache.get(&header.hash()).is_some());
|
||||
assert_eq!(cache.headers.get(&header.hash()).unwrap().hit_count, hit);
|
||||
}
|
||||
|
||||
assert!(cache.get(&header.hash()).is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_zero_hit_eviction_threshold_effectively_disables_cache() {
|
||||
let mut cache = InvalidHeaderCache::new(10, 0);
|
||||
let header = SealedHeader::seal_slow(Header::default());
|
||||
cache.insert(header.block_with_parent());
|
||||
|
||||
assert!(cache.get(&header.hash()).is_none());
|
||||
assert_eq!(cache.headers.len(), 0);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -11,7 +11,7 @@ use alloy_primitives::B256;
|
||||
use alloy_rpc_types_engine::{
|
||||
ForkchoiceState, PayloadStatus, PayloadStatusEnum, PayloadValidationError,
|
||||
};
|
||||
use error::{InsertBlockError, InsertBlockFatalError};
|
||||
use error::{InsertBlockError, InsertBlockFatalError, InsertBlockValidationError};
|
||||
use reth_chain_state::{
|
||||
CanonicalInMemoryState, ComputedTrieData, ExecutedBlock, ExecutionTimingStats,
|
||||
MemoryOverlayStateProvider, NewCanonicalChain,
|
||||
@@ -151,11 +151,15 @@ impl<N: NodePrimitives> EngineApiTreeState<N> {
|
||||
fn new(
|
||||
block_buffer_limit: u32,
|
||||
max_invalid_header_cache_length: u32,
|
||||
invalid_header_hit_eviction_threshold: u8,
|
||||
canonical_block: BlockNumHash,
|
||||
engine_kind: EngineApiKind,
|
||||
) -> Self {
|
||||
Self {
|
||||
invalid_headers: InvalidHeaderCache::new(max_invalid_header_cache_length),
|
||||
invalid_headers: InvalidHeaderCache::new(
|
||||
max_invalid_header_cache_length,
|
||||
invalid_header_hit_eviction_threshold,
|
||||
),
|
||||
buffer: BlockBuffer::new(block_buffer_limit),
|
||||
tree_state: TreeState::new(canonical_block, engine_kind),
|
||||
forkchoice_state_tracker: ForkchoiceStateTracker::default(),
|
||||
@@ -305,6 +309,9 @@ where
|
||||
/// Stored here (not in `ExecutedBlock`) to avoid leaking observability concerns into the block
|
||||
/// type. Entries are removed when blocks are persisted or invalidated.
|
||||
execution_timing_stats: HashMap<B256, Box<ExecutionTimingStats>>,
|
||||
/// Set when an FCU with payload attributes is received, cleared on the next FCU without.
|
||||
/// Suppresses persistence cycles during payload building.
|
||||
building_payload: bool,
|
||||
/// Task runtime for spawning blocking work on named, reusable threads.
|
||||
runtime: reth_tasks::Runtime,
|
||||
}
|
||||
@@ -396,6 +403,7 @@ where
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
execution_timing_stats: HashMap::new(),
|
||||
building_payload: false,
|
||||
runtime,
|
||||
}
|
||||
}
|
||||
@@ -432,6 +440,7 @@ where
|
||||
let state = EngineApiTreeState::new(
|
||||
config.block_buffer_limit(),
|
||||
config.max_invalid_header_cache_length(),
|
||||
config.invalid_header_hit_eviction_threshold(),
|
||||
header.num_hash(),
|
||||
kind,
|
||||
);
|
||||
@@ -1112,6 +1121,8 @@ where
|
||||
) -> ProviderResult<TreeOutcome<OnForkChoiceUpdated>> {
|
||||
trace!(target: "engine::tree", ?attrs, "invoked forkchoice update");
|
||||
|
||||
self.building_payload = attrs.is_some() && self.config.suppress_persistence_during_build();
|
||||
|
||||
// Record metrics
|
||||
self.record_forkchoice_metrics();
|
||||
|
||||
@@ -1500,9 +1511,9 @@ where
|
||||
// Re-prepare overlay for the current canonical head with the new anchor.
|
||||
// Spawn a background task to trigger computation so it's ready when the next payload
|
||||
// arrives.
|
||||
if let Some(overlay) = self.state.tree_state.prepare_canonical_overlay() {
|
||||
if let Some(prepared) = self.state.tree_state.prepare_canonical_overlay() {
|
||||
self.runtime.spawn_blocking_named("prepare-overlay", move || {
|
||||
let _ = overlay.get();
|
||||
let _ = prepared.overlay.get(prepared.anchor_hash);
|
||||
});
|
||||
}
|
||||
|
||||
@@ -1906,9 +1917,37 @@ where
|
||||
self.on_canonical_chain_update(chain_update);
|
||||
}
|
||||
|
||||
self.on_canonicalized_sync_target(target);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Applies the tracked forkchoice state once its sync target head becomes canonical.
|
||||
fn on_canonicalized_sync_target(&mut self, target: B256) {
|
||||
let Some(sync_target_state) = self
|
||||
.state
|
||||
.forkchoice_state_tracker
|
||||
.sync_target_state()
|
||||
.filter(|state| state.head_block_hash == target)
|
||||
else {
|
||||
return;
|
||||
};
|
||||
|
||||
if let Err(outcome) = self.ensure_consistent_forkchoice_state(sync_target_state) {
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
head = %sync_target_state.head_block_hash,
|
||||
safe = %sync_target_state.safe_block_hash,
|
||||
finalized = %sync_target_state.finalized_block_hash,
|
||||
?outcome,
|
||||
"Canonicalized sync target head before safe/finalized could be applied"
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
self.state.forkchoice_state_tracker.promote_sync_target_to_valid(sync_target_state);
|
||||
}
|
||||
|
||||
/// Convenience function to handle an optional tree event.
|
||||
fn on_maybe_tree_event(&mut self, event: Option<TreeEvent>) -> ProviderResult<()> {
|
||||
if let Some(event) = event {
|
||||
@@ -2005,9 +2044,13 @@ where
|
||||
}
|
||||
|
||||
/// Returns true if the canonical chain length minus the last persisted
|
||||
/// block is greater than or equal to the persistence threshold and
|
||||
/// backfill is not running.
|
||||
/// block is greater than or equal to the persistence threshold,
|
||||
/// backfill is not running, and no payload is currently being built.
|
||||
pub const fn should_persist(&self) -> bool {
|
||||
if self.building_payload {
|
||||
return false
|
||||
}
|
||||
|
||||
if !self.backfill_sync_state.is_idle() {
|
||||
// can't persist if backfill is running
|
||||
return false
|
||||
@@ -3009,8 +3052,22 @@ where
|
||||
);
|
||||
let latest_valid_hash = self.latest_valid_hash_for_invalid_payload(block.parent_hash())?;
|
||||
|
||||
// keep track of the invalid header
|
||||
self.state.invalid_headers.insert(block.block_with_parent());
|
||||
// keep track of the invalid header unless the consensus impl considers it transient
|
||||
let is_transient = match &validation_err {
|
||||
InsertBlockValidationError::Consensus(err) => self.consensus.is_transient_error(err),
|
||||
_ => false,
|
||||
};
|
||||
if is_transient {
|
||||
warn!(
|
||||
target: "engine::tree",
|
||||
invalid_hash=%block.hash(),
|
||||
invalid_number=block.number(),
|
||||
%validation_err,
|
||||
"Skipping invalid header cache insert for transient validation error",
|
||||
);
|
||||
} else {
|
||||
self.state.invalid_headers.insert(block.block_with_parent());
|
||||
}
|
||||
self.emit_event(EngineApiEvent::BeaconConsensus(ConsensusEngineEvent::InvalidBlock(
|
||||
Box::new(block),
|
||||
)));
|
||||
|
||||
@@ -7,7 +7,7 @@ use crate::tree::{
|
||||
CacheWaitDurations, CachedStateMetrics, CachedStateMetricsSource, ExecutionCache,
|
||||
PayloadExecutionCache, SavedCache, StateProviderBuilder, TreeConfig, WaitForCaches,
|
||||
};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eip7928::bal::DecodedBal;
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal};
|
||||
use alloy_primitives::B256;
|
||||
use crossbeam_channel::{Receiver as CrossbeamReceiver, Sender as CrossbeamSender};
|
||||
@@ -122,6 +122,12 @@ where
|
||||
sparse_trie_max_hot_accounts: usize,
|
||||
/// Whether sparse trie cache pruning is fully disabled.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Whether to disable BAL-based parallel execution (falls back to tx-based prewarming).
|
||||
disable_bal_parallel_execution: bool,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
/// Whether BAL batched IO is disabled.
|
||||
disable_bal_batch_io: bool,
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
@@ -157,6 +163,9 @@ where
|
||||
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
|
||||
cache_metrics: (!config.disable_cache_metrics())
|
||||
.then(|| CachedStateMetrics::zeroed(CachedStateMetricsSource::Engine)),
|
||||
disable_bal_parallel_execution: config.disable_bal_parallel_execution(),
|
||||
disable_bal_parallel_state_root: config.disable_bal_parallel_state_root(),
|
||||
disable_bal_batch_io: config.disable_bal_batch_io(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -241,7 +250,6 @@ where
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
multiproof_provider_factory: F,
|
||||
config: &TreeConfig,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -264,13 +272,12 @@ where
|
||||
halve_workers,
|
||||
config,
|
||||
);
|
||||
let install_state_hook = bal.is_none();
|
||||
let install_state_hook = env.decoded_bal.is_none();
|
||||
let prewarm_handle = self.spawn_caching_with(
|
||||
env,
|
||||
prewarm_rx,
|
||||
provider_builder,
|
||||
Some(state_root_handle.updates_tx().clone()),
|
||||
bal,
|
||||
);
|
||||
|
||||
PayloadHandle {
|
||||
@@ -291,14 +298,13 @@ where
|
||||
env: ExecutionEnv<Evm>,
|
||||
transactions: I,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> IteratorPayloadHandle<Evm, I, N>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
{
|
||||
let (prewarm_rx, execution_rx) =
|
||||
self.spawn_tx_iterator(transactions, env.transaction_count);
|
||||
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None, bal);
|
||||
let prewarm_handle = self.spawn_caching_with(env, prewarm_rx, provider_builder, None);
|
||||
PayloadHandle {
|
||||
state_root_handle: None,
|
||||
install_state_hook: false,
|
||||
@@ -456,7 +462,7 @@ where
|
||||
level = "debug",
|
||||
target = "engine::tree::payload_processor",
|
||||
skip_all,
|
||||
fields(bal=%bal.is_some())
|
||||
fields(bal=%env.decoded_bal.is_some())
|
||||
)]
|
||||
fn spawn_caching_with<P>(
|
||||
&self,
|
||||
@@ -464,7 +470,6 @@ where
|
||||
transactions: mpsc::Receiver<(usize, impl ExecutableTxFor<Evm> + Clone + Send + 'static)>,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
to_sparse_trie_task: Option<CrossbeamSender<StateRootMessage>>,
|
||||
bal: Option<Arc<BlockAccessList>>,
|
||||
) -> CacheTaskHandle<N::Receipt>
|
||||
where
|
||||
P: BlockReader + StateProviderFactory + StateReader + Clone + 'static,
|
||||
@@ -475,7 +480,7 @@ where
|
||||
let saved_cache = self.disable_state_cache.not().then(|| self.cache_for(env.parent_hash));
|
||||
|
||||
let executed_tx_index = Arc::new(AtomicUsize::new(0));
|
||||
|
||||
let maybe_decoded_bal = env.decoded_bal.clone();
|
||||
// configure prewarming
|
||||
let prewarm_ctx = PrewarmContext {
|
||||
env,
|
||||
@@ -488,6 +493,8 @@ where
|
||||
executed_tx_index: Arc::clone(&executed_tx_index),
|
||||
precompile_cache_disabled: self.precompile_cache_disabled,
|
||||
precompile_cache_map: self.precompile_cache_map.clone(),
|
||||
disable_bal_parallel_state_root: self.disable_bal_parallel_state_root,
|
||||
disable_bal_batch_io: self.disable_bal_batch_io,
|
||||
};
|
||||
|
||||
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
|
||||
@@ -496,14 +503,16 @@ where
|
||||
prewarm_ctx,
|
||||
to_sparse_trie_task,
|
||||
);
|
||||
|
||||
{
|
||||
let to_prewarm_task = to_prewarm_task.clone();
|
||||
let disable_bal_parallel_execution = self.disable_bal_parallel_execution;
|
||||
self.executor.spawn_blocking_named("prewarm", move || {
|
||||
let mode = if skip_prewarm {
|
||||
PrewarmMode::Skipped
|
||||
} else if let Some(bal) = bal {
|
||||
PrewarmMode::BlockAccessList(bal)
|
||||
} else if let Some(decoded_bal) =
|
||||
maybe_decoded_bal.filter(|_| !disable_bal_parallel_execution)
|
||||
{
|
||||
PrewarmMode::BlockAccessList(decoded_bal)
|
||||
} else {
|
||||
PrewarmMode::Transactions(transactions)
|
||||
};
|
||||
@@ -597,6 +606,7 @@ where
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
parent_state_root,
|
||||
chunk_size,
|
||||
);
|
||||
|
||||
@@ -923,6 +933,9 @@ pub struct ExecutionEnv<Evm: ConfigureEvm> {
|
||||
/// Withdrawals included in the block.
|
||||
/// Used to generate prefetch targets for withdrawal addresses.
|
||||
pub withdrawals: Option<Vec<Withdrawal>>,
|
||||
/// Optional decoded BAL for the block.
|
||||
/// Used to validate and optimize execution.
|
||||
pub decoded_bal: Option<Arc<DecodedBal>>,
|
||||
}
|
||||
|
||||
impl<Evm: ConfigureEvm> ExecutionEnv<Evm>
|
||||
@@ -940,6 +953,7 @@ where
|
||||
transaction_count: 0,
|
||||
gas_used: 0,
|
||||
withdrawals: None,
|
||||
decoded_bal: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -956,12 +970,12 @@ mod tests {
|
||||
use rand::Rng;
|
||||
use reth_chainspec::ChainSpec;
|
||||
use reth_db_common::init::init_genesis;
|
||||
use reth_ethereum_primitives::TransactionSigned;
|
||||
use reth_ethereum_primitives::{EthPrimitives, TransactionSigned};
|
||||
use reth_evm::OnStateHook;
|
||||
use reth_evm_ethereum::EthEvmConfig;
|
||||
use reth_primitives_traits::{Account, Recovered, StorageEntry};
|
||||
use reth_provider::{
|
||||
providers::{BlockchainProvider, OverlayStateProviderFactory},
|
||||
providers::{BlockchainProvider, OverlayBuilder, OverlayStateProviderFactory},
|
||||
test_utils::create_test_provider_factory_with_chain_spec,
|
||||
ChainSpecProvider, HashingWriter,
|
||||
};
|
||||
@@ -1236,9 +1250,11 @@ mod tests {
|
||||
std::convert::identity,
|
||||
),
|
||||
StateProviderBuilder::new(provider_factory.clone(), genesis_hash, None),
|
||||
OverlayStateProviderFactory::new(provider_factory, ChangesetCache::new()),
|
||||
OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
OverlayBuilder::<EthPrimitives>::new(genesis_hash, ChangesetCache::new()),
|
||||
),
|
||||
&TreeConfig::default(),
|
||||
None, // No BAL for test
|
||||
);
|
||||
|
||||
let mut state_hook = handle.state_hook().expect("state hook is None");
|
||||
|
||||
@@ -18,7 +18,7 @@ use crate::tree::{
|
||||
StateProviderBuilder,
|
||||
};
|
||||
use alloy_consensus::transaction::TxHashRef;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eip7928::bal::DecodedBal;
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{keccak256, StorageKey, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
@@ -48,7 +48,7 @@ pub enum PrewarmMode<Tx> {
|
||||
/// Prewarm by executing transactions from a stream, each paired with its block index.
|
||||
Transactions(Receiver<(usize, Tx)>),
|
||||
/// Prewarm by prefetching slots from a Block Access List.
|
||||
BlockAccessList(Arc<BlockAccessList>),
|
||||
BlockAccessList(Arc<DecodedBal>),
|
||||
/// Transaction prewarming is skipped (e.g. small blocks where the overhead exceeds the
|
||||
/// benefit). No workers are spawned.
|
||||
Skipped,
|
||||
@@ -331,9 +331,10 @@ where
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor::prewarm", skip_all)]
|
||||
fn run_bal_prewarm(
|
||||
&self,
|
||||
bal: Arc<BlockAccessList>,
|
||||
decoded_bal: Arc<DecodedBal>,
|
||||
actions_tx: Sender<PrewarmTaskEvent<N::Receipt>>,
|
||||
) {
|
||||
let bal = decoded_bal.as_bal();
|
||||
if bal.is_empty() {
|
||||
if let Some(to_sparse_trie_task) = self.to_sparse_trie_task.as_ref() {
|
||||
let _ = to_sparse_trie_task.send(StateRootMessage::FinishedStateUpdates);
|
||||
@@ -355,27 +356,63 @@ where
|
||||
let parent_span = Span::current();
|
||||
let prefetch_parent_span = parent_span.clone();
|
||||
let stream_parent_span = parent_span;
|
||||
let prefetch_bal = Arc::clone(&bal);
|
||||
let stream_bal = Arc::clone(&bal);
|
||||
let prefetch_bal = Arc::clone(&decoded_bal);
|
||||
let stream_bal = Arc::clone(&decoded_bal);
|
||||
let (prefetch_tx, prefetch_rx) = oneshot::channel();
|
||||
let (stream_tx, stream_rx) = oneshot::channel();
|
||||
|
||||
if let Some(to_sparse_trie_task) = to_sparse_trie_task {
|
||||
let stream_ctx = ctx.clone();
|
||||
executor.bal_streaming_pool().spawn(move || {
|
||||
let branch_span = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
parent: &stream_parent_span,
|
||||
"bal_hashed_state_stream",
|
||||
bal_accounts = stream_bal.as_bal().len(),
|
||||
);
|
||||
let provider_parent_span = branch_span.clone();
|
||||
let _span = branch_span.entered();
|
||||
|
||||
stream_bal.as_bal().par_iter().for_each_init(
|
||||
|| {
|
||||
(
|
||||
stream_ctx.clone(),
|
||||
None::<Box<dyn AccountReader>>,
|
||||
provider_parent_span.clone(),
|
||||
)
|
||||
},
|
||||
|(ctx, provider, parent_span), account_changes| {
|
||||
ctx.send_bal_hashed_state(
|
||||
parent_span,
|
||||
provider,
|
||||
account_changes,
|
||||
&to_sparse_trie_task,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
let _ = to_sparse_trie_task.send(StateRootMessage::FinishedStateUpdates);
|
||||
let _ = stream_tx.send(());
|
||||
});
|
||||
} else {
|
||||
let _ = stream_tx.send(());
|
||||
}
|
||||
|
||||
if ctx.saved_cache.is_some() {
|
||||
let prefetch_ctx = ctx.clone();
|
||||
executor.prewarming_pool().spawn(move || {
|
||||
let branch_span = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
parent: &prefetch_parent_span,
|
||||
"bal_prefetch_storage",
|
||||
bal_accounts = prefetch_bal.len(),
|
||||
bal_accounts = prefetch_bal.as_bal().len(),
|
||||
);
|
||||
let provider_parent_span = branch_span.clone();
|
||||
let _span = branch_span.entered();
|
||||
|
||||
prefetch_bal.par_iter().for_each_init(
|
||||
prefetch_bal.as_bal().par_iter().for_each_init(
|
||||
|| {
|
||||
(
|
||||
prefetch_ctx.clone(),
|
||||
ctx.clone(),
|
||||
None::<CachedStateProvider<reth_provider::StateProviderBox, true>>,
|
||||
provider_parent_span.clone(),
|
||||
)
|
||||
@@ -394,36 +431,6 @@ where
|
||||
let _ = prefetch_tx.send(());
|
||||
}
|
||||
|
||||
if let Some(to_sparse_trie_task) = to_sparse_trie_task {
|
||||
executor.bal_streaming_pool().spawn(move || {
|
||||
let branch_span = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
parent: &stream_parent_span,
|
||||
"bal_hashed_state_stream",
|
||||
bal_accounts = stream_bal.len(),
|
||||
);
|
||||
let provider_parent_span = branch_span.clone();
|
||||
let _span = branch_span.entered();
|
||||
|
||||
stream_bal.par_iter().for_each_init(
|
||||
|| (ctx.clone(), None::<Box<dyn AccountReader>>, provider_parent_span.clone()),
|
||||
|(ctx, provider, parent_span), account_changes| {
|
||||
ctx.send_bal_hashed_state(
|
||||
parent_span,
|
||||
provider,
|
||||
account_changes,
|
||||
&to_sparse_trie_task,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
let _ = to_sparse_trie_task.send(StateRootMessage::FinishedStateUpdates);
|
||||
let _ = stream_tx.send(());
|
||||
});
|
||||
} else {
|
||||
let _ = stream_tx.send(());
|
||||
}
|
||||
|
||||
prefetch_rx
|
||||
.blocking_recv()
|
||||
.expect("BAL prefetch task dropped without signaling completion");
|
||||
@@ -536,6 +543,10 @@ where
|
||||
pub precompile_cache_disabled: bool,
|
||||
/// The precompile cache map.
|
||||
pub precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
pub disable_bal_parallel_state_root: bool,
|
||||
/// Whether BAL batched IO is disabled.
|
||||
pub disable_bal_batch_io: bool,
|
||||
}
|
||||
|
||||
/// Per-thread EVM state initialised by [`PrewarmContext::evm_for_ctx`] and stored in
|
||||
@@ -631,6 +642,9 @@ where
|
||||
account_changes: &alloy_eip7928::AccountChanges,
|
||||
to_sparse_trie_task: &CrossbeamSender<StateRootMessage>,
|
||||
) {
|
||||
if self.disable_bal_parallel_state_root {
|
||||
return;
|
||||
}
|
||||
let address = account_changes.address;
|
||||
let mut hashed_address = None;
|
||||
|
||||
@@ -743,7 +757,9 @@ where
|
||||
provider: &mut Option<CachedStateProvider<reth_provider::StateProviderBox, true>>,
|
||||
account: &alloy_eip7928::AccountChanges,
|
||||
) {
|
||||
if account.storage_changes.is_empty() && account.storage_reads.is_empty() {
|
||||
if self.disable_bal_batch_io ||
|
||||
(account.storage_changes.is_empty() && account.storage_reads.is_empty())
|
||||
{
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
@@ -27,8 +27,9 @@ use reth_trie_parallel::{
|
||||
root::ParallelStateRootError,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
errors::SparseTrieResult, ConfigurableSparseTrie, DeferredDrops, LeafUpdate,
|
||||
RevealableSparseTrie, SparseStateTrie, SparseTrie,
|
||||
errors::{SparseStateTrieErrorKind, SparseTrieErrorKind, SparseTrieResult},
|
||||
ConfigurableSparseTrie, DeferredDrops, LeafUpdate, RevealableSparseTrie, SparseStateTrie,
|
||||
SparseTrie,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use tracing::{debug, debug_span, error, instrument, trace_span};
|
||||
@@ -46,6 +47,8 @@ pub(super) struct SparseTrieCacheTask<A = ConfigurableSparseTrie, S = Configurab
|
||||
updates: CrossbeamReceiver<SparseTrieTaskMessage>,
|
||||
/// `SparseStateTrie` used for computing the state root.
|
||||
trie: SparseStateTrie<A, S>,
|
||||
/// The parent block's state root.
|
||||
parent_state_root: B256,
|
||||
/// Handle to the proof worker pools (storage and account).
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
|
||||
@@ -120,6 +123,7 @@ where
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
parent_state_root: B256,
|
||||
chunk_size: usize,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
@@ -138,6 +142,7 @@ where
|
||||
updates: hashed_state_rx,
|
||||
proof_worker_handle,
|
||||
trie,
|
||||
parent_state_root,
|
||||
chunk_size,
|
||||
max_targets_for_chunking: DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
account_updates: Default::default(),
|
||||
@@ -359,10 +364,25 @@ where
|
||||
debug!(target: "engine::root", "All proofs processed, ending calculation");
|
||||
|
||||
let start = Instant::now();
|
||||
let (state_root, trie_updates) =
|
||||
self.trie.root_with_updates(&self.proof_worker_handle).map_err(|e| {
|
||||
ParallelStateRootError::Other(format!("could not calculate state root: {e:?}"))
|
||||
})?;
|
||||
let (state_root, trie_updates) = match self.trie.root_with_updates() {
|
||||
Ok(result) => result,
|
||||
Err(err)
|
||||
if matches!(
|
||||
err.kind(),
|
||||
SparseStateTrieErrorKind::Sparse(SparseTrieErrorKind::Blind)
|
||||
) =>
|
||||
{
|
||||
// A still-blind account trie means this block never changed state, so preserve
|
||||
// the cached parent root instead of fetching and revealing
|
||||
// the unchanged root node.
|
||||
(self.parent_state_root, TrieUpdates::default())
|
||||
}
|
||||
Err(err) => {
|
||||
return Err(ParallelStateRootError::Other(format!(
|
||||
"could not calculate state root: {err:?}"
|
||||
)))
|
||||
}
|
||||
};
|
||||
|
||||
#[cfg(feature = "trie-debug")]
|
||||
let debug_recorders = self.trie.take_debug_recorders();
|
||||
@@ -873,6 +893,13 @@ enum SparseTrieTaskMessage {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{keccak256, Address, B256, U256};
|
||||
use reth_provider::{
|
||||
providers::{OverlayBuilder, OverlayStateProviderFactory},
|
||||
test_utils::create_test_provider_factory,
|
||||
ChainSpecProvider,
|
||||
};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::proof_task::ProofTaskCtx;
|
||||
use reth_trie_sparse::ArenaParallelSparseTrie;
|
||||
|
||||
#[test]
|
||||
@@ -953,4 +980,49 @@ mod tests {
|
||||
assert_eq!(decoded.storage_root, storage_root);
|
||||
assert_eq!(account_rlp_buf, encoded);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn run_returns_parent_root_without_revealing_blind_trie_when_no_state_updates() {
|
||||
let runtime = reth_tasks::Runtime::test();
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let anchor_hash = provider_factory.chain_spec().genesis_hash();
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
OverlayBuilder::<reth_chain_state::EthPrimitives>::new(
|
||||
anchor_hash,
|
||||
ChangesetCache::new(),
|
||||
),
|
||||
);
|
||||
let proof_worker_handle =
|
||||
ProofWorkerHandle::new(&runtime, ProofTaskCtx::new(overlay_factory), false);
|
||||
|
||||
let default_trie = RevealableSparseTrie::blind_from(ConfigurableSparseTrie::Arena(
|
||||
ArenaParallelSparseTrie::default(),
|
||||
));
|
||||
let trie = SparseStateTrie::default()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true);
|
||||
|
||||
let parent_state_root = B256::from([0x55; 32]);
|
||||
let (updates_tx, updates_rx) = crossbeam_channel::unbounded();
|
||||
let mut task = SparseTrieCacheTask::new_with_trie(
|
||||
&runtime,
|
||||
updates_rx,
|
||||
proof_worker_handle,
|
||||
MultiProofTaskMetrics::default(),
|
||||
trie,
|
||||
parent_state_root,
|
||||
1,
|
||||
);
|
||||
|
||||
updates_tx.send(StateRootMessage::FinishedStateUpdates).unwrap();
|
||||
drop(updates_tx);
|
||||
|
||||
let outcome = task.run().expect("state root computation should succeed");
|
||||
|
||||
assert_eq!(outcome.state_root, parent_state_root);
|
||||
assert!(outcome.trie_updates.is_empty());
|
||||
assert!(task.trie.state_trie_ref().is_none(), "blind trie should not be revealed");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -48,7 +48,10 @@ use crate::tree::{
|
||||
PayloadHandle, StateProviderBuilder, StateProviderDatabase, TreeConfig, WaitForCaches,
|
||||
};
|
||||
use alloy_consensus::transaction::{Either, TxHashRef};
|
||||
use alloy_eip7928::{bal::Bal, BlockAccessList};
|
||||
use alloy_eip7928::{
|
||||
bal::{Bal, DecodedBal},
|
||||
BlockAccessList,
|
||||
};
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
use alloy_primitives::{map::B256Set, B256};
|
||||
@@ -77,13 +80,14 @@ use reth_primitives_traits::{
|
||||
RecoveredBlock, SealedBlock, SealedHeader, SignerRecoverable,
|
||||
};
|
||||
use reth_provider::{
|
||||
providers::OverlayStateProviderFactory, BlockExecutionOutput, BlockNumReader, BlockReader,
|
||||
ChangeSetReader, DatabaseProviderFactory, DatabaseProviderROFactory, HashedPostStateProvider,
|
||||
ProviderError, PruneCheckpointReader, StageCheckpointReader, StateProvider,
|
||||
StateProviderFactory, StateReader, StorageChangeSetReader, StorageSettingsCache,
|
||||
providers::{OverlayBuilder, OverlayStateProviderFactory},
|
||||
BlockExecutionOutput, BlockNumReader, BlockReader, ChangeSetReader, DatabaseProviderFactory,
|
||||
DatabaseProviderROFactory, HashedPostStateProvider, ProviderError, PruneCheckpointReader,
|
||||
StageCheckpointReader, StateProvider, StateProviderBox, StateProviderFactory, StateReader,
|
||||
StorageChangeSetReader, StorageSettingsCache,
|
||||
};
|
||||
use reth_revm::db::{states::bundle_state::BundleRetention, BundleAccount, State};
|
||||
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState, StateRoot};
|
||||
use reth_trie::{trie_cursor::TrieCursorFactory, updates::TrieUpdates, HashedPostState};
|
||||
use reth_trie_db::ChangesetCache;
|
||||
use reth_trie_parallel::root::{ParallelStateRoot, ParallelStateRootError};
|
||||
use revm_primitives::{Address, KECCAK_EMPTY};
|
||||
@@ -487,6 +491,12 @@ where
|
||||
.in_scope(|| self.evm_env_for(&input))
|
||||
.map_err(NewPayloadError::other)?;
|
||||
|
||||
// Extract the decoded BAL, if valid and available.
|
||||
let decoded_bal = ensure_ok!(input
|
||||
.try_decoded_access_list()
|
||||
.map_err(|err| { Box::<dyn std::error::Error + Send + Sync>::from(err) }))
|
||||
.map(Arc::new);
|
||||
|
||||
let env = ExecutionEnv {
|
||||
evm_env,
|
||||
hash: input.hash(),
|
||||
@@ -495,6 +505,7 @@ where
|
||||
transaction_count: input.transaction_count(),
|
||||
gas_used: input.gas_used(),
|
||||
withdrawals: input.withdrawals().map(|w| w.to_vec()),
|
||||
decoded_bal,
|
||||
};
|
||||
|
||||
// Plan the strategy used for state root computation.
|
||||
@@ -509,33 +520,25 @@ where
|
||||
// Get an iterator over the transactions in the payload
|
||||
let txs = self.tx_iterator_for(&input)?;
|
||||
|
||||
// Extract the BAL, if valid and available
|
||||
let block_access_list = ensure_ok!(input
|
||||
.block_access_list()
|
||||
.transpose()
|
||||
// Eventually gets converted to a `InsertBlockErrorKind::Other`
|
||||
.map_err(Box::<dyn std::error::Error + Send + Sync>::from))
|
||||
.map(Arc::new);
|
||||
|
||||
// Create lazy overlay from ancestors - this doesn't block, allowing execution to start
|
||||
// before the trie data is ready. The overlay will be computed on first access.
|
||||
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, ctx.state());
|
||||
|
||||
// Create overlay factory for payload processor (StateRootTask path needs it for
|
||||
// multiproofs)
|
||||
let provider_factory = self.provider.clone();
|
||||
let overlay_builder = OverlayBuilder::<N>::new(anchor_hash, self.changeset_cache.clone())
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
OverlayStateProviderFactory::new(provider_factory.clone(), overlay_builder.clone());
|
||||
|
||||
// Spawn the appropriate processor based on strategy
|
||||
let mut handle = ensure_ok!(self.spawn_payload_processor(
|
||||
env.clone(),
|
||||
txs,
|
||||
provider_builder,
|
||||
provider_builder.clone(),
|
||||
overlay_factory.clone(),
|
||||
strategy,
|
||||
block_access_list,
|
||||
));
|
||||
|
||||
// Create optional cache stats for detailed block logging
|
||||
@@ -664,7 +667,7 @@ where
|
||||
let task_result = ensure_ok_post_block!(
|
||||
self.await_state_root_with_timeout(
|
||||
&mut handle,
|
||||
overlay_factory.clone(),
|
||||
provider_builder.clone(),
|
||||
&hashed_state,
|
||||
),
|
||||
block
|
||||
@@ -688,7 +691,9 @@ where
|
||||
// Compare trie updates with serial computation if configured
|
||||
if self.config.always_compare_trie_updates() {
|
||||
let _has_diff = self.compare_trie_updates_with_serial(
|
||||
overlay_factory.clone(),
|
||||
provider_builder.clone(),
|
||||
provider_factory,
|
||||
overlay_builder,
|
||||
&hashed_state,
|
||||
trie_updates.as_ref().clone(),
|
||||
);
|
||||
@@ -727,7 +732,11 @@ where
|
||||
}
|
||||
StateRootStrategy::Parallel => {
|
||||
debug!(target: "engine::tree::payload_validator", "Using parallel state root algorithm");
|
||||
match self.compute_state_root_parallel(overlay_factory.clone(), &hashed_state) {
|
||||
match self.compute_state_root_parallel(
|
||||
provider_factory,
|
||||
overlay_builder,
|
||||
&hashed_state,
|
||||
) {
|
||||
Ok(result) => {
|
||||
let elapsed = root_time.elapsed();
|
||||
info!(
|
||||
@@ -763,7 +772,9 @@ where
|
||||
}
|
||||
|
||||
let (root, updates) = ensure_ok_post_block!(
|
||||
Self::compute_state_root_serial(overlay_factory.clone(), &hashed_state),
|
||||
provider_builder
|
||||
.build()
|
||||
.and_then(|provider| Self::compute_state_root_serial(provider, &hashed_state)),
|
||||
block
|
||||
);
|
||||
|
||||
@@ -1087,7 +1098,8 @@ where
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_validator", skip_all)]
|
||||
fn compute_state_root_parallel(
|
||||
&self,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
provider_factory: P,
|
||||
overlay_builder: OverlayBuilder<N>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> Result<(B256, TrieUpdates), ParallelStateRootError> {
|
||||
let hashed_state = hashed_state.get();
|
||||
@@ -1095,34 +1107,24 @@ where
|
||||
// need to use the prefix sets which were generated from it to indicate to the
|
||||
// ParallelStateRoot which parts of the trie need to be recomputed.
|
||||
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
|
||||
let overlay_factory =
|
||||
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
provider_factory,
|
||||
overlay_builder.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted()),
|
||||
);
|
||||
ParallelStateRoot::new(overlay_factory, prefix_sets, self.runtime.clone())
|
||||
.incremental_root_with_updates()
|
||||
}
|
||||
|
||||
/// Compute state root for the given hashed post state in serial.
|
||||
///
|
||||
/// Uses an overlay factory which provides the state of the parent block, along with the
|
||||
/// [`HashedPostState`] containing the changes of this block, to compute the state root and
|
||||
/// trie updates for this block.
|
||||
/// Uses the same provider construction path as main execution and computes the state root and
|
||||
/// trie updates for this block directly via
|
||||
/// [`reth_provider::StateRootProvider::state_root_with_updates`].
|
||||
fn compute_state_root_serial(
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
state_provider: StateProviderBox,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<(B256, TrieUpdates)> {
|
||||
let hashed_state = hashed_state.get();
|
||||
// The `hashed_state` argument will be taken into account as part of the overlay, but we
|
||||
// need to use the prefix sets which were generated from it to indicate to the
|
||||
// StateRoot which parts of the trie need to be recomputed.
|
||||
let prefix_sets = hashed_state.construct_prefix_sets().freeze();
|
||||
let overlay_factory =
|
||||
overlay_factory.with_extended_hashed_state_overlay(hashed_state.clone_into_sorted());
|
||||
|
||||
let provider = overlay_factory.database_provider_ro()?;
|
||||
|
||||
Ok(StateRoot::new(&provider, &provider)
|
||||
.with_prefix_sets(prefix_sets)
|
||||
.root_with_updates()?)
|
||||
state_provider.state_root_with_updates(hashed_state.get().clone())
|
||||
}
|
||||
|
||||
/// Awaits the state root from the background task, with an optional timeout fallback.
|
||||
@@ -1147,7 +1149,7 @@ where
|
||||
fn await_state_root_with_timeout<Tx, Err, R: Send + Sync + 'static>(
|
||||
&self,
|
||||
handle: &mut PayloadHandle<Tx, Err, R>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
state_provider_builder: StateProviderBuilder<N, P>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
) -> ProviderResult<Result<StateRootComputeOutcome, ParallelStateRootError>> {
|
||||
let Some(timeout) = self.config.state_root_task_timeout() else {
|
||||
@@ -1172,10 +1174,11 @@ where
|
||||
let (seq_tx, seq_rx) =
|
||||
std::sync::mpsc::channel::<ProviderResult<(B256, TrieUpdates)>>();
|
||||
|
||||
let seq_overlay = overlay_factory;
|
||||
let seq_hashed_state = hashed_state.clone();
|
||||
self.payload_processor.executor().spawn_blocking_named("serial-root", move || {
|
||||
let result = Self::compute_state_root_serial(seq_overlay, &seq_hashed_state);
|
||||
let result = state_provider_builder.build().and_then(|provider| {
|
||||
Self::compute_state_root_serial(provider, &seq_hashed_state)
|
||||
});
|
||||
let _ = seq_tx.send(result);
|
||||
});
|
||||
|
||||
@@ -1239,13 +1242,18 @@ where
|
||||
/// updates.
|
||||
fn compare_trie_updates_with_serial(
|
||||
&self,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
state_provider_builder: StateProviderBuilder<N, P>,
|
||||
provider_factory: P,
|
||||
overlay_builder: OverlayBuilder<N>,
|
||||
hashed_state: &LazyHashedPostState,
|
||||
task_trie_updates: TrieUpdates,
|
||||
) -> bool {
|
||||
debug!(target: "engine::tree::payload_validator", "Comparing trie updates with serial computation");
|
||||
|
||||
match Self::compute_state_root_serial(overlay_factory.clone(), hashed_state) {
|
||||
match state_provider_builder
|
||||
.build()
|
||||
.and_then(|provider| Self::compute_state_root_serial(provider, hashed_state))
|
||||
{
|
||||
Ok((serial_root, serial_trie_updates)) => {
|
||||
debug!(
|
||||
target: "engine::tree::payload_validator",
|
||||
@@ -1254,6 +1262,8 @@ where
|
||||
);
|
||||
|
||||
// Get a database provider to use as trie cursor factory
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(provider_factory, overlay_builder);
|
||||
match overlay_factory.database_provider_ro() {
|
||||
Ok(provider) => {
|
||||
match super::trie_updates::compare_trie_updates(
|
||||
@@ -1437,9 +1447,8 @@ where
|
||||
env: ExecutionEnv<Evm>,
|
||||
txs: T,
|
||||
provider_builder: StateProviderBuilder<N, P>,
|
||||
overlay_factory: OverlayStateProviderFactory<P>,
|
||||
overlay_factory: OverlayStateProviderFactory<P, N>,
|
||||
strategy: StateRootStrategy,
|
||||
block_access_list: Option<Arc<BlockAccessList>>,
|
||||
) -> Result<
|
||||
PayloadHandle<
|
||||
impl ExecutableTxFor<Evm> + use<N, P, Evm, V, T>,
|
||||
@@ -1459,7 +1468,6 @@ where
|
||||
provider_builder,
|
||||
overlay_factory,
|
||||
&self.config,
|
||||
block_access_list,
|
||||
);
|
||||
|
||||
// record prewarming initialization duration
|
||||
@@ -1472,12 +1480,8 @@ where
|
||||
}
|
||||
StateRootStrategy::Parallel | StateRootStrategy::Synchronous => {
|
||||
let start = Instant::now();
|
||||
let handle = self.payload_processor.spawn_cache_exclusive(
|
||||
env,
|
||||
txs,
|
||||
provider_builder,
|
||||
block_access_list,
|
||||
);
|
||||
let handle =
|
||||
self.payload_processor.spawn_cache_exclusive(env, txs, provider_builder);
|
||||
|
||||
// Record prewarming initialization duration
|
||||
self.metrics
|
||||
@@ -1563,7 +1567,7 @@ where
|
||||
fn get_parent_lazy_overlay(
|
||||
parent_hash: B256,
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> (Option<LazyOverlay>, B256) {
|
||||
) -> (Option<LazyOverlay<N>>, B256) {
|
||||
// Get blocks leading to the parent to determine the anchor
|
||||
let (anchor_hash, blocks) =
|
||||
state.tree_state.blocks_by_hash(parent_hash).unwrap_or_else(|| (parent_hash, vec![]));
|
||||
@@ -1591,10 +1595,7 @@ where
|
||||
"Creating lazy overlay for in-memory blocks"
|
||||
);
|
||||
|
||||
// Extract deferred trie data handles (non-blocking)
|
||||
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
|
||||
|
||||
(Some(LazyOverlay::new(anchor_hash, handles)), anchor_hash)
|
||||
(Some(LazyOverlay::new(blocks)), anchor_hash)
|
||||
}
|
||||
|
||||
/// Spawns a background task to compute and sort trie data for the executed block.
|
||||
@@ -2026,10 +2027,11 @@ where
|
||||
state: &EngineApiTreeState<N>,
|
||||
) -> Option<StateRootHandle> {
|
||||
let (lazy_overlay, anchor_hash) = Self::get_parent_lazy_overlay(parent_hash, state);
|
||||
let overlay_factory =
|
||||
OverlayStateProviderFactory::new(self.provider.clone(), self.changeset_cache.clone())
|
||||
.with_block_hash(Some(anchor_hash))
|
||||
.with_lazy_overlay(lazy_overlay);
|
||||
let overlay_factory = OverlayStateProviderFactory::new(
|
||||
self.provider.clone(),
|
||||
OverlayBuilder::<N>::new(anchor_hash, self.changeset_cache.clone())
|
||||
.with_lazy_overlay(lazy_overlay),
|
||||
);
|
||||
|
||||
Some(self.payload_processor.spawn_state_root(
|
||||
overlay_factory,
|
||||
@@ -2110,6 +2112,17 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the decoded block access list, if present and successfully decoded.
|
||||
pub fn try_decoded_access_list(&self) -> Result<Option<DecodedBal>, alloy_rlp::Error> {
|
||||
match self {
|
||||
Self::Payload(payload) => payload
|
||||
.block_access_list()
|
||||
.map(|block_access_list| DecodedBal::from_rlp_bytes(block_access_list.clone()))
|
||||
.transpose(),
|
||||
Self::Block(_) => Ok(None),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of transactions in the payload or block.
|
||||
pub fn transaction_count(&self) -> usize
|
||||
where
|
||||
|
||||
@@ -13,7 +13,7 @@ use std::{hash::Hash, sync::Arc};
|
||||
use tracing::error;
|
||||
|
||||
/// Default max cache size for [`PrecompileCache`]
|
||||
const MAX_CACHE_SIZE: u32 = 10_000;
|
||||
const MAX_CACHE_SIZE: u32 = 1024 * 1024;
|
||||
|
||||
/// Stores caches for each precompile.
|
||||
#[derive(Debug, Clone, Default)]
|
||||
@@ -54,6 +54,9 @@ where
|
||||
moka::sync::CacheBuilder::new(MAX_CACHE_SIZE as u64)
|
||||
.initial_capacity(MAX_CACHE_SIZE as usize)
|
||||
.eviction_policy(EvictionPolicy::lru())
|
||||
.weigher(|key: &Bytes, value: &CacheEntry<S>| {
|
||||
(key.len() + value.output.bytes.len()) as u32
|
||||
})
|
||||
.build_with_hasher(Default::default()),
|
||||
)
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use alloy_primitives::{
|
||||
map::{B256Map, B256Set},
|
||||
BlockNumber, B256,
|
||||
};
|
||||
use reth_chain_state::{DeferredTrieData, EthPrimitives, ExecutedBlock, LazyOverlay};
|
||||
use reth_chain_state::{EthPrimitives, ExecutedBlock, LazyOverlay};
|
||||
use reth_primitives_traits::{AlloyBlockHeader, NodePrimitives, SealedHeader};
|
||||
use std::{
|
||||
collections::{btree_map, hash_map, BTreeMap, VecDeque},
|
||||
@@ -43,7 +43,7 @@ pub struct TreeState<N: NodePrimitives = EthPrimitives> {
|
||||
/// This is optimistically prepared after the canonical head changes, so that
|
||||
/// the next payload building on the canonical head can use it immediately
|
||||
/// without recomputing.
|
||||
pub(crate) cached_canonical_overlay: Option<PreparedCanonicalOverlay>,
|
||||
pub(crate) cached_canonical_overlay: Option<PreparedCanonicalOverlay<N>>,
|
||||
}
|
||||
|
||||
impl<N: NodePrimitives> TreeState<N> {
|
||||
@@ -106,10 +106,10 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
/// This should be called after the canonical head changes to optimistically
|
||||
/// prepare the overlay for the next payload that will likely build on it.
|
||||
///
|
||||
/// Returns a clone of the [`LazyOverlay`] so the caller can spawn a background
|
||||
/// task to trigger computation via [`LazyOverlay::get`]. This ensures the overlay
|
||||
/// is actually computed before the next payload arrives.
|
||||
pub(crate) fn prepare_canonical_overlay(&mut self) -> Option<LazyOverlay> {
|
||||
/// Returns a clone of the prepared overlay so the caller can spawn a background
|
||||
/// task to trigger computation via [`LazyOverlay::get`] for the cached anchor.
|
||||
/// This ensures the overlay is actually computed before the next payload arrives.
|
||||
pub(crate) fn prepare_canonical_overlay(&mut self) -> Option<PreparedCanonicalOverlay<N>> {
|
||||
let canonical_hash = self.current_canonical_head.hash;
|
||||
|
||||
// Get blocks leading to the canonical head
|
||||
@@ -119,25 +119,23 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
return None;
|
||||
};
|
||||
|
||||
// Extract deferred trie data handles from blocks (newest to oldest)
|
||||
let handles: Vec<DeferredTrieData> = blocks.iter().map(|b| b.trie_data_handle()).collect();
|
||||
|
||||
let overlay = LazyOverlay::new(anchor_hash, handles);
|
||||
self.cached_canonical_overlay = Some(PreparedCanonicalOverlay {
|
||||
let num_blocks = blocks.len();
|
||||
let prepared = PreparedCanonicalOverlay {
|
||||
parent_hash: canonical_hash,
|
||||
overlay: overlay.clone(),
|
||||
overlay: LazyOverlay::new(blocks),
|
||||
anchor_hash,
|
||||
});
|
||||
};
|
||||
self.cached_canonical_overlay = Some(prepared.clone());
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
%canonical_hash,
|
||||
%anchor_hash,
|
||||
num_blocks = blocks.len(),
|
||||
num_blocks,
|
||||
"Prepared cached canonical overlay"
|
||||
);
|
||||
|
||||
Some(overlay)
|
||||
Some(prepared)
|
||||
}
|
||||
|
||||
/// Returns the cached overlay if it matches the requested parent hash and anchor.
|
||||
@@ -148,7 +146,7 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
&self,
|
||||
parent_hash: B256,
|
||||
expected_anchor: B256,
|
||||
) -> Option<&PreparedCanonicalOverlay> {
|
||||
) -> Option<&PreparedCanonicalOverlay<N>> {
|
||||
self.cached_canonical_overlay.as_ref().filter(|cached| {
|
||||
cached.parent_hash == parent_hash && cached.anchor_hash == expected_anchor
|
||||
})
|
||||
@@ -429,10 +427,10 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
/// the next payload (which typically builds on the canonical head) to reuse
|
||||
/// the pre-computed overlay immediately without re-traversing in-memory blocks.
|
||||
///
|
||||
/// The overlay captures deferred trie data handles from all in-memory blocks
|
||||
/// The overlay captures executed blocks from all in-memory blocks
|
||||
/// between the canonical head and the persisted anchor. When a new payload
|
||||
/// arrives building on the canonical head, this cached overlay can be used
|
||||
/// directly instead of calling `blocks_by_hash` and collecting handles again.
|
||||
/// directly instead of calling `blocks_by_hash` again.
|
||||
///
|
||||
/// # Invalidation
|
||||
///
|
||||
@@ -440,16 +438,16 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
/// - Persistence completes (anchor changes)
|
||||
/// - The canonical head changes to a different block
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PreparedCanonicalOverlay {
|
||||
pub struct PreparedCanonicalOverlay<N: NodePrimitives = EthPrimitives> {
|
||||
/// The block hash for which this overlay is prepared as a parent.
|
||||
///
|
||||
/// When a payload arrives with this parent hash, the overlay can be reused.
|
||||
pub parent_hash: B256,
|
||||
/// The pre-computed lazy overlay containing deferred trie data handles.
|
||||
/// The pre-computed lazy overlay containing executed blocks for the canonical segment.
|
||||
///
|
||||
/// This is computed optimistically after `set_canonical_head` so subsequent
|
||||
/// payloads don't need to re-collect the handles.
|
||||
pub overlay: LazyOverlay,
|
||||
/// This is computed optimistically after `set_canonical_head` so subsequent payloads don't
|
||||
/// need to walk the in-memory chain again.
|
||||
pub overlay: LazyOverlay<N>,
|
||||
/// The anchor hash (persisted ancestor) this overlay is based on.
|
||||
///
|
||||
/// Used to verify the overlay is still valid (anchor hasn't changed due to persistence).
|
||||
|
||||
@@ -184,11 +184,18 @@ impl TestHarness {
|
||||
let payload_validator = MockEngineValidator;
|
||||
|
||||
let (from_tree_tx, from_tree_rx) = unbounded_channel();
|
||||
let tree_config =
|
||||
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true);
|
||||
|
||||
let header = chain_spec.genesis_header().clone();
|
||||
let header = SealedHeader::seal_slow(header);
|
||||
let engine_api_tree_state =
|
||||
EngineApiTreeState::new(10, 10, header.num_hash(), EngineApiKind::Ethereum);
|
||||
let engine_api_tree_state = EngineApiTreeState::new(
|
||||
10,
|
||||
10,
|
||||
tree_config.invalid_header_hit_eviction_threshold(),
|
||||
header.num_hash(),
|
||||
EngineApiKind::Ethereum,
|
||||
);
|
||||
let canonical_in_memory_state = CanonicalInMemoryState::with_head(header, None, None);
|
||||
|
||||
let (to_payload_service, _payload_command_rx) = unbounded_channel();
|
||||
@@ -217,8 +224,7 @@ impl TestHarness {
|
||||
persistence_handle,
|
||||
PersistenceState { last_persisted_block: BlockNumHash::default(), rx: None },
|
||||
payload_builder,
|
||||
// always assume enough parallelism for tests
|
||||
TreeConfig::default().with_legacy_state_root(false).with_has_enough_parallelism(true),
|
||||
tree_config,
|
||||
EngineApiKind::Ethereum,
|
||||
evm_config,
|
||||
changeset_cache,
|
||||
@@ -2248,3 +2254,65 @@ fn test_on_valid_downloaded_head_sync_target_returns_make_canonical() {
|
||||
other => panic!("Expected MakeCanonical for head block, got: {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
/// Tests that canonicalizing a downloaded sync target head also applies the tracked finalized
|
||||
/// block from the original `SYNCING` forkchoice state.
|
||||
#[test]
|
||||
fn test_canonicalizing_downloaded_sync_target_head_updates_finalized() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = MAINNET.clone();
|
||||
let mut test_harness = TestHarness::new(chain_spec);
|
||||
|
||||
let blocks: Vec<_> = test_harness.block_builder.get_executed_blocks(0..3).collect();
|
||||
let genesis = &blocks[0];
|
||||
let finalized_block = &blocks[1];
|
||||
let head_block = &blocks[2];
|
||||
|
||||
test_harness = test_harness.with_blocks(vec![
|
||||
genesis.clone(),
|
||||
finalized_block.clone(),
|
||||
head_block.clone(),
|
||||
]);
|
||||
|
||||
let finalized_num_hash = finalized_block.recovered_block().num_hash();
|
||||
let head_num_hash = head_block.recovered_block().num_hash();
|
||||
|
||||
test_harness.tree.state.tree_state.set_canonical_head(genesis.recovered_block().num_hash());
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: head_num_hash.hash,
|
||||
safe_block_hash: head_num_hash.hash,
|
||||
finalized_block_hash: finalized_num_hash.hash,
|
||||
};
|
||||
test_harness
|
||||
.tree
|
||||
.state
|
||||
.forkchoice_state_tracker
|
||||
.set_latest(fcu_state, ForkchoiceStatus::Syncing);
|
||||
|
||||
let event = test_harness
|
||||
.tree
|
||||
.on_valid_downloaded_block(head_num_hash)
|
||||
.unwrap()
|
||||
.expect("expected canonicalization event for sync target head");
|
||||
|
||||
test_harness.tree.on_tree_event(event).unwrap();
|
||||
|
||||
assert_eq!(test_harness.tree.state.tree_state.canonical_block_hash(), head_num_hash.hash);
|
||||
assert_eq!(
|
||||
test_harness.tree.canonical_in_memory_state.get_finalized_num_hash(),
|
||||
Some(finalized_num_hash),
|
||||
"Finalized block from the syncing FCU should be applied once the head becomes canonical"
|
||||
);
|
||||
assert_eq!(
|
||||
test_harness.tree.canonical_in_memory_state.get_safe_num_hash(),
|
||||
Some(head_num_hash),
|
||||
"Safe block from the syncing FCU should be applied once the head becomes canonical"
|
||||
);
|
||||
assert_eq!(
|
||||
test_harness.tree.state.forkchoice_state_tracker.last_valid_state(),
|
||||
Some(fcu_state)
|
||||
);
|
||||
assert!(test_harness.tree.state.forkchoice_state_tracker.sync_target_state().is_none());
|
||||
}
|
||||
|
||||
@@ -287,7 +287,7 @@ where
|
||||
let tx_recovered =
|
||||
tx.try_into_recovered().map_err(|_| ProviderError::SenderRecoveryError)?;
|
||||
let gas_used = match builder.execute_transaction(tx_recovered) {
|
||||
Ok(gas_used) => gas_used,
|
||||
Ok(gas_used) => gas_used.tx_gas_used(),
|
||||
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
|
||||
hash,
|
||||
error,
|
||||
|
||||
@@ -2,8 +2,8 @@
|
||||
//! and injecting them into era1 files with `Era1Writer`.
|
||||
|
||||
use crate::calculate_td_by_number;
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_primitives::{BlockNumber, B256, U256};
|
||||
use alloy_consensus::{BlockHeader, Sealable, TxReceipt};
|
||||
use alloy_primitives::{BlockNumber, U256};
|
||||
use eyre::{eyre, Result};
|
||||
use reth_era::{
|
||||
common::file_ops::{EraFileId, StreamWriter},
|
||||
@@ -13,7 +13,7 @@ use reth_era::{
|
||||
types::{
|
||||
execution::{
|
||||
Accumulator, BlockTuple, CompressedBody, CompressedHeader, CompressedReceipts,
|
||||
TotalDifficulty, MAX_BLOCKS_PER_ERA1,
|
||||
HeaderRecord, TotalDifficulty, MAX_BLOCKS_PER_ERA1,
|
||||
},
|
||||
group::{BlockIndex, Era1Id},
|
||||
},
|
||||
@@ -139,17 +139,21 @@ where
|
||||
|
||||
let headers = provider.headers_range(start_block..=end_block)?;
|
||||
|
||||
// Extract first 4 bytes of last block's state root as historical identifier
|
||||
let historical_root = headers
|
||||
.last()
|
||||
.map(|header| {
|
||||
let state_root = header.state_root();
|
||||
[state_root[0], state_root[1], state_root[2], state_root[3]]
|
||||
// Pre-compute accumulator from headers to determine filename
|
||||
let mut precompute_td = total_difficulty;
|
||||
let header_records: Vec<HeaderRecord> = headers
|
||||
.iter()
|
||||
.map(|h| {
|
||||
precompute_td += h.difficulty();
|
||||
HeaderRecord { block_hash: h.hash_slow(), total_difficulty: precompute_td }
|
||||
})
|
||||
.unwrap_or([0u8; 4]);
|
||||
.collect();
|
||||
let accumulator = Accumulator::from_header_records(&header_records)
|
||||
.map_err(|e| eyre!("Failed to compute accumulator: {e}"))?;
|
||||
let file_hash: [u8; 4] = accumulator.root[..4].try_into().unwrap();
|
||||
|
||||
let era1_id = Era1Id::new(&config.network, start_block, block_count as u32)
|
||||
.with_hash(historical_root);
|
||||
let era1_id =
|
||||
Era1Id::new(&config.network, start_block, block_count as u32).with_hash(file_hash);
|
||||
|
||||
let era1_id = if config.max_blocks_per_file == MAX_BLOCKS_PER_ERA1 as u64 {
|
||||
era1_id
|
||||
@@ -166,7 +170,6 @@ where
|
||||
let mut offsets = Vec::<i64>::with_capacity(block_count);
|
||||
let mut position = VERSION_ENTRY_SIZE as i64;
|
||||
let mut blocks_written = 0;
|
||||
let mut final_header_data = Vec::new();
|
||||
|
||||
for (i, header) in headers.into_iter().enumerate() {
|
||||
let expected_block_number = start_block + i as u64;
|
||||
@@ -178,11 +181,6 @@ where
|
||||
&mut total_difficulty,
|
||||
)?;
|
||||
|
||||
// Save last block's header data for accumulator
|
||||
if expected_block_number == end_block {
|
||||
final_header_data = compressed_header.data.clone();
|
||||
}
|
||||
|
||||
let difficulty = TotalDifficulty::new(total_difficulty);
|
||||
|
||||
let header_size = compressed_header.data.len() + ENTRY_HEADER_SIZE;
|
||||
@@ -218,10 +216,12 @@ where
|
||||
}
|
||||
}
|
||||
if blocks_written > 0 {
|
||||
let accumulator_hash =
|
||||
B256::from_slice(&final_header_data[0..32.min(final_header_data.len())]);
|
||||
let accumulator = Accumulator::new(accumulator_hash);
|
||||
let block_index = BlockIndex::new(start_block, offsets);
|
||||
// Convert absolute offsets to relative (measured from block-index entry start)
|
||||
let accumulator_entry_size = (ENTRY_HEADER_SIZE + 32) as i64;
|
||||
let block_index_position = position + accumulator_entry_size;
|
||||
let relative_offsets: Vec<i64> =
|
||||
offsets.iter().map(|&abs| abs - block_index_position).collect();
|
||||
let block_index = BlockIndex::new(start_block, relative_offsets);
|
||||
|
||||
writer.write_accumulator(&accumulator)?;
|
||||
writer.write_block_index(&block_index)?;
|
||||
@@ -310,7 +310,9 @@ where
|
||||
|
||||
let compressed_header = CompressedHeader::from_header(&header)?;
|
||||
let compressed_body = CompressedBody::from_body(&body)?;
|
||||
let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts)
|
||||
let receipts_with_bloom: Vec<_> =
|
||||
receipts.iter().map(|r| TxReceipt::with_bloom_ref(r)).collect();
|
||||
let compressed_receipts = CompressedReceipts::from_encodable_list(&receipts_with_bloom)
|
||||
.map_err(|e| eyre!("Failed to compress receipts: {}", e))?;
|
||||
|
||||
Ok((compressed_header, compressed_body, compressed_receipts))
|
||||
|
||||
@@ -24,6 +24,7 @@ snap.workspace = true
|
||||
# ssz encoding and decoding
|
||||
ethereum_ssz.workspace = true
|
||||
ethereum_ssz_derive.workspace = true
|
||||
sha2.workspace = true
|
||||
|
||||
[dev-dependencies]
|
||||
eyre.workspace = true
|
||||
|
||||
@@ -76,6 +76,7 @@ use crate::{
|
||||
use alloy_consensus::{Block, BlockBody, Header};
|
||||
use alloy_primitives::{B256, U256};
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
use sha2::{Digest, Sha256};
|
||||
use snap::{read::FrameDecoder, write::FrameEncoder};
|
||||
use std::{
|
||||
io::{Read, Write},
|
||||
@@ -493,6 +494,73 @@ impl Accumulator {
|
||||
|
||||
Ok(Self { root: B256::from(root) })
|
||||
}
|
||||
|
||||
/// Compute the accumulator from a list of header records.
|
||||
///
|
||||
/// Implements `hash_tree_root(List[HeaderRecord, 8192])` per the ERA1 spec:
|
||||
/// - Each leaf is `sha256(block_hash || total_difficulty_le_bytes32)`
|
||||
/// - Leaves are padded to `MAX_BLOCKS_PER_ERA1` (8192) with zero hashes
|
||||
/// - Binary Merkle tree is computed bottom-up
|
||||
/// - Final root is `sha256(merkle_root || le_bytes32(actual_count))`
|
||||
///
|
||||
/// Returns `Err` if `records` exceeds [`MAX_BLOCKS_PER_ERA1`].
|
||||
pub fn from_header_records(records: &[HeaderRecord]) -> Result<Self, E2sError> {
|
||||
let capacity = MAX_BLOCKS_PER_ERA1;
|
||||
|
||||
if records.len() > capacity {
|
||||
return Err(E2sError::Ssz(format!(
|
||||
"Too many header records: got {}, max {}",
|
||||
records.len(),
|
||||
capacity
|
||||
)));
|
||||
}
|
||||
|
||||
// Compute leaf hash for each header record
|
||||
let mut leaves = Vec::with_capacity(capacity);
|
||||
for record in records {
|
||||
let mut data = [0u8; 64];
|
||||
data[..32].copy_from_slice(record.block_hash.as_slice());
|
||||
data[32..].copy_from_slice(&record.total_difficulty.to_le_bytes::<32>());
|
||||
leaves.push(<[u8; 32]>::from(Sha256::digest(data)));
|
||||
}
|
||||
|
||||
// Pad to capacity with zero hashes
|
||||
leaves.resize(capacity, [0u8; 32]);
|
||||
|
||||
// Binary Merkle tree bottom-up (capacity is always a power of two)
|
||||
while leaves.len() > 1 {
|
||||
let mut next_level = Vec::with_capacity(leaves.len() / 2);
|
||||
for pair in leaves.chunks_exact(2) {
|
||||
let mut data = [0u8; 64];
|
||||
data[..32].copy_from_slice(&pair[0]);
|
||||
data[32..].copy_from_slice(&pair[1]);
|
||||
next_level.push(<[u8; 32]>::from(Sha256::digest(data)));
|
||||
}
|
||||
leaves = next_level;
|
||||
}
|
||||
|
||||
let merkle_root = leaves[0];
|
||||
|
||||
// mix_in_length: sha256(merkle_root || le_bytes32(actual_length))
|
||||
let mut mix = [0u8; 64];
|
||||
mix[..32].copy_from_slice(&merkle_root);
|
||||
let length = records.len() as u64;
|
||||
mix[32..40].copy_from_slice(&length.to_le_bytes());
|
||||
// remaining bytes stay zero (uint256 LE padding)
|
||||
|
||||
Ok(Self { root: B256::from(<[u8; 32]>::from(Sha256::digest(mix))) })
|
||||
}
|
||||
}
|
||||
|
||||
/// A header record used to compute the ERA1 accumulator.
|
||||
///
|
||||
/// Per the ERA1 spec: `header-record := { block-hash: Bytes32, total-difficulty: Uint256 }`
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct HeaderRecord {
|
||||
/// The block hash (keccak256 of RLP-encoded header)
|
||||
pub block_hash: B256,
|
||||
/// The cumulative total difficulty at this block
|
||||
pub total_difficulty: U256,
|
||||
}
|
||||
|
||||
/// A block tuple in an Era1 file, containing all components for a single block
|
||||
@@ -691,6 +759,44 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accumulator_from_header_records_known_vectors() {
|
||||
// Known-answer vectors computed from the SSZ spec:
|
||||
// hash_tree_root(List[HeaderRecord, 8192])
|
||||
let expected_empty: B256 =
|
||||
"4a8c3a07c8d23adc5bac61157555c3c784d53d9bc110c1370809bd23cd93777d".parse().unwrap();
|
||||
let expected_single_zero: B256 =
|
||||
"81fd641249670887a731386e756a7a1538dc781b1b0bf016889045d350812817".parse().unwrap();
|
||||
let expected_single_nonzero: B256 =
|
||||
"ada35c48d81117f4fd588554cd4c4752356336e84cb41106dea1ceb4cfac8799".parse().unwrap();
|
||||
|
||||
// Empty list
|
||||
let acc_empty = Accumulator::from_header_records(&[]).unwrap();
|
||||
assert_eq!(acc_empty.root, expected_empty);
|
||||
|
||||
// Single record with zero values
|
||||
let records = vec![HeaderRecord { block_hash: B256::ZERO, total_difficulty: U256::ZERO }];
|
||||
let acc = Accumulator::from_header_records(&records).unwrap();
|
||||
assert_eq!(acc.root, expected_single_zero);
|
||||
|
||||
// Single record with non-zero values
|
||||
let records2 = vec![HeaderRecord {
|
||||
block_hash: B256::from([1u8; 32]),
|
||||
total_difficulty: U256::from(100u64),
|
||||
}];
|
||||
let acc2 = Accumulator::from_header_records(&records2).unwrap();
|
||||
assert_eq!(acc2.root, expected_single_nonzero);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_accumulator_rejects_oversized_input() {
|
||||
let records = vec![
|
||||
HeaderRecord { block_hash: B256::ZERO, total_difficulty: U256::ZERO };
|
||||
MAX_BLOCKS_PER_ERA1 + 1
|
||||
];
|
||||
assert!(Accumulator::from_header_records(&records).is_err());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_receipt_list_compression() {
|
||||
let receipts = create_test_receipts();
|
||||
|
||||
@@ -102,8 +102,8 @@ pub struct Era1Id {
|
||||
/// Number of blocks in the file
|
||||
pub block_count: u32,
|
||||
|
||||
/// Optional hash identifier for this file
|
||||
/// First 4 bytes of the last historical root in the last state in the era file
|
||||
/// Optional hash identifier for this file.
|
||||
/// First 4 bytes of the accumulator root hash.
|
||||
pub hash: Option<[u8; 4]>,
|
||||
|
||||
/// Whether to include era count in filename
|
||||
|
||||
@@ -491,7 +491,6 @@ mod tests {
|
||||
fn parse_env_filter_directives() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
|
||||
unsafe { std::env::set_var("RUST_LOG", "info,evm=debug") };
|
||||
let reth = Cli::try_parse_args_from([
|
||||
"reth",
|
||||
"init",
|
||||
|
||||
@@ -48,7 +48,7 @@ tokio.workspace = true
|
||||
|
||||
# revm with required ethereum features
|
||||
# Note: this must be kept to ensure all features are properly enabled/forwarded
|
||||
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit"] }
|
||||
revm = { workspace = true, features = ["secp256k1", "blst", "c-kzg", "memory_limit", "p256-aws-lc-rs"] }
|
||||
|
||||
# misc
|
||||
eyre.workspace = true
|
||||
|
||||
@@ -12,7 +12,7 @@ use alloy_rpc_types_eth::TransactionRequest;
|
||||
use rand::{rngs::StdRng, Rng, SeedableRng};
|
||||
use reth_chainspec::{ChainSpecBuilder, EthChainSpec, MAINNET};
|
||||
use reth_e2e_test_utils::setup_engine;
|
||||
use reth_network::types::NatResolver;
|
||||
use reth_network::{types::NatResolver, PeersInfo};
|
||||
use reth_node_builder::{NodeBuilder, NodeHandle};
|
||||
use reth_node_core::{
|
||||
args::{NetworkArgs, RpcServerArgs},
|
||||
@@ -375,3 +375,47 @@ async fn test_admin_external_ip() -> eyre::Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_admin_node_info_uses_discv5_port_when_discv4_is_disabled() -> eyre::Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let runtime = Runtime::test();
|
||||
|
||||
let genesis: Genesis = serde_json::from_str(include_str!("../assets/genesis.json")).unwrap();
|
||||
let chain_spec =
|
||||
Arc::new(ChainSpecBuilder::default().chain(MAINNET.chain).genesis(genesis).build());
|
||||
|
||||
let mut network = NetworkArgs::default().with_unused_ports();
|
||||
network.bootnodes = Some(Vec::new());
|
||||
network.discovery.disable_dns_discovery = true;
|
||||
network.discovery.disable_discv4_discovery = true;
|
||||
network = network.with_nat_resolver(NatResolver::ExternalIp("127.0.0.1".parse().unwrap()));
|
||||
|
||||
let node_config = NodeConfig::test()
|
||||
.with_chain(chain_spec)
|
||||
.with_network(network)
|
||||
.with_rpc(RpcServerArgs::default().with_unused_ports().with_http());
|
||||
|
||||
let NodeHandle { node, node_exit_future: _ } = NodeBuilder::new(node_config)
|
||||
.testing_node(runtime)
|
||||
.node(EthereumNode::default())
|
||||
.launch()
|
||||
.await?;
|
||||
|
||||
assert!(node.network.discv4().is_none());
|
||||
let discv5_port = node.network.discv5().expect("discv5 should be enabled").local_port();
|
||||
|
||||
let local_record = node.network.local_node_record();
|
||||
let local_enr = node.network.local_enr();
|
||||
let info = node.add_ons_handle.admin_api().node_info().await.unwrap();
|
||||
|
||||
assert_eq!(local_record.udp_port, discv5_port);
|
||||
assert_eq!(local_enr.udp4(), Some(discv5_port));
|
||||
assert_eq!(info.ports.discovery, discv5_port);
|
||||
assert_eq!(info.ports.listener, local_record.tcp_port);
|
||||
assert_eq!(info.enode, local_record.to_string());
|
||||
assert!(info.enode.contains(&format!("?discport={discv5_port}")));
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -21,6 +21,7 @@ use reth_consensus_common::validation::MAX_RLP_BLOCK_SIZE;
|
||||
use reth_errors::{BlockExecutionError, BlockValidationError, ConsensusError};
|
||||
use reth_ethereum_primitives::{EthPrimitives, TransactionSigned};
|
||||
use reth_evm::{
|
||||
block::TxResult,
|
||||
execute::{BlockBuilder, BlockBuilderOutcome, BlockExecutor},
|
||||
ConfigureEvm, Evm, NextBlockEnvAttributes,
|
||||
};
|
||||
@@ -37,7 +38,7 @@ use reth_transaction_pool::{
|
||||
BestTransactions, BestTransactionsAttributes, PoolTransaction, TransactionPool,
|
||||
ValidPoolTransaction,
|
||||
};
|
||||
use revm::context_interface::Block as _;
|
||||
use revm::context_interface::{Block as _, Cfg as _};
|
||||
use std::sync::Arc;
|
||||
use tracing::{debug, trace, warn};
|
||||
|
||||
@@ -178,8 +179,13 @@ where
|
||||
));
|
||||
}
|
||||
let state = StateProviderDatabase::new(state_provider.as_ref());
|
||||
let mut db =
|
||||
State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build();
|
||||
let chain_spec = client.chain_spec();
|
||||
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(attributes.timestamp());
|
||||
let mut db = State::builder()
|
||||
.with_database(cached_reads.as_db_mut(state))
|
||||
.with_bundle_update()
|
||||
.with_bal_builder_if(is_amsterdam)
|
||||
.build();
|
||||
|
||||
let mut builder = evm_config
|
||||
.builder_for_next_block(
|
||||
@@ -198,11 +204,12 @@ where
|
||||
)
|
||||
.map_err(PayloadBuilderError::other)?;
|
||||
|
||||
let chain_spec = client.chain_spec();
|
||||
|
||||
debug!(target: "payload_builder", id=%payload_id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload");
|
||||
let mut cumulative_gas_used = 0;
|
||||
let mut cumulative_tx_gas_used = 0;
|
||||
let mut block_regular_gas_used = 0;
|
||||
let mut block_state_gas_used = 0;
|
||||
let block_gas_limit: u64 = builder.evm_mut().block().gas_limit();
|
||||
let tx_gas_limit_cap = builder.evm_mut().cfg_env().tx_gas_limit_cap();
|
||||
let base_fee = builder.evm_mut().block().basefee();
|
||||
|
||||
let mut best_txs = best_txs(BestTransactionsAttributes::new(
|
||||
@@ -247,13 +254,34 @@ where
|
||||
|
||||
while let Some(pool_tx) = best_txs.next() {
|
||||
// ensure we still have capacity for this transaction
|
||||
if cumulative_gas_used + pool_tx.gas_limit() > block_gas_limit {
|
||||
let exceeds_gas_limit = if is_amsterdam {
|
||||
let regular_available_gas = block_gas_limit.saturating_sub(block_regular_gas_used);
|
||||
let state_available_gas = block_gas_limit.saturating_sub(block_state_gas_used);
|
||||
let regular_tx_gas_limit = pool_tx.gas_limit().min(tx_gas_limit_cap);
|
||||
|
||||
if regular_tx_gas_limit > regular_available_gas {
|
||||
Some((regular_tx_gas_limit, regular_available_gas))
|
||||
} else if pool_tx.gas_limit() > state_available_gas {
|
||||
Some((pool_tx.gas_limit(), state_available_gas))
|
||||
} else {
|
||||
None
|
||||
}
|
||||
} else {
|
||||
let block_available_gas = block_gas_limit.saturating_sub(cumulative_tx_gas_used);
|
||||
(pool_tx.gas_limit() > block_available_gas)
|
||||
.then_some((pool_tx.gas_limit(), block_available_gas))
|
||||
};
|
||||
|
||||
if let Some((transaction_gas_limit, block_available_gas)) = exceeds_gas_limit {
|
||||
// we can't fit this transaction into the block, so we need to mark it as invalid
|
||||
// which also removes all dependent transaction from the iterator before we can
|
||||
// continue
|
||||
best_txs.mark_invalid(
|
||||
&pool_tx,
|
||||
&InvalidPoolTransactionError::ExceedsGasLimit(pool_tx.gas_limit(), block_gas_limit),
|
||||
&InvalidPoolTransactionError::ExceedsGasLimit(
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
),
|
||||
);
|
||||
continue
|
||||
}
|
||||
@@ -338,8 +366,11 @@ where
|
||||
let miner_fee = tx.effective_tip_per_gas(base_fee);
|
||||
let tx_hash = *tx.tx_hash();
|
||||
|
||||
let gas_used = match builder.execute_transaction(tx) {
|
||||
Ok(gas_used) => gas_used,
|
||||
let mut tx_regular_gas_used = 0;
|
||||
let gas_output = match builder.execute_transaction_with_result_closure(tx, |result| {
|
||||
tx_regular_gas_used = result.result().result.gas().block_regular_gas_used();
|
||||
}) {
|
||||
Ok(gas_output) => gas_output,
|
||||
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
|
||||
error, ..
|
||||
})) => {
|
||||
@@ -359,6 +390,24 @@ where
|
||||
}
|
||||
continue
|
||||
}
|
||||
// The executor is the source of truth for block gas availability. Keep this
|
||||
// non-fatal in case local builder accounting diverges from executor rules.
|
||||
Err(BlockExecutionError::Validation(
|
||||
BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas {
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
},
|
||||
)) => {
|
||||
trace!(target: "payload_builder", %transaction_gas_limit, %block_available_gas, ?tx_hash, "skipping transaction exceeding block gas limit");
|
||||
best_txs.mark_invalid(
|
||||
&pool_tx,
|
||||
&InvalidPoolTransactionError::ExceedsGasLimit(
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
),
|
||||
);
|
||||
continue
|
||||
}
|
||||
// this is an error that we should treat as fatal for this attempt
|
||||
Err(err) => return Err(PayloadBuilderError::evm(err)),
|
||||
};
|
||||
@@ -376,9 +425,12 @@ where
|
||||
block_transactions_rlp_length += tx_rlp_len;
|
||||
|
||||
// update and add to total fees
|
||||
let gas_used = gas_output.tx_gas_used();
|
||||
let miner_fee = miner_fee.expect("fee is always valid; execution succeeded");
|
||||
total_fees += U256::from(miner_fee) * U256::from(gas_used);
|
||||
cumulative_gas_used += gas_used;
|
||||
cumulative_tx_gas_used += gas_used;
|
||||
block_regular_gas_used += tx_regular_gas_used;
|
||||
block_state_gas_used += gas_output.state_gas_used();
|
||||
|
||||
// Add blob tx sidecar to the payload.
|
||||
if let Some(sidecar) = blob_tx_sidecar {
|
||||
|
||||
@@ -1,8 +1,11 @@
|
||||
//! Helper aliases when working with [`ConfigureEvm`] and the traits in this crate.
|
||||
|
||||
use crate::ConfigureEvm;
|
||||
use alloy_evm::{block::BlockExecutorFactory, Database, EvmEnv, EvmFactory};
|
||||
use revm::{inspector::NoOpInspector, Inspector};
|
||||
use alloy_evm::{
|
||||
block::{BlockExecutorFactory, BlockExecutorFor},
|
||||
Database, EvmEnv, EvmFactory,
|
||||
};
|
||||
use revm::{database::State, inspector::NoOpInspector, Inspector};
|
||||
|
||||
/// Helper to access [`EvmFactory`] for a given [`ConfigureEvm`].
|
||||
pub type EvmFactoryFor<Evm> =
|
||||
@@ -33,6 +36,10 @@ pub type TxEnvFor<Evm> = <EvmFactoryFor<Evm> as EvmFactory>::Tx;
|
||||
pub type ExecutionCtxFor<'a, Evm> =
|
||||
<<Evm as ConfigureEvm>::BlockExecutorFactory as BlockExecutorFactory>::ExecutionCtx<'a>;
|
||||
|
||||
/// Helper to access [`alloy_evm::block::BlockExecutor`] for a given [`ConfigureEvm`].
|
||||
pub type BlockExecutorForEvm<'a, Evm, DB, I = NoOpInspector> =
|
||||
BlockExecutorFor<'a, <Evm as ConfigureEvm>::BlockExecutorFactory, &'a mut State<DB>, I>;
|
||||
|
||||
/// Type alias for [`EvmEnv`] for a given [`ConfigureEvm`].
|
||||
pub type EvmEnvFor<Evm> = EvmEnv<SpecFor<Evm>, BlockEnvFor<Evm>>;
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::{ConfigureEvm, Database, OnStateHook, TxEnvFor};
|
||||
use alloc::{boxed::Box, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::{BlockHeader, Header};
|
||||
use alloy_eips::eip2718::WithEncoded;
|
||||
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
|
||||
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory, GasOutput};
|
||||
use alloy_evm::{
|
||||
block::{CommitChanges, ExecutableTxParts},
|
||||
Evm, EvmEnv, EvmFactory, RecoveredTx, ToTxEnv,
|
||||
@@ -327,7 +327,7 @@ pub trait BlockBuilder {
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
f: impl FnOnce(&<Self::Executor as BlockExecutor>::Result) -> CommitChanges,
|
||||
) -> Result<Option<u64>, BlockExecutionError>;
|
||||
) -> Result<Option<GasOutput>, BlockExecutionError>;
|
||||
|
||||
/// Invokes [`BlockExecutor::execute_transaction_with_result_closure`] and saves the
|
||||
/// transaction in internal state.
|
||||
@@ -335,7 +335,7 @@ pub trait BlockBuilder {
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
f: impl FnOnce(&<Self::Executor as BlockExecutor>::Result),
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
) -> Result<GasOutput, BlockExecutionError> {
|
||||
self.execute_transaction_with_commit_condition(tx, |res| {
|
||||
f(res);
|
||||
CommitChanges::Yes
|
||||
@@ -348,7 +348,7 @@ pub trait BlockBuilder {
|
||||
fn execute_transaction(
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
) -> Result<u64, BlockExecutionError> {
|
||||
) -> Result<GasOutput, BlockExecutionError> {
|
||||
self.execute_transaction_with_result_closure(tx, |_| ())
|
||||
}
|
||||
|
||||
@@ -460,13 +460,13 @@ where
|
||||
&mut self,
|
||||
tx: impl ExecutorTx<Self::Executor>,
|
||||
f: impl FnOnce(&<Self::Executor as BlockExecutor>::Result) -> CommitChanges,
|
||||
) -> Result<Option<u64>, BlockExecutionError> {
|
||||
) -> Result<Option<GasOutput>, BlockExecutionError> {
|
||||
let (tx_env, tx) = tx.into_parts();
|
||||
if let Some(gas_used) =
|
||||
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
|
||||
{
|
||||
self.transactions.push(tx);
|
||||
Ok(Some(gas_used.tx_gas_used()))
|
||||
Ok(Some(gas_used))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
@@ -20,10 +20,7 @@ extern crate alloc;
|
||||
use crate::execute::{BasicBlockBuilder, Executor};
|
||||
use alloc::vec::Vec;
|
||||
use alloy_eips::eip4895::Withdrawals;
|
||||
use alloy_evm::{
|
||||
block::{BlockExecutorFactory, BlockExecutorFor},
|
||||
precompiles::PrecompilesMap,
|
||||
};
|
||||
use alloy_evm::{block::BlockExecutorFactory, precompiles::PrecompilesMap};
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use core::{error::Error, fmt::Debug};
|
||||
use execute::{BasicBlockExecutor, BlockAssembler, BlockBuilder};
|
||||
@@ -312,7 +309,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
|
||||
&'a self,
|
||||
evm: EvmFor<Self, &'a mut State<DB>, I>,
|
||||
ctx: <Self::BlockExecutorFactory as BlockExecutorFactory>::ExecutionCtx<'a>,
|
||||
) -> impl BlockExecutorFor<'a, Self::BlockExecutorFactory, &'a mut State<DB>, I>
|
||||
) -> BlockExecutorForEvm<'a, Self, DB, I>
|
||||
where
|
||||
DB: Database,
|
||||
I: InspectorFor<Self, &'a mut State<DB>> + 'a,
|
||||
@@ -325,8 +322,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
|
||||
&'a self,
|
||||
db: &'a mut State<DB>,
|
||||
block: &'a SealedBlock<<Self::Primitives as NodePrimitives>::Block>,
|
||||
) -> Result<impl BlockExecutorFor<'a, Self::BlockExecutorFactory, &'a mut State<DB>>, Self::Error>
|
||||
{
|
||||
) -> Result<BlockExecutorForEvm<'a, Self, DB>, Self::Error> {
|
||||
let evm = self.evm_for_block(db, block.header())?;
|
||||
let ctx = self.context_for_block(block)?;
|
||||
Ok(self.create_executor(evm, ctx))
|
||||
@@ -352,10 +348,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
|
||||
evm: EvmFor<Self, &'a mut State<DB>, I>,
|
||||
parent: &'a SealedHeader<HeaderTy<Self::Primitives>>,
|
||||
ctx: <Self::BlockExecutorFactory as BlockExecutorFactory>::ExecutionCtx<'a>,
|
||||
) -> impl BlockBuilder<
|
||||
Primitives = Self::Primitives,
|
||||
Executor: BlockExecutorFor<'a, Self::BlockExecutorFactory, &'a mut State<DB>, I>,
|
||||
>
|
||||
) -> impl BlockBuilder<Primitives = Self::Primitives, Executor = BlockExecutorForEvm<'a, Self, DB, I>>
|
||||
where
|
||||
DB: Database,
|
||||
I: InspectorFor<Self, &'a mut State<DB>> + 'a,
|
||||
@@ -404,10 +397,7 @@ pub trait ConfigureEvm: Clone + Debug + Send + Sync + Unpin {
|
||||
parent: &'a SealedHeader<<Self::Primitives as NodePrimitives>::BlockHeader>,
|
||||
attributes: Self::NextBlockEnvCtx,
|
||||
) -> Result<
|
||||
impl BlockBuilder<
|
||||
Primitives = Self::Primitives,
|
||||
Executor: BlockExecutorFor<'a, Self::BlockExecutorFactory, &'a mut State<DB>>,
|
||||
>,
|
||||
impl BlockBuilder<Primitives = Self::Primitives, Executor = BlockExecutorForEvm<'a, Self, DB>>,
|
||||
Self::Error,
|
||||
> {
|
||||
let evm_env = self.next_evm_env(parent, &attributes)?;
|
||||
|
||||
@@ -16,10 +16,13 @@ workspace = true
|
||||
metrics.workspace = true
|
||||
metrics-derive.workspace = true
|
||||
|
||||
# reth
|
||||
reth-primitives-traits = { workspace = true, optional = true }
|
||||
|
||||
# async
|
||||
tokio = { workspace = true, features = ["full"], optional = true }
|
||||
futures = { workspace = true, optional = true }
|
||||
tokio-util = { workspace = true, optional = true }
|
||||
|
||||
[features]
|
||||
common = ["tokio", "futures", "tokio-util"]
|
||||
common = ["tokio", "futures", "tokio-util", "reth-primitives-traits"]
|
||||
|
||||
@@ -4,8 +4,13 @@
|
||||
use crate::Metrics;
|
||||
use futures::Stream;
|
||||
use metrics::Counter;
|
||||
use reth_primitives_traits::InMemorySize;
|
||||
use std::{
|
||||
pin::Pin,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
},
|
||||
task::{ready, Context, Poll},
|
||||
};
|
||||
use tokio::sync::mpsc::{
|
||||
@@ -399,3 +404,147 @@ struct MeteredPollSenderMetrics {
|
||||
/// Number of delayed message deliveries caused by a full channel
|
||||
back_pressure_total: Counter,
|
||||
}
|
||||
|
||||
/// Shared state for tracking memory budget across sender and receiver.
|
||||
///
|
||||
/// `used` is a pure accounting counter — it does not gate access to any other
|
||||
/// shared memory, so all operations on it use [`Ordering::Relaxed`]. Cross-thread
|
||||
/// publication of message contents is handled by the underlying mpsc channel.
|
||||
#[derive(Debug)]
|
||||
struct MemoryBudget {
|
||||
/// Current number of bytes used by buffered messages.
|
||||
used: AtomicUsize,
|
||||
/// Maximum allowed bytes.
|
||||
max_bytes: usize,
|
||||
}
|
||||
|
||||
/// Guard that releases memory budget when dropped.
|
||||
///
|
||||
/// Holds the size of the message and a reference to the shared budget counter.
|
||||
/// When dropped, it atomically decreases the used counter.
|
||||
#[derive(Debug)]
|
||||
struct BudgetGuard {
|
||||
size: usize,
|
||||
budget: Arc<MemoryBudget>,
|
||||
}
|
||||
|
||||
impl Drop for BudgetGuard {
|
||||
fn drop(&mut self) {
|
||||
self.budget.used.fetch_sub(self.size, Ordering::Relaxed);
|
||||
}
|
||||
}
|
||||
|
||||
/// Message envelope that holds the memory budget while the message sits in the channel.
|
||||
///
|
||||
/// The guard is dropped (releasing the budget) as soon as the receiver dequeues
|
||||
/// the message via [`MemoryBoundedReceiver::recv`] / [`MemoryBoundedReceiver::poll_recv`],
|
||||
/// so the budget tracks bytes *currently in the channel queue*, not bytes in flight
|
||||
/// downstream of the receiver.
|
||||
#[derive(Debug)]
|
||||
struct Budgeted<T> {
|
||||
msg: T,
|
||||
_guard: BudgetGuard,
|
||||
}
|
||||
|
||||
/// A sender that enforces a byte budget before enqueueing messages.
|
||||
///
|
||||
/// Uses a shared atomic counter to track memory usage. Each message's size is added
|
||||
/// to the counter on send and subtracted when the message is dequeued by the receiver.
|
||||
///
|
||||
/// The current call sites (specifically [`crate::common::mpsc::MemoryBoundedSender`] used
|
||||
/// for the `NetworkManager → TransactionsManager` channel) have a single producer driven
|
||||
/// from a single `poll`, so the `fetch_add → check → fetch_sub-on-overflow` reservation
|
||||
/// pattern can never race with itself. The atomic is still used so the receiver can
|
||||
/// release budget from a different task.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MemoryBoundedSender<T: InMemorySize> {
|
||||
/// The underlying unbounded metered sender
|
||||
inner: UnboundedMeteredSender<Budgeted<T>>,
|
||||
/// Shared memory budget tracker
|
||||
budget: Arc<MemoryBudget>,
|
||||
}
|
||||
|
||||
impl<T: InMemorySize> MemoryBoundedSender<T> {
|
||||
/// Tries to send a message if there is sufficient budget.
|
||||
///
|
||||
/// Returns `TrySendError::Full` if insufficient budget is available.
|
||||
pub fn try_send(&self, msg: T) -> Result<(), TrySendError<T>> {
|
||||
let size = msg.size();
|
||||
|
||||
// Reserve budget: add first, check after
|
||||
let prev = self.budget.used.fetch_add(size, Ordering::Relaxed);
|
||||
if prev.saturating_add(size) > self.budget.max_bytes {
|
||||
// Over budget, undo
|
||||
self.budget.used.fetch_sub(size, Ordering::Relaxed);
|
||||
return Err(TrySendError::Full(msg));
|
||||
}
|
||||
|
||||
let guard = BudgetGuard { size, budget: Arc::clone(&self.budget) };
|
||||
let budgeted = Budgeted { msg, _guard: guard };
|
||||
|
||||
self.inner.send(budgeted).map_err(|e| {
|
||||
// Guard will be dropped here, releasing the budget
|
||||
TrySendError::Closed(e.0.msg)
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
/// A receiver for memory-bounded messages.
|
||||
///
|
||||
/// On receive, the budget reserved for the message is released immediately and the
|
||||
/// inner `T` is yielded — callers do not need to opt into any wrapper type.
|
||||
#[derive(Debug)]
|
||||
pub struct MemoryBoundedReceiver<T> {
|
||||
/// The underlying unbounded metered receiver
|
||||
inner: UnboundedMeteredReceiver<Budgeted<T>>,
|
||||
}
|
||||
|
||||
impl<T> MemoryBoundedReceiver<T> {
|
||||
/// Receives the next message, returning `None` if the channel is closed.
|
||||
///
|
||||
/// Releases the message's reserved budget before returning.
|
||||
pub async fn recv(&mut self) -> Option<T> {
|
||||
self.inner.recv().await.map(unwrap_budgeted)
|
||||
}
|
||||
|
||||
/// Polls to receive the next message on this channel.
|
||||
///
|
||||
/// Releases the message's reserved budget before returning.
|
||||
pub fn poll_recv(&mut self, cx: &mut Context<'_>) -> Poll<Option<T>> {
|
||||
self.inner.poll_recv(cx).map(|opt| opt.map(unwrap_budgeted))
|
||||
}
|
||||
}
|
||||
|
||||
/// Releases the budget guard and returns the inner message.
|
||||
fn unwrap_budgeted<T>(b: Budgeted<T>) -> T {
|
||||
// Destructuring binds `_guard` so it is dropped when this function returns,
|
||||
// which runs `BudgetGuard::drop` and releases the reserved bytes.
|
||||
let Budgeted { msg, _guard } = b;
|
||||
msg
|
||||
}
|
||||
|
||||
impl<T> Stream for MemoryBoundedReceiver<T> {
|
||||
type Item = T;
|
||||
|
||||
fn poll_next(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll<Option<Self::Item>> {
|
||||
self.poll_recv(cx)
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a new memory-bounded channel with the given byte budget.
|
||||
///
|
||||
/// The budget tracks bytes currently buffered in the channel; it is reserved on
|
||||
/// [`MemoryBoundedSender::try_send`] and released as soon as the receiver dequeues
|
||||
/// the message.
|
||||
pub fn memory_bounded_channel<T: InMemorySize>(
|
||||
max_bytes: usize,
|
||||
scope: &'static str,
|
||||
) -> (MemoryBoundedSender<T>, MemoryBoundedReceiver<T>) {
|
||||
let (tx, rx) = metered_unbounded_channel(scope);
|
||||
let budget = Arc::new(MemoryBudget { used: AtomicUsize::new(0), max_bytes });
|
||||
|
||||
let sender = MemoryBoundedSender { inner: tx, budget };
|
||||
let receiver = MemoryBoundedReceiver { inner: rx };
|
||||
|
||||
(sender, receiver)
|
||||
}
|
||||
|
||||
@@ -47,9 +47,7 @@ use secp256k1::SecretKey;
|
||||
use std::{
|
||||
cell::RefCell,
|
||||
collections::{btree_map, hash_map::Entry, BTreeMap, HashMap, VecDeque},
|
||||
fmt,
|
||||
future::poll_fn,
|
||||
io,
|
||||
fmt, io,
|
||||
net::{IpAddr, Ipv4Addr, SocketAddr, SocketAddrV4},
|
||||
pin::Pin,
|
||||
rc::Rc,
|
||||
@@ -243,17 +241,56 @@ impl Discv4 {
|
||||
/// ```
|
||||
pub async fn bind(
|
||||
local_address: SocketAddr,
|
||||
local_node_record: NodeRecord,
|
||||
secret_key: SecretKey,
|
||||
config: Discv4Config,
|
||||
) -> io::Result<(Self, Discv4Service)> {
|
||||
let socket = Arc::new(UdpSocket::bind(local_address).await?);
|
||||
trace!(target: "discv4", local_addr=?socket.local_addr(), "opened UDP socket");
|
||||
let (tx, rx) = mpsc::channel(config.udp_ingress_message_buffer);
|
||||
|
||||
Self::bind_with_socket(socket, Some(tx), rx, local_node_record, secret_key, config)
|
||||
}
|
||||
|
||||
/// Creates a new `Discv4` instance using a pre-bound shared socket. No receive loop is
|
||||
/// spawned; instead returns an [`IngressHandler`] that should be used to forward raw packets
|
||||
/// received by the socket owner (e.g. discv5 unrecognized frames).
|
||||
pub fn bind_shared(
|
||||
socket: Arc<UdpSocket>,
|
||||
local_node_record: NodeRecord,
|
||||
secret_key: SecretKey,
|
||||
config: Discv4Config,
|
||||
) -> io::Result<(Self, Discv4Service, IngressHandler)> {
|
||||
let (tx, rx) = mpsc::channel(config.udp_ingress_message_buffer);
|
||||
let local_id = local_node_record.id;
|
||||
let (discv4, service) =
|
||||
Self::bind_with_socket(socket, None, rx, local_node_record, secret_key, config)?;
|
||||
|
||||
let handler = IngressHandler::new(tx, local_id);
|
||||
|
||||
Ok((discv4, service, handler))
|
||||
}
|
||||
|
||||
fn bind_with_socket(
|
||||
socket: Arc<UdpSocket>,
|
||||
ingress_tx: Option<IngressSender>,
|
||||
ingress_rx: IngressReceiver,
|
||||
mut local_node_record: NodeRecord,
|
||||
secret_key: SecretKey,
|
||||
config: Discv4Config,
|
||||
) -> io::Result<(Self, Discv4Service)> {
|
||||
let socket = UdpSocket::bind(local_address).await?;
|
||||
let local_addr = socket.local_addr()?;
|
||||
local_node_record.udp_port = local_addr.port();
|
||||
trace!(target: "discv4", ?local_addr,"opened UDP socket");
|
||||
|
||||
let mut service =
|
||||
Discv4Service::new(socket, local_addr, local_node_record, secret_key, config);
|
||||
let mut service = Discv4Service::new(
|
||||
socket,
|
||||
ingress_tx,
|
||||
ingress_rx,
|
||||
local_addr,
|
||||
local_node_record,
|
||||
secret_key,
|
||||
config,
|
||||
);
|
||||
|
||||
// resolve the external address immediately
|
||||
service.resolve_external_ip();
|
||||
@@ -520,20 +557,25 @@ pub struct Discv4Service {
|
||||
|
||||
impl Discv4Service {
|
||||
/// Create a new instance for a bound [`UdpSocket`].
|
||||
///
|
||||
/// If `ingress_tx` is `Some`, the receive loop is spawned to read from the socket. If `None`,
|
||||
/// the caller feeds packets into `ingress_rx` externally (shared socket mode).
|
||||
pub(crate) fn new(
|
||||
socket: UdpSocket,
|
||||
socket: Arc<UdpSocket>,
|
||||
ingress_tx: Option<IngressSender>,
|
||||
ingress_rx: IngressReceiver,
|
||||
local_address: SocketAddr,
|
||||
local_node_record: NodeRecord,
|
||||
secret_key: SecretKey,
|
||||
config: Discv4Config,
|
||||
) -> Self {
|
||||
let socket = Arc::new(socket);
|
||||
let (ingress_tx, ingress_rx) = mpsc::channel(config.udp_ingress_message_buffer);
|
||||
let (egress_tx, egress_rx) = mpsc::channel(config.udp_egress_message_buffer);
|
||||
let mut tasks = JoinSet::<()>::new();
|
||||
|
||||
let udp = Arc::clone(&socket);
|
||||
tasks.spawn(receive_loop(udp, ingress_tx, local_node_record.id));
|
||||
if let Some(ingress_tx) = ingress_tx {
|
||||
let udp = Arc::clone(&socket);
|
||||
tasks.spawn(receive_loop(udp, ingress_tx, local_node_record.id));
|
||||
}
|
||||
|
||||
let udp = Arc::clone(&socket);
|
||||
tasks.spawn(send_loop(udp, egress_rx));
|
||||
@@ -947,7 +989,7 @@ impl Discv4Service {
|
||||
let key = kad_key(peer_id);
|
||||
match self.kbuckets.entry(&key) {
|
||||
BucketEntry::Present(entry, _) => Some(f(entry.value())),
|
||||
BucketEntry::Pending(mut entry, _) => Some(f(entry.value())),
|
||||
BucketEntry::Pending(entry, _) => Some(f(entry.value())),
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
@@ -973,7 +1015,9 @@ impl Discv4Service {
|
||||
kbucket::Entry::Present(mut entry, _) => {
|
||||
entry.value_mut().update_with_enr(last_enr_seq)
|
||||
}
|
||||
kbucket::Entry::Pending(mut entry, _) => entry.value().update_with_enr(last_enr_seq),
|
||||
kbucket::Entry::Pending(mut entry, _) => {
|
||||
entry.value_mut().update_with_enr(last_enr_seq)
|
||||
}
|
||||
_ => return,
|
||||
};
|
||||
|
||||
@@ -1025,8 +1069,8 @@ impl Discv4Service {
|
||||
}
|
||||
kbucket::Entry::Pending(mut entry, mut status) => {
|
||||
// endpoint is now proven
|
||||
entry.value().establish_proof();
|
||||
entry.value().update_with_enr(last_enr_seq);
|
||||
entry.value_mut().establish_proof();
|
||||
entry.value_mut().update_with_enr(last_enr_seq);
|
||||
|
||||
if !status.is_connected() {
|
||||
status.state = ConnectionState::Connected;
|
||||
@@ -1158,7 +1202,7 @@ impl Discv4Service {
|
||||
} else {
|
||||
is_proven = entry.value().has_endpoint_proof;
|
||||
}
|
||||
entry.value().update_with_enr(ping.enr_sq)
|
||||
entry.value_mut().update_with_enr(ping.enr_sq)
|
||||
}
|
||||
kbucket::Entry::Absent(entry) => {
|
||||
let mut node = NodeEntry::new(record);
|
||||
@@ -1388,7 +1432,7 @@ impl Discv4Service {
|
||||
(entry.value().record, id)
|
||||
}
|
||||
kbucket::Entry::Pending(mut entry, _) => {
|
||||
let id = entry.value().update_with_fork_id(fork_id);
|
||||
let id = entry.value_mut().update_with_fork_id(fork_id);
|
||||
(entry.value().record, id)
|
||||
}
|
||||
_ => return,
|
||||
@@ -1538,7 +1582,7 @@ impl Discv4Service {
|
||||
}
|
||||
}
|
||||
}
|
||||
BucketEntry::Pending(mut entry, _) => {
|
||||
BucketEntry::Pending(entry, _) => {
|
||||
if entry.value().has_endpoint_proof {
|
||||
if entry
|
||||
.value()
|
||||
@@ -1642,7 +1686,7 @@ impl Discv4Service {
|
||||
entry.value().find_node_failures
|
||||
}
|
||||
kbucket::Entry::Pending(mut entry, _) => {
|
||||
entry.value().inc_failed_request();
|
||||
entry.value_mut().inc_failed_request();
|
||||
entry.value().find_node_failures
|
||||
}
|
||||
_ => continue,
|
||||
@@ -1962,80 +2006,100 @@ const MAX_INCOMING_PACKETS_PER_MINUTE_BY_IP: usize = 60usize;
|
||||
|
||||
/// Continuously awaits new incoming messages and sends them back through the channel.
|
||||
///
|
||||
/// The receive loop enforce primitive rate limiting for ips to prevent message spams from
|
||||
/// individual IPs
|
||||
/// The receive loop enforces primitive rate limiting for IPs to prevent message spams from
|
||||
/// individual IPs.
|
||||
pub(crate) async fn receive_loop(udp: Arc<UdpSocket>, tx: IngressSender, local_id: PeerId) {
|
||||
let send = |event: IngressEvent| async {
|
||||
let _ = tx.send(event).await.map_err(|err| {
|
||||
debug!(
|
||||
target: "discv4",
|
||||
%err,
|
||||
"failed send incoming packet",
|
||||
)
|
||||
});
|
||||
};
|
||||
|
||||
let mut cache = ReceiveCache::default();
|
||||
|
||||
// tick at half the rate of the limit
|
||||
let tick = MAX_INCOMING_PACKETS_PER_MINUTE_BY_IP / 2;
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(tick as u64));
|
||||
|
||||
let mut handler = IngressHandler::new(tx, local_id);
|
||||
let mut buf = [0; MAX_PACKET_SIZE];
|
||||
loop {
|
||||
let res = udp.recv_from(&mut buf).await;
|
||||
match res {
|
||||
Err(err) => {
|
||||
debug!(target: "discv4", %err, "Failed to read datagram.");
|
||||
send(IngressEvent::RecvError(err)).await;
|
||||
handler.send(IngressEvent::RecvError(err)).await;
|
||||
}
|
||||
Ok((read, remote_addr)) => {
|
||||
// rate limit incoming packets by IP
|
||||
if cache.inc_ip(remote_addr.ip()) > MAX_INCOMING_PACKETS_PER_MINUTE_BY_IP {
|
||||
trace!(target: "discv4", ?remote_addr, "Too many incoming packets from IP.");
|
||||
continue
|
||||
}
|
||||
|
||||
let packet = &buf[..read];
|
||||
match Message::decode(packet) {
|
||||
Ok(packet) => {
|
||||
if packet.node_id == local_id {
|
||||
// received our own message
|
||||
debug!(target: "discv4", ?remote_addr, "Received own packet.");
|
||||
continue
|
||||
}
|
||||
|
||||
// skip if we've already received the same packet
|
||||
if cache.contains_packet(packet.hash) {
|
||||
debug!(target: "discv4", ?remote_addr, "Received duplicate packet.");
|
||||
continue
|
||||
}
|
||||
|
||||
send(IngressEvent::Packet(remote_addr, packet)).await;
|
||||
}
|
||||
Err(err) => {
|
||||
trace!(target: "discv4", %err,"Failed to decode packet");
|
||||
send(IngressEvent::BadPacket(remote_addr, err, packet.to_vec())).await
|
||||
}
|
||||
}
|
||||
handler.handle_packet(&buf[..read], remote_addr).await;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// reset the tracked ips if the interval has passed
|
||||
if poll_fn(|cx| match interval.poll_tick(cx) {
|
||||
Poll::Ready(_) => Poll::Ready(true),
|
||||
Poll::Pending => Poll::Ready(false),
|
||||
})
|
||||
.await
|
||||
{
|
||||
cache.tick_ips(tick);
|
||||
/// Handles decoding, rate-limiting, and deduplication of incoming discv4 packets.
|
||||
///
|
||||
/// Used by both the standalone receive loop and the shared-port mode via
|
||||
/// [`Discv4::bind_shared`].
|
||||
#[derive(Debug)]
|
||||
pub struct IngressHandler {
|
||||
tx: IngressSender,
|
||||
local_id: PeerId,
|
||||
tick: usize,
|
||||
tick_interval: Duration,
|
||||
cache: ReceiveCache,
|
||||
last_tick: Instant,
|
||||
}
|
||||
|
||||
impl IngressHandler {
|
||||
fn new(tx: IngressSender, local_id: PeerId) -> Self {
|
||||
let tick = MAX_INCOMING_PACKETS_PER_MINUTE_BY_IP / 2;
|
||||
Self {
|
||||
tx,
|
||||
local_id,
|
||||
tick,
|
||||
tick_interval: Duration::from_secs(tick as u64),
|
||||
cache: ReceiveCache::default(),
|
||||
last_tick: Instant::now(),
|
||||
}
|
||||
}
|
||||
|
||||
async fn send(&self, event: IngressEvent) {
|
||||
let _ = self.tx.send(event).await.map_err(|err| {
|
||||
debug!(target: "discv4", %err, "failed send incoming packet");
|
||||
});
|
||||
}
|
||||
|
||||
/// Handles an incoming raw packet: decodes, rate-limits, deduplicates, and forwards to the
|
||||
/// discv4 service. Used in shared-port mode to process unrecognized frames from discv5.
|
||||
pub async fn handle_packet(&mut self, data: &[u8], src: SocketAddr) {
|
||||
if self.last_tick.elapsed() >= self.tick_interval {
|
||||
self.cache.tick_ips(self.tick);
|
||||
self.last_tick = Instant::now();
|
||||
}
|
||||
|
||||
// rate limit incoming packets by IP
|
||||
if self.cache.inc_ip(src.ip()) > MAX_INCOMING_PACKETS_PER_MINUTE_BY_IP {
|
||||
trace!(target: "discv4", ?src, "Too many incoming packets from IP.");
|
||||
return
|
||||
}
|
||||
|
||||
let event = match Message::decode(data) {
|
||||
Ok(packet) => {
|
||||
if packet.node_id == self.local_id {
|
||||
debug!(target: "discv4", ?src, "Received own packet.");
|
||||
return
|
||||
}
|
||||
|
||||
if self.cache.contains_packet(packet.hash) {
|
||||
debug!(target: "discv4", ?src, "Received duplicate packet.");
|
||||
return
|
||||
}
|
||||
|
||||
IngressEvent::Packet(src, packet)
|
||||
}
|
||||
Err(err) => {
|
||||
trace!(target: "discv4", %err, "Failed to decode packet");
|
||||
IngressEvent::BadPacket(src, err, data.to_vec())
|
||||
}
|
||||
};
|
||||
|
||||
self.send(event).await;
|
||||
}
|
||||
}
|
||||
|
||||
/// A cache for received packets and their source address.
|
||||
///
|
||||
/// This is used to discard duplicated packets and rate limit messages from the same source.
|
||||
#[derive(Debug)]
|
||||
struct ReceiveCache {
|
||||
/// keeps track of how many messages we've received from a given IP address since the last
|
||||
/// tick.
|
||||
|
||||
@@ -308,6 +308,18 @@ impl Config {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a mutable reference to the inner [`discv5::Config`]. This allows overriding
|
||||
/// the listen config after the config has been built.
|
||||
pub const fn discv5_config_mut(&mut self) -> &mut discv5::Config {
|
||||
&mut self.discv5_config
|
||||
}
|
||||
|
||||
/// Returns `true` if any socket in the discv5 listen config matches the given address.
|
||||
pub fn has_matching_socket(&self, addr: SocketAddr) -> bool {
|
||||
ipv4(&self.discv5_config.listen_config).is_some_and(|v4| SocketAddr::V4(v4) == addr) ||
|
||||
ipv6(&self.discv5_config.listen_config).is_some_and(|v6| SocketAddr::V6(v6) == addr)
|
||||
}
|
||||
|
||||
/// Inserts a new boot node to the list of boot nodes.
|
||||
pub fn insert_boot_node(&mut self, boot_node: BootNode) {
|
||||
self.bootstrap_nodes.insert(boot_node);
|
||||
@@ -333,11 +345,11 @@ impl Config {
|
||||
/// socket, if both IPv4 and v6 are configured. This socket will be advertised to peers in the
|
||||
/// local [`Enr`](discv5::enr::Enr).
|
||||
pub fn discovery_socket(&self) -> SocketAddr {
|
||||
match self.discv5_config.listen_config {
|
||||
ListenConfig::Ipv4 { ip, port } => (ip, port).into(),
|
||||
ListenConfig::Ipv6 { ip, port } => (ip, port).into(),
|
||||
ListenConfig::DualStack { ipv6, ipv6_port, .. } => (ipv6, ipv6_port).into(),
|
||||
}
|
||||
// Prefer v6 when both are configured (matches original `DualStack` behavior).
|
||||
ipv6(&self.discv5_config.listen_config)
|
||||
.map(SocketAddr::V6)
|
||||
.or_else(|| ipv4(&self.discv5_config.listen_config).map(SocketAddr::V4))
|
||||
.unwrap_or_else(|| SocketAddr::from((std::net::Ipv4Addr::UNSPECIFIED, 0)))
|
||||
}
|
||||
|
||||
/// Returns the `RLPx` (TCP) socket contained in the [`discv5::Config`]. This socket will be
|
||||
@@ -348,24 +360,32 @@ impl Config {
|
||||
}
|
||||
|
||||
/// Returns the IPv4 discovery socket if one is configured.
|
||||
pub const fn ipv4(listen_config: &ListenConfig) -> Option<SocketAddrV4> {
|
||||
pub fn ipv4(listen_config: &ListenConfig) -> Option<SocketAddrV4> {
|
||||
match listen_config {
|
||||
ListenConfig::Ipv4 { ip, port } |
|
||||
ListenConfig::DualStack { ipv4: ip, ipv4_port: port, .. } => {
|
||||
Some(SocketAddrV4::new(*ip, *port))
|
||||
}
|
||||
ListenConfig::Ipv6 { .. } => None,
|
||||
ListenConfig::FromSockets { ipv4: Some(s), .. } => match s.local_addr().ok()? {
|
||||
SocketAddr::V4(addr) => Some(addr),
|
||||
SocketAddr::V6(_) => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the IPv6 discovery socket if one is configured.
|
||||
pub const fn ipv6(listen_config: &ListenConfig) -> Option<SocketAddrV6> {
|
||||
pub fn ipv6(listen_config: &ListenConfig) -> Option<SocketAddrV6> {
|
||||
match listen_config {
|
||||
ListenConfig::Ipv4 { .. } => None,
|
||||
ListenConfig::Ipv6 { ip, port } |
|
||||
ListenConfig::DualStack { ipv6: ip, ipv6_port: port, .. } => {
|
||||
Some(SocketAddrV6::new(*ip, *port, 0, 0))
|
||||
}
|
||||
ListenConfig::FromSockets { ipv6: Some(s), .. } => match s.local_addr().ok()? {
|
||||
SocketAddr::V6(addr) => Some(addr),
|
||||
SocketAddr::V4(_) => None,
|
||||
},
|
||||
_ => None,
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -18,7 +18,6 @@ use std::{
|
||||
|
||||
use ::enr::Enr;
|
||||
use alloy_primitives::bytes::Bytes;
|
||||
use discv5::ListenConfig;
|
||||
use enr::{discv4_id_to_discv5_id, EnrCombinedKeyWrapper};
|
||||
use futures::future::join_all;
|
||||
use itertools::Itertools;
|
||||
@@ -225,7 +224,7 @@ impl Discv5 {
|
||||
bootstrap_lookup_interval,
|
||||
bootstrap_lookup_countdown,
|
||||
metrics.clone(),
|
||||
discv5.clone(),
|
||||
Arc::downgrade(&discv5),
|
||||
);
|
||||
|
||||
Ok((
|
||||
@@ -247,7 +246,9 @@ impl Discv5 {
|
||||
match update {
|
||||
discv5::Event::SocketUpdated(_) | discv5::Event::TalkRequest(_) |
|
||||
// `Discovered` not unique discovered peers
|
||||
discv5::Event::Discovered(_) => None,
|
||||
discv5::Event::Discovered(_) |
|
||||
// Unrecognized frames are handled separately by the discovery layer
|
||||
discv5::Event::UnrecognizedFrame(_) => None,
|
||||
discv5::Event::NodeInserted { .. } => {
|
||||
|
||||
// node has been inserted into kbuckets
|
||||
@@ -472,39 +473,33 @@ pub fn build_local_enr(
|
||||
|
||||
let Config { discv5_config, fork, tcp_socket, other_enr_kv_pairs, .. } = config;
|
||||
|
||||
let socket = match discv5_config.listen_config {
|
||||
ListenConfig::Ipv4 { ip, port } => {
|
||||
if ip != Ipv4Addr::UNSPECIFIED {
|
||||
builder.ip4(ip);
|
||||
}
|
||||
builder.udp4(port);
|
||||
builder.tcp4(tcp_socket.port());
|
||||
let socket = {
|
||||
let v4 = crate::config::ipv4(&discv5_config.listen_config);
|
||||
let v6 = crate::config::ipv6(&discv5_config.listen_config);
|
||||
|
||||
(ip, port).into()
|
||||
}
|
||||
ListenConfig::Ipv6 { ip, port } => {
|
||||
if ip != Ipv6Addr::UNSPECIFIED {
|
||||
builder.ip6(ip);
|
||||
if let Some(addr) = v4 {
|
||||
if *addr.ip() != Ipv4Addr::UNSPECIFIED {
|
||||
builder.ip4(*addr.ip());
|
||||
}
|
||||
builder.udp6(port);
|
||||
builder.udp4(addr.port());
|
||||
}
|
||||
if let Some(addr) = v6 {
|
||||
if *addr.ip() != Ipv6Addr::UNSPECIFIED {
|
||||
builder.ip6(*addr.ip());
|
||||
}
|
||||
builder.udp6(addr.port());
|
||||
}
|
||||
// Advertise tcp4 when v4 is configured, else tcp6.
|
||||
if v4.is_some() {
|
||||
builder.tcp4(tcp_socket.port());
|
||||
} else if v6.is_some() {
|
||||
builder.tcp6(tcp_socket.port());
|
||||
|
||||
(ip, port).into()
|
||||
}
|
||||
ListenConfig::DualStack { ipv4, ipv4_port, ipv6, ipv6_port } => {
|
||||
if ipv4 != Ipv4Addr::UNSPECIFIED {
|
||||
builder.ip4(ipv4);
|
||||
}
|
||||
builder.udp4(ipv4_port);
|
||||
builder.tcp4(tcp_socket.port());
|
||||
|
||||
if ipv6 != Ipv6Addr::UNSPECIFIED {
|
||||
builder.ip6(ipv6);
|
||||
}
|
||||
builder.udp6(ipv6_port);
|
||||
|
||||
(ipv6, ipv6_port).into()
|
||||
}
|
||||
// Prefer v6 when both are configured
|
||||
v6.map(SocketAddr::V6)
|
||||
.or_else(|| v4.map(SocketAddr::V4))
|
||||
.unwrap_or_else(|| SocketAddr::from((Ipv4Addr::UNSPECIFIED, 0)))
|
||||
};
|
||||
|
||||
let rlpx_ip_mode = if tcp_socket.is_ipv4() { IpMode::Ip4 } else { IpMode::Ip6 };
|
||||
@@ -573,14 +568,19 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
bootstrap_lookup_interval: u64,
|
||||
bootstrap_lookup_countdown: u64,
|
||||
metrics: Discv5Metrics,
|
||||
discv5: Arc<discv5::Discv5>,
|
||||
discv5: std::sync::Weak<discv5::Discv5>,
|
||||
) {
|
||||
let local_node_id = discv5.local_enr().node_id();
|
||||
let lookup_interval = Duration::from_secs(lookup_interval);
|
||||
let metrics = metrics.discovered_peers;
|
||||
let mut kbucket_index = MAX_KBUCKET_INDEX;
|
||||
let pulse_lookup_interval = Duration::from_secs(bootstrap_lookup_interval);
|
||||
task::spawn(async move {
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
let local_node_id = discv5_handle.local_enr().node_id();
|
||||
drop(discv5_handle);
|
||||
|
||||
// make many fast lookup queries at bootstrap, trying to fill kbuckets at furthest
|
||||
// log2distance from local node
|
||||
for i in (0..bootstrap_lookup_countdown).rev() {
|
||||
@@ -593,7 +593,12 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
"starting bootstrap boost lookup query"
|
||||
);
|
||||
|
||||
lookup(target, &discv5, &metrics).await;
|
||||
{
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
lookup(target, &discv5_handle, &metrics).await;
|
||||
}
|
||||
|
||||
tokio::time::sleep(pulse_lookup_interval).await;
|
||||
}
|
||||
@@ -610,7 +615,12 @@ pub fn spawn_populate_kbuckets_bg(
|
||||
"starting periodic lookup query"
|
||||
);
|
||||
|
||||
lookup(target, &discv5, &metrics).await;
|
||||
{
|
||||
let Some(discv5_handle) = discv5.upgrade() else {
|
||||
return;
|
||||
};
|
||||
lookup(target, &discv5_handle, &metrics).await;
|
||||
}
|
||||
|
||||
if kbucket_index > DEFAULT_MIN_TARGET_KBUCKET_INDEX {
|
||||
// try to populate bucket one step closer
|
||||
@@ -696,10 +706,13 @@ mod test {
|
||||
#![allow(deprecated)]
|
||||
use super::*;
|
||||
use ::enr::{CombinedKey, EnrKey};
|
||||
use discv5::ListenConfig;
|
||||
use rand_08::thread_rng;
|
||||
use reth_chainspec::MAINNET;
|
||||
use reth_tracing::init_test_tracing;
|
||||
use std::env;
|
||||
use std::{
|
||||
net::UdpSocket,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::trace;
|
||||
|
||||
fn discv5_noop() -> Discv5 {
|
||||
@@ -738,6 +751,61 @@ mod test {
|
||||
Discv5::start(&secret_key, discv5_config).await.expect("should build discv5")
|
||||
}
|
||||
|
||||
async fn start_discovery_node_with_key(
|
||||
secret_key: &SecretKey,
|
||||
udp_port_discv5: u16,
|
||||
) -> Result<(Discv5, mpsc::Receiver<discv5::Event>), Error> {
|
||||
let discv5_addr: SocketAddr = format!("127.0.0.1:{udp_port_discv5}").parse().unwrap();
|
||||
let rlpx_addr: SocketAddr = "127.0.0.1:30303".parse().unwrap();
|
||||
|
||||
let discv5_listen_config = ListenConfig::from(discv5_addr);
|
||||
let discv5_config = Config::builder(rlpx_addr)
|
||||
.discv5_config(discv5::ConfigBuilder::new(discv5_listen_config).build())
|
||||
.build();
|
||||
|
||||
Discv5::start(secret_key, discv5_config).await
|
||||
}
|
||||
|
||||
fn unused_udp_port() -> u16 {
|
||||
UdpSocket::bind("127.0.0.1:0").unwrap().local_addr().unwrap().port()
|
||||
}
|
||||
|
||||
async fn wait_for_udp_port_release(port: u16, timeout: Duration) {
|
||||
let deadline = Instant::now() + timeout;
|
||||
|
||||
loop {
|
||||
match UdpSocket::bind(("127.0.0.1", port)) {
|
||||
Ok(socket) => {
|
||||
drop(socket);
|
||||
return;
|
||||
}
|
||||
Err(err) if Instant::now() < deadline => {
|
||||
trace!(target: "net::discv5::test", %port, %err, "waiting for discv5 port release");
|
||||
tokio::time::sleep(Duration::from_millis(10)).await;
|
||||
}
|
||||
Err(err) => panic!("discv5 did not release port {port} before timeout: {err}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discv5_releases_port_on_drop() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let secret_key = SecretKey::new(&mut thread_rng());
|
||||
let port = unused_udp_port();
|
||||
|
||||
let (node, updates) =
|
||||
start_discovery_node_with_key(&secret_key, port).await.expect("should start discv5");
|
||||
drop(updates);
|
||||
drop(node);
|
||||
|
||||
wait_for_udp_port_release(port, Duration::from_secs(1)).await;
|
||||
|
||||
let restarted = start_discovery_node_with_key(&secret_key, port).await;
|
||||
assert!(restarted.is_ok(), "discv5 failed to rebind dropped port: {restarted:?}");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn discv5() {
|
||||
reth_tracing::init_test_tracing();
|
||||
@@ -937,11 +1005,6 @@ mod test {
|
||||
|
||||
#[test]
|
||||
fn get_fork_id_with_different_network_stack_ids() {
|
||||
unsafe {
|
||||
env::set_var("RUST_LOG", "net::discv5=trace");
|
||||
}
|
||||
init_test_tracing();
|
||||
|
||||
let fork_id = MAINNET.latest_fork_id();
|
||||
let sk = SecretKey::new(&mut thread_rng());
|
||||
|
||||
|
||||
@@ -7,13 +7,14 @@ use alloy_primitives::{
|
||||
Bytes, TxHash, B256, U128,
|
||||
};
|
||||
use alloy_rlp::{
|
||||
Decodable, Encodable, RlpDecodable, RlpDecodableWrapper, RlpEncodable, RlpEncodableWrapper,
|
||||
Decodable, Encodable, Header, RlpDecodable, RlpDecodableWrapper, RlpEncodable,
|
||||
RlpEncodableWrapper,
|
||||
};
|
||||
use core::{fmt::Debug, mem};
|
||||
use derive_more::{Constructor, Deref, DerefMut, From, IntoIterator};
|
||||
use reth_codecs_derive::{add_arbitrary_tests, generate_tests};
|
||||
use reth_ethereum_primitives::TransactionSigned;
|
||||
use reth_primitives_traits::{Block, SignedTransaction};
|
||||
use reth_primitives_traits::{Block, InMemorySize, SignedTransaction};
|
||||
|
||||
/// This informs peers of new blocks that have appeared on the network.
|
||||
#[derive(
|
||||
@@ -143,6 +144,53 @@ impl<T> From<Transactions<T>> for Vec<T> {
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Decodable + InMemorySize> Transactions<T> {
|
||||
/// Decodes the RLP list of transactions, stopping once the cumulative
|
||||
/// [`InMemorySize`] of decoded transactions exceeds `memory_budget` bytes.
|
||||
/// Any remaining transactions in the payload are skipped.
|
||||
pub fn decode_with_memory_budget(
|
||||
buf: &mut &[u8],
|
||||
memory_budget: usize,
|
||||
) -> alloy_rlp::Result<Self> {
|
||||
decode_list_with_memory_budget(buf, memory_budget).map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
/// Decodes an RLP list, stopping once the cumulative [`InMemorySize`] of decoded items exceeds
|
||||
/// `memory_budget` bytes. Any remaining items in the payload are skipped.
|
||||
pub fn decode_list_with_memory_budget<T: Decodable + InMemorySize>(
|
||||
buf: &mut &[u8],
|
||||
memory_budget: usize,
|
||||
) -> alloy_rlp::Result<Vec<T>> {
|
||||
let header = Header::decode(buf)?;
|
||||
if !header.list {
|
||||
return Err(alloy_rlp::Error::UnexpectedString);
|
||||
}
|
||||
if buf.len() < header.payload_length {
|
||||
return Err(alloy_rlp::Error::InputTooShort);
|
||||
}
|
||||
|
||||
let (payload, rest) = buf.split_at(header.payload_length);
|
||||
let mut payload = payload;
|
||||
|
||||
let mut txs = Vec::new();
|
||||
let mut total_size = 0usize;
|
||||
|
||||
while !payload.is_empty() {
|
||||
let item = T::decode(&mut payload)?;
|
||||
total_size = total_size.saturating_add(item.size());
|
||||
|
||||
if total_size > memory_budget {
|
||||
break;
|
||||
}
|
||||
|
||||
txs.push(item);
|
||||
}
|
||||
|
||||
*buf = rest;
|
||||
Ok(txs)
|
||||
}
|
||||
|
||||
/// Same as [`Transactions`] but this is intended as egress message send from local to _many_ peers.
|
||||
///
|
||||
/// The list of transactions is constructed on per-peers basis, but the underlying transaction
|
||||
@@ -837,6 +885,19 @@ pub struct BlockRangeUpdate {
|
||||
pub latest_hash: B256,
|
||||
}
|
||||
|
||||
impl InMemorySize for NewPooledTransactionHashes {
|
||||
fn size(&self) -> usize {
|
||||
match self {
|
||||
Self::Eth66(msg) => msg.0.len() * core::mem::size_of::<B256>(),
|
||||
Self::Eth68(msg) => {
|
||||
msg.types.len() * core::mem::size_of::<u8>() +
|
||||
msg.sizes.len() * core::mem::size_of::<usize>() +
|
||||
msg.hashes.len() * core::mem::size_of::<B256>()
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! All capability related types
|
||||
|
||||
use crate::{EthMessageID, EthVersion};
|
||||
use crate::{EthMessageID, EthVersion, SnapVersion};
|
||||
use alloc::{borrow::Cow, string::String, vec::Vec};
|
||||
use alloy_primitives::bytes::Bytes;
|
||||
use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
|
||||
@@ -85,6 +85,11 @@ impl Capability {
|
||||
Self::new_static("eth", version as usize)
|
||||
}
|
||||
|
||||
/// Returns the corresponding snap capability for the given version.
|
||||
pub const fn snap(version: SnapVersion) -> Self {
|
||||
Self::new_static("snap", version as usize)
|
||||
}
|
||||
|
||||
/// Returns the [`EthVersion::Eth66`] capability.
|
||||
pub const fn eth_66() -> Self {
|
||||
Self::eth(EthVersion::Eth66)
|
||||
@@ -115,6 +120,16 @@ impl Capability {
|
||||
Self::eth(EthVersion::Eth71)
|
||||
}
|
||||
|
||||
/// Returns the `snap/1` capability.
|
||||
pub const fn snap_1() -> Self {
|
||||
Self::snap(SnapVersion::V1)
|
||||
}
|
||||
|
||||
/// Returns the `snap/2` capability.
|
||||
pub const fn snap_2() -> Self {
|
||||
Self::snap(SnapVersion::V2)
|
||||
}
|
||||
|
||||
/// Whether this is eth v66 protocol.
|
||||
#[inline]
|
||||
pub fn is_eth_v66(&self) -> bool {
|
||||
|
||||
@@ -28,6 +28,15 @@ use core::fmt::Debug;
|
||||
// https://github.com/ethereum/go-ethereum/blob/30602163d5d8321fbc68afdcbbaf2362b2641bde/eth/protocols/eth/protocol.go#L50
|
||||
pub const MAX_MESSAGE_SIZE: usize = 10 * 1024 * 1024;
|
||||
|
||||
/// Multiplier applied to `max_message_size` to derive the in-memory budget for decoding
|
||||
/// `Transactions` and `PooledTransactions` messages.
|
||||
///
|
||||
/// Decoded transactions expand relative to their RLP encoding due to struct overhead and heap
|
||||
/// allocations. With many peers in flight this can cause significant memory pressure, so we
|
||||
/// stop decoding once the cumulative in-memory size of decoded transactions exceeds
|
||||
/// `max_message_size * TX_MEMORY_BUDGET_MULTIPLIER`. Remaining transactions are silently dropped.
|
||||
pub const TX_MEMORY_BUDGET_MULTIPLIER: usize = 2;
|
||||
|
||||
/// Error when sending/receiving a message
|
||||
#[derive(thiserror::Error, Debug)]
|
||||
pub enum MessageError {
|
||||
@@ -87,6 +96,19 @@ impl<N: NetworkPrimitives> ProtocolMessage<N> {
|
||||
///
|
||||
/// This will enforce decoding according to the given [`EthVersion`] of the connection.
|
||||
pub fn decode_message(version: EthVersion, buf: &mut &[u8]) -> Result<Self, MessageError> {
|
||||
Self::decode_message_with_tx_memory_budget(version, buf, usize::MAX)
|
||||
}
|
||||
|
||||
/// Like [`Self::decode_message`], but caps the cumulative in-memory size of decoded
|
||||
/// transactions in `Transactions` and `PooledTransactions` messages. Once exceeded,
|
||||
/// remaining transactions are silently dropped.
|
||||
///
|
||||
/// Use [`TX_MEMORY_BUDGET_MULTIPLIER`] to derive a reasonable default.
|
||||
pub fn decode_message_with_tx_memory_budget(
|
||||
version: EthVersion,
|
||||
buf: &mut &[u8],
|
||||
tx_memory_budget: usize,
|
||||
) -> Result<Self, MessageError> {
|
||||
let message_type = EthMessageID::decode(buf)?;
|
||||
|
||||
// For EIP-7642 (https://github.com/ethereum/EIPs/blob/master/EIPS/eip-7642.md):
|
||||
@@ -103,7 +125,9 @@ impl<N: NetworkPrimitives> ProtocolMessage<N> {
|
||||
EthMessageID::NewBlock => {
|
||||
EthMessage::NewBlock(Box::new(N::NewBlockPayload::decode(buf)?))
|
||||
}
|
||||
EthMessageID::Transactions => EthMessage::Transactions(Transactions::decode(buf)?),
|
||||
EthMessageID::Transactions => EthMessage::Transactions(
|
||||
Transactions::decode_with_memory_budget(buf, tx_memory_budget)?,
|
||||
),
|
||||
EthMessageID::NewPooledTransactionHashes => {
|
||||
if version >= EthVersion::Eth68 {
|
||||
EthMessage::NewPooledTransactionHashes68(NewPooledTransactionHashes68::decode(
|
||||
@@ -123,7 +147,9 @@ impl<N: NetworkPrimitives> ProtocolMessage<N> {
|
||||
EthMessage::GetPooledTransactions(RequestPair::decode(buf)?)
|
||||
}
|
||||
EthMessageID::PooledTransactions => {
|
||||
EthMessage::PooledTransactions(RequestPair::decode(buf)?)
|
||||
EthMessage::PooledTransactions(RequestPair::decode_with(buf, |buf| {
|
||||
PooledTransactions::decode_with_memory_budget(buf, tx_memory_budget)
|
||||
})?)
|
||||
}
|
||||
EthMessageID::GetNodeData => {
|
||||
if version >= EthVersion::Eth67 {
|
||||
@@ -732,6 +758,25 @@ impl<T> RequestPair<T> {
|
||||
let Self { request_id, message } = self;
|
||||
RequestPair { request_id, message: f(message) }
|
||||
}
|
||||
|
||||
/// Decodes the request id and then decodes the message payload using `decode_msg`.
|
||||
pub fn decode_with<F>(buf: &mut &[u8], decode_msg: F) -> alloy_rlp::Result<Self>
|
||||
where
|
||||
F: FnOnce(&mut &[u8]) -> alloy_rlp::Result<T>,
|
||||
{
|
||||
let header = Header::decode(buf)?;
|
||||
|
||||
let initial_length = buf.len();
|
||||
let request_id = u64::decode(buf)?;
|
||||
let message = decode_msg(buf)?;
|
||||
|
||||
let consumed_len = initial_length - buf.len();
|
||||
if consumed_len != header.payload_length {
|
||||
return Err(alloy_rlp::Error::UnexpectedLength)
|
||||
}
|
||||
|
||||
Ok(Self { request_id, message })
|
||||
}
|
||||
}
|
||||
|
||||
/// Allows messages with request ids to be serialized into RLP bytes.
|
||||
|
||||
@@ -3,13 +3,41 @@
|
||||
//! facilitating the exchange of Ethereum state snapshots between peers
|
||||
//! Reference: [Ethereum Snapshot Protocol](https://github.com/ethereum/devp2p/blob/master/caps/snap.md#protocol-messages)
|
||||
//!
|
||||
//! Current version: snap/1
|
||||
//! This module currently includes snap/1 plus preparatory snap/2 message definitions.
|
||||
|
||||
use crate::BlockAccessLists;
|
||||
use alloc::vec::Vec;
|
||||
use alloy_primitives::{Bytes, B256};
|
||||
use alloy_rlp::{Decodable, Encodable, RlpDecodable, RlpEncodable};
|
||||
use reth_codecs_derive::add_arbitrary_tests;
|
||||
|
||||
/// Supported SNAP protocol versions.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq, Default, Hash)]
|
||||
#[cfg_attr(feature = "serde", derive(serde::Serialize, serde::Deserialize))]
|
||||
#[repr(u8)]
|
||||
pub enum SnapVersion {
|
||||
/// The original snapshot protocol.
|
||||
#[default]
|
||||
V1 = 1,
|
||||
/// BAL-based healing as proposed by EIP-8189.
|
||||
V2 = 2,
|
||||
}
|
||||
|
||||
impl SnapVersion {
|
||||
/// Returns the number of messages supported by this version.
|
||||
pub const fn message_count(self) -> u8 {
|
||||
match self {
|
||||
Self::V1 => 8,
|
||||
Self::V2 => 10,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the highest supported message id for this version.
|
||||
pub const fn max_message_id(self) -> u8 {
|
||||
self.message_count() - 1
|
||||
}
|
||||
}
|
||||
|
||||
/// Message IDs for the snap sync protocol
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum SnapMessageId {
|
||||
@@ -27,9 +55,21 @@ pub enum SnapMessageId {
|
||||
/// Response for the number of requested contract codes.
|
||||
ByteCodes = 0x05,
|
||||
/// Request of the number of state (either account or storage) Merkle trie nodes by path.
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
GetTrieNodes = 0x06,
|
||||
/// Response for the number of requested state trie nodes.
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
TrieNodes = 0x07,
|
||||
/// Request BALs for a list of block hashes.
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
GetBlockAccessLists = 0x08,
|
||||
/// Response containing BALs for the requested block hashes.
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
BlockAccessLists = 0x09,
|
||||
}
|
||||
|
||||
/// Request for a range of accounts from the state trie.
|
||||
@@ -187,6 +227,30 @@ pub struct TrieNodesMessage {
|
||||
pub nodes: Vec<Bytes>,
|
||||
}
|
||||
|
||||
/// Request BALs for the given block hashes.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct GetBlockAccessListsMessage {
|
||||
/// Request ID to match up responses with.
|
||||
pub request_id: u64,
|
||||
/// Block hashes to retrieve BALs for.
|
||||
pub block_hashes: Vec<B256>,
|
||||
/// Soft limit at which to stop returning data (in bytes).
|
||||
pub response_bytes: u64,
|
||||
}
|
||||
|
||||
/// Response containing one BAL per requested block hash.
|
||||
#[derive(Debug, Clone, PartialEq, Eq, RlpEncodable, RlpDecodable)]
|
||||
#[cfg_attr(any(test, feature = "arbitrary"), derive(arbitrary::Arbitrary))]
|
||||
#[add_arbitrary_tests(rlp)]
|
||||
pub struct BlockAccessListsMessage {
|
||||
/// ID of the request this is a response for.
|
||||
pub request_id: u64,
|
||||
/// Raw BAL payloads in request order.
|
||||
pub block_access_lists: BlockAccessLists,
|
||||
}
|
||||
|
||||
/// Represents all types of messages in the snap sync protocol.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub enum SnapProtocolMessage {
|
||||
@@ -203,9 +267,21 @@ pub enum SnapProtocolMessage {
|
||||
/// Response with contract codes - see [`ByteCodesMessage`]
|
||||
ByteCodes(ByteCodesMessage),
|
||||
/// Request for trie nodes - see [`GetTrieNodesMessage`]
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
GetTrieNodes(GetTrieNodesMessage),
|
||||
/// Response with trie nodes - see [`TrieNodesMessage`]
|
||||
///
|
||||
/// Only valid for `snap/1`. Replaced by BAL-based healing in `snap/2`.
|
||||
TrieNodes(TrieNodesMessage),
|
||||
/// Request for block access lists - see [`GetBlockAccessListsMessage`]
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
GetBlockAccessLists(GetBlockAccessListsMessage),
|
||||
/// Response with block access lists - see [`BlockAccessListsMessage`]
|
||||
///
|
||||
/// Only valid for `snap/2`.
|
||||
BlockAccessLists(BlockAccessListsMessage),
|
||||
}
|
||||
|
||||
impl SnapProtocolMessage {
|
||||
@@ -222,6 +298,8 @@ impl SnapProtocolMessage {
|
||||
Self::ByteCodes(_) => SnapMessageId::ByteCodes,
|
||||
Self::GetTrieNodes(_) => SnapMessageId::GetTrieNodes,
|
||||
Self::TrieNodes(_) => SnapMessageId::TrieNodes,
|
||||
Self::GetBlockAccessLists(_) => SnapMessageId::GetBlockAccessLists,
|
||||
Self::BlockAccessLists(_) => SnapMessageId::BlockAccessLists,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -241,6 +319,8 @@ impl SnapProtocolMessage {
|
||||
Self::ByteCodes(msg) => msg.encode(&mut buf),
|
||||
Self::GetTrieNodes(msg) => msg.encode(&mut buf),
|
||||
Self::TrieNodes(msg) => msg.encode(&mut buf),
|
||||
Self::GetBlockAccessLists(msg) => msg.encode(&mut buf),
|
||||
Self::BlockAccessLists(msg) => msg.encode(&mut buf),
|
||||
}
|
||||
|
||||
Bytes::from(buf)
|
||||
@@ -314,6 +394,20 @@ impl SnapProtocolMessage {
|
||||
TrieNodes,
|
||||
TrieNodesMessage
|
||||
);
|
||||
decode_snap_message_variant!(
|
||||
message_id,
|
||||
buf,
|
||||
SnapMessageId::GetBlockAccessLists,
|
||||
GetBlockAccessLists,
|
||||
GetBlockAccessListsMessage
|
||||
);
|
||||
decode_snap_message_variant!(
|
||||
message_id,
|
||||
buf,
|
||||
SnapMessageId::BlockAccessLists,
|
||||
BlockAccessLists,
|
||||
BlockAccessListsMessage
|
||||
);
|
||||
|
||||
Err(alloy_rlp::Error::Custom("Unknown message ID"))
|
||||
}
|
||||
@@ -344,6 +438,9 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn test_all_message_roundtrips() {
|
||||
assert_eq!(SnapVersion::V1.message_count(), 8);
|
||||
assert_eq!(SnapVersion::V2.message_count(), 10);
|
||||
|
||||
test_roundtrip(SnapProtocolMessage::GetAccountRange(GetAccountRangeMessage {
|
||||
request_id: 42,
|
||||
root_hash: b256_from_u64(123),
|
||||
@@ -404,6 +501,20 @@ mod tests {
|
||||
request_id: 42,
|
||||
nodes: vec![Bytes::from(vec![1, 2, 3])],
|
||||
}));
|
||||
|
||||
test_roundtrip(SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage {
|
||||
request_id: 42,
|
||||
block_hashes: vec![b256_from_u64(123), b256_from_u64(456)],
|
||||
response_bytes: 4096,
|
||||
}));
|
||||
|
||||
test_roundtrip(SnapProtocolMessage::BlockAccessLists(BlockAccessListsMessage {
|
||||
request_id: 42,
|
||||
block_access_lists: BlockAccessLists(vec![
|
||||
Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]),
|
||||
Bytes::from_static(&[0xc1, alloy_rlp::EMPTY_LIST_CODE]),
|
||||
]),
|
||||
}));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -1,12 +1,14 @@
|
||||
//! Implements the `GetPooledTransactions` and `PooledTransactions` message types.
|
||||
|
||||
use crate::broadcast::decode_list_with_memory_budget;
|
||||
use alloc::vec::Vec;
|
||||
use alloy_consensus::transaction::PooledTransaction;
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::{RlpDecodableWrapper, RlpEncodableWrapper};
|
||||
use alloy_rlp::{Decodable, RlpDecodableWrapper, RlpEncodableWrapper};
|
||||
use derive_more::{Constructor, Deref, IntoIterator};
|
||||
use reth_codecs_derive::add_arbitrary_tests;
|
||||
use reth_primitives_traits::InMemorySize;
|
||||
|
||||
/// A list of transaction hashes that the peer would like transaction bodies for.
|
||||
#[derive(
|
||||
@@ -37,6 +39,12 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl InMemorySize for GetPooledTransactions {
|
||||
fn size(&self) -> usize {
|
||||
self.0.len() * core::mem::size_of::<B256>()
|
||||
}
|
||||
}
|
||||
|
||||
/// The response to [`GetPooledTransactions`], containing the transaction bodies associated with
|
||||
/// the requested hashes.
|
||||
///
|
||||
@@ -62,6 +70,18 @@ pub struct PooledTransactions<T = PooledTransaction>(
|
||||
pub Vec<T>,
|
||||
);
|
||||
|
||||
impl<T: Decodable + InMemorySize> PooledTransactions<T> {
|
||||
/// Decodes the RLP list of transactions, stopping once the cumulative
|
||||
/// [`InMemorySize`] of decoded transactions exceeds `memory_budget` bytes.
|
||||
/// Any remaining transactions in the payload are skipped.
|
||||
pub fn decode_with_memory_budget(
|
||||
buf: &mut &[u8],
|
||||
memory_budget: usize,
|
||||
) -> alloy_rlp::Result<Self> {
|
||||
decode_list_with_memory_budget(buf, memory_budget).map(Self)
|
||||
}
|
||||
}
|
||||
|
||||
impl<T: Encodable2718> PooledTransactions<T> {
|
||||
/// Returns an iterator over the transaction hashes in this response.
|
||||
pub fn hashes(&self) -> impl Iterator<Item = B256> + '_ {
|
||||
|
||||
@@ -5,9 +5,9 @@
|
||||
|
||||
use super::message::MAX_MESSAGE_SIZE;
|
||||
use crate::{
|
||||
message::{EthBroadcastMessage, ProtocolBroadcastMessage},
|
||||
message::{EthBroadcastMessage, ProtocolBroadcastMessage, TX_MEMORY_BUDGET_MULTIPLIER},
|
||||
EthMessage, EthMessageID, EthNetworkPrimitives, EthVersion, NetworkPrimitives, ProtocolMessage,
|
||||
RawCapabilityMessage, SnapMessageId, SnapProtocolMessage,
|
||||
RawCapabilityMessage, SnapProtocolMessage, SnapVersion,
|
||||
};
|
||||
use alloy_rlp::{Bytes, BytesMut, Encodable};
|
||||
use core::fmt::Debug;
|
||||
@@ -74,6 +74,18 @@ where
|
||||
Self { eth_snap: EthSnapStreamInner::new(eth_version), inner: stream }
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with an explicit snap version.
|
||||
pub const fn new_with_snap_version(
|
||||
stream: S,
|
||||
eth_version: EthVersion,
|
||||
snap_version: SnapVersion,
|
||||
) -> Self {
|
||||
Self {
|
||||
eth_snap: EthSnapStreamInner::new_with_snap_version(eth_version, snap_version),
|
||||
inner: stream,
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size.
|
||||
pub const fn with_max_message_size(
|
||||
stream: S,
|
||||
@@ -86,12 +98,35 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size and snap version.
|
||||
pub const fn with_max_message_size_and_snap_version(
|
||||
stream: S,
|
||||
eth_version: EthVersion,
|
||||
snap_version: SnapVersion,
|
||||
max_message_size: usize,
|
||||
) -> Self {
|
||||
Self {
|
||||
eth_snap: EthSnapStreamInner::with_max_message_size_and_snap_version(
|
||||
eth_version,
|
||||
snap_version,
|
||||
max_message_size,
|
||||
),
|
||||
inner: stream,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the eth version
|
||||
#[inline]
|
||||
pub const fn eth_version(&self) -> EthVersion {
|
||||
self.eth_snap.eth_version()
|
||||
}
|
||||
|
||||
/// Returns the snap version.
|
||||
#[inline]
|
||||
pub const fn snap_version(&self) -> SnapVersion {
|
||||
self.eth_snap.snap_version()
|
||||
}
|
||||
|
||||
/// Returns the underlying stream
|
||||
#[inline]
|
||||
pub const fn inner(&self) -> &S {
|
||||
@@ -193,13 +228,13 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
/// Stream handling combined eth and snap protocol logic
|
||||
/// Snap version is not critical to specify yet,
|
||||
/// Only one version, snap/1, does exist.
|
||||
/// Stream handling combined eth and snap protocol logic.
|
||||
#[derive(Debug, Clone)]
|
||||
struct EthSnapStreamInner<N> {
|
||||
/// Eth protocol version
|
||||
eth_version: EthVersion,
|
||||
/// Snap protocol version
|
||||
snap_version: SnapVersion,
|
||||
/// Maximum allowed ETH/Snap message size.
|
||||
max_message_size: usize,
|
||||
/// Type marker
|
||||
@@ -212,12 +247,26 @@ where
|
||||
{
|
||||
/// Create a new eth and snap protocol stream
|
||||
const fn new(eth_version: EthVersion) -> Self {
|
||||
Self::with_max_message_size(eth_version, MAX_MESSAGE_SIZE)
|
||||
Self::new_with_snap_version(eth_version, SnapVersion::V1)
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with an explicit snap version.
|
||||
const fn new_with_snap_version(eth_version: EthVersion, snap_version: SnapVersion) -> Self {
|
||||
Self::with_max_message_size_and_snap_version(eth_version, snap_version, MAX_MESSAGE_SIZE)
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size.
|
||||
const fn with_max_message_size(eth_version: EthVersion, max_message_size: usize) -> Self {
|
||||
Self { eth_version, max_message_size, _pd: PhantomData }
|
||||
Self::with_max_message_size_and_snap_version(eth_version, SnapVersion::V1, max_message_size)
|
||||
}
|
||||
|
||||
/// Create a new eth and snap protocol stream with a custom max message size and snap version.
|
||||
const fn with_max_message_size_and_snap_version(
|
||||
eth_version: EthVersion,
|
||||
snap_version: SnapVersion,
|
||||
max_message_size: usize,
|
||||
) -> Self {
|
||||
Self { eth_version, snap_version, max_message_size, _pd: PhantomData }
|
||||
}
|
||||
|
||||
#[inline]
|
||||
@@ -225,6 +274,11 @@ where
|
||||
self.eth_version
|
||||
}
|
||||
|
||||
#[inline]
|
||||
const fn snap_version(&self) -> SnapVersion {
|
||||
self.snap_version
|
||||
}
|
||||
|
||||
/// Decode a message from the stream
|
||||
fn decode_message(&self, bytes: BytesMut) -> Result<EthSnapMessage<N>, EthSnapStreamError> {
|
||||
if bytes.len() > self.max_message_size {
|
||||
@@ -244,7 +298,11 @@ where
|
||||
// See also <https://github.com/paradigmxyz/reth/blob/main/crates/net/eth-wire/src/capability.rs#L272-L283>.
|
||||
if message_id <= EthMessageID::max(self.eth_version) {
|
||||
let mut buf = bytes.as_ref();
|
||||
match ProtocolMessage::decode_message(self.eth_version, &mut buf) {
|
||||
match ProtocolMessage::decode_message_with_tx_memory_budget(
|
||||
self.eth_version,
|
||||
&mut buf,
|
||||
self.max_message_size * TX_MEMORY_BUDGET_MULTIPLIER,
|
||||
) {
|
||||
Ok(protocol_msg) => {
|
||||
if matches!(protocol_msg.message, EthMessage::Status(_)) {
|
||||
return Err(EthSnapStreamError::StatusNotInHandshake);
|
||||
@@ -256,8 +314,9 @@ where
|
||||
}
|
||||
}
|
||||
} else if message_id > EthMessageID::max(self.eth_version) &&
|
||||
message_id <=
|
||||
EthMessageID::message_count(self.eth_version) + SnapMessageId::TrieNodes as u8
|
||||
message_id <
|
||||
EthMessageID::message_count(self.eth_version) +
|
||||
self.snap_version.message_count()
|
||||
{
|
||||
// Checks for multiplexed snap message IDs :
|
||||
// - message_id > EthMessageID::max() : ensures it's not an eth message
|
||||
@@ -313,8 +372,8 @@ mod tests {
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::Encodable;
|
||||
use reth_eth_wire_types::{
|
||||
message::RequestPair, GetAccountRangeMessage, GetBlockAccessLists, GetBlockHeaders,
|
||||
HeadersDirection,
|
||||
message::RequestPair, BlockAccessLists, BlockAccessListsMessage, GetAccountRangeMessage,
|
||||
GetBlockAccessLists, GetBlockAccessListsMessage, GetBlockHeaders, HeadersDirection,
|
||||
};
|
||||
|
||||
// Helper to create eth message and its bytes
|
||||
@@ -352,6 +411,22 @@ mod tests {
|
||||
(snap_msg, BytesMut::from(&encoded[..]))
|
||||
}
|
||||
|
||||
fn create_snap2_message() -> (SnapProtocolMessage, BytesMut) {
|
||||
let snap_msg = SnapProtocolMessage::GetBlockAccessLists(GetBlockAccessListsMessage {
|
||||
request_id: 1,
|
||||
block_hashes: vec![B256::default()],
|
||||
response_bytes: 1000,
|
||||
});
|
||||
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
|
||||
EthVersion::Eth67,
|
||||
SnapVersion::V2,
|
||||
);
|
||||
let encoded = inner.encode_snap_message(snap_msg.clone());
|
||||
|
||||
(snap_msg, BytesMut::from(&encoded[..]))
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_eth_message_roundtrip() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
|
||||
@@ -412,6 +487,25 @@ mod tests {
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snap2_protocol() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
|
||||
EthVersion::Eth67,
|
||||
SnapVersion::V2,
|
||||
);
|
||||
let (snap_msg, snap_bytes) = create_snap2_message();
|
||||
|
||||
let encoded_bytes = inner.encode_snap_message(snap_msg.clone());
|
||||
assert!(!encoded_bytes.is_empty());
|
||||
|
||||
let decoded_result = inner.decode_message(snap_bytes.clone());
|
||||
assert!(matches!(decoded_result, Ok(EthSnapMessage::Snap(_))));
|
||||
|
||||
if let Ok(EthSnapMessage::Snap(decoded_msg)) = inner.decode_message(snap_bytes) {
|
||||
assert_eq!(decoded_msg, snap_msg);
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_message_id_boundaries() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
|
||||
@@ -475,4 +569,24 @@ mod tests {
|
||||
};
|
||||
assert_eq!(decoded_eth, eth_msg);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_snap1_rejects_snap2_message_ids() {
|
||||
let inner = EthSnapStreamInner::<EthNetworkPrimitives>::new(EthVersion::Eth67);
|
||||
let snap2_msg = SnapProtocolMessage::BlockAccessLists(BlockAccessListsMessage {
|
||||
request_id: 1,
|
||||
block_access_lists: BlockAccessLists(vec![alloy_primitives::Bytes::from_static(&[
|
||||
alloy_rlp::EMPTY_LIST_CODE,
|
||||
])]),
|
||||
});
|
||||
|
||||
let encoded = EthSnapStreamInner::<EthNetworkPrimitives>::new_with_snap_version(
|
||||
EthVersion::Eth67,
|
||||
SnapVersion::V2,
|
||||
)
|
||||
.encode_snap_message(snap2_msg);
|
||||
|
||||
let decoded = inner.decode_message(BytesMut::from(&encoded[..]));
|
||||
assert!(matches!(decoded, Err(EthSnapStreamError::UnknownMessageId(_))));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -7,7 +7,10 @@
|
||||
use crate::{
|
||||
errors::{EthHandshakeError, EthStreamError},
|
||||
handshake::EthereumEthHandshake,
|
||||
message::{EthBroadcastMessage, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE},
|
||||
message::{
|
||||
EthBroadcastMessage, ProtocolBroadcastMessage, MAX_MESSAGE_SIZE,
|
||||
TX_MEMORY_BUDGET_MULTIPLIER,
|
||||
},
|
||||
p2pstream::HANDSHAKE_TIMEOUT,
|
||||
CanDisconnect, DisconnectReason, EthMessage, EthNetworkPrimitives, EthVersion, ProtocolMessage,
|
||||
UnifiedStatus,
|
||||
@@ -16,7 +19,7 @@ use alloy_primitives::bytes::{Bytes, BytesMut};
|
||||
use alloy_rlp::Encodable;
|
||||
use futures::{ready, Sink, SinkExt};
|
||||
use pin_project::pin_project;
|
||||
use reth_eth_wire_types::{NetworkPrimitives, RawCapabilityMessage};
|
||||
use reth_eth_wire_types::{EthMessageID, NetworkPrimitives, RawCapabilityMessage};
|
||||
use reth_ethereum_forks::ForkFilter;
|
||||
use std::{
|
||||
future::Future,
|
||||
@@ -108,6 +111,9 @@ pub struct EthStreamInner<N> {
|
||||
version: EthVersion,
|
||||
/// Maximum allowed ETH message size.
|
||||
max_message_size: usize,
|
||||
/// When true, `NewBlock` (0x07) and `NewBlockHashes` (0x01) messages are rejected before RLP
|
||||
/// decoding to avoid any memory impact for non-PoW networks.
|
||||
reject_block_announcements: bool,
|
||||
_pd: std::marker::PhantomData<N>,
|
||||
}
|
||||
|
||||
@@ -122,7 +128,12 @@ where
|
||||
|
||||
/// Creates a new [`EthStreamInner`] with the given eth version and message size limit.
|
||||
pub const fn with_max_message_size(version: EthVersion, max_message_size: usize) -> Self {
|
||||
Self { version, max_message_size, _pd: std::marker::PhantomData }
|
||||
Self {
|
||||
version,
|
||||
max_message_size,
|
||||
reject_block_announcements: false,
|
||||
_pd: std::marker::PhantomData,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the eth version
|
||||
@@ -131,13 +142,30 @@ where
|
||||
self.version
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding.
|
||||
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
self.reject_block_announcements = reject;
|
||||
}
|
||||
|
||||
/// Decodes incoming bytes into an [`EthMessage`].
|
||||
pub fn decode_message(&self, bytes: BytesMut) -> Result<EthMessage<N>, EthStreamError> {
|
||||
if bytes.len() > self.max_message_size {
|
||||
return Err(EthStreamError::MessageTooBig(bytes.len()));
|
||||
}
|
||||
|
||||
let msg = match ProtocolMessage::decode_message(self.version, &mut bytes.as_ref()) {
|
||||
if self.reject_block_announcements &&
|
||||
let Some(&id) = bytes.first() &&
|
||||
(id == EthMessageID::NewBlock.to_u8() || id == EthMessageID::NewBlockHashes.to_u8())
|
||||
{
|
||||
return Err(EthStreamError::UnsupportedMessage { message_id: id });
|
||||
}
|
||||
|
||||
let msg = match ProtocolMessage::decode_message_with_tx_memory_budget(
|
||||
self.version,
|
||||
&mut bytes.as_ref(),
|
||||
self.max_message_size * TX_MEMORY_BUDGET_MULTIPLIER,
|
||||
) {
|
||||
Ok(m) => m,
|
||||
Err(err) => {
|
||||
let msg = if bytes.len() > 50 {
|
||||
@@ -208,6 +236,12 @@ impl<S, N: NetworkPrimitives> EthStream<S, N> {
|
||||
self.eth.version()
|
||||
}
|
||||
|
||||
/// Sets whether to reject block announcement messages (`NewBlock`, `NewBlockHashes`) before
|
||||
/// RLP decoding.
|
||||
pub const fn set_reject_block_announcements(&mut self, reject: bool) {
|
||||
self.eth.set_reject_block_announcements(reject);
|
||||
}
|
||||
|
||||
/// Returns the underlying stream.
|
||||
#[inline]
|
||||
pub const fn inner(&self) -> &S {
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! A Protocol defines a P2P subprotocol in an `RLPx` connection
|
||||
|
||||
use crate::{Capability, EthMessageID, EthVersion};
|
||||
use crate::{Capability, EthMessageID, EthVersion, SnapVersion};
|
||||
|
||||
/// Type that represents a [Capability] and the number of messages it uses.
|
||||
///
|
||||
@@ -30,6 +30,13 @@ impl Protocol {
|
||||
Self::new(cap, messages)
|
||||
}
|
||||
|
||||
/// Returns the corresponding snap capability for the given version.
|
||||
pub const fn snap(version: SnapVersion) -> Self {
|
||||
let cap = Capability::snap(version);
|
||||
let messages = version.message_count();
|
||||
Self::new(cap, messages)
|
||||
}
|
||||
|
||||
/// Returns the [`EthVersion::Eth66`] capability.
|
||||
pub const fn eth_66() -> Self {
|
||||
Self::eth(EthVersion::Eth66)
|
||||
@@ -45,6 +52,16 @@ impl Protocol {
|
||||
Self::eth(EthVersion::Eth68)
|
||||
}
|
||||
|
||||
/// Returns the `snap/1` capability.
|
||||
pub const fn snap_1() -> Self {
|
||||
Self::snap(SnapVersion::V1)
|
||||
}
|
||||
|
||||
/// Returns the `snap/2` capability.
|
||||
pub const fn snap_2() -> Self {
|
||||
Self::snap(SnapVersion::V2)
|
||||
}
|
||||
|
||||
/// Consumes the type and returns a tuple of the [Capability] and number of messages.
|
||||
#[inline]
|
||||
pub(crate) fn split(self) -> (Capability, u8) {
|
||||
@@ -86,5 +103,7 @@ mod tests {
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth69).messages(), 18);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth70).messages(), 18);
|
||||
assert_eq!(Protocol::eth(EthVersion::Eth71).messages(), 20);
|
||||
assert_eq!(Protocol::snap(SnapVersion::V1).messages(), 8);
|
||||
assert_eq!(Protocol::snap(SnapVersion::V2).messages(), 10);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -50,6 +50,7 @@ reth-ethereum-primitives.workspace = true
|
||||
futures.workspace = true
|
||||
pin-project.workspace = true
|
||||
tokio = { workspace = true, features = ["io-util", "net", "macros", "rt-multi-thread", "time"] }
|
||||
socket2 = { workspace = true, features = ["all"] }
|
||||
tokio-stream.workspace = true
|
||||
tokio-util = { workspace = true, features = ["codec"] }
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
use crate::{
|
||||
eth_requests::EthRequestHandler,
|
||||
metrics::NETWORK_POOL_TRANSACTIONS_SCOPE,
|
||||
transactions::{
|
||||
config::{
|
||||
AnnouncementFilteringPolicy, StrictEthAnnouncementFilter, TransactionPropagationKind,
|
||||
@@ -12,7 +13,9 @@ use crate::{
|
||||
NetworkHandle, NetworkManager,
|
||||
};
|
||||
use reth_eth_wire::{EthNetworkPrimitives, NetworkPrimitives};
|
||||
use reth_metrics::common::mpsc::memory_bounded_channel;
|
||||
use reth_network_api::test_utils::PeersHandleProvider;
|
||||
use reth_storage_api::BalProvider;
|
||||
use reth_transaction_pool::TransactionPool;
|
||||
use tokio::sync::mpsc;
|
||||
|
||||
@@ -63,7 +66,10 @@ impl<Tx, Eth, N: NetworkPrimitives> NetworkBuilder<Tx, Eth, N> {
|
||||
pub fn request_handler<Client>(
|
||||
self,
|
||||
client: Client,
|
||||
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N> {
|
||||
) -> NetworkBuilder<Tx, EthRequestHandler<Client, N>, N>
|
||||
where
|
||||
Client: BalProvider,
|
||||
{
|
||||
let Self { mut network, transactions, .. } = self;
|
||||
let (tx, rx) = mpsc::channel(ETH_REQUEST_CHANNEL_CAPACITY);
|
||||
network.set_eth_request_handler(tx);
|
||||
@@ -118,7 +124,10 @@ impl<Tx, Eth, N: NetworkPrimitives> NetworkBuilder<Tx, Eth, N> {
|
||||
announcement_policy: A,
|
||||
) -> NetworkBuilder<TransactionsManager<Pool, N>, Eth, N> {
|
||||
let Self { mut network, request_handler, .. } = self;
|
||||
let (tx, rx) = mpsc::unbounded_channel();
|
||||
let (tx, rx) = memory_bounded_channel(
|
||||
transactions_manager_config.tx_channel_memory_limit_bytes,
|
||||
NETWORK_POOL_TRANSACTIONS_SCOPE,
|
||||
);
|
||||
network.set_transactions(tx);
|
||||
let handle = network.handle().clone();
|
||||
let policies = NetworkPolicies::new(propagation_policy, announcement_policy);
|
||||
|
||||
@@ -20,7 +20,9 @@ use reth_eth_wire_types::message::MAX_MESSAGE_SIZE;
|
||||
use reth_ethereum_forks::{ForkFilter, Head};
|
||||
use reth_network_peers::{mainnet_nodes, pk2id, sepolia_nodes, PeerId, TrustedPeer};
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_storage_api::{noop::NoopProvider, BlockNumReader, BlockReader, HeaderProvider};
|
||||
use reth_storage_api::{
|
||||
noop::NoopProvider, BalProvider, BlockNumReader, BlockReader, HeaderProvider,
|
||||
};
|
||||
use reth_tasks::Runtime;
|
||||
use secp256k1::SECP256K1;
|
||||
use std::{collections::HashSet, net::SocketAddr, sync::Arc};
|
||||
@@ -157,7 +159,8 @@ where
|
||||
impl<C, N> NetworkConfig<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
|
||||
C: BalProvider
|
||||
+ BlockReader<Block = N::Block, Receipt = N::Receipt, Header = N::BlockHeader>
|
||||
+ HeaderProvider
|
||||
+ Clone
|
||||
+ Unpin
|
||||
|
||||
@@ -23,7 +23,7 @@ use std::{
|
||||
sync::Arc,
|
||||
task::{ready, Context, Poll},
|
||||
};
|
||||
use tokio::{sync::mpsc, task::JoinHandle};
|
||||
use tokio::{net::UdpSocket, sync::mpsc, task::JoinHandle};
|
||||
use tokio_stream::{wrappers::ReceiverStream, Stream};
|
||||
use tracing::{debug, trace};
|
||||
|
||||
@@ -54,6 +54,9 @@ pub struct Discovery {
|
||||
discv5: Option<Discv5>,
|
||||
/// All KAD table updates from the discv5 service.
|
||||
discv5_updates: Option<ReceiverStream<discv5::Event>>,
|
||||
/// Background task that, in shared-port mode, drains `UnrecognizedFrame`s from discv5 and
|
||||
/// feeds them into the discv4 ingress so packets advance without polling `Discovery`.
|
||||
_discv5_forwarder: Option<JoinHandle<()>>,
|
||||
/// Handler to interact with the DNS discovery service
|
||||
_dns_discovery: Option<DnsDiscoveryHandle>,
|
||||
/// Updates from the DNS discovery service.
|
||||
@@ -76,39 +79,138 @@ impl Discovery {
|
||||
discovery_v4_addr: SocketAddr,
|
||||
sk: SecretKey,
|
||||
discv4_config: Option<Discv4Config>,
|
||||
discv5_config: Option<reth_discv5::Config>, // contains discv5 listen address
|
||||
mut discv5_config: Option<reth_discv5::Config>, // contains discv5 listen address
|
||||
dns_discovery_config: Option<DnsDiscoveryConfig>,
|
||||
) -> Result<Self, NetworkError> {
|
||||
// setup discv4 with the discovery address and tcp port
|
||||
let local_enr =
|
||||
NodeRecord::from_secret_key(discovery_v4_addr, &sk).with_tcp_port(tcp_addr.port());
|
||||
|
||||
let discv4_future = async {
|
||||
let Some(disc_config) = discv4_config else { return Ok((None, None, None)) };
|
||||
let (discv4, mut discv4_service) =
|
||||
Discv4::bind(discovery_v4_addr, local_enr, sk, disc_config).await.map_err(
|
||||
|err| {
|
||||
// For IPv6 we set IPV6_V6ONLY=true so an IPv4 sibling socket on the same port doesn't
|
||||
// clash with the IPv6 one (Linux's default of V6ONLY=0 has IPv6 also claim the IPv4
|
||||
// port via mapped addresses), matching how discv5 binds its `DualStack` sockets.
|
||||
let bind_socket = async |addr: SocketAddr| {
|
||||
let result = match addr {
|
||||
SocketAddr::V4(_) => UdpSocket::bind(addr).await,
|
||||
SocketAddr::V6(_) => {
|
||||
use socket2::{Domain, Protocol, Socket, Type};
|
||||
(|| {
|
||||
let socket = Socket::new(Domain::IPV6, Type::DGRAM, Some(Protocol::UDP))?;
|
||||
socket.set_only_v6(true)?;
|
||||
socket.set_nonblocking(true)?;
|
||||
socket.bind(&addr.into())?;
|
||||
UdpSocket::from_std(socket.into())
|
||||
})()
|
||||
}
|
||||
};
|
||||
result
|
||||
.map(Arc::new)
|
||||
.map_err(|err| NetworkError::from_io_error(err, ServiceKind::Discovery(addr)))
|
||||
};
|
||||
|
||||
// In shared-port mode, bind the shared socket and start discv4 without its own receive
|
||||
// loop. Unrecognized frames from discv5 will be forwarded to the ingress handler.
|
||||
let (discv4, discv4_updates, _discv4_service, discv4_ingress, shared_socket) =
|
||||
if let Some(config) = discv4_config {
|
||||
if let Some(discv5_config) = &mut discv5_config &&
|
||||
discv5_config.has_matching_socket(discovery_v4_addr)
|
||||
{
|
||||
let socket = bind_socket(discovery_v4_addr).await?;
|
||||
|
||||
let (discv4, mut discv4_service, ingress) = Discv4::bind_shared(
|
||||
socket.clone(),
|
||||
local_enr,
|
||||
sk,
|
||||
config,
|
||||
)
|
||||
.map_err(|err| {
|
||||
NetworkError::from_io_error(err, ServiceKind::Discovery(discovery_v4_addr))
|
||||
},
|
||||
)?;
|
||||
let discv4_updates = discv4_service.update_stream();
|
||||
// spawn the service
|
||||
let discv4_service = discv4_service.spawn();
|
||||
})?;
|
||||
|
||||
debug!(target:"net", ?discovery_v4_addr, "started discovery v4");
|
||||
let discv4_updates = discv4_service.update_stream();
|
||||
let discv4_service = discv4_service.spawn();
|
||||
debug!(target:"net", ?discovery_v4_addr, "started discovery v4 (shared port)");
|
||||
(
|
||||
Some(discv4),
|
||||
Some(discv4_updates),
|
||||
Some(discv4_service),
|
||||
Some(ingress),
|
||||
Some(socket),
|
||||
)
|
||||
} else {
|
||||
let (discv4, mut discv4_service) =
|
||||
Discv4::bind(discovery_v4_addr, local_enr, sk, config).await.map_err(
|
||||
|err| {
|
||||
NetworkError::from_io_error(
|
||||
err,
|
||||
ServiceKind::Discovery(discovery_v4_addr),
|
||||
)
|
||||
},
|
||||
)?;
|
||||
let discv4_updates = discv4_service.update_stream();
|
||||
// spawn the service
|
||||
let discv4_service = discv4_service.spawn();
|
||||
|
||||
Ok((Some(discv4), Some(discv4_updates), Some(discv4_service)))
|
||||
};
|
||||
debug!(target:"net", ?discovery_v4_addr, "started discovery v4");
|
||||
|
||||
(Some(discv4), Some(discv4_updates), Some(discv4_service), None, None)
|
||||
}
|
||||
} else {
|
||||
(None, None, None, None, None)
|
||||
};
|
||||
|
||||
// Start discv5, wiring in the shared socket if in shared-port mode.
|
||||
let (discv5, discv5_updates) = if let Some(mut config) = discv5_config {
|
||||
if let Some(socket) = shared_socket {
|
||||
let discv5_cfg = config.discv5_config_mut();
|
||||
|
||||
// The shared socket covers discv4's address family; bind the opposite family
|
||||
// only if discv5 was configured for dual-stack.
|
||||
let (mut ipv4, mut ipv6) = (None, None);
|
||||
if discovery_v4_addr.is_ipv4() {
|
||||
ipv4 = Some(socket);
|
||||
if let Some(addr) = reth_discv5::config::ipv6(&discv5_cfg.listen_config) {
|
||||
ipv6 = Some(bind_socket(SocketAddr::V6(addr)).await?);
|
||||
}
|
||||
} else {
|
||||
ipv6 = Some(socket);
|
||||
if let Some(addr) = reth_discv5::config::ipv4(&discv5_cfg.listen_config) {
|
||||
ipv4 = Some(bind_socket(SocketAddr::V4(addr)).await?);
|
||||
}
|
||||
}
|
||||
|
||||
discv5_cfg.listen_config = discv5::ListenConfig::FromSockets { ipv4, ipv6 };
|
||||
}
|
||||
|
||||
let discv5_future = async {
|
||||
let Some(config) = discv5_config else { return Ok::<_, NetworkError>((None, None)) };
|
||||
let (discv5, discv5_updates) = Discv5::start(&sk, config).await?;
|
||||
debug!(target:"net", discovery_v5_enr=? discv5.local_enr(), "started discovery v5");
|
||||
Ok((Some(discv5), Some(discv5_updates.into())))
|
||||
debug!(target:"net", discovery_v5_enr=?discv5.local_enr(), "started discovery v5");
|
||||
(Some(discv5), Some(discv5_updates))
|
||||
} else {
|
||||
(None, None)
|
||||
};
|
||||
|
||||
let ((discv4, discv4_updates, _discv4_service), (discv5, discv5_updates)) =
|
||||
tokio::try_join!(discv4_future, discv5_future)?;
|
||||
// In shared-port mode, spawn a task that peels `UnrecognizedFrame` events off the discv5
|
||||
// update stream and feeds them into discv4's ingress. Other events are forwarded through
|
||||
// a new channel that `Discovery::poll` reads. This keeps both protocols moving without
|
||||
// requiring the main `Discovery::poll` loop to be driven for packets to be routed.
|
||||
let (discv5_updates, _discv5_forwarder) = match (discv4_ingress, discv5_updates) {
|
||||
(Some(mut ingress), Some(mut updates)) => {
|
||||
let (tx, rx) = mpsc::channel(updates.max_capacity());
|
||||
let handle = tokio::spawn(async move {
|
||||
while let Some(event) = updates.recv().await {
|
||||
if let discv5::Event::UnrecognizedFrame(frame) = &event {
|
||||
ingress.handle_packet(&frame.packet, frame.src_address).await;
|
||||
continue;
|
||||
}
|
||||
if tx.send(event).await.is_err() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
(Some(ReceiverStream::new(rx)), Some(handle))
|
||||
}
|
||||
(_, updates) => (updates.map(ReceiverStream::new), None),
|
||||
};
|
||||
|
||||
// setup DNS discovery
|
||||
let (_dns_discovery, dns_discovery_updates, _dns_disc_service) =
|
||||
@@ -132,6 +234,7 @@ impl Discovery {
|
||||
_discv4_service,
|
||||
discv5,
|
||||
discv5_updates,
|
||||
_discv5_forwarder,
|
||||
discovered_nodes: LruMap::new(DEFAULT_MAX_CAPACITY_DISCOVERED_PEERS_CACHE),
|
||||
queued_events: Default::default(),
|
||||
_dns_disc_service,
|
||||
@@ -309,6 +412,9 @@ impl Drop for Discovery {
|
||||
if let Some(handle) = self._discv4_service.take() {
|
||||
handle.abort();
|
||||
}
|
||||
if let Some(handle) = self._discv5_forwarder.take() {
|
||||
handle.abort();
|
||||
}
|
||||
if let Some(handle) = self._dns_disc_service.take() {
|
||||
handle.abort();
|
||||
}
|
||||
@@ -342,10 +448,11 @@ impl Discovery {
|
||||
},
|
||||
discv4: Default::default(),
|
||||
discv4_updates: Default::default(),
|
||||
_discv4_service: Default::default(),
|
||||
_discv5_forwarder: None,
|
||||
discv5: None,
|
||||
discv5_updates: None,
|
||||
queued_events: Default::default(),
|
||||
_discv4_service: Default::default(),
|
||||
_dns_discovery: None,
|
||||
dns_discovery_updates: None,
|
||||
_dns_disc_service: None,
|
||||
@@ -487,4 +594,179 @@ mod tests {
|
||||
assert_eq!(1, node_1.discovered_nodes.len());
|
||||
assert_eq!(1, node_2.discovered_nodes.len());
|
||||
}
|
||||
|
||||
/// Starts a discovery node with discv4 and discv5 sharing the same UDP port.
|
||||
async fn start_shared_port_node(port: u16) -> Discovery {
|
||||
let secret_key = SecretKey::new(&mut rand_08::thread_rng());
|
||||
let disc_addr: SocketAddr = format!("127.0.0.1:{port}").parse().unwrap();
|
||||
// Use a non-zero TCP port so the node record isn't filtered out by
|
||||
// `on_node_record_update` (which drops peers with tcp port == 0).
|
||||
let tcp_addr: SocketAddr = "127.0.0.1:30303".parse().unwrap();
|
||||
|
||||
let discv4_config = Discv4ConfigBuilder::default().external_ip_resolver(None).build();
|
||||
|
||||
let discv5_listen_config = discv5::ListenConfig::from(disc_addr);
|
||||
let discv5_config = reth_discv5::Config::builder(tcp_addr)
|
||||
.discv5_config(discv5::ConfigBuilder::new(discv5_listen_config).build())
|
||||
.build();
|
||||
|
||||
// Both protocols use the same address, triggering shared-port mode
|
||||
Discovery::new(
|
||||
tcp_addr,
|
||||
disc_addr,
|
||||
secret_key,
|
||||
Some(discv4_config),
|
||||
Some(discv5_config),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("should start with shared port")
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_shared_port_setup() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
// Use port 0 so the OS picks a free port
|
||||
let node = start_shared_port_node(0).await;
|
||||
|
||||
// Both protocols should be active
|
||||
assert!(node.discv4.is_some(), "discv4 should be running");
|
||||
assert!(node.discv5.is_some(), "discv5 should be running");
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_shared_port_discv5_discovery() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let mut node_1 = start_shared_port_node(0).await;
|
||||
let mut node_2 = start_shared_port_node(0).await;
|
||||
|
||||
let discv5_enr_1 = node_1.discv5.as_ref().unwrap().with_discv5(|discv5| discv5.local_enr());
|
||||
let discv5_enr_2 = node_2.discv5.as_ref().unwrap().with_discv5(|discv5| discv5.local_enr());
|
||||
|
||||
let peer_id_1 = enr_to_discv4_id(&discv5_enr_1).unwrap();
|
||||
let peer_id_2 = enr_to_discv4_id(&discv5_enr_2).unwrap();
|
||||
|
||||
// Add node_2's ENR to node_1's discv5 kbuckets and trigger a ping to establish a session.
|
||||
// send_ping awaits the PONG, so the handshake completes before we poll the Discovery
|
||||
// stream. The discv5 service runs its own background task.
|
||||
node_1.add_discv5_node(EnrCombinedKeyWrapper(discv5_enr_2.clone()).into()).unwrap();
|
||||
node_1
|
||||
.discv5
|
||||
.as_ref()
|
||||
.unwrap()
|
||||
.with_discv5(|discv5| discv5.send_ping(discv5_enr_2))
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
// Both SessionEstablished events should now be buffered in the update channels.
|
||||
// Drive both nodes concurrently to collect them.
|
||||
let mut event_1 = None;
|
||||
let mut event_2 = None;
|
||||
let timeout = tokio::time::sleep(std::time::Duration::from_secs(5));
|
||||
tokio::pin!(timeout);
|
||||
loop {
|
||||
tokio::select! {
|
||||
ev = node_1.next(), if event_1.is_none() => {
|
||||
event_1 = ev;
|
||||
}
|
||||
ev = node_2.next(), if event_2.is_none() => {
|
||||
event_2 = ev;
|
||||
}
|
||||
_ = &mut timeout => {
|
||||
panic!("timed out waiting for discv5 discovery events");
|
||||
}
|
||||
}
|
||||
if event_1.is_some() && event_2.is_some() {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
assert!(matches!(
|
||||
event_1.unwrap(),
|
||||
DiscoveryEvent::NewNode(DiscoveredEvent::EventQueued { peer_id, .. })
|
||||
if peer_id == peer_id_2
|
||||
));
|
||||
assert!(matches!(
|
||||
event_2.unwrap(),
|
||||
DiscoveryEvent::NewNode(DiscoveredEvent::EventQueued { peer_id, .. })
|
||||
if peer_id == peer_id_1
|
||||
));
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_shared_port_discv4_discovery() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let mut node_1 = start_shared_port_node(0).await;
|
||||
let mut node_2 = start_shared_port_node(0).await;
|
||||
|
||||
let enr_1 = node_1.discv4.as_ref().unwrap().node_record();
|
||||
let enr_2 = node_2.discv4.as_ref().unwrap().node_record();
|
||||
|
||||
// Introduce node_2 to node_1 via discv4
|
||||
node_1.add_discv4_node(enr_2);
|
||||
|
||||
// Both nodes should discover each other via discv4 ping/pong
|
||||
let event_1 = node_1.next().await.unwrap();
|
||||
let event_2 = node_2.next().await.unwrap();
|
||||
|
||||
assert_eq!(
|
||||
DiscoveryEvent::NewNode(DiscoveredEvent::EventQueued {
|
||||
peer_id: enr_2.id,
|
||||
addr: PeerAddr::new(enr_2.tcp_addr(), Some(enr_2.udp_addr())),
|
||||
fork_id: None
|
||||
}),
|
||||
event_1
|
||||
);
|
||||
assert_eq!(
|
||||
DiscoveryEvent::NewNode(DiscoveredEvent::EventQueued {
|
||||
peer_id: enr_1.id,
|
||||
addr: PeerAddr::new(enr_1.tcp_addr(), Some(enr_1.udp_addr())),
|
||||
fork_id: None
|
||||
}),
|
||||
event_2
|
||||
);
|
||||
}
|
||||
|
||||
/// Verifies that shared-port mode binds correctly when discv5 is configured for dual-stack.
|
||||
/// On Linux this exercises the IPv6 V6ONLY path: without it, the IPv4 sibling would clash
|
||||
/// with the IPv6 socket bound to the same port.
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_shared_port_dual_stack() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
// Find a port that's free on the v4 wildcard so we can use it for both v4 and v6.
|
||||
let probe = UdpSocket::bind("0.0.0.0:0").await.expect("probe bind");
|
||||
let port = probe.local_addr().unwrap().port();
|
||||
drop(probe);
|
||||
|
||||
let secret_key = SecretKey::new(&mut rand_08::thread_rng());
|
||||
let v4_addr: SocketAddr = format!("0.0.0.0:{port}").parse().unwrap();
|
||||
let tcp_addr: SocketAddr = "0.0.0.0:30303".parse().unwrap();
|
||||
|
||||
let discv4_config = Discv4ConfigBuilder::default().external_ip_resolver(None).build();
|
||||
|
||||
let discv5_listen_config = discv5::ListenConfig::DualStack {
|
||||
ipv4: std::net::Ipv4Addr::UNSPECIFIED,
|
||||
ipv4_port: port,
|
||||
ipv6: std::net::Ipv6Addr::UNSPECIFIED,
|
||||
ipv6_port: port,
|
||||
};
|
||||
let discv5_config = reth_discv5::Config::builder(tcp_addr)
|
||||
.discv5_config(discv5::ConfigBuilder::new(discv5_listen_config).build())
|
||||
.build();
|
||||
|
||||
Discovery::new(
|
||||
tcp_addr,
|
||||
v4_addr,
|
||||
secret_key,
|
||||
Some(discv4_config),
|
||||
Some(discv5_config),
|
||||
None,
|
||||
)
|
||||
.await
|
||||
.expect("discovery should start with shared port + dual-stack");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -18,7 +18,7 @@ use reth_network_api::test_utils::PeersHandle;
|
||||
use reth_network_p2p::error::RequestResult;
|
||||
use reth_network_peers::PeerId;
|
||||
use reth_primitives_traits::Block;
|
||||
use reth_storage_api::{BlockReader, HeaderProvider};
|
||||
use reth_storage_api::{BalProvider, BlockReader, GetBlockAccessListLimit, HeaderProvider};
|
||||
use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
@@ -46,6 +46,11 @@ pub const MAX_HEADERS_SERVE: usize = 1024;
|
||||
/// `SOFT_RESPONSE_LIMIT`.
|
||||
pub const MAX_BODIES_SERVE: usize = 1024;
|
||||
|
||||
/// Maximum number of block access lists to serve.
|
||||
///
|
||||
/// Used to limit lookups.
|
||||
pub const MAX_BLOCK_ACCESS_LISTS_SERVE: usize = 1024;
|
||||
|
||||
/// Maximum size of replies to data retrievals: 2MB
|
||||
pub const SOFT_RESPONSE_LIMIT: usize = 2 * 1024 * 1024;
|
||||
|
||||
@@ -282,27 +287,6 @@ where
|
||||
let _ = response.send(Ok(Receipts70 { last_block_incomplete, receipts }));
|
||||
}
|
||||
|
||||
/// Handles [`GetBlockAccessLists`] queries.
|
||||
///
|
||||
/// EIP-8159 defines the final `BlockAccessLists` response semantics:
|
||||
/// <https://eips.ethereum.org/EIPS/eip-8159>
|
||||
fn on_block_access_lists_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
request: GetBlockAccessLists,
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
) {
|
||||
// TODO: BAL serving is not fully implemented yet. Per EIP-8159, unavailable BALs are
|
||||
// returned as empty BAL entries while preserving request order, so we currently return
|
||||
// one RLP-encoded empty BAL (`0xc0`) per requested hash.
|
||||
let access_lists = request
|
||||
.0
|
||||
.into_iter()
|
||||
.map(|_| Bytes::from_static(&[alloy_rlp::EMPTY_LIST_CODE]))
|
||||
.collect();
|
||||
let _ = response.send(Ok(BlockAccessLists(access_lists)));
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn get_receipts_response<T, F>(&self, request: GetReceipts, transform_fn: F) -> Vec<Vec<T>>
|
||||
where
|
||||
@@ -332,13 +316,57 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<C, N> EthRequestHandler<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BalProvider,
|
||||
{
|
||||
/// Handles [`GetBlockAccessLists`] queries.
|
||||
///
|
||||
/// EIP-8159 defines the final `BlockAccessLists` response semantics:
|
||||
/// <https://eips.ethereum.org/EIPS/eip-8159>
|
||||
fn on_block_access_lists_request(
|
||||
&self,
|
||||
_peer_id: PeerId,
|
||||
mut request: GetBlockAccessLists,
|
||||
response: oneshot::Sender<RequestResult<BlockAccessLists>>,
|
||||
) {
|
||||
request.0.truncate(MAX_BLOCK_ACCESS_LISTS_SERVE);
|
||||
|
||||
let limit = GetBlockAccessListLimit::ResponseSizeSoftLimit(SOFT_RESPONSE_LIMIT);
|
||||
let access_lists = self
|
||||
.client
|
||||
.bal_store()
|
||||
.get_by_hashes_with_limit(&request.0, limit)
|
||||
.unwrap_or_else(|_| empty_block_access_lists_with_limit(request.0.len(), limit));
|
||||
let _ = response.send(Ok(BlockAccessLists(access_lists)));
|
||||
}
|
||||
}
|
||||
|
||||
/// Builds the error fallback response while still enforcing the BAL response soft limit.
|
||||
fn empty_block_access_lists_with_limit(count: usize, limit: GetBlockAccessListLimit) -> Vec<Bytes> {
|
||||
let mut out = Vec::with_capacity(count);
|
||||
let mut size = 0;
|
||||
for _ in 0..count {
|
||||
let bal = Bytes::from_static(&[0xc0]);
|
||||
size += bal.len();
|
||||
out.push(bal);
|
||||
|
||||
if limit.exceeds(size) {
|
||||
break
|
||||
}
|
||||
}
|
||||
out
|
||||
}
|
||||
|
||||
/// An endless future.
|
||||
///
|
||||
/// This should be spawned or used as part of `tokio::select!`.
|
||||
impl<C, N> Future for EthRequestHandler<C, N>
|
||||
where
|
||||
N: NetworkPrimitives,
|
||||
C: BlockReader<Block = N::Block, Receipt = N::Receipt>
|
||||
C: BalProvider
|
||||
+ BlockReader<Block = N::Block, Receipt = N::Receipt>
|
||||
+ HeaderProvider<Header = N::BlockHeader>
|
||||
+ Unpin,
|
||||
{
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user