mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
9 Commits
bal-devnet
...
yk/shared-
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
c67534d872 | ||
|
|
cb7ba92e3d | ||
|
|
78ba601e47 | ||
|
|
ed90ebe0da | ||
|
|
f7b45a8585 | ||
|
|
64e5a6fdab | ||
|
|
9c826df006 | ||
|
|
1d1398bd6a | ||
|
|
d17405f563 |
@@ -1,6 +0,0 @@
|
||||
---
|
||||
reth-trie-common: minor
|
||||
reth-trie: minor
|
||||
---
|
||||
|
||||
Added `contains_range` method to `PrefixSet` for checking if any key falls within a half-open range. Added prefix set support to `ProofCalculator` via `with_prefix_set`, enabling stale cached hash invalidation and branch collapse detection when keys are inserted or removed; propagated storage prefix sets through `SyncAccountValueEncoder`.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-trie-sparse: minor
|
||||
---
|
||||
|
||||
Fixed a bug in `ArenaParallelSparseTrie` where subtrie updates that would completely empty a subtrie were incorrectly dispatched to parallel workers instead of being processed inline, preventing correct branch collapse detection when blinded siblings are present. Refactored the `SparseTrie` test suite to accept a `fn() -> T` factory instead of requiring `T: Default`, enabling a new `arena_parallel_sparse_trie_always_parallel` test variant that exercises all tests with parallelism thresholds set to 1. Added `test_branch_collapse_multi_empty_subtries_blinded_remaining` to cover the case where removing multiple revealed leaves empties their subtries and leaves a single blinded sibling requiring a proof.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Fixed another branch collapse edge case where `check_subtrie_collapse_needs_proof` incorrectly compared removal count against total update count (including `Touched` entries), causing it to skip proof requests for blinded siblings and panic when the subtrie emptied. Added a regression test covering the removals + `Touched` + blinded sibling scenario.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-engine-tree: patch
|
||||
---
|
||||
|
||||
Added idle-time pre-computation of account trie upper hashes in the sparse trie payload processor when no pending proof results are available.
|
||||
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-engine-tree: patch
|
||||
---
|
||||
|
||||
Downgraded per-transaction prewarm span from `debug_span!` to `trace_span!` to reduce noise in debug-level logging.
|
||||
@@ -1,7 +0,0 @@
|
||||
---
|
||||
reth-engine-primitives: patch
|
||||
reth-engine-tree: patch
|
||||
reth-node-core: patch
|
||||
---
|
||||
|
||||
Removed `--engine.enable-arena-sparse-trie` CLI flag and made the arena-based sparse trie the default implementation. The hash-map-based `ParallelSparseTrie` variant is no longer selectable.
|
||||
106
.github/scripts/bench-job-summary.js
vendored
106
.github/scripts/bench-job-summary.js
vendored
@@ -1,106 +0,0 @@
|
||||
// Generates a rich GitHub Actions job summary for reth-bench results.
|
||||
//
|
||||
// Reads from environment:
|
||||
// BENCH_WORK_DIR – Directory containing summary.json
|
||||
// BENCH_PR – PR number (may be empty)
|
||||
// BENCH_ACTOR – GitHub user who triggered the bench
|
||||
// BENCH_CORES – CPU core limit (0 = all)
|
||||
// BENCH_WARMUP_BLOCKS – Number of warmup blocks
|
||||
// BENCH_SAMPLY – 'true' if samply profiling was enabled
|
||||
// BENCH_ABBA – 'true' if ABBA interleaved order was used
|
||||
//
|
||||
// Usage from actions/github-script:
|
||||
// const jobSummary = require('./.github/scripts/bench-job-summary.js');
|
||||
// await jobSummary({ core, context, chartSha, grafanaUrl, runId });
|
||||
|
||||
const fs = require('fs');
|
||||
const { verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
|
||||
|
||||
module.exports = async function ({ core, context, chartSha, grafanaUrl, runId }) {
|
||||
let summary;
|
||||
try {
|
||||
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
|
||||
} catch (e) {
|
||||
await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
|
||||
return;
|
||||
}
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const prNumber = process.env.BENCH_PR;
|
||||
const actor = process.env.BENCH_ACTOR;
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
|
||||
const { emoji, label } = verdict(summary.changes);
|
||||
const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
|
||||
const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
|
||||
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
|
||||
|
||||
// Header & metadata
|
||||
const metaParts = [];
|
||||
if (prNumber) metaParts.push(`**[PR #${prNumber}](https://github.com/${repo}/pull/${prNumber})**`);
|
||||
metaParts.push(`triggered by @${actor}`);
|
||||
|
||||
let md = `# ${emoji} ${label}\n\n`;
|
||||
md += metaParts.join(' · ') + '\n\n';
|
||||
md += `**Baseline:** ${baselineLink}\n`;
|
||||
md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
|
||||
md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
|
||||
|
||||
// Main comparison table
|
||||
const rows = metricRows(summary);
|
||||
md += `| Metric | Baseline | Feature | Change |\n`;
|
||||
md += `|--------|----------|---------|--------|\n`;
|
||||
for (const r of rows) {
|
||||
md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
|
||||
}
|
||||
md += '\n';
|
||||
|
||||
// Wait time breakdown
|
||||
const wtRows = waitTimeRows(summary);
|
||||
if (wtRows.length > 0) {
|
||||
md += `### Wait Time Breakdown\n\n`;
|
||||
md += `| Metric | Baseline | Feature |\n`;
|
||||
md += `|--------|----------|--------|\n`;
|
||||
for (const r of wtRows) {
|
||||
md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
|
||||
}
|
||||
md += '\n';
|
||||
}
|
||||
|
||||
// Charts
|
||||
if (chartSha) {
|
||||
const prNum = prNumber || '0';
|
||||
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`;
|
||||
const charts = [
|
||||
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
|
||||
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
|
||||
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
|
||||
];
|
||||
md += `### Charts\n\n`;
|
||||
for (const chart of charts) {
|
||||
md += `<details><summary>${chart.label}</summary>\n\n`;
|
||||
md += `\n\n`;
|
||||
md += `</details>\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
// Samply profiles
|
||||
const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
|
||||
const samplyLinks = Object.entries(samplyUrls).map(([run, url]) => `- **${run}**: [Firefox Profiler](${url})`);
|
||||
if (samplyLinks.length > 0) {
|
||||
md += `### Samply Profiles\n\n${samplyLinks.join('\n')}\n\n`;
|
||||
}
|
||||
|
||||
// Grafana
|
||||
if (grafanaUrl) {
|
||||
md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
|
||||
}
|
||||
|
||||
// Node errors
|
||||
try {
|
||||
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
|
||||
if (errors.trim()) md += '\n' + errors + '\n';
|
||||
} catch {}
|
||||
|
||||
await core.summary.addRaw(md).write();
|
||||
};
|
||||
45
.github/scripts/bench-reth-run.sh
vendored
45
.github/scripts/bench-reth-run.sh
vendored
@@ -120,18 +120,9 @@ RETH_ARGS=(
|
||||
--no-persist-peers
|
||||
)
|
||||
|
||||
# Gate flag on binary support (older baselines may not have it).
|
||||
# Uses --help which exits immediately via clap without node init.
|
||||
SYNC_STATE_IDLE=false
|
||||
if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then
|
||||
RETH_ARGS+=(--debug.startup-sync-state-idle)
|
||||
SYNC_STATE_IDLE=true
|
||||
fi
|
||||
|
||||
# Big blocks mode requires the testing API, skip-invalid-transactions, and
|
||||
# skip-gas-limit-ramp-check + gas-limit override to avoid the 6800-block ramp.
|
||||
# Big blocks mode requires the testing API and skip-invalid-transactions
|
||||
if [ "$BIG_BLOCKS" = "true" ]; then
|
||||
RETH_ARGS+=(--http.api eth,net,web3,reth,testing --rpc.max-request-size max --testing.skip-invalid-transactions --testing.skip-gas-limit-ramp-check --testing.gas-limit 1000000000)
|
||||
RETH_ARGS+=(--http.api eth,net,web3,reth,testing --testing.skip-invalid-transactions)
|
||||
fi
|
||||
|
||||
# Append per-label extra node args (baseline or feature)
|
||||
@@ -203,7 +194,7 @@ for i in $(seq 1 60); do
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
|
||||
> /dev/null 2>&1; then
|
||||
echo "reth (${LABEL}) RPC is up after ${i}s"
|
||||
echo "reth (${LABEL}) is ready after ${i}s"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 60 ]; then
|
||||
@@ -214,29 +205,6 @@ for i in $(seq 1 60); do
|
||||
sleep 1
|
||||
done
|
||||
|
||||
# Wait for the pipeline to finish (eth_syncing returns false) so the
|
||||
# engine is in live mode and can accept newPayload calls.
|
||||
# Only possible when --debug.startup-sync-state-idle is supported.
|
||||
if [ "$SYNC_STATE_IDLE" = "true" ]; then
|
||||
for i in $(seq 1 300); do
|
||||
SYNC_RESULT=$(curl -sf http://127.0.0.1:8545 -X POST \
|
||||
-H 'Content-Type: application/json' \
|
||||
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' 2>/dev/null || true)
|
||||
if [ -n "$SYNC_RESULT" ] && jq -e '.result == false' <<< "$SYNC_RESULT" > /dev/null 2>&1; then
|
||||
echo "reth (${LABEL}) pipeline finished after ${i}s, engine is live"
|
||||
break
|
||||
fi
|
||||
if [ "$i" -eq 300 ]; then
|
||||
echo "::error::reth (${LABEL}) pipeline did not finish within 300s"
|
||||
cat "$LOG"
|
||||
exit 1
|
||||
fi
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
echo "reth (${LABEL}) binary does not support --debug.startup-sync-state-idle, skipping sync wait"
|
||||
fi
|
||||
|
||||
# Run reth-bench with high priority but as the current user so output
|
||||
# files are not root-owned (avoids EACCES on next checkout).
|
||||
BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
|
||||
@@ -251,8 +219,12 @@ if [ -n "${BENCH_WAIT_TIME:-}" ]; then
|
||||
fi
|
||||
|
||||
if [ "$BIG_BLOCKS" = "true" ]; then
|
||||
# Big blocks mode: replay pre-generated payloads
|
||||
# Big blocks mode: replay pre-generated payloads with gas ramp
|
||||
BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
|
||||
# Count gas ramp blocks for reporting
|
||||
GAS_RAMP_COUNT=$(find "$BIG_BLOCKS_DIR/gas-ramp-dir" -name '*.json' | wc -l)
|
||||
echo "$GAS_RAMP_COUNT" > "$OUTPUT_DIR/gas_ramp_blocks.txt"
|
||||
echo "Gas ramp blocks: $GAS_RAMP_COUNT"
|
||||
|
||||
# Start tracy-capture so profile only covers the benchmark
|
||||
if [ "${BENCH_TRACY:-off}" != "off" ]; then
|
||||
@@ -265,6 +237,7 @@ if [ "$BIG_BLOCKS" = "true" ]; then
|
||||
echo "Running big blocks benchmark (replay-payloads)..."
|
||||
$BENCH_NICE "$RETH_BENCH" replay-payloads \
|
||||
"${EXTRA_BENCH_ARGS[@]}" \
|
||||
--gas-ramp-dir "$BIG_BLOCKS_DIR/gas-ramp-dir" \
|
||||
--payload-dir "$BIG_BLOCKS_DIR/payloads" \
|
||||
--engine-rpc-url http://127.0.0.1:8551 \
|
||||
--jwt-secret "$DATADIR/jwt.hex" \
|
||||
|
||||
129
.github/scripts/bench-reth-snapshot.sh
vendored
129
.github/scripts/bench-reth-snapshot.sh
vendored
@@ -1,17 +1,15 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Downloads the latest snapshot into the schelk volume using
|
||||
# `reth download` with progress reporting to the GitHub PR comment.
|
||||
# Downloads the latest nightly snapshot into the schelk volume with
|
||||
# progress reporting to the GitHub PR comment.
|
||||
#
|
||||
# Skips the download if the manifest content hasn't changed since
|
||||
# the last successful download (checked via SHA-256 of the manifest).
|
||||
# Skips the download if the local ETag marker matches the remote one.
|
||||
#
|
||||
# Usage: bench-reth-snapshot.sh [--check]
|
||||
# --check Only check if a download is needed; exits 0 if up-to-date, 10 if not.
|
||||
# --check Only check if a download is needed; exits 0 if up-to-date, 1 if not.
|
||||
#
|
||||
# Required env:
|
||||
# SCHELK_MOUNT – schelk mount point (e.g. /reth-bench)
|
||||
# BENCH_RETH_BINARY – path to the reth binary
|
||||
# GITHUB_TOKEN – token for GitHub API calls (only for download)
|
||||
# BENCH_COMMENT_ID – PR comment ID to update (optional)
|
||||
# BENCH_REPO – owner/repo (e.g. paradigmxyz/reth)
|
||||
@@ -20,65 +18,52 @@
|
||||
# BENCH_CONFIG – config summary line
|
||||
set -euo pipefail
|
||||
|
||||
MC="mc"
|
||||
BUCKET="minio/reth-snapshots"
|
||||
MANIFEST_PATH="reth-1-minimal-stable/manifest.json"
|
||||
BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous.tar.zst"
|
||||
DATADIR="$SCHELK_MOUNT/datadir"
|
||||
HASH_FILE="$HOME/.reth-bench-snapshot-hash"
|
||||
ETAG_FILE="$HOME/.reth-bench-snapshot-etag"
|
||||
|
||||
# Fetch manifest and compute content hash for reliable freshness check
|
||||
MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
|
||||
echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
|
||||
exit 2
|
||||
}
|
||||
REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
|
||||
# Get remote metadata via JSON for reliable parsing
|
||||
MC_STAT=$(mc stat --json "$BUCKET" 2>/dev/null || true)
|
||||
REMOTE_ETAG=$(echo "$MC_STAT" | jq -r '.etag // empty')
|
||||
if [ -z "$REMOTE_ETAG" ]; then
|
||||
echo "::warning::Failed to get ETag from mc stat, will re-download"
|
||||
REMOTE_ETAG="unknown-$(date +%s)"
|
||||
fi
|
||||
|
||||
LOCAL_HASH=""
|
||||
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
|
||||
LOCAL_ETAG=""
|
||||
[ -f "$ETAG_FILE" ] && LOCAL_ETAG=$(cat "$ETAG_FILE")
|
||||
|
||||
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
|
||||
echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
if [ "$REMOTE_ETAG" = "$LOCAL_ETAG" ]; then
|
||||
echo "Snapshot is up-to-date (ETag: ${REMOTE_ETAG})"
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 0
|
||||
fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
|
||||
echo "Snapshot needs update (local: ${LOCAL_ETAG:-<none>}, remote: ${REMOTE_ETAG})"
|
||||
if [ "${1:-}" = "--check" ]; then
|
||||
exit 10
|
||||
fi
|
||||
|
||||
RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
|
||||
if [ ! -x "$RETH" ]; then
|
||||
echo "::error::reth binary not found or not executable at $RETH"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Resolve the MinIO HTTP endpoint from the mc alias so reth can
|
||||
# fetch archives over HTTP (the manifest's embedded base_url points
|
||||
# to the cluster-internal address which is unreachable from runners).
|
||||
MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
|
||||
if [ -z "$MINIO_ENDPOINT" ]; then
|
||||
echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
|
||||
# Get compressed size for progress tracking
|
||||
TOTAL_BYTES=$(echo "$MC_STAT" | jq -r '.size // empty')
|
||||
if [ -z "$TOTAL_BYTES" ] || [ "$TOTAL_BYTES" = "0" ]; then
|
||||
echo "::error::Failed to get snapshot size from mc stat"
|
||||
exit 1
|
||||
fi
|
||||
BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/reth-1-minimal-stable"
|
||||
|
||||
# Rewrite manifest's base_url with the runner-reachable endpoint
|
||||
MANIFEST_TMP=$(mktemp --suffix=.json)
|
||||
trap 'rm -f -- "$MANIFEST_TMP"' EXIT
|
||||
echo "$MANIFEST_CONTENT" \
|
||||
| jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
|
||||
echo "Snapshot size: $TOTAL_BYTES bytes ($(numfmt --to=iec "$TOTAL_BYTES"))"
|
||||
|
||||
# Prepare mount
|
||||
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
|
||||
sudo schelk mount -y
|
||||
sudo rm -rf "$DATADIR"
|
||||
sudo mkdir -p "$DATADIR"
|
||||
# reth download runs as current user (not root), needs write access
|
||||
sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
|
||||
|
||||
update_comment() {
|
||||
local status="$1"
|
||||
local pct="$1"
|
||||
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
|
||||
local status="Building binaries & downloading snapshot… ${pct}%"
|
||||
local body
|
||||
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
|
||||
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
|
||||
@@ -90,31 +75,53 @@ update_comment() {
|
||||
> /dev/null 2>&1 || true
|
||||
}
|
||||
|
||||
update_comment "Downloading snapshot…"
|
||||
# Track compressed bytes flowing through the pipe
|
||||
DL_BYTES_FILE=$(mktemp)
|
||||
echo 0 > "$DL_BYTES_FILE"
|
||||
|
||||
# Download using reth download (manifest-path with rewritten base_url)
|
||||
"$RETH" download \
|
||||
--manifest-path "$MANIFEST_TMP" \
|
||||
-y \
|
||||
--minimal \
|
||||
--datadir "$DATADIR"
|
||||
# Start progress reporter in background
|
||||
(
|
||||
while true; do
|
||||
sleep 10
|
||||
CURRENT=$(cat "$DL_BYTES_FILE" 2>/dev/null || echo 0)
|
||||
if [ "$TOTAL_BYTES" -gt 0 ]; then
|
||||
PCT=$(( CURRENT * 100 / TOTAL_BYTES ))
|
||||
[ "$PCT" -gt 100 ] && PCT=100
|
||||
echo "Snapshot download: $(numfmt --to=iec "$CURRENT") / $(numfmt --to=iec "$TOTAL_BYTES") (${PCT}%)"
|
||||
update_comment "$PCT"
|
||||
fi
|
||||
done
|
||||
) &
|
||||
PROGRESS_PID=$!
|
||||
trap 'kill $PROGRESS_PID 2>/dev/null || true; rm -f "$DL_BYTES_FILE"' EXIT
|
||||
|
||||
update_comment "Downloading snapshot… done"
|
||||
# Download and extract; python byte counter tracks compressed bytes received
|
||||
mc cat "$BUCKET" | python3 -c "
|
||||
import sys
|
||||
count = 0
|
||||
while True:
|
||||
data = sys.stdin.buffer.read(1048576)
|
||||
if not data:
|
||||
break
|
||||
count += len(data)
|
||||
sys.stdout.buffer.write(data)
|
||||
with open('$DL_BYTES_FILE', 'w') as f:
|
||||
f.write(str(count))
|
||||
" | pzstd -d -p 6 | sudo tar -xf - -C "$DATADIR"
|
||||
|
||||
# Stop progress reporter
|
||||
kill $PROGRESS_PID 2>/dev/null || true
|
||||
wait $PROGRESS_PID 2>/dev/null || true
|
||||
|
||||
update_comment "100"
|
||||
echo "Snapshot download complete"
|
||||
|
||||
# Sanity check: verify expected directories exist
|
||||
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
|
||||
echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
|
||||
ls -la "$DATADIR" || true
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Promote the new snapshot to become the schelk baseline (virgin volume).
|
||||
# This copies changed blocks from scratch → virgin so that future
|
||||
# `schelk recover` calls restore to this new state.
|
||||
sync
|
||||
sudo schelk promote -y
|
||||
|
||||
# Save manifest hash
|
||||
echo "$REMOTE_HASH" > "$HASH_FILE"
|
||||
echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"
|
||||
# Save ETag marker
|
||||
echo "$REMOTE_ETAG" > "$ETAG_FILE"
|
||||
echo "Snapshot promoted to schelk baseline (ETag: ${REMOTE_ETAG})"
|
||||
|
||||
2
.github/scripts/bench-reth-summary.py
vendored
2
.github/scripts/bench-reth-summary.py
vendored
@@ -472,6 +472,7 @@ def main():
|
||||
parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
|
||||
parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
|
||||
parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode")
|
||||
parser.add_argument("--gas-ramp-blocks", type=int, default=0, help="Number of gas ramp blocks (big blocks mode)")
|
||||
parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL for this benchmark run")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -553,6 +554,7 @@ def main():
|
||||
summary = {
|
||||
"blocks": paired_stats["blocks"],
|
||||
"big_blocks": args.big_blocks,
|
||||
"gas_ramp_blocks": args.gas_ramp_blocks,
|
||||
"baseline": {
|
||||
"name": baseline_name,
|
||||
"ref": baseline_ref,
|
||||
|
||||
132
.github/scripts/bench-scheduled-refs.sh
vendored
132
.github/scripts/bench-scheduled-refs.sh
vendored
@@ -1,132 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Resolves baseline and feature refs for nightly regression benchmark runs.
|
||||
#
|
||||
# Queries the latest successful scheduled docker.yml run via GitHub API
|
||||
# to find the commit that built the nightly Docker image. Compares with
|
||||
# the last successful feature ref (from GH Actions cache) to determine
|
||||
# baseline, detect staleness, and decide whether to skip.
|
||||
#
|
||||
# Usage: bench-nightly-refs.sh [--force]
|
||||
#
|
||||
# Outputs (via GITHUB_OUTPUT):
|
||||
# baseline-ref — commit SHA for baseline
|
||||
# feature-ref — commit SHA for feature (current nightly)
|
||||
# should-skip — "true" if no new nightly since last run
|
||||
# is-stale — "true" if latest nightly build is >24h old
|
||||
# stale-age-hours — age of the nightly build in hours (only if stale)
|
||||
# nightly-created — ISO timestamp of the nightly build
|
||||
#
|
||||
# Reads:
|
||||
# .nightly-state/last-feature-ref (from GH Actions cache, may not exist)
|
||||
#
|
||||
# Requires: gh (GitHub CLI), jq, date
|
||||
set -euo pipefail
|
||||
|
||||
FORCE="${1:-false}"
|
||||
REPO="${GITHUB_REPOSITORY:-paradigmxyz/reth}"
|
||||
|
||||
# --- Step 1: Query latest successful scheduled docker.yml run ---
|
||||
echo "::group::Querying latest nightly docker build"
|
||||
|
||||
RUNS_JSON=$(gh run list \
|
||||
-R "$REPO" \
|
||||
--workflow=docker.yml \
|
||||
--event=schedule \
|
||||
--status=completed \
|
||||
--limit 5 \
|
||||
--json headSha,createdAt,conclusion)
|
||||
|
||||
# Find the most recent successful run
|
||||
LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
|
||||
|
||||
if [ -z "$LATEST" ]; then
|
||||
echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
|
||||
echo "Runs found: $RUNS_JSON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
|
||||
CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt')
|
||||
echo "Latest nightly commit: $FEATURE_REF"
|
||||
echo "Built at: $CREATED_AT"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 2: Staleness check ---
|
||||
echo "::group::Checking staleness"
|
||||
NOW_EPOCH=$(date +%s)
|
||||
# Handle both GNU date (-d) and BSD date (-j -f) for cross-platform compat
|
||||
CREATED_EPOCH=$(date -d "$CREATED_AT" +%s 2>/dev/null || \
|
||||
date -j -f "%Y-%m-%dT%H:%M:%SZ" "$CREATED_AT" +%s 2>/dev/null || \
|
||||
date -j -f "%Y-%m-%dT%T%z" "$CREATED_AT" +%s 2>/dev/null || \
|
||||
{ echo "::error::Cannot parse date: $CREATED_AT"; exit 1; })
|
||||
|
||||
AGE_SECONDS=$(( NOW_EPOCH - CREATED_EPOCH ))
|
||||
AGE_HOURS=$(( AGE_SECONDS / 3600 ))
|
||||
IS_STALE="false"
|
||||
|
||||
if [ "$AGE_HOURS" -gt 24 ]; then
|
||||
IS_STALE="true"
|
||||
echo "::warning::STALE NIGHTLY: Build is ${AGE_HOURS}h old (>24h threshold)"
|
||||
echo "This indicates the nightly docker build failed — no new image was produced"
|
||||
else
|
||||
echo "Nightly build age: ${AGE_HOURS}h (within 24h threshold)"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 3: Read last successful feature ref from cache ---
|
||||
echo "::group::Reading cached state"
|
||||
LAST_FEATURE_REF=""
|
||||
STATE_FILE=".nightly-state/last-feature-ref"
|
||||
if [ -f "$STATE_FILE" ]; then
|
||||
LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
|
||||
echo "Previous feature ref: $LAST_FEATURE_REF"
|
||||
else
|
||||
echo "No cached state found (first run)"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 4: Determine baseline and skip logic ---
|
||||
echo "::group::Resolving refs"
|
||||
SHOULD_SKIP="false"
|
||||
BASELINE_REF="$FEATURE_REF" # default for first run
|
||||
|
||||
if [ "$IS_STALE" = "true" ]; then
|
||||
# Stale = error path, don't skip (will alert and fail downstream)
|
||||
SHOULD_SKIP="false"
|
||||
BASELINE_REF="${LAST_FEATURE_REF:-$FEATURE_REF}"
|
||||
echo "Stale nightly detected — will alert and fail"
|
||||
elif [ -z "$LAST_FEATURE_REF" ]; then
|
||||
# First run: baseline = feature (self-comparison to establish baseline)
|
||||
BASELINE_REF="$FEATURE_REF"
|
||||
echo "First run — will benchmark nightly against itself to establish baseline"
|
||||
elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
|
||||
# No new nightly since last successful run
|
||||
if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
|
||||
echo "No new nightly, but force=true — running anyway"
|
||||
BASELINE_REF="$LAST_FEATURE_REF"
|
||||
else
|
||||
SHOULD_SKIP="true"
|
||||
echo "No new nightly since last run — will skip"
|
||||
fi
|
||||
else
|
||||
# Normal case: new nightly available
|
||||
BASELINE_REF="$LAST_FEATURE_REF"
|
||||
echo "New nightly detected"
|
||||
fi
|
||||
|
||||
echo "Baseline: $BASELINE_REF"
|
||||
echo "Feature: $FEATURE_REF"
|
||||
echo "Skip: $SHOULD_SKIP"
|
||||
echo "Stale: $IS_STALE"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 5: Write outputs ---
|
||||
{
|
||||
echo "baseline-ref=$BASELINE_REF"
|
||||
echo "feature-ref=$FEATURE_REF"
|
||||
echo "should-skip=$SHOULD_SKIP"
|
||||
echo "is-stale=$IS_STALE"
|
||||
echo "stale-age-hours=$AGE_HOURS"
|
||||
echo "nightly-created=$CREATED_AT"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
104
.github/scripts/bench-slack-notify.js
vendored
104
.github/scripts/bench-slack-notify.js
vendored
@@ -18,7 +18,6 @@
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { fmtChange, fmtMs, verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
|
||||
|
||||
const SLACK_API = 'https://slack.com/api/chat.postMessage';
|
||||
|
||||
@@ -62,17 +61,41 @@ function cell(text) {
|
||||
return { type: 'raw_text', text: s || ' ' };
|
||||
}
|
||||
|
||||
// Slack shortcodes for verdict (Block Kit header doesn't support unicode emoji)
|
||||
const SLACK_VERDICT = {
|
||||
'⚠️': ':warning:',
|
||||
'❌': ':x:',
|
||||
'✅': ':white_check_mark:',
|
||||
'⚪': ':white_circle:',
|
||||
};
|
||||
|
||||
function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls }) {
|
||||
const { emoji, label } = verdict(summary.changes);
|
||||
const headerEmoji = SLACK_VERDICT[emoji] || emoji;
|
||||
const b = summary.baseline.stats;
|
||||
const f = summary.feature.stats;
|
||||
const c = summary.changes;
|
||||
|
||||
const sigEmoji = { good: '\u2705', bad: '\u274c', neutral: '\u26aa' };
|
||||
|
||||
function fmtMs(v) { return v.toFixed(2) + 'ms'; }
|
||||
function fmtMgas(v) { return v.toFixed(2); }
|
||||
function fmtS(v) { return v.toFixed(2) + 's'; }
|
||||
function fmtChange(ch) {
|
||||
if (!ch.pct && !ch.ci_pct) return ' ';
|
||||
const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
|
||||
const ciStr = ch.ci_pct ? ` (\u00b1${ch.ci_pct.toFixed(2)}%)` : '';
|
||||
return `${pctStr}${ciStr} ${sigEmoji[ch.sig]}`;
|
||||
}
|
||||
|
||||
// Overall result for header
|
||||
const vals = Object.values(c);
|
||||
const hasBad = vals.some(v => v.sig === 'bad');
|
||||
const hasGood = vals.some(v => v.sig === 'good');
|
||||
let headerEmoji, headerResult;
|
||||
if (hasBad && hasGood) {
|
||||
headerEmoji = ':warning:';
|
||||
headerResult = 'Mixed Results';
|
||||
} else if (hasBad) {
|
||||
headerEmoji = ':x:';
|
||||
headerResult = 'Regression';
|
||||
} else if (hasGood) {
|
||||
headerEmoji = ':white_check_mark:';
|
||||
headerResult = 'Improvement';
|
||||
} else {
|
||||
headerEmoji = ':white_circle:';
|
||||
headerResult = 'No Difference';
|
||||
}
|
||||
|
||||
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
@@ -97,7 +120,19 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
|
||||
if (fl1) featureLine += ` | <${fl1}|Samply 1>`;
|
||||
if (fl2) featureLine += ` | <${fl2}|Samply 2>`;
|
||||
|
||||
const countsLine = blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | ');
|
||||
const cores = process.env.BENCH_CORES || '0';
|
||||
const countsParts = [];
|
||||
if (summary.big_blocks) {
|
||||
const gasRamp = summary.gas_ramp_blocks || 0;
|
||||
if (gasRamp > 0) countsParts.push(`*Gas Ramp:* ${gasRamp}`);
|
||||
countsParts.push(`*Big Blocks:* ${summary.blocks}`);
|
||||
} else {
|
||||
const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
|
||||
if (warmup) countsParts.push(`*Warmup:* ${warmup}`);
|
||||
countsParts.push(`*Blocks:* ${summary.blocks}`);
|
||||
}
|
||||
if (cores !== '0') countsParts.push(`*Cores:* ${cores}`);
|
||||
const countsLine = countsParts.join(' | ');
|
||||
|
||||
const baselineArgs = process.env.BENCH_BASELINE_ARGS || '';
|
||||
const featureArgs = process.env.BENCH_FEATURE_ARGS || '';
|
||||
@@ -124,17 +159,10 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
|
||||
},
|
||||
];
|
||||
|
||||
// Build table rows from shared metricRows
|
||||
const rows = metricRows(summary);
|
||||
const tableRows = [
|
||||
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
|
||||
...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
|
||||
];
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: `${headerEmoji} ${label}`, emoji: true },
|
||||
text: { type: 'plain_text', text: `${headerEmoji} ${headerResult}`, emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
@@ -148,7 +176,16 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
|
||||
{ align: 'right' },
|
||||
{ align: 'right' },
|
||||
],
|
||||
rows: tableRows,
|
||||
rows: [
|
||||
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
|
||||
[cell('Mean'), cell(fmtMs(b.mean_ms)), cell(fmtMs(f.mean_ms)), cell(fmtChange(c.mean))],
|
||||
[cell('StdDev'), cell(fmtMs(b.stddev_ms)), cell(fmtMs(f.stddev_ms)), cell(' ')],
|
||||
[cell('P50'), cell(fmtMs(b.p50_ms)), cell(fmtMs(f.p50_ms)), cell(fmtChange(c.p50))],
|
||||
[cell('P90'), cell(fmtMs(b.p90_ms)), cell(fmtMs(f.p90_ms)), cell(fmtChange(c.p90))],
|
||||
[cell('P99'), cell(fmtMs(b.p99_ms)), cell(fmtMs(f.p99_ms)), cell(fmtChange(c.p99))],
|
||||
[cell('Mgas/s'), cell(fmtMgas(b.mean_mgas_s)), cell(fmtMgas(f.mean_mgas_s)), cell(fmtChange(c.mgas_s))],
|
||||
[cell('Wall Clock'), cell(fmtS(b.wall_clock_s)), cell(fmtS(f.wall_clock_s)), cell(fmtChange(c.wall_clock))],
|
||||
],
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
@@ -158,12 +195,16 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
|
||||
|
||||
// Wait times as a separate table block (sent as threaded reply due to Slack one-table limit)
|
||||
const threadBlocks = [];
|
||||
const wtRows = waitTimeRows(summary);
|
||||
if (wtRows.length > 0) {
|
||||
const waitTableRows = [
|
||||
const waitTimes = summary.wait_times || {};
|
||||
const waitKeys = Object.keys(waitTimes);
|
||||
if (waitKeys.length > 0) {
|
||||
const waitRows = [
|
||||
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
|
||||
...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
|
||||
];
|
||||
for (const key of waitKeys) {
|
||||
const wt = waitTimes[key];
|
||||
waitRows.push([cell(wt.title), cell(fmtMs(wt.baseline.mean_ms)), cell(fmtMs(wt.feature.mean_ms))]);
|
||||
}
|
||||
threadBlocks.push({
|
||||
type: 'table',
|
||||
column_settings: [
|
||||
@@ -171,7 +212,7 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
|
||||
{ align: 'right' },
|
||||
{ align: 'right' },
|
||||
],
|
||||
rows: waitTableRows,
|
||||
rows: waitRows,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -233,7 +274,16 @@ async function success({ core, context }) {
|
||||
const jobUrl = process.env.BENCH_JOB_URL ||
|
||||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
|
||||
// Load samply profile URLs (files exist when samply profiling was enabled)
|
||||
const samplyUrls = {};
|
||||
for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
|
||||
try {
|
||||
const url = fs.readFileSync(
|
||||
path.join(process.env.BENCH_WORK_DIR, run, 'samply-profile-url.txt'), 'utf8'
|
||||
).trim();
|
||||
if (url) samplyUrls[run] = url;
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
|
||||
const actorSlackId = slackUsers[actor];
|
||||
|
||||
95
.github/scripts/bench-utils.js
vendored
95
.github/scripts/bench-utils.js
vendored
@@ -1,95 +0,0 @@
|
||||
// Shared utilities for reth-bench result rendering.
|
||||
//
|
||||
// Used by bench-job-summary.js and bench-slack-notify.js.
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const SIG_EMOJI = { good: '✅', bad: '❌', neutral: '⚪' };
|
||||
|
||||
function fmtMs(v) { return v.toFixed(2) + 'ms'; }
|
||||
function fmtMgas(v) { return v.toFixed(2); }
|
||||
function fmtS(v) { return v.toFixed(2) + 's'; }
|
||||
|
||||
function fmtChange(ch) {
|
||||
if (!ch || (!ch.pct && !ch.ci_pct)) return '';
|
||||
const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
|
||||
const ciStr = ch.ci_pct ? ` (±${ch.ci_pct.toFixed(2)}%)` : '';
|
||||
return `${pctStr}${ciStr} ${SIG_EMOJI[ch.sig]}`;
|
||||
}
|
||||
|
||||
function verdict(changes) {
|
||||
const vals = Object.values(changes);
|
||||
const hasBad = vals.some(v => v.sig === 'bad');
|
||||
const hasGood = vals.some(v => v.sig === 'good');
|
||||
if (hasBad && hasGood) return { emoji: '⚠️', label: 'Mixed Results' };
|
||||
if (hasBad) return { emoji: '❌', label: 'Regression' };
|
||||
if (hasGood) return { emoji: '✅', label: 'Improvement' };
|
||||
return { emoji: '⚪', label: 'No Difference' };
|
||||
}
|
||||
|
||||
function loadSamplyUrls(workDir) {
|
||||
const urls = {};
|
||||
for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
|
||||
try {
|
||||
const url = fs.readFileSync(path.join(workDir, run, 'samply-profile-url.txt'), 'utf8').trim();
|
||||
if (url) urls[run] = url;
|
||||
} catch {}
|
||||
}
|
||||
return urls;
|
||||
}
|
||||
|
||||
function blocksLabel(summary) {
|
||||
const parts = [];
|
||||
if (summary.big_blocks) {
|
||||
parts.push({ key: 'Big Blocks', value: summary.blocks });
|
||||
} else {
|
||||
const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
|
||||
if (warmup) parts.push({ key: 'Warmup', value: warmup });
|
||||
parts.push({ key: 'Blocks', value: summary.blocks });
|
||||
}
|
||||
const cores = process.env.BENCH_CORES || '0';
|
||||
if (cores !== '0') parts.push({ key: 'Cores', value: cores });
|
||||
return parts;
|
||||
}
|
||||
|
||||
// The 7 metric rows shared by all renderers.
|
||||
// Returns an array of { label, baseline, feature, change } objects.
|
||||
function metricRows(summary) {
|
||||
const b = summary.baseline.stats;
|
||||
const f = summary.feature.stats;
|
||||
const c = summary.changes;
|
||||
return [
|
||||
{ label: 'Mean', baseline: fmtMs(b.mean_ms), feature: fmtMs(f.mean_ms), change: fmtChange(c.mean) },
|
||||
{ label: 'StdDev', baseline: fmtMs(b.stddev_ms), feature: fmtMs(f.stddev_ms), change: '' },
|
||||
{ label: 'P50', baseline: fmtMs(b.p50_ms), feature: fmtMs(f.p50_ms), change: fmtChange(c.p50) },
|
||||
{ label: 'P90', baseline: fmtMs(b.p90_ms), feature: fmtMs(f.p90_ms), change: fmtChange(c.p90) },
|
||||
{ label: 'P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) },
|
||||
{ label: 'Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
|
||||
{ label: 'Wall Clock', baseline: fmtS(b.wall_clock_s), feature: fmtS(f.wall_clock_s), change: fmtChange(c.wall_clock) },
|
||||
];
|
||||
}
|
||||
|
||||
// Wait time rows: one row per metric showing mean values.
|
||||
function waitTimeRows(summary) {
|
||||
const waitTimes = summary.wait_times || {};
|
||||
const rows = [];
|
||||
for (const key of Object.keys(waitTimes)) {
|
||||
const wt = waitTimes[key];
|
||||
rows.push({ title: wt.title, baseline: fmtMs(wt.baseline.mean_ms), feature: fmtMs(wt.feature.mean_ms) });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
SIG_EMOJI,
|
||||
fmtMs,
|
||||
fmtMgas,
|
||||
fmtS,
|
||||
fmtChange,
|
||||
verdict,
|
||||
loadSamplyUrls,
|
||||
blocksLabel,
|
||||
metricRows,
|
||||
waitTimeRows,
|
||||
};
|
||||
2
.github/scripts/hive/Dockerfile
vendored
2
.github/scripts/hive/Dockerfile
vendored
@@ -3,7 +3,7 @@
|
||||
#
|
||||
# We'll use cargo-chef to speed up the build
|
||||
#
|
||||
FROM lukemathwalker/cargo-chef:latest-rust-1.94.0-trixie AS chef
|
||||
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
|
||||
WORKDIR /app
|
||||
|
||||
# Install system dependencies
|
||||
|
||||
11
.github/scripts/hive/build_simulators.sh
vendored
11
.github/scripts/hive/build_simulators.sh
vendored
@@ -9,17 +9,10 @@ go build .
|
||||
|
||||
./hive -client reth # first builds and caches the client
|
||||
|
||||
# Run each hive command in the background for each simulator and wait
|
||||
# Run each hive command in the background for each simulator and wait
|
||||
echo "Building images"
|
||||
# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
|
||||
./hive -client reth --sim "ethereum/eels/consume-engine" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
|
||||
--sim.buildarg branch=err-map-3 \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/eels/consume-rlp" \
|
||||
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
|
||||
--sim.buildarg branch=err-map-3 \
|
||||
--sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
|
||||
./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &
|
||||
|
||||
82
.github/scripts/hive/expected_failures.yaml
vendored
82
.github/scripts/hive/expected_failures.yaml
vendored
@@ -16,19 +16,15 @@ rpc-compat:
|
||||
# syncing mode, the test expects syncing to be false on start
|
||||
- eth_syncing/check-syncing (reth)
|
||||
|
||||
engine-withdrawals: []
|
||||
engine-withdrawals: [ ]
|
||||
|
||||
engine-api: []
|
||||
engine-api: [ ]
|
||||
|
||||
engine-cancun:
|
||||
- Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
|
||||
# the test fails with older versions of the code for which it passed before, probably related to changes
|
||||
# in hive or its dependencies
|
||||
- Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
|
||||
engine-cancun: [ ]
|
||||
|
||||
sync: []
|
||||
sync: [ ]
|
||||
|
||||
engine-auth: []
|
||||
engine-auth: [ ]
|
||||
|
||||
# EIP-7610 related tests (Revert creation in case of non-empty storage):
|
||||
#
|
||||
@@ -46,10 +42,6 @@ engine-auth: []
|
||||
#
|
||||
# System contract tests (already fixed and deployed):
|
||||
#
|
||||
# tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout and test_invalid_log_length
|
||||
# System contract is already fixed and deployed; tests cover scenarios where contract is
|
||||
# malformed which can't happen retroactively. No point in adding checks.
|
||||
#
|
||||
# tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment
|
||||
# tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment
|
||||
# Post-fork system contract deployment tests. Should fix for spec compliance but not realistic
|
||||
@@ -58,44 +50,8 @@ eels/consume-engine:
|
||||
- tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test_engine-zero_nonce]-reth
|
||||
- tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth
|
||||
- tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
|
||||
- tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth
|
||||
- tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_False]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_True]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_True]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_False]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
|
||||
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
@@ -146,20 +102,6 @@ eels/consume-engine:
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_engine_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-empty-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
|
||||
|
||||
# Blob limit tests:
|
||||
#
|
||||
@@ -254,17 +196,3 @@ eels/consume-rlp:
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-initcode-with-deploy]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-sstore-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_from_state_test]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
|
||||
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-empty-initcode]-reth
|
||||
|
||||
8
.github/scripts/hive/run_simulator.sh
vendored
8
.github/scripts/hive/run_simulator.sh
vendored
@@ -13,13 +13,7 @@ if [[ "${sim}" == *"eels"* ]]; then
|
||||
fi
|
||||
|
||||
run_hive() {
|
||||
hive \
|
||||
--sim "${sim}" \
|
||||
--sim.limit "${limit}" \
|
||||
--sim.limit.exact=false \
|
||||
--sim.parallelism "${parallelism}" \
|
||||
--client reth \
|
||||
2>&1 | tee /tmp/log || true
|
||||
hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism "${parallelism}" --client reth 2>&1 | tee /tmp/log || true
|
||||
}
|
||||
|
||||
check_log() {
|
||||
|
||||
872
.github/workflows/bench-scheduled.yml
vendored
872
.github/workflows/bench-scheduled.yml
vendored
@@ -1,872 +0,0 @@
|
||||
# Nightly regression benchmark.
|
||||
#
|
||||
# Compares the previous nightly build against the current nightly build to
|
||||
# detect performance regressions. Runs daily after docker.yml produces a new
|
||||
# nightly image at 01:00 UTC.
|
||||
#
|
||||
# State is persisted between runs via GitHub Actions cache: each successful
|
||||
# run saves the feature commit SHA so the next run knows what to compare against.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 5 * * *" # 06:30 UTC daily
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force:
|
||||
description: "Force run even if no new nightly (bypass skip logic)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
no_slack:
|
||||
description: "Suppress Slack notifications"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
name: bench-scheduled
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job 1: Resolve nightly refs, check staleness, manage state
|
||||
# ---------------------------------------------------------------------------
|
||||
resolve-refs:
|
||||
name: resolve-refs
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
|
||||
feature-ref: ${{ steps.refs.outputs.feature-ref }}
|
||||
should-skip: ${{ steps.refs.outputs.should-skip }}
|
||||
is-stale: ${{ steps.refs.outputs.is-stale }}
|
||||
stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
|
||||
nightly-created: ${{ steps.refs.outputs.nightly-created }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
sparse-checkout-cone-mode: true
|
||||
|
||||
- name: Restore nightly state
|
||||
id: state-cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: .nightly-state
|
||||
key: bench-scheduled-state-dummy
|
||||
restore-keys: |
|
||||
bench-scheduled-state-
|
||||
|
||||
- name: Resolve nightly refs
|
||||
id: refs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
FORCE="${{ inputs.force || 'false' }}"
|
||||
.github/scripts/bench-scheduled-refs.sh "$FORCE"
|
||||
|
||||
- name: Alert on stale nightly
|
||||
if: steps.refs.outputs.is-stale == 'true'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
if (!token || !channel) {
|
||||
core.warning('Slack credentials not set, skipping stale nightly alert');
|
||||
return;
|
||||
}
|
||||
|
||||
const ageHours = '${{ steps.refs.outputs.stale-age-hours }}';
|
||||
const created = '${{ steps.refs.outputs.nightly-created }}';
|
||||
const featureRef = '${{ steps.refs.outputs.feature-ref }}';
|
||||
const shortSha = featureRef.slice(0, 8);
|
||||
const repo = '${{ github.repository }}';
|
||||
const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: ':rotating_light: Nightly Regression: nightly build is stale', emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: [
|
||||
'*Nightly regression did not run* — nightly build is stale',
|
||||
'',
|
||||
`The latest nightly image was built from a commit that is *${ageHours}h old* (threshold: 24h).`,
|
||||
`This means today's nightly docker build likely failed and no new image was produced.`,
|
||||
'',
|
||||
`Stale commit: \`${shortSha}\` (built at ${created})`,
|
||||
'',
|
||||
'*Action required:* Check the <https://github.com/' + repo + '/actions/workflows/docker.yml|docker.yml> workflow for failures.',
|
||||
].join('\n'),
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: [
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
|
||||
url: runUrl,
|
||||
action_id: 'ci_button',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const resp = await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel,
|
||||
blocks,
|
||||
text: 'Nightly regression: nightly build is stale',
|
||||
unfurl_links: false,
|
||||
}),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.ok) {
|
||||
core.warning(`Slack API error: ${JSON.stringify(data)}`);
|
||||
}
|
||||
|
||||
- name: Fail on stale nightly
|
||||
if: steps.refs.outputs.is-stale == 'true'
|
||||
run: |
|
||||
echo "::error::Nightly build is stale (>24h old). Aborting."
|
||||
exit 1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job 2: Run the benchmark
|
||||
# ---------------------------------------------------------------------------
|
||||
bench-scheduled:
|
||||
needs: resolve-refs
|
||||
if: |
|
||||
needs.resolve-refs.outputs.should-skip != 'true' &&
|
||||
needs.resolve-refs.outputs.is-stale != 'true'
|
||||
name: bench-scheduled
|
||||
runs-on: [self-hosted, Linux, X64, available]
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
SCHELK_MOUNT: /reth-bench
|
||||
BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
|
||||
BENCH_PR: ""
|
||||
BENCH_ACTOR: "nightly-regression"
|
||||
BENCH_BLOCKS: "2000"
|
||||
BENCH_WARMUP_BLOCKS: "500"
|
||||
BENCH_SAMPLY: "false"
|
||||
BENCH_CORES: "0"
|
||||
BENCH_BIG_BLOCKS: "false"
|
||||
BENCH_RETH_NEW_PAYLOAD: "true"
|
||||
BENCH_WAIT_TIME: ""
|
||||
BENCH_BASELINE_ARGS: ""
|
||||
BENCH_FEATURE_ARGS: ""
|
||||
BENCH_ABBA: "true"
|
||||
BENCH_COMMENT_ID: ""
|
||||
BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
|
||||
BENCH_METRICS_ADDR: "127.0.0.1:9100"
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
|
||||
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
|
||||
BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
|
||||
FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
steps:
|
||||
- name: Clean up previous bench-work
|
||||
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
|
||||
- name: Resolve job URL
|
||||
id: job-url
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
});
|
||||
const job = jobs.jobs.find(j => j.name === 'bench-scheduled');
|
||||
const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
core.exportVariable('BENCH_JOB_URL', jobUrl);
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
continue-on-error: true
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
run: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
|
||||
# apt packages
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
python3 make jq zstd curl dmsetup \
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
fi
|
||||
|
||||
# Configure git auth for private repos
|
||||
git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
|
||||
# thin-provisioning-tools (era_invalidate, required by schelk)
|
||||
if ! command -v era_invalidate &>/dev/null; then
|
||||
git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
|
||||
sudo make -C /tmp/tpt install
|
||||
rm -rf /tmp/tpt
|
||||
fi
|
||||
|
||||
# schelk (snapshot rollback tool, invoked via sudo)
|
||||
if ! sudo sh -c 'command -v schelk' &>/dev/null; then
|
||||
cargo install --git https://github.com/tempoxyz/schelk --locked
|
||||
sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
|
||||
fi
|
||||
|
||||
- name: Check dependencies
|
||||
run: |
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
echo "::error::Missing required tools: ${missing[*]}"
|
||||
exit 1
|
||||
fi
|
||||
echo "All dependencies found"
|
||||
|
||||
- name: Resolve display names
|
||||
id: refs
|
||||
run: |
|
||||
BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
|
||||
FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
|
||||
echo "baseline-name=nightly-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-name=nightly-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
id: build
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
run: |
|
||||
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
|
||||
FEATURE_DIR="$(cd ../reth-feature && pwd)"
|
||||
|
||||
.github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "$BASELINE_REF" &
|
||||
PID_BASELINE=$!
|
||||
.github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "$FEATURE_REF" &
|
||||
PID_FEATURE=$!
|
||||
|
||||
FAIL=0
|
||||
wait $PID_BASELINE || FAIL=1
|
||||
wait $PID_FEATURE || FAIL=1
|
||||
if [ $FAIL -ne 0 ]; then
|
||||
echo "::error::One or more build tasks failed"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
sudo cpupower frequency-set -g performance || true
|
||||
# Disable turbo boost (Intel and AMD paths)
|
||||
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
|
||||
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
|
||||
sudo swapoff -a || true
|
||||
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
|
||||
# Disable SMT (hyperthreading)
|
||||
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
|
||||
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
|
||||
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
|
||||
if [ "$current" != "$first" ]; then
|
||||
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
|
||||
fi
|
||||
done
|
||||
echo "Online CPUs: $(nproc)"
|
||||
# Disable transparent huge pages
|
||||
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
||||
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
||||
done || true
|
||||
# Prevent deep C-states
|
||||
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
||||
# Move all IRQs to core 0
|
||||
for irq in /proc/irq/*/smp_affinity_list; do
|
||||
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
||||
done
|
||||
# Stop noisy background services
|
||||
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
|
||||
echo "=== Benchmark environment ==="
|
||||
uname -r
|
||||
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
|
||||
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
|
||||
free -h
|
||||
|
||||
- name: Pre-flight cleanup
|
||||
run: |
|
||||
sudo pkill -9 reth || true
|
||||
sleep 1
|
||||
if mountpoint -q "$SCHELK_MOUNT"; then
|
||||
sudo umount -l "$SCHELK_MOUNT" || true
|
||||
sudo schelk recover -y || true
|
||||
fi
|
||||
rm -rf "$BENCH_WORK_DIR"
|
||||
mkdir -p "$BENCH_WORK_DIR"
|
||||
|
||||
- name: Start metrics proxy
|
||||
run: |
|
||||
BENCH_ID="nightly-${{ github.run_id }}"
|
||||
BENCH_REFERENCE_EPOCH=$(date +%s)
|
||||
echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
|
||||
echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
|
||||
|
||||
LABELS_FILE="/tmp/bench-metrics-labels.json"
|
||||
echo '{}' > "$LABELS_FILE"
|
||||
echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
|
||||
|
||||
python3 .github/scripts/bench-metrics-proxy.py \
|
||||
--labels "$LABELS_FILE" \
|
||||
--upstream "http://${BENCH_METRICS_ADDR}/" \
|
||||
--subnet 10.10.0.0/24 \
|
||||
--port 9090 &
|
||||
PROXY_PID=$!
|
||||
echo "BENCH_METRICS_PROXY_PID=${PROXY_PID}" >> "$GITHUB_ENV"
|
||||
echo "Metrics proxy started (PID $PROXY_PID)"
|
||||
|
||||
# Interleaved run order (B-F-F-B) to reduce systematic bias
|
||||
- name: "Run benchmark: baseline (1/2)"
|
||||
id: run-baseline-1
|
||||
run: |
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
|
||||
|
||||
- name: "Run benchmark: feature (1/2)"
|
||||
id: run-feature-1
|
||||
run: |
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
|
||||
|
||||
- name: "Run benchmark: feature (2/2)"
|
||||
id: run-feature-2
|
||||
run: |
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
|
||||
|
||||
- name: "Run benchmark: baseline (2/2)"
|
||||
id: run-baseline-2
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
|
||||
|
||||
- name: Stop metrics proxy & generate Grafana URL
|
||||
id: metrics
|
||||
if: "!cancelled()"
|
||||
run: |
|
||||
kill "$BENCH_METRICS_PROXY_PID" 2>/dev/null || true
|
||||
|
||||
LAST_RUN_DURATION=$(( $(date +%s) - BENCH_LAST_RUN_START ))
|
||||
FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
|
||||
TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
|
||||
GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=ef57fux92e9z4e&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
|
||||
echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "Grafana URL: ${GRAFANA_URL}"
|
||||
|
||||
- name: Scan logs for errors
|
||||
if: "!cancelled()"
|
||||
run: |
|
||||
ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
|
||||
found=false
|
||||
for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
|
||||
LOG="$BENCH_WORK_DIR/$run_dir/node.log"
|
||||
if [ ! -f "$LOG" ]; then continue; fi
|
||||
|
||||
panics=$(grep -c -E 'panicked at' "$LOG" || true)
|
||||
errors=$(grep -c ' ERROR ' "$LOG" || true)
|
||||
|
||||
if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
|
||||
if [ "$found" = false ]; then
|
||||
printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
|
||||
found=true
|
||||
fi
|
||||
printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
|
||||
if [ "$panics" -gt 0 ]; then
|
||||
printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
|
||||
grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
|
||||
printf '```\n' >> "$ERRORS_FILE"
|
||||
fi
|
||||
if [ "$errors" -gt 0 ]; then
|
||||
printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
|
||||
grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
|
||||
printf '```\n' >> "$ERRORS_FILE"
|
||||
fi
|
||||
printf '\n</details>\n\n' >> "$ERRORS_FILE"
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Parse results
|
||||
id: results
|
||||
if: success()
|
||||
env:
|
||||
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
||||
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
||||
run: |
|
||||
SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
|
||||
|
||||
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
|
||||
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
|
||||
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
|
||||
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
|
||||
|
||||
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
|
||||
if [ -n "$GRAFANA_URL" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
|
||||
|
||||
- name: Generate charts
|
||||
if: success()
|
||||
env:
|
||||
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
||||
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
||||
run: |
|
||||
CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
|
||||
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
|
||||
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
|
||||
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
|
||||
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
|
||||
CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
|
||||
CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
|
||||
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
|
||||
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
|
||||
# shellcheck disable=SC2086
|
||||
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
|
||||
|
||||
- name: Upload results
|
||||
if: "!cancelled()"
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: bench-scheduled-results
|
||||
path: ${{ env.BENCH_WORK_DIR }}
|
||||
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success()
|
||||
run: |
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="nightly/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
|
||||
true
|
||||
else
|
||||
git init "${TMP_DIR}"
|
||||
git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
|
||||
fi
|
||||
|
||||
mkdir -p "${TMP_DIR}/${CHART_DIR}"
|
||||
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
|
||||
git -C "${TMP_DIR}" add "${CHART_DIR}"
|
||||
git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
||||
commit -m "nightly bench charts for run ${RUN_ID}"
|
||||
git -C "${TMP_DIR}" push origin HEAD:main
|
||||
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
rm -rf "${TMP_DIR}"
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const { verdict, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
|
||||
|
||||
let summary;
|
||||
try {
|
||||
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
|
||||
} catch (e) {
|
||||
await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
|
||||
return;
|
||||
}
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
const { emoji, label } = verdict(summary.changes);
|
||||
const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
|
||||
const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
|
||||
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
|
||||
|
||||
let md = `# ${emoji} Nightly Regression: ${label}\n\n`;
|
||||
md += `**Baseline:** ${baselineLink}\n`;
|
||||
md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
|
||||
md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
|
||||
|
||||
const rows = metricRows(summary);
|
||||
md += `| Metric | Baseline | Feature | Change |\n`;
|
||||
md += `|--------|----------|---------|--------|\n`;
|
||||
for (const r of rows) {
|
||||
md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
|
||||
}
|
||||
md += '\n';
|
||||
|
||||
const wtRows = waitTimeRows(summary);
|
||||
if (wtRows.length > 0) {
|
||||
md += `### Wait Time Breakdown\n\n`;
|
||||
md += `| Metric | Baseline | Feature |\n`;
|
||||
md += `|--------|----------|--------|\n`;
|
||||
for (const r of wtRows) {
|
||||
md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
|
||||
}
|
||||
md += '\n';
|
||||
}
|
||||
|
||||
// Charts
|
||||
const chartSha = '${{ steps.push-charts.outputs.sha }}';
|
||||
if (chartSha) {
|
||||
const runId = '${{ github.run_id }}';
|
||||
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`;
|
||||
const charts = [
|
||||
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
|
||||
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
|
||||
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
|
||||
];
|
||||
md += `### Charts\n\n`;
|
||||
for (const chart of charts) {
|
||||
md += `<details><summary>${chart.label}</summary>\n\n`;
|
||||
md += `\n\n`;
|
||||
md += `</details>\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}';
|
||||
if (grafanaUrl) {
|
||||
md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
|
||||
}
|
||||
|
||||
try {
|
||||
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
|
||||
if (errors.trim()) md += '\n' + errors + '\n';
|
||||
} catch {}
|
||||
|
||||
await core.summary.addRaw(md).write();
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && env.BENCH_NO_SLACK != 'true'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const { verdict, fmtChange, fmtMs, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
|
||||
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
if (!token || !channel) {
|
||||
core.info('Slack credentials not set, skipping notification');
|
||||
return;
|
||||
}
|
||||
|
||||
let summary;
|
||||
try {
|
||||
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
|
||||
} catch (e) {
|
||||
core.warning('Could not read summary.json for Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
// Only notify on significant changes (regression OR improvement)
|
||||
const changes = summary.changes || {};
|
||||
const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
|
||||
if (!hasSignificant) {
|
||||
core.info('No significant changes detected, skipping nightly Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
const SLACK_VERDICT = {
|
||||
'⚠️': ':warning:',
|
||||
'❌': ':x:',
|
||||
'✅': ':white_check_mark:',
|
||||
'⚪': ':white_circle:',
|
||||
};
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const { emoji, label } = verdict(changes);
|
||||
const headerEmoji = SLACK_VERDICT[emoji] || emoji;
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
|
||||
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
|
||||
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
|
||||
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
|
||||
|
||||
const sectionText = [
|
||||
'*Nightly Regression*',
|
||||
'',
|
||||
`*Baseline:* ${baselineLink}`,
|
||||
`*Feature:* ${featureLink}`,
|
||||
blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '),
|
||||
].join('\n');
|
||||
|
||||
const rows = metricRows(summary);
|
||||
const tableRows = [
|
||||
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
|
||||
...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
|
||||
];
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: `${headerEmoji} Nightly: ${label}`, emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: sectionText },
|
||||
},
|
||||
{
|
||||
type: 'table',
|
||||
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }, { align: 'right' }],
|
||||
rows: tableRows,
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: [
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
|
||||
url: jobUrl,
|
||||
action_id: 'ci_button',
|
||||
},
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
|
||||
url: diffUrl,
|
||||
action_id: 'diff_button',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const text = `Nightly Regression: ${summary.baseline.name} vs ${summary.feature.name}`;
|
||||
const resp = await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ channel, blocks, text, unfurl_links: false }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.ok) {
|
||||
core.warning(`Slack API error: ${JSON.stringify(data)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Post wait time breakdown as threaded reply
|
||||
const wtRows = waitTimeRows(summary);
|
||||
if (data.ts && wtRows.length > 0) {
|
||||
const waitTableRows = [
|
||||
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
|
||||
...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
|
||||
];
|
||||
await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel,
|
||||
thread_ts: data.ts,
|
||||
blocks: [{
|
||||
type: 'table',
|
||||
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }],
|
||||
rows: waitTableRows,
|
||||
}],
|
||||
text: 'Wait time breakdown',
|
||||
unfurl_links: false,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure()
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
if (!token || !channel) return;
|
||||
|
||||
const steps_status = [
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
|
||||
['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
|
||||
];
|
||||
const failed = steps_status.find(([, o]) => o === 'failure');
|
||||
const failedStep = failed ? failed[0] : 'unknown step';
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: ':rotating_light: Nightly Bench Failed', emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*Nightly regression* failed while *${failedStep}*` },
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: [{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'View Logs :github:', emoji: true },
|
||||
url: jobUrl,
|
||||
action_id: 'ci_button',
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel,
|
||||
blocks,
|
||||
text: `Nightly bench failed while ${failedStep}`,
|
||||
unfurl_links: false,
|
||||
}),
|
||||
});
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job 3: Save state on success
|
||||
# ---------------------------------------------------------------------------
|
||||
save-state:
|
||||
needs: [resolve-refs, bench-scheduled]
|
||||
if: success()
|
||||
name: save-state
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Write state file
|
||||
run: |
|
||||
mkdir -p .nightly-state
|
||||
echo "${{ needs.resolve-refs.outputs.feature-ref }}" > .nightly-state/last-feature-ref
|
||||
|
||||
- name: Save nightly state
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: .nightly-state
|
||||
key: bench-scheduled-state-${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
106
.github/workflows/bench.yml
vendored
106
.github/workflows/bench.yml
vendored
@@ -71,14 +71,11 @@ on:
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
otlp:
|
||||
description: "Export OTLP traces and logs"
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
BASELINE: base
|
||||
SEED: reth
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
BENCH_RUNNERS: 2
|
||||
|
||||
@@ -113,7 +110,6 @@ jobs:
|
||||
baseline-args: ${{ steps.args.outputs.baseline-args }}
|
||||
feature-args: ${{ steps.args.outputs.feature-args }}
|
||||
abba: ${{ steps.args.outputs.abba }}
|
||||
otlp: ${{ steps.args.outputs.otlp }}
|
||||
comment-id: ${{ steps.ack.outputs.comment-id }}
|
||||
steps:
|
||||
- name: Check org membership
|
||||
@@ -155,7 +151,6 @@ jobs:
|
||||
bigBlocks = blocks === 'big' ? 'true' : 'false';
|
||||
var rethNewPayload = '${{ github.event.inputs.reth_newPayload }}' !== 'false' ? 'true' : 'false';
|
||||
var abba = '${{ github.event.inputs.abba }}' !== 'false' ? 'true' : 'false';
|
||||
var otlp = '${{ github.event.inputs.otlp }}' !== 'false' ? 'true' : 'false';
|
||||
var waitTime = '${{ github.event.inputs.wait_time }}' || '';
|
||||
var baselineNodeArgs = '${{ github.event.inputs.baseline_args }}' || '';
|
||||
var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || '';
|
||||
@@ -182,10 +177,10 @@ jobs:
|
||||
const intOrKeywordArgs = new Map([['blocks', new Set(['big'])]]);
|
||||
const refArgs = new Set(['baseline', 'feature']);
|
||||
const boolArgs = new Set(['samply', 'no-slack']);
|
||||
const boolDefaultTrue = new Set(['reth_newPayload', 'abba', 'otlp']);
|
||||
const boolDefaultTrue = new Set(['reth_newPayload', 'abba']);
|
||||
const durationArgs = new Set(['wait-time']);
|
||||
const stringArgs = new Set(['baseline-args', 'feature-args']);
|
||||
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const unknown = [];
|
||||
const invalid = [];
|
||||
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
|
||||
@@ -255,7 +250,7 @@ jobs:
|
||||
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
|
||||
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
|
||||
if (errors.length) {
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [otlp=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
@@ -275,7 +270,6 @@ jobs:
|
||||
bigBlocks = blocks === 'big' ? 'true' : 'false';
|
||||
var rethNewPayload = defaults.reth_newPayload;
|
||||
var abba = defaults.abba;
|
||||
var otlp = defaults.otlp;
|
||||
var waitTime = defaults['wait-time'];
|
||||
var baselineNodeArgs = defaults['baseline-args'];
|
||||
var featureNodeArgs = defaults['feature-args'];
|
||||
@@ -314,7 +308,6 @@ jobs:
|
||||
core.setOutput('baseline-args', baselineNodeArgs);
|
||||
core.setOutput('feature-args', featureNodeArgs);
|
||||
core.setOutput('abba', abba);
|
||||
core.setOutput('otlp', otlp);
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
@@ -382,8 +375,6 @@ jobs:
|
||||
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
|
||||
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
|
||||
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
|
||||
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
|
||||
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
|
||||
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
|
||||
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
|
||||
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
|
||||
@@ -391,7 +382,7 @@ jobs:
|
||||
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
|
||||
const { data: comment } = await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
@@ -426,8 +417,6 @@ jobs:
|
||||
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
|
||||
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
|
||||
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
|
||||
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
|
||||
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
|
||||
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
|
||||
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
|
||||
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
|
||||
@@ -435,7 +424,7 @@ jobs:
|
||||
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
|
||||
@@ -505,12 +494,11 @@ jobs:
|
||||
BENCH_BASELINE_ARGS: ${{ needs.reth-bench-ack.outputs.baseline-args }}
|
||||
BENCH_FEATURE_ARGS: ${{ needs.reth-bench-ack.outputs.feature-args }}
|
||||
BENCH_ABBA: ${{ needs.reth-bench-ack.outputs.abba }}
|
||||
BENCH_OTLP: ${{ needs.reth-bench-ack.outputs.otlp }}
|
||||
BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
|
||||
BENCH_NO_SLACK: ${{ needs.reth-bench-ack.outputs.no-slack }}
|
||||
BENCH_METRICS_ADDR: "127.0.0.1:9100"
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_TRACES_ENDPOINT || '' }}
|
||||
BENCH_OTLP_LOGS_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_LOGS_ENDPOINT || '' }}
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
|
||||
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
|
||||
steps:
|
||||
- name: Clean up previous bench-work
|
||||
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
@@ -571,8 +559,6 @@ jobs:
|
||||
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
|
||||
const abbaEnabled = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
|
||||
const otlpEnabled = (process.env.BENCH_OTLP || 'true') !== 'false';
|
||||
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
|
||||
const waitTimeVal = process.env.BENCH_WAIT_TIME || '';
|
||||
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
|
||||
const baselineArgsVal = process.env.BENCH_BASELINE_ARGS || '';
|
||||
@@ -580,7 +566,7 @@ jobs:
|
||||
const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
|
||||
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
|
||||
|
||||
const { buildBody } = require('./.github/scripts/bench-update-status.js');
|
||||
await github.rest.issues.updateComment({
|
||||
@@ -726,16 +712,11 @@ jobs:
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: |
|
||||
set +e
|
||||
.github/scripts/bench-reth-snapshot.sh --check
|
||||
rc=$?
|
||||
set -e
|
||||
case "$rc" in
|
||||
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
|
||||
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
|
||||
*) echo "::error::Snapshot check failed (exit $rc)"
|
||||
exit "$rc" ;;
|
||||
esac
|
||||
if .github/scripts/bench-reth-snapshot.sh --check; then
|
||||
echo "needed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "needed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Update status (snapshot needed)
|
||||
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
|
||||
@@ -744,7 +725,7 @@ jobs:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
const s = require('./.github/scripts/bench-update-status.js');
|
||||
await s({github, context, status: 'Building binaries (snapshot update pending)...'});
|
||||
await s({github, context, status: 'Building binaries & downloading snapshot...'});
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
@@ -764,11 +745,12 @@ jobs:
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries
|
||||
- name: Build binaries and download snapshot in parallel
|
||||
id: build
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
SNAPSHOT_NEEDED: ${{ steps.snapshot-check.outputs.needed }}
|
||||
run: |
|
||||
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
|
||||
FEATURE_DIR="$(cd ../reth-feature && pwd)"
|
||||
@@ -778,23 +760,21 @@ jobs:
|
||||
.github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "${{ steps.refs.outputs.feature-ref }}" &
|
||||
PID_FEATURE=$!
|
||||
|
||||
PID_SNAPSHOT=
|
||||
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
|
||||
.github/scripts/bench-reth-snapshot.sh &
|
||||
PID_SNAPSHOT=$!
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
wait $PID_BASELINE || FAIL=1
|
||||
wait $PID_FEATURE || FAIL=1
|
||||
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
|
||||
if [ $FAIL -ne 0 ]; then
|
||||
echo "::error::One or more build tasks failed"
|
||||
echo "::error::One or more parallel tasks failed (builds / snapshot download)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Download snapshot
|
||||
id: snapshot-download
|
||||
if: steps.snapshot-check.outputs.needed == 'true'
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
|
||||
run: .github/scripts/bench-reth-snapshot.sh
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
@@ -851,15 +831,15 @@ jobs:
|
||||
run: |
|
||||
set -euo pipefail
|
||||
MC="mc --config-dir /home/ubuntu/.mc"
|
||||
BUCKET="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.tar.zst"
|
||||
BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous-big-blocks.tar.zst"
|
||||
BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
|
||||
rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
|
||||
echo "Downloading big blocks from $BUCKET..."
|
||||
$MC cat "$BUCKET" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
|
||||
echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
|
||||
# Verify expected directory structure
|
||||
if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
|
||||
echo "::error::Big blocks archive missing expected payloads/ directory"
|
||||
if [ ! -d "$BIG_BLOCKS_DIR/gas-ramp-dir" ] || [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
|
||||
echo "::error::Big blocks archive missing expected gas-ramp-dir/ or payloads/ directories"
|
||||
ls -laR "$BIG_BLOCKS_DIR"
|
||||
exit 1
|
||||
fi
|
||||
@@ -1071,6 +1051,11 @@ jobs:
|
||||
fi
|
||||
if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --big-blocks"
|
||||
# Read gas ramp blocks count from first baseline run (same for all runs)
|
||||
GAS_RAMP_FILE="$BENCH_WORK_DIR/baseline-1/gas_ramp_blocks.txt"
|
||||
if [ -f "$GAS_RAMP_FILE" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --gas-ramp-blocks $(cat "$GAS_RAMP_FILE" | tr -d '[:space:]')"
|
||||
fi
|
||||
fi
|
||||
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
|
||||
if [ -n "$GRAFANA_URL" ]; then
|
||||
@@ -1210,22 +1195,11 @@ jobs:
|
||||
comment_id: parseInt(ackCommentId),
|
||||
body,
|
||||
});
|
||||
} else {
|
||||
// No PR — write results to job summary
|
||||
await core.summary.addRaw(body).write();
|
||||
}
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const jobSummary = require('./.github/scripts/bench-job-summary.js');
|
||||
await jobSummary({
|
||||
core,
|
||||
context,
|
||||
chartSha: '${{ steps.push-charts.outputs.sha }}',
|
||||
grafanaUrl: '${{ steps.metrics.outputs.grafana-url }}',
|
||||
runId: '${{ github.run_id }}',
|
||||
});
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && env.BENCH_NO_SLACK != 'true'
|
||||
uses: actions/github-script@v8
|
||||
@@ -1245,8 +1219,7 @@ jobs:
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const steps_status = [
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
@@ -1281,8 +1254,7 @@ jobs:
|
||||
script: |
|
||||
const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const steps_status = [
|
||||
['building binaries', '${{ steps.build.outcome }}'],
|
||||
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
|
||||
['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
|
||||
|
||||
2
.github/workflows/docker-test.yml
vendored
2
.github/workflows/docker-test.yml
vendored
@@ -79,4 +79,4 @@ jobs:
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: ${{ inputs.artifact_name }}
|
||||
path: ./artifacts
|
||||
path: ./artifacts
|
||||
|
||||
71
.github/workflows/hive.yml
vendored
71
.github/workflows/hive.yml
vendored
@@ -5,10 +5,7 @@ name: hive
|
||||
on:
|
||||
workflow_dispatch:
|
||||
schedule:
|
||||
- cron: "0 */6 * * *"
|
||||
pull_request:
|
||||
branches:
|
||||
- "**"
|
||||
- cron: "0 0 * * *"
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
@@ -18,26 +15,23 @@ concurrency:
|
||||
cancel-in-progress: true
|
||||
|
||||
jobs:
|
||||
prepare-reth-stable:
|
||||
uses: ./.github/workflows/prepare-reth.yml
|
||||
build-reth:
|
||||
uses: ./.github/workflows/docker-test.yml
|
||||
with:
|
||||
image_tag: ghcr.io/paradigmxyz/reth:latest
|
||||
binary_name: reth
|
||||
cargo_features: "asm-keccak"
|
||||
artifact_name: "reth-stable"
|
||||
hive_target: hive
|
||||
artifact_name: "reth"
|
||||
secrets: inherit
|
||||
|
||||
prepare-hive:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on:
|
||||
group: Reth
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: Soubhik-10/hive # rm later and use ethereum/hive
|
||||
ref: master
|
||||
repository: ethereum/hive
|
||||
path: hivetests
|
||||
|
||||
- name: Get hive commit hash
|
||||
@@ -83,7 +77,6 @@ jobs:
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
storage: [stable]
|
||||
# ethereum/rpc to be deprecated:
|
||||
# https://github.com/ethereum/hive/pull/1117
|
||||
scenario:
|
||||
@@ -124,28 +117,26 @@ jobs:
|
||||
- sim: ethereum/rpc-compat
|
||||
include:
|
||||
- eth_blockNumber
|
||||
# - eth_call
|
||||
# - eth_chainId
|
||||
# - eth_createAccessList
|
||||
# - eth_estimateGas
|
||||
# - eth_feeHistory
|
||||
# - eth_getBalance
|
||||
# - eth_getBlockBy
|
||||
# - eth_getBlockTransactionCountBy
|
||||
# - eth_getCode
|
||||
# - eth_getProof
|
||||
# - eth_getStorage
|
||||
# - eth_getTransactionBy
|
||||
# - eth_getTransactionCount
|
||||
# - eth_getTransactionReceipt
|
||||
# - eth_sendRawTransaction
|
||||
# - eth_syncing
|
||||
# # debug_ rpc methods
|
||||
# - debug_
|
||||
- eth_call
|
||||
- eth_chainId
|
||||
- eth_createAccessList
|
||||
- eth_estimateGas
|
||||
- eth_feeHistory
|
||||
- eth_getBalance
|
||||
- eth_getBlockBy
|
||||
- eth_getBlockTransactionCountBy
|
||||
- eth_getCode
|
||||
- eth_getProof
|
||||
- eth_getStorage
|
||||
- eth_getTransactionBy
|
||||
- eth_getTransactionCount
|
||||
- eth_getTransactionReceipt
|
||||
- eth_sendRawTransaction
|
||||
- eth_syncing
|
||||
# debug_ rpc methods
|
||||
- debug_
|
||||
|
||||
# consume-engine
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-engine
|
||||
@@ -166,8 +157,6 @@ jobs:
|
||||
limit: .*tests/paris.*
|
||||
|
||||
# consume-rlp
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/amsterdam.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/osaka.*
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
@@ -187,9 +176,9 @@ jobs:
|
||||
- sim: ethereum/eels/consume-rlp
|
||||
limit: .*tests/paris.*
|
||||
needs:
|
||||
- prepare-reth-stable
|
||||
- build-reth
|
||||
- prepare-hive
|
||||
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
name: ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
|
||||
# Use larger runners for eels tests to avoid OOM runner crashes
|
||||
runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
|
||||
permissions:
|
||||
@@ -208,7 +197,7 @@ jobs:
|
||||
- name: Download reth image
|
||||
uses: actions/download-artifact@v8
|
||||
with:
|
||||
name: reth-${{ matrix.storage }}
|
||||
name: reth
|
||||
path: /tmp
|
||||
|
||||
- name: Load Docker images
|
||||
@@ -222,7 +211,7 @@ jobs:
|
||||
- name: Checkout hive tests
|
||||
uses: actions/checkout@v6
|
||||
with:
|
||||
repository: Soubhik-10/hive
|
||||
repository: ethereum/hive
|
||||
ref: master
|
||||
path: hivetests
|
||||
|
||||
|
||||
61
.github/workflows/prepare-reth.yml
vendored
61
.github/workflows/prepare-reth.yml
vendored
@@ -1,61 +0,0 @@
|
||||
name: Prepare Reth Image
|
||||
|
||||
on:
|
||||
workflow_call:
|
||||
inputs:
|
||||
image_tag:
|
||||
required: true
|
||||
type: string
|
||||
description: "Docker image tag to use"
|
||||
binary_name:
|
||||
required: false
|
||||
type: string
|
||||
default: "reth"
|
||||
description: "Binary name to build (reth or op-reth)"
|
||||
cargo_features:
|
||||
required: false
|
||||
type: string
|
||||
default: "asm-keccak"
|
||||
description: "Cargo features to enable"
|
||||
cargo_package:
|
||||
required: false
|
||||
type: string
|
||||
description: "Optional cargo package path"
|
||||
artifact_name:
|
||||
required: false
|
||||
type: string
|
||||
default: "artifacts"
|
||||
description: "Name for the uploaded artifact"
|
||||
|
||||
jobs:
|
||||
prepare-reth:
|
||||
if: github.repository == 'paradigmxyz/reth'
|
||||
timeout-minutes: 45
|
||||
runs-on: depot-ubuntu-latest-4
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- run: mkdir artifacts
|
||||
|
||||
- name: Set up Docker Buildx
|
||||
uses: docker/setup-buildx-action@v3
|
||||
|
||||
- name: Build and export reth image
|
||||
uses: docker/build-push-action@v6
|
||||
with:
|
||||
context: .
|
||||
file: .github/scripts/hive/Dockerfile
|
||||
tags: ${{ inputs.image_tag }}
|
||||
outputs: type=docker,dest=./artifacts/reth_image.tar
|
||||
build-args: |
|
||||
CARGO_BIN=${{ inputs.binary_name }}
|
||||
MANIFEST_PATH=${{ inputs.cargo_package }}
|
||||
FEATURES=${{ inputs.cargo_features }}
|
||||
cache-from: type=gha
|
||||
cache-to: type=gha,mode=max
|
||||
|
||||
- name: Upload reth image
|
||||
id: upload
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: ${{ inputs.artifact_name }}
|
||||
path: ./artifacts
|
||||
4
.github/workflows/release.yml
vendored
4
.github/workflows/release.yml
vendored
@@ -227,14 +227,14 @@ jobs:
|
||||
|
||||
- name: Upload artifact
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
|
||||
path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
|
||||
|
||||
- name: Upload signature
|
||||
if: ${{ github.event.inputs.dry_run != 'true' }}
|
||||
uses: actions/upload-artifact@v7
|
||||
uses: actions/upload-artifact@v6
|
||||
with:
|
||||
name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
|
||||
path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
|
||||
|
||||
345
Cargo.lock
generated
345
Cargo.lock
generated
@@ -106,9 +106,9 @@ checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923"
|
||||
|
||||
[[package]]
|
||||
name = "alloy-chains"
|
||||
version = "0.2.32"
|
||||
version = "0.2.31"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9247f0a399ef71aeb68f497b2b8fb348014f742b50d3b83b1e00dfe1b7d64b3d"
|
||||
checksum = "6d9d22005bf31b018f31ef9ecadb5d2c39cf4f6acc8db0456f72c815f3d7f757"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rlp",
|
||||
@@ -122,7 +122,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-consensus"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0c0dc44157867da82c469c13186015b86abef209bf0e41625e4b68bac61d728"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"alloy-primitives",
|
||||
@@ -149,7 +150,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-consensus-any"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ba4cdb42df3871cd6b346d6a938ec2ba69a9a0f49d1f82714bc5c48349268434"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -163,7 +165,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-contract"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ca63b7125a981415898ffe2a2a696c83696c9c6bdb1671c8a912946bbd8e49e7"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-dyn-abi",
|
||||
@@ -195,7 +198,7 @@ dependencies = [
|
||||
"itoa",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"winnow 0.7.15",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -260,7 +263,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-eips"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9f7ef09f21bd1e9cb8a686f168cb4a206646804567f0889eadb8dcc4c9288c8"
|
||||
dependencies = [
|
||||
"alloy-eip2124",
|
||||
"alloy-eip2930",
|
||||
@@ -286,7 +290,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-evm"
|
||||
version = "0.29.2"
|
||||
source = "git+https://github.com/alloy-rs/evm?branch=bal-devnet2#5ca2989a1fc18079ca29387c61c5e5b6d6150030"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cb6ba2dafd6327f78f2b59ae539bd5c39c57a01dc76763e92942619d934a7bb"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -308,7 +313,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-genesis"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7c9cf3b99f46615fbf7dc1add0c96553abb7bf88fc9ec70dfbe7ad0b47ba7fe8"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"alloy-primitives",
|
||||
@@ -361,7 +367,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-json-rpc"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ff42cd777eea61f370c0b10f2648a1c81e0b783066cd7269228aa993afd487f7"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-sol-types",
|
||||
@@ -375,7 +382,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-network"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8cbca04f9b410fdc51aaaf88433cbac761213905a65fe832058bcf6690585762"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-consensus-any",
|
||||
@@ -400,7 +408,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-network-primitives"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42d6d15e069a8b11f56bef2eccbad2a873c6dd4d4c81d04dda29710f5ea52f04"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -412,7 +421,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-node-bindings"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "091dc8117d84de3a9ac7ec97f2c4d83987e24d485b478d26aa1ec455d7d52f7d"
|
||||
dependencies = [
|
||||
"alloy-genesis",
|
||||
"alloy-hardforks 0.2.13",
|
||||
@@ -476,7 +486,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-provider"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d181c8cc7cf4805d7e589bf4074d56d55064fa1a979f005a45a62b047616d870"
|
||||
dependencies = [
|
||||
"alloy-chains",
|
||||
"alloy-consensus",
|
||||
@@ -520,7 +531,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-pubsub"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e8bd82953194dec221aa4cbbbb0b1e2df46066fe9d0333ac25b43a311e122d13"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-primitives",
|
||||
@@ -563,7 +575,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-client"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f2792758a93ae32a32e9047c843d536e1448044f78422d71bf7d7c05149e103f"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-primitives",
|
||||
@@ -588,7 +601,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7bdcbf9dfd5eea8bfeb078b1d906da8cd3a39c4d4dbe7a628025648e323611f6"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-engine",
|
||||
@@ -600,7 +614,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-admin"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "42325c117af3a9e49013f881c1474168db57978e02085fc9853a1c89e0562740"
|
||||
dependencies = [
|
||||
"alloy-genesis",
|
||||
"alloy-primitives",
|
||||
@@ -611,7 +626,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-anvil"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0a3100b76987c1b1dc81f3abe592b7edc29e92b1242067a69d65e0030b35cf9"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -622,7 +638,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-any"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "dd720b63f82b457610f2eaaf1f32edf44efffe03ae25d537632e7d23e7929e1a"
|
||||
dependencies = [
|
||||
"alloy-consensus-any",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -632,7 +649,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-beacon"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4a22e13215866f5dfd5d3278f4c41f1fad9410dc68ce39022f58593c873c26f8"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"alloy-primitives",
|
||||
@@ -651,7 +669,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-debug"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e1b21e1ad18ff1b31ff1030e046462ab8168cf8894e6778cd805c8bdfe2bd649"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"derive_more",
|
||||
@@ -662,7 +681,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-engine"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e4ac61f03f1edabccde1c687b5b25fff28f183afee64eaa2e767def3929e4457"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -682,7 +702,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-eth"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b2dc411f13092f237d2bf6918caf80977fc2f51485f9b90cb2a2f956912c8c9"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-consensus-any",
|
||||
@@ -703,7 +724,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-mev"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fe85bf3be739126aa593dca9fb3ab13ca93fa7873e6f2247be64d7f2cb15f34a"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -717,7 +739,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-trace"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1ad79f1e27e161943b5a4f99fe5534ef0849876214be411e0032c12f38e94daa"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -730,7 +753,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-rpc-types-txpool"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d459f902a2313737bc66d18ed094c25d2aeb268b74d98c26bbbda2aa44182ab0"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -741,7 +765,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-serde"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2ce1e0dbf7720eee747700e300c99aac01b1a95bb93f493a01e78ee28bb1a37"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"arbitrary",
|
||||
@@ -752,7 +777,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-signer"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2425c6f314522c78e8198979c8cbf6769362be4da381d4152ea8eefce383535d"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"async-trait",
|
||||
@@ -766,7 +792,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-signer-local"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3ecb71ee53d8d9c3fa7bac17542c8116ebc7a9726c91b1bf333ec3d04f5a789"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-network",
|
||||
@@ -836,7 +863,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a6df77fea9d6a2a75c0ef8d2acbdfd92286cc599983d3175ccdc170d3433d249"
|
||||
dependencies = [
|
||||
"serde",
|
||||
"winnow 0.7.15",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -854,7 +881,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-transport"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fa186e560d523d196580c48bf00f1bf62e63041f28ecf276acc22f8b27bb9f53"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"auto_impl",
|
||||
@@ -876,7 +904,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-transport-http"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "aa501ad58dd20acddbfebc65b52e60f05ebf97c52fa40d1b35e91f5e2da0ad0e"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-transport",
|
||||
@@ -891,7 +920,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-transport-ipc"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2ef85688e5ac2da72afc804e0a1f153a1f309f05a864b1998bbbed7804dbaab"
|
||||
dependencies = [
|
||||
"alloy-json-rpc",
|
||||
"alloy-pubsub",
|
||||
@@ -910,7 +940,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-transport-ws"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b9f00445db69d63298e2b00a0ea1d859f00e6424a3144ffc5eba9c31da995e16"
|
||||
dependencies = [
|
||||
"alloy-pubsub",
|
||||
"alloy-transport",
|
||||
@@ -946,7 +977,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-tx-macros"
|
||||
version = "1.7.3"
|
||||
source = "git+https://github.com/alloy-rs/alloy?branch=bal-devnet2#1fd12c4107dcf3ad3fbbc404004a710bf91db3e6"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6fa0c53e8c1e1ef4d01066b01c737fb62fc9397ab52c6e7bb5669f97d281b9bc"
|
||||
dependencies = [
|
||||
"darling 0.21.3",
|
||||
"proc-macro2",
|
||||
@@ -1699,6 +1731,15 @@ dependencies = [
|
||||
"generic-array",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "block2"
|
||||
version = "0.6.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cdeb9d870516001442e364c5220d3574d2da8dc765554b4a617230d33fa58ef5"
|
||||
dependencies = [
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "blst"
|
||||
version = "0.3.16"
|
||||
@@ -1854,20 +1895,19 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "borsh"
|
||||
version = "1.6.1"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfd1e3f8955a5d7de9fab72fc8373fade9fb8a703968cb200ae3dc6cf08e185a"
|
||||
checksum = "d1da5ab77c1437701eeff7c88d968729e7766172279eab0676857b3d63af7a6f"
|
||||
dependencies = [
|
||||
"borsh-derive",
|
||||
"bytes",
|
||||
"cfg_aliases",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "borsh-derive"
|
||||
version = "1.6.1"
|
||||
version = "1.6.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bfcfdc083699101d5a7965e49925975f2f55060f94f9a05e7187be95d530ca59"
|
||||
checksum = "0686c856aa6aac0c4498f936d7d6a02df690f614c03e4d906d1018062b5c5e2c"
|
||||
dependencies = [
|
||||
"once_cell",
|
||||
"proc-macro-crate",
|
||||
@@ -2735,6 +2775,17 @@ dependencies = [
|
||||
"cipher",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "ctrlc"
|
||||
version = "3.5.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e0b1fab2ae45819af2d0731d60f2afe17227ebb1a1538a236da84c93e9a60162"
|
||||
dependencies = [
|
||||
"dispatch2",
|
||||
"nix",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "curve25519-dalek"
|
||||
version = "4.1.3"
|
||||
@@ -3127,6 +3178,18 @@ dependencies = [
|
||||
"zeroize",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "dispatch2"
|
||||
version = "0.3.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e0e367e4e7da84520dedcac1901e4da967309406d1e51017ae1abfb97adbd38"
|
||||
dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
"block2",
|
||||
"libc",
|
||||
"objc2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
@@ -3140,9 +3203,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "doctest-file"
|
||||
version = "1.1.1"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2db04e74f0a9a93103b50e90b96024c9b2bdca8bce6a632ec71b88736d3d359"
|
||||
checksum = "aac81fa3e28d21450aa4d2ac065992ba96a1d7303efbce51a95f4fd175b67562"
|
||||
|
||||
[[package]]
|
||||
name = "document-features"
|
||||
@@ -3722,6 +3785,19 @@ dependencies = [
|
||||
"tokio",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "example-migrate-trie-to-packed"
|
||||
version = "0.0.0"
|
||||
dependencies = [
|
||||
"clap",
|
||||
"eyre",
|
||||
"reth-db",
|
||||
"reth-db-api",
|
||||
"reth-trie-common",
|
||||
"reth-trie-db",
|
||||
"serde_json",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "example-network"
|
||||
version = "0.0.0"
|
||||
@@ -5039,9 +5115,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "instability"
|
||||
version = "0.3.12"
|
||||
version = "0.3.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5eb2d60ef19920a3a9193c3e371f726ec1dafc045dac788d0fb3704272458971"
|
||||
checksum = "357b7205c6cd18dd2c86ed312d1e70add149aea98e7ef72b9fdf0270e555c11d"
|
||||
dependencies = [
|
||||
"darling 0.23.0",
|
||||
"indoc",
|
||||
@@ -6170,9 +6246,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num_enum"
|
||||
version = "0.7.6"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5d0bca838442ec211fa11de3a8b0e0e8f3a4522575b5c4c06ed722e005036f26"
|
||||
checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c"
|
||||
dependencies = [
|
||||
"num_enum_derive",
|
||||
"rustversion",
|
||||
@@ -6180,9 +6256,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "num_enum_derive"
|
||||
version = "0.7.6"
|
||||
version = "0.7.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "680998035259dcfcafe653688bf2aa6d3e2dc05e98be6ab46afb089dc84f1df8"
|
||||
checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7"
|
||||
dependencies = [
|
||||
"proc-macro-crate",
|
||||
"proc-macro2",
|
||||
@@ -6214,6 +6290,15 @@ dependencies = [
|
||||
"smallvec",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2"
|
||||
version = "0.6.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a12a8ed07aefc768292f076dc3ac8c48f3781c8f2d5851dd3d98950e8c5a89f"
|
||||
dependencies = [
|
||||
"objc2-encode",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-core-foundation"
|
||||
version = "0.3.2"
|
||||
@@ -6223,6 +6308,12 @@ dependencies = [
|
||||
"bitflags 2.11.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "objc2-encode"
|
||||
version = "4.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "ef25abbcd74fb2609453eb695bd2f860d389e457f67dc17cafc8b8cbc89d0c33"
|
||||
|
||||
[[package]]
|
||||
name = "objc2-io-kit"
|
||||
version = "0.3.2"
|
||||
@@ -6266,9 +6357,9 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy"
|
||||
version = "0.23.1"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e9b8fee21003dd4f076563de9b9d26f8c97840157ef78593cd7f262c5ca99848"
|
||||
checksum = "a95dd0974d5e60ffe9342a70cc0033d299244fab01cb16a958eb7352ddba1fa7"
|
||||
dependencies = [
|
||||
"op-alloy-consensus",
|
||||
"op-alloy-network",
|
||||
@@ -6279,8 +6370,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-consensus"
|
||||
version = "0.23.1"
|
||||
source = "git+https://github.com/alloy-rs/op-alloy?branch=bal-devnet2#d3c6e661ce58697327f53508f9514a75ca68eb60"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fadcb964b0aa645d12e952d470c7585d23286d8bcf1ac41589410b6724216b68"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -6298,8 +6390,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-network"
|
||||
version = "0.23.1"
|
||||
source = "git+https://github.com/alloy-rs/op-alloy?branch=bal-devnet2#d3c6e661ce58697327f53508f9514a75ca68eb60"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8ea44162d493219cc678aaca1253d46c3aa73aa361326dfa9d406f086dfa135"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-network",
|
||||
@@ -6313,9 +6406,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-provider"
|
||||
version = "0.23.1"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6753d90efbaa8ea8bcb89c1737408ca85fa60d7adb875049d3f382c063666f86"
|
||||
checksum = "83aa8dc34bdf077c8e6d48ff75beff4ac14b428d982c9722483ccd7473c0e114"
|
||||
dependencies = [
|
||||
"alloy-network",
|
||||
"alloy-primitives",
|
||||
@@ -6328,8 +6421,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-rpc-types"
|
||||
version = "0.23.1"
|
||||
source = "git+https://github.com/alloy-rs/op-alloy?branch=bal-devnet2#d3c6e661ce58697327f53508f9514a75ca68eb60"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "be9d16ec9f7810e0623a37edc293d1c05fe9c58a5647f6973fdd93e0b4795015"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -6346,8 +6440,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-rpc-types-engine"
|
||||
version = "0.23.1"
|
||||
source = "git+https://github.com/alloy-rs/op-alloy?branch=bal-devnet2#d3c6e661ce58697327f53508f9514a75ca68eb60"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c0ac5c5ddcbffadcd097d278c717d34849bcc629f6e4f8514eb000ec862def8"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
@@ -6368,7 +6463,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "op-revm"
|
||||
version = "17.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a98f3a512a7e02a1dcf1242b57302d83657b265a665d50ad98d0b158efaf2c"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"revm",
|
||||
@@ -7556,8 +7652,10 @@ dependencies = [
|
||||
name = "reth-bench"
|
||||
version = "1.11.3"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
"alloy-json-rpc",
|
||||
"alloy-network",
|
||||
"alloy-primitives",
|
||||
"alloy-provider",
|
||||
"alloy-pubsub",
|
||||
@@ -7577,9 +7675,11 @@ dependencies = [
|
||||
"op-alloy-consensus",
|
||||
"op-alloy-rpc-types-engine",
|
||||
"reqwest",
|
||||
"reth-chainspec",
|
||||
"reth-cli-runner",
|
||||
"reth-cli-util",
|
||||
"reth-engine-primitives",
|
||||
"reth-ethereum-primitives",
|
||||
"reth-fs-util",
|
||||
"reth-node-api",
|
||||
"reth-node-core",
|
||||
@@ -7595,6 +7695,33 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reth-bench-compare"
|
||||
version = "1.11.3"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-provider",
|
||||
"alloy-rpc-client",
|
||||
"alloy-rpc-types-eth",
|
||||
"alloy-transport-ws",
|
||||
"chrono",
|
||||
"clap",
|
||||
"csv",
|
||||
"ctrlc",
|
||||
"eyre",
|
||||
"nix",
|
||||
"reth-chainspec",
|
||||
"reth-cli-runner",
|
||||
"reth-cli-util",
|
||||
"reth-node-core",
|
||||
"reth-tracing",
|
||||
"serde",
|
||||
"serde_json",
|
||||
"shlex",
|
||||
"tokio",
|
||||
"tracing",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "reth-chain-state"
|
||||
version = "1.11.3"
|
||||
@@ -7835,7 +7962,6 @@ name = "reth-consensus"
|
||||
version = "1.11.3"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eip7928",
|
||||
"alloy-primitives",
|
||||
"auto_impl",
|
||||
"reth-execution-types",
|
||||
@@ -8259,7 +8385,6 @@ dependencies = [
|
||||
"eyre",
|
||||
"fixed-cache",
|
||||
"futures",
|
||||
"itertools 0.14.0",
|
||||
"metrics",
|
||||
"metrics-util",
|
||||
"moka",
|
||||
@@ -9514,7 +9639,6 @@ version = "1.11.3"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
"alloy-genesis",
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-engine",
|
||||
"assert_matches",
|
||||
@@ -9793,6 +9917,7 @@ dependencies = [
|
||||
"reth-node-core",
|
||||
"reth-node-ethereum",
|
||||
"reth-payload-builder",
|
||||
"reth-payload-primitives",
|
||||
"reth-primitives-traits",
|
||||
"reth-provider",
|
||||
"reth-rpc",
|
||||
@@ -10537,7 +10662,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm"
|
||||
version = "36.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0abc15d09cd211e9e73410ada10134069c794d4bcdb787dfc16a1bf0939849c"
|
||||
dependencies = [
|
||||
"revm-bytecode",
|
||||
"revm-context",
|
||||
@@ -10555,7 +10681,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-bytecode"
|
||||
version = "9.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e86e468df3cf5cf59fa7ef71a3e9ccabb76bb336401ea2c0674f563104cf3c5e"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"phf",
|
||||
@@ -10566,7 +10693,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-context"
|
||||
version = "15.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9eb1f0a76b14d684a444fc52f7bf6b7564bf882599d91ee62e76d602e7a187c7"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"cfg-if",
|
||||
@@ -10582,7 +10710,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-context-interface"
|
||||
version = "16.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc256b27743e2912ca16899568e6652a372eb5d1d573e6edb16c7836b16cf487"
|
||||
dependencies = [
|
||||
"alloy-eip2930",
|
||||
"alloy-eip7702",
|
||||
@@ -10597,7 +10726,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-database"
|
||||
version = "12.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c0a7d6da41061f2c50f99a2632571026b23684b5449ff319914151f4449b6c8"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"revm-bytecode",
|
||||
@@ -10610,7 +10740,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-database-interface"
|
||||
version = "10.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd497a38a79057b94a049552cb1f925ad15078bc1a479c132aeeebd1d2ccc768"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"either",
|
||||
@@ -10623,7 +10754,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-handler"
|
||||
version = "17.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f1eed729ca9b228ae98688f352235871e9b8be3d568d488e4070f64c56e9d3d"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"derive-where",
|
||||
@@ -10641,7 +10773,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-inspector"
|
||||
version = "17.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf5102391706513689f91cb3cb3d97b5f13a02e8647e6e9cb7620877ef84847"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"either",
|
||||
@@ -10657,9 +10790,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "revm-inspectors"
|
||||
version = "0.36.1"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9487362b728f80dd2033ef5f4d0688453435bbe7caa721fa7e3b8fa25d89242b"
|
||||
checksum = "cfb0f462c8a3d9989d3dbc62d7cca4dfecd7072cfa5d563ab90ced60590ed1da"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -10678,7 +10811,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-interpreter"
|
||||
version = "34.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf22f80612bb8f58fd1f578750281f2afadb6c93835b14ae6a4d6b75ca26f445"
|
||||
dependencies = [
|
||||
"revm-bytecode",
|
||||
"revm-context-interface",
|
||||
@@ -10690,7 +10824,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-precompile"
|
||||
version = "32.1.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2ec11f45deec71e4945e1809736bb20d454285f9167ab53c5159dae1deb603f"
|
||||
dependencies = [
|
||||
"ark-bls12-381",
|
||||
"ark-bn254",
|
||||
@@ -10713,7 +10848,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-primitives"
|
||||
version = "22.1.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bcfb5ce6cf18b118932bcdb7da05cd9c250f2cb9f64131396b55f3fe3537c35"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"num_enum",
|
||||
@@ -10724,7 +10860,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-state"
|
||||
version = "10.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=fa5a6914398c9b178422efc1edd1d2ab33dad923#fa5a6914398c9b178422efc1edd1d2ab33dad923"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29404707763da607e5d6e4771cb203998c28159279c2f64cc32de08d2814651"
|
||||
dependencies = [
|
||||
"alloy-eip7928",
|
||||
"bitflags 2.11.0",
|
||||
@@ -11558,8 +11695,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
[[package]]
|
||||
name = "slotmap"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"
|
||||
source = "git+https://github.com/DaniPopes/slotmap.git?branch=dani%2Fshrink-methods#09fbd360968f9ecef19fc9559037cf869d33858b"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
@@ -12166,11 +12302,7 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tungstenite 0.28.0",
|
||||
]
|
||||
|
||||
@@ -12201,7 +12333,7 @@ dependencies = [
|
||||
"toml_datetime 0.7.5+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"toml_writer",
|
||||
"winnow 0.7.15",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12215,39 +12347,39 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "toml_datetime"
|
||||
version = "1.0.1+spec-1.1.0"
|
||||
version = "1.0.0+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9b320e741db58cac564e26c607d3cc1fdc4a88fd36c879568c07856ed83ff3e9"
|
||||
checksum = "32c2555c699578a4f59f0cc68e5116c8d7cabbd45e1409b989d4be085b53f13e"
|
||||
dependencies = [
|
||||
"serde_core",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_edit"
|
||||
version = "0.25.5+spec-1.1.0"
|
||||
version = "0.25.4+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "8ca1a40644a28bce036923f6a431df0b34236949d111cc07cb6dca830c9ef2e1"
|
||||
checksum = "7193cbd0ce53dc966037f54351dbbcf0d5a642c7f0038c382ef9e677ce8c13f2"
|
||||
dependencies = [
|
||||
"indexmap 2.13.0",
|
||||
"toml_datetime 1.0.1+spec-1.1.0",
|
||||
"toml_datetime 1.0.0+spec-1.1.0",
|
||||
"toml_parser",
|
||||
"winnow 1.0.0",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_parser"
|
||||
version = "1.0.10+spec-1.1.0"
|
||||
version = "1.0.9+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7df25b4befd31c4816df190124375d5a20c6b6921e2cad937316de3fccd63420"
|
||||
checksum = "702d4415e08923e7e1ef96cd5727c0dfed80b4d2fa25db9647fe5eb6f7c5a4c4"
|
||||
dependencies = [
|
||||
"winnow 1.0.0",
|
||||
"winnow",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "toml_writer"
|
||||
version = "1.0.7+spec-1.1.0"
|
||||
version = "1.0.6+spec-1.1.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f17aaa1c6e3dc22b1da4b6bba97d066e354c7945cac2f7852d4e4e7ca7a6b56d"
|
||||
checksum = "ab16f14aed21ee8bfd8ec22513f7287cd4a91aa92e44edfe2c17ddd004e92607"
|
||||
|
||||
[[package]]
|
||||
name = "tonic"
|
||||
@@ -12625,8 +12757,6 @@ dependencies = [
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.2",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1",
|
||||
"thiserror 2.0.18",
|
||||
"utf-8",
|
||||
@@ -13643,15 +13773,6 @@ dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winnow"
|
||||
version = "1.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a90e88e4667264a994d34e6d1ab2d26d398dcdca8b7f52bec8668957517fc7d8"
|
||||
dependencies = [
|
||||
"memchr",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "winreg"
|
||||
version = "0.50.0"
|
||||
@@ -13837,18 +13958,18 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy"
|
||||
version = "0.8.46"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "5c5030500cb2d66bdfbb4ebc9563be6ce7005a4b5d0f26be0c523870fe372ca6"
|
||||
checksum = "f2578b716f8a7a858b7f02d5bd870c14bf4ddbbcf3a4c05414ba6503640505e3"
|
||||
dependencies = [
|
||||
"zerocopy-derive",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zerocopy-derive"
|
||||
version = "0.8.46"
|
||||
version = "0.8.42"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a5f86989a046a79640b9d8867c823349a139367bda96549794fcc3313ce91f4e"
|
||||
checksum = "7e6cc098ea4d3bd6246687de65af3f920c430e236bee1e3bf2e441463f08a02f"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
|
||||
93
Cargo.toml
93
Cargo.toml
@@ -10,6 +10,7 @@ exclude = [".github/"]
|
||||
[workspace]
|
||||
members = [
|
||||
"bin/reth-bench/",
|
||||
"bin/reth-bench-compare/",
|
||||
"bin/reth/",
|
||||
"crates/storage/rpc-provider/",
|
||||
"crates/chain-state/",
|
||||
@@ -137,6 +138,7 @@ members = [
|
||||
"examples/exex-subscription",
|
||||
"examples/exex-test",
|
||||
"examples/full-contract-state",
|
||||
"examples/migrate-trie-to-packed",
|
||||
"examples/manual-p2p/",
|
||||
"examples/network-txpool/",
|
||||
"examples/network/",
|
||||
@@ -322,6 +324,7 @@ reth = { path = "bin/reth" }
|
||||
reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" }
|
||||
reth-basic-payload-builder = { path = "crates/payload/basic" }
|
||||
reth-bench = { path = "bin/reth-bench" }
|
||||
reth-bench-compare = { path = "bin/reth-bench-compare" }
|
||||
reth-chain-state = { path = "crates/chain-state" }
|
||||
reth-chainspec = { path = "crates/chainspec", default-features = false }
|
||||
reth-cli = { path = "crates/cli/cli" }
|
||||
@@ -441,16 +444,17 @@ revm-state = { version = "10.0.0", default-features = false }
|
||||
revm-primitives = { version = "22.1.0", default-features = false }
|
||||
revm-interpreter = { version = "34.0.0", default-features = false }
|
||||
revm-database-interface = { version = "10.0.0", default-features = false }
|
||||
op-revm = { version = "17.0.0", default-features = false }
|
||||
revm-inspectors = "0.36.0"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.0"
|
||||
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-sol-types = { version = "1.5.0", default-features = false }
|
||||
alloy-dyn-abi = "1.5.6"
|
||||
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.3", default-features = false, features = ["rlp"] }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.29.2", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
@@ -486,13 +490,9 @@ alloy-transport-ipc = { version = "1.7.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.7.3", default-features = false }
|
||||
|
||||
# op
|
||||
alloy-op-hardforks = "0.4.4"
|
||||
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
|
||||
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
|
||||
op-alloy-network = { version = "0.23.1", default-features = false }
|
||||
op-alloy-consensus = { version = "0.23.1", default-features = false }
|
||||
op-alloy-rpc-jsonrpsee = { version = "0.23.1", default-features = false }
|
||||
op-alloy-flz = { version = "0.13.1", default-features = false }
|
||||
op-alloy-rpc-types = { version = "0.24.0", default-features = false }
|
||||
op-alloy-rpc-types-engine = { version = "0.24.0", default-features = false }
|
||||
op-alloy-consensus = { version = "0.24.0", default-features = false }
|
||||
|
||||
# misc
|
||||
either = { version = "1.15.0", default-features = false }
|
||||
@@ -531,21 +531,21 @@ quanta = "0.12"
|
||||
paste = "1.0"
|
||||
rand = "0.9"
|
||||
rayon = "1.7"
|
||||
thread-priority = "3.0.0"
|
||||
rustc-hash = { version = "2.0", default-features = false }
|
||||
schnellru = "0.2"
|
||||
serde = { version = "1.0", default-features = false }
|
||||
serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros"] }
|
||||
sha2 = { version = "0.10", default-features = false }
|
||||
shellexpand = "3.0.0"
|
||||
shlex = "1.3"
|
||||
slotmap = "1"
|
||||
# https://github.com/orlp/slotmap/pull/148
|
||||
slotmap = { git = "https://github.com/DaniPopes/slotmap.git", branch = "dani/shrink-methods" }
|
||||
smallvec = "1"
|
||||
strum = { version = "0.27", default-features = false }
|
||||
strum_macros = "0.27"
|
||||
syn = "2.0"
|
||||
thiserror = { version = "2.0.0", default-features = false }
|
||||
thread-priority = "3.0.0"
|
||||
tar = "0.4.44"
|
||||
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
|
||||
tracing-appender = "0.2"
|
||||
@@ -756,68 +756,3 @@ ipnet = "2.11"
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "9bc2dba" }
|
||||
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
|
||||
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
|
||||
|
||||
# =============================================================================
|
||||
# BAL devnet patches (EIP-7778, EIP-7928 block access lists)
|
||||
# =============================================================================
|
||||
|
||||
# revm staging patches
|
||||
revm = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-database = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-state = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-context = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
op-revm = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", branch = "staging" }
|
||||
|
||||
# alloy-evm bal-devnet2 patches
|
||||
alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "bal-devnet2" }
|
||||
|
||||
# alloy bal-devnet2 patches
|
||||
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-consensus-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
alloy-tx-macros = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
|
||||
|
||||
# op-alloy bal-devnet2 patches
|
||||
op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
|
||||
op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
|
||||
op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
|
||||
op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
|
||||
|
||||
2
Makefile
2
Makefile
@@ -29,7 +29,7 @@ EF_TESTS_URL := https://github.com/ethereum/tests/archive/refs/tags/$(EF_TESTS_T
|
||||
EF_TESTS_DIR := ./testing/ef-tests/ethereum-tests
|
||||
|
||||
# The release tag of https://github.com/ethereum/execution-spec-tests to use for EEST tests
|
||||
EEST_TESTS_TAG := bal@v5.0.0
|
||||
EEST_TESTS_TAG := v4.5.0
|
||||
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
|
||||
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
|
||||
|
||||
|
||||
99
bin/reth-bench-compare/Cargo.toml
Normal file
99
bin/reth-bench-compare/Cargo.toml
Normal file
@@ -0,0 +1,99 @@
|
||||
[package]
|
||||
name = "reth-bench-compare"
|
||||
version.workspace = true
|
||||
edition.workspace = true
|
||||
rust-version.workspace = true
|
||||
license.workspace = true
|
||||
homepage.workspace = true
|
||||
repository.workspace = true
|
||||
description = "Automated reth benchmark comparison between git references"
|
||||
|
||||
[lints]
|
||||
workspace = true
|
||||
|
||||
[[bin]]
|
||||
name = "reth-bench-compare"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
# reth
|
||||
reth-cli-runner.workspace = true
|
||||
reth-cli-util.workspace = true
|
||||
reth-node-core.workspace = true
|
||||
reth-tracing.workspace = true
|
||||
reth-chainspec.workspace = true
|
||||
|
||||
# alloy
|
||||
alloy-provider = { workspace = true, features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
|
||||
alloy-rpc-types-eth.workspace = true
|
||||
alloy-transport-ws.workspace = true
|
||||
alloy-primitives.workspace = true
|
||||
|
||||
# CLI and argument parsing
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
eyre.workspace = true
|
||||
|
||||
# Async runtime
|
||||
tokio = { workspace = true, features = ["full"] }
|
||||
tracing.workspace = true
|
||||
|
||||
# Serialization
|
||||
serde = { workspace = true, features = ["derive"] }
|
||||
serde_json.workspace = true
|
||||
|
||||
# Time handling
|
||||
chrono = { workspace = true, features = ["serde"] }
|
||||
|
||||
# CSV handling
|
||||
csv.workspace = true
|
||||
|
||||
# Process management
|
||||
ctrlc.workspace = true
|
||||
shlex.workspace = true
|
||||
|
||||
[target.'cfg(unix)'.dependencies]
|
||||
nix = { version = "0.31", features = ["signal", "process"] }
|
||||
|
||||
[features]
|
||||
default = ["jemalloc"]
|
||||
|
||||
asm-keccak = [
|
||||
"reth-node-core/asm-keccak",
|
||||
"alloy-primitives/asm-keccak",
|
||||
]
|
||||
|
||||
jemalloc = [
|
||||
"reth-cli-util/jemalloc",
|
||||
"reth-node-core/jemalloc",
|
||||
]
|
||||
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
|
||||
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
|
||||
tracy = [
|
||||
"reth-node-core/tracy",
|
||||
"reth-tracing/tracy",
|
||||
]
|
||||
|
||||
min-error-logs = [
|
||||
"tracing/release_max_level_error",
|
||||
"reth-node-core/min-error-logs",
|
||||
]
|
||||
min-warn-logs = [
|
||||
"tracing/release_max_level_warn",
|
||||
"reth-node-core/min-warn-logs",
|
||||
]
|
||||
min-info-logs = [
|
||||
"tracing/release_max_level_info",
|
||||
"reth-node-core/min-info-logs",
|
||||
]
|
||||
min-debug-logs = [
|
||||
"tracing/release_max_level_debug",
|
||||
"reth-node-core/min-debug-logs",
|
||||
]
|
||||
min-trace-logs = [
|
||||
"tracing/release_max_level_trace",
|
||||
"reth-node-core/min-trace-logs",
|
||||
]
|
||||
|
||||
# no-op feature flag for CI matrices
|
||||
ethereum = []
|
||||
50
bin/reth-bench-compare/README.md
Normal file
50
bin/reth-bench-compare/README.md
Normal file
@@ -0,0 +1,50 @@
|
||||
# reth-bench-compare
|
||||
|
||||
Compare reth performance between two git references.
|
||||
|
||||
## Usage
|
||||
|
||||
```bash
|
||||
reth-bench-compare \
|
||||
--baseline-ref main \
|
||||
--feature-ref my-feature \
|
||||
--blocks 100 \
|
||||
--wait-for-persistence
|
||||
```
|
||||
|
||||
## Arguments
|
||||
|
||||
| Argument | Description | Default | Required |
|
||||
|----------|-------------|---------|----------|
|
||||
| `--baseline-ref <REF>` | Git reference for baseline | - | Yes |
|
||||
| `--feature-ref <REF>` | Git reference to compare | - | Yes |
|
||||
| `--blocks <N>` | Number of blocks to benchmark | `100` | No |
|
||||
| `--chain <CHAIN>` | Chain to benchmark | `mainnet` | No |
|
||||
| `--datadir <PATH>` | Data directory path | OS-specific | No |
|
||||
| `--rpc-url <URL>` | RPC endpoint for block data | Chain default | No |
|
||||
| `--output-dir <PATH>` | Output directory | `./reth-bench-compare` | No |
|
||||
| `--wait-for-persistence` | Wait for block persistence | `false` | No |
|
||||
| `--persistence-threshold <N>` | Wait after every N+1 blocks | `2` | No |
|
||||
| `--wait-time <DURATION>` | Fixed delay (legacy) | - | No |
|
||||
| `--warmup-blocks <N>` | Cache warmup blocks | Same as `--blocks` | No |
|
||||
| `--draw` | Generate charts (needs Python/uv) | `false` | No |
|
||||
| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
|
||||
| `-vvvv` | Debug logging | Info | No |
|
||||
| `--features <FEATURES>` | Extra Rust features for both builds | - | No |
|
||||
| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
|
||||
| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
|
||||
| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |
|
||||
| `--baseline-rustflags <FLAGS>` | RUSTFLAGS for baseline only | Inherits `--rustflags` | No |
|
||||
| `--feature-rustflags <FLAGS>` | RUSTFLAGS for feature only | Inherits `--rustflags` | No |
|
||||
| `--baseline-args <ARGS>` | Extra args for baseline node | - | No |
|
||||
| `--feature-args <ARGS>` | Extra args for feature node | - | No |
|
||||
| `--metrics-port <PORT>` | Metrics endpoint port | `5005` | No |
|
||||
| `--sudo` | Run with elevated privileges | `false` | No |
|
||||
|
||||
## Output
|
||||
|
||||
Results in `./reth-bench-compare/results/<timestamp>/`:
|
||||
- `comparison_report.json` - Metrics comparison
|
||||
- `per_block_comparison.csv` - Per-block statistics
|
||||
- `baseline/` and `feature/` - Individual run results
|
||||
- `latency_comparison.png` - Chart (if `--draw` used)
|
||||
307
bin/reth-bench-compare/src/benchmark.rs
Normal file
307
bin/reth-bench-compare/src/benchmark.rs
Normal file
@@ -0,0 +1,307 @@
|
||||
//! Benchmark execution using reth-bench.
|
||||
|
||||
use crate::cli::Args;
|
||||
use eyre::{eyre, Result, WrapErr};
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{Arc, Mutex},
|
||||
};
|
||||
use tokio::{
|
||||
fs::File as AsyncFile,
|
||||
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
|
||||
process::Command,
|
||||
};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Manages benchmark execution using reth-bench
|
||||
pub(crate) struct BenchmarkRunner {
|
||||
rpc_url: String,
|
||||
jwt_secret: String,
|
||||
wait_time: Option<String>,
|
||||
wait_for_persistence: bool,
|
||||
persistence_threshold: Option<u64>,
|
||||
warmup_blocks: u64,
|
||||
}
|
||||
|
||||
impl BenchmarkRunner {
|
||||
/// Create a new `BenchmarkRunner` from CLI arguments
|
||||
pub(crate) fn new(args: &Args) -> Self {
|
||||
Self {
|
||||
rpc_url: args.get_rpc_url(),
|
||||
jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(),
|
||||
wait_time: args.wait_time.clone(),
|
||||
wait_for_persistence: args.wait_for_persistence,
|
||||
persistence_threshold: args.persistence_threshold,
|
||||
warmup_blocks: args.get_warmup_blocks(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Clear filesystem caches (page cache, dentries, and inodes)
|
||||
pub(crate) async fn clear_fs_caches() -> Result<()> {
|
||||
info!("Clearing filesystem caches...");
|
||||
|
||||
// First sync to ensure all pending writes are flushed
|
||||
let sync_output =
|
||||
Command::new("sync").output().await.wrap_err("Failed to execute sync command")?;
|
||||
|
||||
if !sync_output.status.success() {
|
||||
return Err(eyre!("sync command failed"));
|
||||
}
|
||||
|
||||
// Drop caches - requires sudo/root permissions
|
||||
// 3 = drop pagecache, dentries, and inodes
|
||||
let drop_caches_cmd = Command::new("sudo")
|
||||
.args(["-n", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"])
|
||||
.output()
|
||||
.await;
|
||||
|
||||
match drop_caches_cmd {
|
||||
Ok(output) if output.status.success() => {
|
||||
info!("Successfully cleared filesystem caches");
|
||||
Ok(())
|
||||
}
|
||||
Ok(output) => {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
if stderr.contains("sudo: a password is required") {
|
||||
warn!("Unable to clear filesystem caches: sudo password required");
|
||||
warn!(
|
||||
"For optimal benchmarking, configure passwordless sudo for cache clearing:"
|
||||
);
|
||||
warn!(" echo '$USER ALL=(ALL) NOPASSWD: /bin/sh -c echo\\\\ [0-9]\\\\ \\\\>\\\\ /proc/sys/vm/drop_caches' | sudo tee /etc/sudoers.d/drop_caches");
|
||||
Ok(())
|
||||
} else {
|
||||
Err(eyre!("Failed to clear filesystem caches: {}", stderr))
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
warn!("Unable to clear filesystem caches: {}", e);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Run a warmup benchmark for cache warming
|
||||
pub(crate) async fn run_warmup(&self, from_block: u64) -> Result<()> {
|
||||
let to_block = from_block + self.warmup_blocks;
|
||||
info!(
|
||||
"Running warmup benchmark from block {} to {} ({} blocks)",
|
||||
from_block, to_block, self.warmup_blocks
|
||||
);
|
||||
|
||||
// Build the reth-bench command for warmup (no output flag)
|
||||
let mut cmd = Command::new("reth-bench");
|
||||
cmd.args([
|
||||
"new-payload-fcu",
|
||||
"--rpc-url",
|
||||
&self.rpc_url,
|
||||
"--jwt-secret",
|
||||
&self.jwt_secret,
|
||||
"--from",
|
||||
&from_block.to_string(),
|
||||
"--to",
|
||||
&to_block.to_string(),
|
||||
"--wait-time=0ms", // Warmup should avoid persistence waits.
|
||||
]);
|
||||
|
||||
cmd.env("RUST_LOG_STYLE", "never")
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
|
||||
// Set process group for consistent signal handling
|
||||
#[cfg(unix)]
|
||||
{
|
||||
cmd.process_group(0);
|
||||
}
|
||||
|
||||
debug!("Executing warmup reth-bench command: {:?}", cmd);
|
||||
|
||||
// Execute the warmup benchmark
|
||||
let mut child = cmd.spawn().wrap_err("Failed to start warmup reth-bench process")?;
|
||||
|
||||
// Stream output at debug level
|
||||
if let Some(stdout) = child.stdout.take() {
|
||||
tokio::spawn(async move {
|
||||
let reader = BufReader::new(stdout);
|
||||
let mut lines = reader.lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[WARMUP] {}", line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(stderr) = child.stderr.take() {
|
||||
tokio::spawn(async move {
|
||||
let reader = BufReader::new(stderr);
|
||||
let mut lines = reader.lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[WARMUP] {}", line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let status = child.wait().await.wrap_err("Failed to wait for warmup reth-bench")?;
|
||||
|
||||
if !status.success() {
|
||||
return Err(eyre!("Warmup reth-bench failed with exit code: {:?}", status.code()));
|
||||
}
|
||||
|
||||
info!("Warmup completed successfully");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Run a benchmark for the specified block range
|
||||
pub(crate) async fn run_benchmark(
|
||||
&self,
|
||||
from_block: u64,
|
||||
to_block: u64,
|
||||
output_dir: &Path,
|
||||
) -> Result<()> {
|
||||
info!(
|
||||
"Running benchmark from block {} to {} (output: {:?})",
|
||||
from_block, to_block, output_dir
|
||||
);
|
||||
|
||||
// Ensure output directory exists
|
||||
std::fs::create_dir_all(output_dir)
|
||||
.wrap_err_with(|| format!("Failed to create output directory: {output_dir:?}"))?;
|
||||
|
||||
// Create log file path for reth-bench output
|
||||
let log_file_path = output_dir.join("reth_bench.log");
|
||||
info!("reth-bench logs will be saved to: {:?}", log_file_path);
|
||||
|
||||
// Build the reth-bench command
|
||||
let mut cmd = Command::new("reth-bench");
|
||||
cmd.args([
|
||||
"new-payload-fcu",
|
||||
"--rpc-url",
|
||||
&self.rpc_url,
|
||||
"--jwt-secret",
|
||||
&self.jwt_secret,
|
||||
"--from",
|
||||
&from_block.to_string(),
|
||||
"--to",
|
||||
&to_block.to_string(),
|
||||
"--output",
|
||||
&output_dir.to_string_lossy(),
|
||||
]);
|
||||
|
||||
// Configure wait mode: both can be used together
|
||||
// When both are set: wait at least wait_time, and also wait for persistence if needed
|
||||
if let Some(ref wait_time) = self.wait_time {
|
||||
cmd.args(["--wait-time", wait_time]);
|
||||
}
|
||||
if self.wait_for_persistence {
|
||||
cmd.arg("--wait-for-persistence");
|
||||
|
||||
// Add persistence threshold if specified
|
||||
if let Some(threshold) = self.persistence_threshold {
|
||||
cmd.args(["--persistence-threshold", &threshold.to_string()]);
|
||||
}
|
||||
}
|
||||
|
||||
cmd.env("RUST_LOG_STYLE", "never")
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true);
|
||||
|
||||
// Set process group for consistent signal handling
|
||||
#[cfg(unix)]
|
||||
{
|
||||
cmd.process_group(0);
|
||||
}
|
||||
|
||||
// Debug log the command
|
||||
debug!("Executing reth-bench command: {:?}", cmd);
|
||||
|
||||
// Execute the benchmark
|
||||
let mut child = cmd.spawn().wrap_err("Failed to start reth-bench process")?;
|
||||
|
||||
// Capture stdout and stderr for error reporting
|
||||
let stdout_lines = Arc::new(Mutex::new(Vec::new()));
|
||||
let stderr_lines = Arc::new(Mutex::new(Vec::new()));
|
||||
|
||||
// Stream stdout with prefix at debug level, capture for error reporting, and write to log
|
||||
// file
|
||||
if let Some(stdout) = child.stdout.take() {
|
||||
let stdout_lines_clone = stdout_lines.clone();
|
||||
let log_file = AsyncFile::create(&log_file_path)
|
||||
.await
|
||||
.wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
|
||||
tokio::spawn(async move {
|
||||
let reader = BufReader::new(stdout);
|
||||
let mut lines = reader.lines();
|
||||
let mut log_file = log_file;
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[RETH-BENCH] {}", line);
|
||||
if let Ok(mut captured) = stdout_lines_clone.lock() {
|
||||
captured.push(line.clone());
|
||||
}
|
||||
// Write to log file (reth-bench output already has timestamps if needed)
|
||||
let log_line = format!("{}\n", line);
|
||||
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
|
||||
debug!("Failed to write to log file: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Stream stderr with prefix at debug level, capture for error reporting, and write to log
|
||||
// file
|
||||
if let Some(stderr) = child.stderr.take() {
|
||||
let stderr_lines_clone = stderr_lines.clone();
|
||||
let log_file = AsyncFile::options()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&log_file_path)
|
||||
.await
|
||||
.wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
|
||||
tokio::spawn(async move {
|
||||
let reader = BufReader::new(stderr);
|
||||
let mut lines = reader.lines();
|
||||
let mut log_file = log_file;
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[RETH-BENCH] {}", line);
|
||||
if let Ok(mut captured) = stderr_lines_clone.lock() {
|
||||
captured.push(line.clone());
|
||||
}
|
||||
// Write to log file (reth-bench output already has timestamps if needed)
|
||||
let log_line = format!("{}\n", line);
|
||||
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
|
||||
debug!("Failed to write to log file: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
let status = child.wait().await.wrap_err("Failed to wait for reth-bench")?;
|
||||
|
||||
if !status.success() {
|
||||
// Print all captured output when command fails
|
||||
error!("reth-bench failed with exit code: {:?}", status.code());
|
||||
|
||||
if let Ok(stdout) = stdout_lines.lock() &&
|
||||
!stdout.is_empty()
|
||||
{
|
||||
error!("reth-bench stdout:");
|
||||
for line in stdout.iter() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if let Ok(stderr) = stderr_lines.lock() &&
|
||||
!stderr.is_empty()
|
||||
{
|
||||
error!("reth-bench stderr:");
|
||||
for line in stderr.iter() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
return Err(eyre!("reth-bench failed with exit code: {:?}", status.code()));
|
||||
}
|
||||
|
||||
info!("Benchmark completed");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
1028
bin/reth-bench-compare/src/cli.rs
Normal file
1028
bin/reth-bench-compare/src/cli.rs
Normal file
File diff suppressed because it is too large
Load Diff
723
bin/reth-bench-compare/src/comparison.rs
Normal file
723
bin/reth-bench-compare/src/comparison.rs
Normal file
@@ -0,0 +1,723 @@
|
||||
//! Results comparison and report generation.
|
||||
|
||||
use crate::cli::Args;
|
||||
use chrono::{DateTime, Utc};
|
||||
use csv::Reader;
|
||||
use eyre::{eyre, Result, WrapErr};
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::HashMap,
|
||||
fs,
|
||||
path::{Path, PathBuf},
|
||||
};
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// Manages comparison between baseline and feature reference results
|
||||
pub(crate) struct ComparisonGenerator {
|
||||
output_dir: PathBuf,
|
||||
timestamp: String,
|
||||
baseline_ref_name: String,
|
||||
feature_ref_name: String,
|
||||
baseline_results: Option<BenchmarkResults>,
|
||||
feature_results: Option<BenchmarkResults>,
|
||||
baseline_command: Option<String>,
|
||||
feature_command: Option<String>,
|
||||
}
|
||||
|
||||
/// Represents the results from a single benchmark run
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct BenchmarkResults {
|
||||
pub ref_name: String,
|
||||
pub combined_latency_data: Vec<CombinedLatencyRow>,
|
||||
pub summary: BenchmarkSummary,
|
||||
pub start_timestamp: Option<DateTime<Utc>>,
|
||||
pub end_timestamp: Option<DateTime<Utc>>,
|
||||
}
|
||||
|
||||
/// Combined latency CSV row structure
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub(crate) struct CombinedLatencyRow {
|
||||
pub block_number: u64,
|
||||
#[serde(default)]
|
||||
pub transaction_count: Option<u64>,
|
||||
pub gas_used: u64,
|
||||
pub new_payload_latency: u128,
|
||||
}
|
||||
|
||||
/// Total gas CSV row structure
|
||||
#[derive(Debug, Clone, Deserialize, Serialize)]
|
||||
pub(crate) struct TotalGasRow {
|
||||
pub block_number: u64,
|
||||
#[serde(default)]
|
||||
pub transaction_count: Option<u64>,
|
||||
pub gas_used: u64,
|
||||
pub time: u128,
|
||||
}
|
||||
|
||||
/// Summary statistics for a benchmark run.
|
||||
///
|
||||
/// Latencies are derived from per-block `engine_newPayload` timings (converted from µs to ms):
|
||||
/// - `mean_new_payload_latency_ms`: arithmetic mean latency across blocks.
|
||||
/// - `median_new_payload_latency_ms`: p50 latency across blocks.
|
||||
/// - `p90_new_payload_latency_ms` / `p99_new_payload_latency_ms`: tail latencies across blocks.
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
pub(crate) struct BenchmarkSummary {
|
||||
pub total_blocks: u64,
|
||||
pub total_gas_used: u64,
|
||||
pub total_duration_ms: u128,
|
||||
pub mean_new_payload_latency_ms: f64,
|
||||
pub median_new_payload_latency_ms: f64,
|
||||
pub p90_new_payload_latency_ms: f64,
|
||||
pub p99_new_payload_latency_ms: f64,
|
||||
pub gas_per_second: f64,
|
||||
pub blocks_per_second: f64,
|
||||
pub min_block_number: u64,
|
||||
pub max_block_number: u64,
|
||||
}
|
||||
|
||||
/// Comparison report between two benchmark runs
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct ComparisonReport {
|
||||
pub timestamp: String,
|
||||
pub baseline: RefInfo,
|
||||
pub feature: RefInfo,
|
||||
pub comparison_summary: ComparisonSummary,
|
||||
pub per_block_comparisons: Vec<BlockComparison>,
|
||||
}
|
||||
|
||||
/// Information about a reference in the comparison
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct RefInfo {
|
||||
pub ref_name: String,
|
||||
pub summary: BenchmarkSummary,
|
||||
pub start_timestamp: Option<DateTime<Utc>>,
|
||||
pub end_timestamp: Option<DateTime<Utc>>,
|
||||
pub reth_command: Option<String>,
|
||||
}
|
||||
|
||||
/// Summary of the comparison between references.
|
||||
///
|
||||
/// Percent deltas are `(feature - baseline) / baseline * 100`:
|
||||
/// - `new_payload_latency_mean_change_percent`: percent changes of the per-block means.
|
||||
/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective
|
||||
/// per-block percentiles.
|
||||
/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the
|
||||
/// mean and median of per-block percent deltas (feature vs baseline), capturing block-level
|
||||
/// drift.
|
||||
/// - `per_block_latency_change_std_dev_percent`: standard deviation of per-block percent changes,
|
||||
/// measuring consistency of performance changes across blocks.
|
||||
/// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time
|
||||
/// across the run.
|
||||
///
|
||||
/// Positive means slower/higher; negative means faster/lower.
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct ComparisonSummary {
|
||||
pub per_block_latency_change_mean_percent: f64,
|
||||
pub per_block_latency_change_median_percent: f64,
|
||||
pub per_block_latency_change_std_dev_percent: f64,
|
||||
pub new_payload_total_latency_change_percent: f64,
|
||||
pub new_payload_latency_mean_change_percent: f64,
|
||||
pub new_payload_latency_p50_change_percent: f64,
|
||||
pub new_payload_latency_p90_change_percent: f64,
|
||||
pub new_payload_latency_p99_change_percent: f64,
|
||||
pub gas_per_second_change_percent: f64,
|
||||
pub blocks_per_second_change_percent: f64,
|
||||
}
|
||||
|
||||
/// Per-block comparison data
|
||||
#[derive(Debug, Serialize)]
|
||||
pub(crate) struct BlockComparison {
|
||||
pub block_number: u64,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub transaction_count: Option<u64>,
|
||||
pub gas_used: u64,
|
||||
pub baseline_new_payload_latency: u128,
|
||||
pub feature_new_payload_latency: u128,
|
||||
pub new_payload_latency_change_percent: f64,
|
||||
}
|
||||
|
||||
impl ComparisonGenerator {
|
||||
/// Create a new comparison generator
|
||||
pub(crate) fn new(args: &Args) -> Self {
|
||||
let now: DateTime<Utc> = Utc::now();
|
||||
let timestamp = now.format("%Y%m%d_%H%M%S").to_string();
|
||||
|
||||
Self {
|
||||
output_dir: args.output_dir_path(),
|
||||
timestamp,
|
||||
baseline_ref_name: args.baseline_ref.clone(),
|
||||
feature_ref_name: args.feature_ref.clone(),
|
||||
baseline_results: None,
|
||||
feature_results: None,
|
||||
baseline_command: None,
|
||||
feature_command: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Get the output directory for a specific reference
|
||||
pub(crate) fn get_ref_output_dir(&self, ref_type: &str) -> PathBuf {
|
||||
self.output_dir.join("results").join(&self.timestamp).join(ref_type)
|
||||
}
|
||||
|
||||
/// Get the main output directory for this comparison run
|
||||
pub(crate) fn get_output_dir(&self) -> PathBuf {
|
||||
self.output_dir.join("results").join(&self.timestamp)
|
||||
}
|
||||
|
||||
/// Add benchmark results for a reference
|
||||
pub(crate) fn add_ref_results(&mut self, ref_type: &str, output_path: &Path) -> Result<()> {
|
||||
let ref_name = match ref_type {
|
||||
"baseline" => &self.baseline_ref_name,
|
||||
"feature" => &self.feature_ref_name,
|
||||
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
|
||||
};
|
||||
|
||||
let results = self.load_benchmark_results(ref_name, output_path)?;
|
||||
|
||||
match ref_type {
|
||||
"baseline" => self.baseline_results = Some(results),
|
||||
"feature" => self.feature_results = Some(results),
|
||||
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
|
||||
}
|
||||
|
||||
info!("Loaded benchmark results for {} reference", ref_type);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the benchmark run timestamps for a reference
|
||||
pub(crate) fn set_ref_timestamps(
|
||||
&mut self,
|
||||
ref_type: &str,
|
||||
start: DateTime<Utc>,
|
||||
end: DateTime<Utc>,
|
||||
) -> Result<()> {
|
||||
match ref_type {
|
||||
"baseline" => {
|
||||
if let Some(ref mut results) = self.baseline_results {
|
||||
results.start_timestamp = Some(start);
|
||||
results.end_timestamp = Some(end);
|
||||
} else {
|
||||
return Err(eyre!("Baseline results not loaded yet"));
|
||||
}
|
||||
}
|
||||
"feature" => {
|
||||
if let Some(ref mut results) = self.feature_results {
|
||||
results.start_timestamp = Some(start);
|
||||
results.end_timestamp = Some(end);
|
||||
} else {
|
||||
return Err(eyre!("Feature results not loaded yet"));
|
||||
}
|
||||
}
|
||||
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Set the reth command for a reference
|
||||
pub(crate) fn set_ref_command(&mut self, ref_type: &str, command: String) -> Result<()> {
|
||||
match ref_type {
|
||||
"baseline" => {
|
||||
self.baseline_command = Some(command);
|
||||
}
|
||||
"feature" => {
|
||||
self.feature_command = Some(command);
|
||||
}
|
||||
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Generate the final comparison report
|
||||
pub(crate) async fn generate_comparison_report(&self) -> Result<()> {
|
||||
info!("Generating comparison report...");
|
||||
|
||||
let baseline =
|
||||
self.baseline_results.as_ref().ok_or_else(|| eyre!("Baseline results not loaded"))?;
|
||||
|
||||
let feature =
|
||||
self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?;
|
||||
|
||||
let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?;
|
||||
let comparison_summary = self.calculate_comparison_summary(
|
||||
&baseline.summary,
|
||||
&feature.summary,
|
||||
&per_block_comparisons,
|
||||
)?;
|
||||
|
||||
let report = ComparisonReport {
|
||||
timestamp: self.timestamp.clone(),
|
||||
baseline: RefInfo {
|
||||
ref_name: baseline.ref_name.clone(),
|
||||
summary: baseline.summary.clone(),
|
||||
start_timestamp: baseline.start_timestamp,
|
||||
end_timestamp: baseline.end_timestamp,
|
||||
reth_command: self.baseline_command.clone(),
|
||||
},
|
||||
feature: RefInfo {
|
||||
ref_name: feature.ref_name.clone(),
|
||||
summary: feature.summary.clone(),
|
||||
start_timestamp: feature.start_timestamp,
|
||||
end_timestamp: feature.end_timestamp,
|
||||
reth_command: self.feature_command.clone(),
|
||||
},
|
||||
comparison_summary,
|
||||
per_block_comparisons,
|
||||
};
|
||||
|
||||
// Write reports
|
||||
self.write_comparison_reports(&report).await?;
|
||||
|
||||
// Print summary to console
|
||||
self.print_comparison_summary(&report);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Load benchmark results from CSV files
|
||||
fn load_benchmark_results(
|
||||
&self,
|
||||
ref_name: &str,
|
||||
output_path: &Path,
|
||||
) -> Result<BenchmarkResults> {
|
||||
let combined_latency_path = output_path.join("combined_latency.csv");
|
||||
let total_gas_path = output_path.join("total_gas.csv");
|
||||
|
||||
let combined_latency_data = self.load_combined_latency_csv(&combined_latency_path)?;
|
||||
let total_gas_data = self.load_total_gas_csv(&total_gas_path)?;
|
||||
|
||||
let summary = self.calculate_summary(&combined_latency_data, &total_gas_data)?;
|
||||
|
||||
Ok(BenchmarkResults {
|
||||
ref_name: ref_name.to_string(),
|
||||
combined_latency_data,
|
||||
summary,
|
||||
start_timestamp: None,
|
||||
end_timestamp: None,
|
||||
})
|
||||
}
|
||||
|
||||
/// Load combined latency CSV data
|
||||
fn load_combined_latency_csv(&self, path: &Path) -> Result<Vec<CombinedLatencyRow>> {
|
||||
let mut reader = Reader::from_path(path)
|
||||
.wrap_err_with(|| format!("Failed to open combined latency CSV: {path:?}"))?;
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for result in reader.deserialize() {
|
||||
let row: CombinedLatencyRow = result
|
||||
.wrap_err_with(|| format!("Failed to parse combined latency row in {path:?}"))?;
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
if rows.is_empty() {
|
||||
return Err(eyre!("No data found in combined latency CSV: {:?}", path));
|
||||
}
|
||||
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Load total gas CSV data
|
||||
fn load_total_gas_csv(&self, path: &Path) -> Result<Vec<TotalGasRow>> {
|
||||
let mut reader = Reader::from_path(path)
|
||||
.wrap_err_with(|| format!("Failed to open total gas CSV: {path:?}"))?;
|
||||
|
||||
let mut rows = Vec::new();
|
||||
for result in reader.deserialize() {
|
||||
let row: TotalGasRow =
|
||||
result.wrap_err_with(|| format!("Failed to parse total gas row in {path:?}"))?;
|
||||
rows.push(row);
|
||||
}
|
||||
|
||||
if rows.is_empty() {
|
||||
return Err(eyre!("No data found in total gas CSV: {:?}", path));
|
||||
}
|
||||
|
||||
Ok(rows)
|
||||
}
|
||||
|
||||
/// Calculate summary statistics for a benchmark run.
|
||||
///
|
||||
/// Computes latency statistics from per-block `new_payload_latency` values in `combined_data`
|
||||
/// (converting from µs to ms), and throughput metrics using the total run duration from
|
||||
/// `total_gas_data`. Percentiles (p50/p90/p99) use linear interpolation on sorted latencies.
|
||||
fn calculate_summary(
|
||||
&self,
|
||||
combined_data: &[CombinedLatencyRow],
|
||||
total_gas_data: &[TotalGasRow],
|
||||
) -> Result<BenchmarkSummary> {
|
||||
if combined_data.is_empty() || total_gas_data.is_empty() {
|
||||
return Err(eyre!("Cannot calculate summary for empty data"));
|
||||
}
|
||||
|
||||
let total_blocks = combined_data.len() as u64;
|
||||
let total_gas_used: u64 = combined_data.iter().map(|r| r.gas_used).sum();
|
||||
|
||||
let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds
|
||||
|
||||
let latencies_ms: Vec<f64> =
|
||||
combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).collect();
|
||||
let mean_new_payload_latency_ms: f64 =
|
||||
latencies_ms.iter().sum::<f64>() / total_blocks as f64;
|
||||
|
||||
let mut sorted_latencies_ms = latencies_ms;
|
||||
sorted_latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
|
||||
let median_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.5);
|
||||
let p90_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.9);
|
||||
let p99_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.99);
|
||||
|
||||
let total_duration_seconds = total_duration_ms as f64 / 1000.0;
|
||||
let gas_per_second = if total_duration_seconds > f64::EPSILON {
|
||||
total_gas_used as f64 / total_duration_seconds
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let blocks_per_second = if total_duration_seconds > f64::EPSILON {
|
||||
total_blocks as f64 / total_duration_seconds
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
let min_block_number = combined_data.first().unwrap().block_number;
|
||||
let max_block_number = combined_data.last().unwrap().block_number;
|
||||
|
||||
Ok(BenchmarkSummary {
|
||||
total_blocks,
|
||||
total_gas_used,
|
||||
total_duration_ms,
|
||||
mean_new_payload_latency_ms,
|
||||
median_new_payload_latency_ms,
|
||||
p90_new_payload_latency_ms,
|
||||
p99_new_payload_latency_ms,
|
||||
gas_per_second,
|
||||
blocks_per_second,
|
||||
min_block_number,
|
||||
max_block_number,
|
||||
})
|
||||
}
|
||||
|
||||
/// Calculate comparison summary between baseline and feature
|
||||
fn calculate_comparison_summary(
|
||||
&self,
|
||||
baseline: &BenchmarkSummary,
|
||||
feature: &BenchmarkSummary,
|
||||
per_block_comparisons: &[BlockComparison],
|
||||
) -> Result<ComparisonSummary> {
|
||||
let calc_percent_change = |baseline: f64, feature: f64| -> f64 {
|
||||
if baseline.abs() > f64::EPSILON {
|
||||
((feature - baseline) / baseline) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
};
|
||||
|
||||
// Calculate per-block statistics. "Per-block" means: for each block, compute the percent
|
||||
// change (feature - baseline) / baseline * 100, then calculate statistics across those
|
||||
// per-block percent changes. This captures how consistently the feature performs relative
|
||||
// to baseline across all blocks.
|
||||
let per_block_percent_changes: Vec<f64> =
|
||||
per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect();
|
||||
let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
per_block_percent_changes.iter().sum::<f64>() / per_block_percent_changes.len() as f64
|
||||
};
|
||||
let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() {
|
||||
0.0
|
||||
} else {
|
||||
let mut sorted = per_block_percent_changes.clone();
|
||||
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
|
||||
percentile(&sorted, 0.5)
|
||||
};
|
||||
let per_block_latency_change_std_dev_percent =
|
||||
calculate_std_dev(&per_block_percent_changes, per_block_latency_change_mean_percent);
|
||||
|
||||
let baseline_total_latency_ms =
|
||||
baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64;
|
||||
let feature_total_latency_ms =
|
||||
feature.mean_new_payload_latency_ms * feature.total_blocks as f64;
|
||||
let new_payload_total_latency_change_percent =
|
||||
calc_percent_change(baseline_total_latency_ms, feature_total_latency_ms);
|
||||
|
||||
Ok(ComparisonSummary {
|
||||
per_block_latency_change_mean_percent,
|
||||
per_block_latency_change_median_percent,
|
||||
per_block_latency_change_std_dev_percent,
|
||||
new_payload_total_latency_change_percent,
|
||||
new_payload_latency_mean_change_percent: calc_percent_change(
|
||||
baseline.mean_new_payload_latency_ms,
|
||||
feature.mean_new_payload_latency_ms,
|
||||
),
|
||||
new_payload_latency_p50_change_percent: calc_percent_change(
|
||||
baseline.median_new_payload_latency_ms,
|
||||
feature.median_new_payload_latency_ms,
|
||||
),
|
||||
new_payload_latency_p90_change_percent: calc_percent_change(
|
||||
baseline.p90_new_payload_latency_ms,
|
||||
feature.p90_new_payload_latency_ms,
|
||||
),
|
||||
new_payload_latency_p99_change_percent: calc_percent_change(
|
||||
baseline.p99_new_payload_latency_ms,
|
||||
feature.p99_new_payload_latency_ms,
|
||||
),
|
||||
gas_per_second_change_percent: calc_percent_change(
|
||||
baseline.gas_per_second,
|
||||
feature.gas_per_second,
|
||||
),
|
||||
blocks_per_second_change_percent: calc_percent_change(
|
||||
baseline.blocks_per_second,
|
||||
feature.blocks_per_second,
|
||||
),
|
||||
})
|
||||
}
|
||||
|
||||
/// Calculate per-block comparisons
|
||||
fn calculate_per_block_comparisons(
|
||||
&self,
|
||||
baseline: &BenchmarkResults,
|
||||
feature: &BenchmarkResults,
|
||||
) -> Result<Vec<BlockComparison>> {
|
||||
let mut baseline_map: HashMap<u64, &CombinedLatencyRow> = HashMap::new();
|
||||
for row in &baseline.combined_latency_data {
|
||||
baseline_map.insert(row.block_number, row);
|
||||
}
|
||||
|
||||
let mut comparisons = Vec::new();
|
||||
for feature_row in &feature.combined_latency_data {
|
||||
if let Some(baseline_row) = baseline_map.get(&feature_row.block_number) {
|
||||
let calc_percent_change = |baseline: u128, feature: u128| -> f64 {
|
||||
if baseline > 0 {
|
||||
((feature as f64 - baseline as f64) / baseline as f64) * 100.0
|
||||
} else {
|
||||
0.0
|
||||
}
|
||||
};
|
||||
|
||||
let comparison = BlockComparison {
|
||||
block_number: feature_row.block_number,
|
||||
transaction_count: feature_row.transaction_count,
|
||||
gas_used: feature_row.gas_used,
|
||||
baseline_new_payload_latency: baseline_row.new_payload_latency,
|
||||
feature_new_payload_latency: feature_row.new_payload_latency,
|
||||
new_payload_latency_change_percent: calc_percent_change(
|
||||
baseline_row.new_payload_latency,
|
||||
feature_row.new_payload_latency,
|
||||
),
|
||||
};
|
||||
comparisons.push(comparison);
|
||||
} else {
|
||||
warn!("Block {} not found in baseline data", feature_row.block_number);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(comparisons)
|
||||
}
|
||||
|
||||
/// Write comparison reports to files
|
||||
async fn write_comparison_reports(&self, report: &ComparisonReport) -> Result<()> {
|
||||
let report_dir = self.output_dir.join("results").join(&self.timestamp);
|
||||
fs::create_dir_all(&report_dir)
|
||||
.wrap_err_with(|| format!("Failed to create report directory: {report_dir:?}"))?;
|
||||
|
||||
// Write JSON report
|
||||
let json_path = report_dir.join("comparison_report.json");
|
||||
let json_content = serde_json::to_string_pretty(report)
|
||||
.wrap_err("Failed to serialize comparison report to JSON")?;
|
||||
fs::write(&json_path, json_content)
|
||||
.wrap_err_with(|| format!("Failed to write JSON report: {json_path:?}"))?;
|
||||
|
||||
// Write CSV report for per-block comparisons
|
||||
let csv_path = report_dir.join("per_block_comparison.csv");
|
||||
let mut writer = csv::Writer::from_path(&csv_path)
|
||||
.wrap_err_with(|| format!("Failed to create CSV writer: {csv_path:?}"))?;
|
||||
|
||||
for comparison in &report.per_block_comparisons {
|
||||
writer.serialize(comparison).wrap_err("Failed to write comparison row to CSV")?;
|
||||
}
|
||||
writer.flush().wrap_err("Failed to flush CSV writer")?;
|
||||
|
||||
info!("Comparison reports written to: {:?}", report_dir);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Print comparison summary to console
|
||||
fn print_comparison_summary(&self, report: &ComparisonReport) {
|
||||
// Parse and format timestamp nicely
|
||||
let formatted_timestamp = if let Ok(dt) = chrono::DateTime::parse_from_str(
|
||||
&format!("{} +0000", report.timestamp.replace('_', " ")),
|
||||
"%Y%m%d %H%M%S %z",
|
||||
) {
|
||||
dt.format("%Y-%m-%d %H:%M:%S UTC").to_string()
|
||||
} else {
|
||||
// Fallback to original if parsing fails
|
||||
report.timestamp.clone()
|
||||
};
|
||||
|
||||
println!("\n=== BENCHMARK COMPARISON SUMMARY ===");
|
||||
println!("Timestamp: {formatted_timestamp}");
|
||||
println!("Baseline: {}", report.baseline.ref_name);
|
||||
println!("Feature: {}", report.feature.ref_name);
|
||||
println!();
|
||||
|
||||
let summary = &report.comparison_summary;
|
||||
|
||||
println!("Performance Changes:");
|
||||
println!(
|
||||
" NewPayload Latency per-block mean change: {:+.2}%",
|
||||
summary.per_block_latency_change_mean_percent
|
||||
);
|
||||
println!(
|
||||
" NewPayload Latency per-block median change: {:+.2}%",
|
||||
summary.per_block_latency_change_median_percent
|
||||
);
|
||||
println!(
|
||||
" NewPayload Latency per-block std dev: {:.2}%",
|
||||
summary.per_block_latency_change_std_dev_percent
|
||||
);
|
||||
println!(
|
||||
" Total newPayload time change: {:+.2}%",
|
||||
summary.new_payload_total_latency_change_percent
|
||||
);
|
||||
println!(
|
||||
" NewPayload Latency mean: {:+.2}%",
|
||||
summary.new_payload_latency_mean_change_percent
|
||||
);
|
||||
println!(
|
||||
" NewPayload Latency p50: {:+.2}%",
|
||||
summary.new_payload_latency_p50_change_percent
|
||||
);
|
||||
println!(
|
||||
" NewPayload Latency p90: {:+.2}%",
|
||||
summary.new_payload_latency_p90_change_percent
|
||||
);
|
||||
println!(
|
||||
" NewPayload Latency p99: {:+.2}%",
|
||||
summary.new_payload_latency_p99_change_percent
|
||||
);
|
||||
println!(
|
||||
" Gas/Second: {:+.2}%",
|
||||
summary.gas_per_second_change_percent
|
||||
);
|
||||
println!(
|
||||
" Blocks/Second: {:+.2}%",
|
||||
summary.blocks_per_second_change_percent
|
||||
);
|
||||
println!();
|
||||
|
||||
println!("Baseline Summary:");
|
||||
let baseline = &report.baseline.summary;
|
||||
println!(
|
||||
" Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
|
||||
baseline.total_blocks,
|
||||
baseline.min_block_number,
|
||||
baseline.max_block_number,
|
||||
baseline.total_gas_used,
|
||||
baseline.total_duration_ms as f64 / 1000.0
|
||||
);
|
||||
println!(" NewPayload latency (ms):");
|
||||
println!(
|
||||
" mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
|
||||
baseline.mean_new_payload_latency_ms,
|
||||
baseline.median_new_payload_latency_ms,
|
||||
baseline.p90_new_payload_latency_ms,
|
||||
baseline.p99_new_payload_latency_ms
|
||||
);
|
||||
if let (Some(start), Some(end)) =
|
||||
(&report.baseline.start_timestamp, &report.baseline.end_timestamp)
|
||||
{
|
||||
println!(
|
||||
" Started: {}, Ended: {}",
|
||||
start.format("%Y-%m-%d %H:%M:%S UTC"),
|
||||
end.format("%Y-%m-%d %H:%M:%S UTC")
|
||||
);
|
||||
}
|
||||
if let Some(ref cmd) = report.baseline.reth_command {
|
||||
println!(" Command: {}", cmd);
|
||||
}
|
||||
println!();
|
||||
|
||||
println!("Feature Summary:");
|
||||
let feature = &report.feature.summary;
|
||||
println!(
|
||||
" Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
|
||||
feature.total_blocks,
|
||||
feature.min_block_number,
|
||||
feature.max_block_number,
|
||||
feature.total_gas_used,
|
||||
feature.total_duration_ms as f64 / 1000.0
|
||||
);
|
||||
println!(" NewPayload latency (ms):");
|
||||
println!(
|
||||
" mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
|
||||
feature.mean_new_payload_latency_ms,
|
||||
feature.median_new_payload_latency_ms,
|
||||
feature.p90_new_payload_latency_ms,
|
||||
feature.p99_new_payload_latency_ms
|
||||
);
|
||||
if let (Some(start), Some(end)) =
|
||||
(&report.feature.start_timestamp, &report.feature.end_timestamp)
|
||||
{
|
||||
println!(
|
||||
" Started: {}, Ended: {}",
|
||||
start.format("%Y-%m-%d %H:%M:%S UTC"),
|
||||
end.format("%Y-%m-%d %H:%M:%S UTC")
|
||||
);
|
||||
}
|
||||
if let Some(ref cmd) = report.feature.reth_command {
|
||||
println!(" Command: {}", cmd);
|
||||
}
|
||||
println!();
|
||||
}
|
||||
}
|
||||
|
||||
/// Calculate standard deviation from a set of values and their mean.
|
||||
///
|
||||
/// Computes the population standard deviation using the formula:
|
||||
/// `sqrt(sum((x - mean)²) / n)`
|
||||
///
|
||||
/// Returns 0.0 for empty input.
|
||||
fn calculate_std_dev(values: &[f64], mean: f64) -> f64 {
|
||||
if values.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
let variance = values
|
||||
.iter()
|
||||
.map(|x| {
|
||||
let diff = x - mean;
|
||||
diff * diff
|
||||
})
|
||||
.sum::<f64>() /
|
||||
values.len() as f64;
|
||||
|
||||
variance.sqrt()
|
||||
}
|
||||
|
||||
/// Calculate percentile using linear interpolation on a sorted slice.
|
||||
///
|
||||
/// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls
|
||||
/// between two indices, linearly interpolates between those values. For example, with 100 values,
|
||||
/// p90 computes rank = 0.9 × 99 = 89.1, then returns `values[89] × 0.9 + values[90] × 0.1`.
|
||||
///
|
||||
/// Returns 0.0 for empty input.
|
||||
fn percentile(sorted_values: &[f64], percentile: f64) -> f64 {
|
||||
if sorted_values.is_empty() {
|
||||
return 0.0;
|
||||
}
|
||||
|
||||
let clamped = percentile.clamp(0.0, 1.0);
|
||||
let max_index = sorted_values.len() - 1;
|
||||
let rank = clamped * max_index as f64;
|
||||
let lower = rank.floor() as usize;
|
||||
let upper = rank.ceil() as usize;
|
||||
|
||||
if lower == upper {
|
||||
sorted_values[lower]
|
||||
} else {
|
||||
let weight = rank - lower as f64;
|
||||
sorted_values[lower].mul_add(1.0 - weight, sorted_values[upper] * weight)
|
||||
}
|
||||
}
|
||||
305
bin/reth-bench-compare/src/compilation.rs
Normal file
305
bin/reth-bench-compare/src/compilation.rs
Normal file
@@ -0,0 +1,305 @@
|
||||
//! Compilation operations for reth and reth-bench.
|
||||
|
||||
use crate::git::GitManager;
|
||||
use eyre::{eyre, Result, WrapErr};
|
||||
use std::{fs, path::PathBuf, process::Command};
|
||||
use tracing::{debug, error, info, warn};
|
||||
|
||||
/// Manages compilation operations for reth components
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct CompilationManager {
|
||||
repo_root: String,
|
||||
output_dir: PathBuf,
|
||||
git_manager: GitManager,
|
||||
}
|
||||
|
||||
impl CompilationManager {
|
||||
/// Create a new `CompilationManager`
|
||||
pub(crate) const fn new(
|
||||
repo_root: String,
|
||||
output_dir: PathBuf,
|
||||
git_manager: GitManager,
|
||||
) -> Result<Self> {
|
||||
Ok(Self { repo_root, output_dir, git_manager })
|
||||
}
|
||||
|
||||
/// Get the path to the cached binary using explicit commit hash
|
||||
pub(crate) fn get_cached_binary_path_for_commit(&self, commit: &str) -> PathBuf {
|
||||
let identifier = &commit[..8]; // Use first 8 chars of commit
|
||||
self.output_dir.join("bin").join(format!("reth_{identifier}"))
|
||||
}
|
||||
|
||||
/// Compile reth using cargo build and cache the binary
|
||||
pub(crate) fn compile_reth(&self, commit: &str, features: &str, rustflags: &str) -> Result<()> {
|
||||
// Validate that current git commit matches the expected commit
|
||||
let current_commit = self.git_manager.get_current_commit()?;
|
||||
if current_commit != commit {
|
||||
return Err(eyre!(
|
||||
"Git commit mismatch! Expected: {}, but currently at: {}",
|
||||
&commit[..8],
|
||||
¤t_commit[..8]
|
||||
));
|
||||
}
|
||||
|
||||
let cached_path = self.get_cached_binary_path_for_commit(commit);
|
||||
|
||||
// Check if cached binary already exists (since path contains commit hash, it's valid)
|
||||
if cached_path.exists() {
|
||||
info!("Using cached binary (commit: {})", &commit[..8]);
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!("No cached binary found, compiling (commit: {})...", &commit[..8]);
|
||||
|
||||
let binary_name = "reth";
|
||||
|
||||
info!(
|
||||
"Compiling {} with profiling configuration (commit: {})...",
|
||||
binary_name,
|
||||
&commit[..8]
|
||||
);
|
||||
|
||||
let mut cmd = Command::new("cargo");
|
||||
cmd.arg("build").arg("--profile").arg("profiling");
|
||||
|
||||
cmd.arg("--features").arg(features);
|
||||
info!("Using features: {features}");
|
||||
|
||||
cmd.current_dir(&self.repo_root);
|
||||
|
||||
// Set RUSTFLAGS
|
||||
cmd.env("RUSTFLAGS", rustflags);
|
||||
info!("Using RUSTFLAGS: {rustflags}");
|
||||
|
||||
info!("Compiling {binary_name} with {cmd:?}");
|
||||
|
||||
let output = cmd.output().wrap_err("Failed to execute cargo build command")?;
|
||||
|
||||
// Print stdout and stderr with prefixes at debug level
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
for line in stdout.lines() {
|
||||
if !line.trim().is_empty() {
|
||||
debug!("[CARGO] {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
for line in stderr.lines() {
|
||||
if !line.trim().is_empty() {
|
||||
debug!("[CARGO] {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if !output.status.success() {
|
||||
// Print all output when compilation fails
|
||||
error!("Cargo build failed with exit code: {:?}", output.status.code());
|
||||
|
||||
if !stdout.trim().is_empty() {
|
||||
error!("Cargo stdout:");
|
||||
for line in stdout.lines() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if !stderr.trim().is_empty() {
|
||||
error!("Cargo stderr:");
|
||||
for line in stderr.lines() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
return Err(eyre!("Compilation failed with exit code: {:?}", output.status.code()));
|
||||
}
|
||||
|
||||
info!("{} compilation completed", binary_name);
|
||||
|
||||
// Copy the compiled binary to cache
|
||||
let source_path =
|
||||
PathBuf::from(&self.repo_root).join(format!("target/profiling/{}", binary_name));
|
||||
if !source_path.exists() {
|
||||
return Err(eyre!("Compiled binary not found at {:?}", source_path));
|
||||
}
|
||||
|
||||
// Create bin directory if it doesn't exist
|
||||
let bin_dir = self.output_dir.join("bin");
|
||||
fs::create_dir_all(&bin_dir).wrap_err("Failed to create bin directory")?;
|
||||
|
||||
// Copy binary to cache
|
||||
fs::copy(&source_path, &cached_path).wrap_err("Failed to copy binary to cache")?;
|
||||
|
||||
// Make the cached binary executable
|
||||
#[cfg(unix)]
|
||||
{
|
||||
use std::os::unix::fs::PermissionsExt;
|
||||
let mut perms = fs::metadata(&cached_path)?.permissions();
|
||||
perms.set_mode(0o755);
|
||||
fs::set_permissions(&cached_path, perms)?;
|
||||
}
|
||||
|
||||
info!("Cached compiled binary at: {:?}", cached_path);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Check if reth-bench is available in PATH
|
||||
pub(crate) fn is_reth_bench_available(&self) -> bool {
|
||||
match Command::new("which").arg("reth-bench").output() {
|
||||
Ok(output) => {
|
||||
if output.status.success() {
|
||||
let path = String::from_utf8_lossy(&output.stdout);
|
||||
info!("Found reth-bench: {}", path.trim());
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Check if samply is available in PATH
|
||||
pub(crate) fn is_samply_available(&self) -> bool {
|
||||
match Command::new("which").arg("samply").output() {
|
||||
Ok(output) => {
|
||||
if output.status.success() {
|
||||
let path = String::from_utf8_lossy(&output.stdout);
|
||||
info!("Found samply: {}", path.trim());
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
}
|
||||
Err(_) => false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Install samply using cargo
|
||||
pub(crate) fn install_samply(&self) -> Result<()> {
|
||||
info!("Installing samply via cargo...");
|
||||
|
||||
let mut cmd = Command::new("cargo");
|
||||
cmd.args(["install", "--locked", "samply"]);
|
||||
|
||||
info!("Installing samply with {cmd:?}");
|
||||
|
||||
let output = cmd.output().wrap_err("Failed to execute cargo install samply command")?;
|
||||
|
||||
// Print stdout and stderr with prefixes at debug level
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
for line in stdout.lines() {
|
||||
if !line.trim().is_empty() {
|
||||
debug!("[CARGO-SAMPLY] {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
for line in stderr.lines() {
|
||||
if !line.trim().is_empty() {
|
||||
debug!("[CARGO-SAMPLY] {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if !output.status.success() {
|
||||
// Print all output when installation fails
|
||||
error!("Cargo install samply failed with exit code: {:?}", output.status.code());
|
||||
|
||||
if !stdout.trim().is_empty() {
|
||||
error!("Cargo stdout:");
|
||||
for line in stdout.lines() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if !stderr.trim().is_empty() {
|
||||
error!("Cargo stderr:");
|
||||
for line in stderr.lines() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
return Err(eyre!(
|
||||
"samply installation failed with exit code: {:?}",
|
||||
output.status.code()
|
||||
));
|
||||
}
|
||||
|
||||
info!("Samply installation completed");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensure samply is available, installing if necessary
|
||||
pub(crate) fn ensure_samply_available(&self) -> Result<()> {
|
||||
if self.is_samply_available() {
|
||||
Ok(())
|
||||
} else {
|
||||
warn!("samply not found in PATH, installing...");
|
||||
self.install_samply()
|
||||
}
|
||||
}
|
||||
|
||||
/// Ensure reth-bench is available, compiling if necessary
|
||||
pub(crate) fn ensure_reth_bench_available(&self) -> Result<()> {
|
||||
if self.is_reth_bench_available() {
|
||||
Ok(())
|
||||
} else {
|
||||
warn!("reth-bench not found in PATH, compiling and installing...");
|
||||
self.compile_reth_bench()
|
||||
}
|
||||
}
|
||||
|
||||
/// Compile and install reth-bench using `make install-reth-bench`
|
||||
pub(crate) fn compile_reth_bench(&self) -> Result<()> {
|
||||
info!("Compiling and installing reth-bench...");
|
||||
|
||||
let mut cmd = Command::new("make");
|
||||
cmd.arg("install-reth-bench").current_dir(&self.repo_root);
|
||||
|
||||
info!("Compiling reth-bench with {cmd:?}");
|
||||
|
||||
let output = cmd.output().wrap_err("Failed to execute make install-reth-bench command")?;
|
||||
|
||||
// Print stdout and stderr with prefixes at debug level
|
||||
let stdout = String::from_utf8_lossy(&output.stdout);
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
|
||||
for line in stdout.lines() {
|
||||
if !line.trim().is_empty() {
|
||||
debug!("[MAKE-BENCH] {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
for line in stderr.lines() {
|
||||
if !line.trim().is_empty() {
|
||||
debug!("[MAKE-BENCH] {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if !output.status.success() {
|
||||
// Print all output when compilation fails
|
||||
error!("Make install-reth-bench failed with exit code: {:?}", output.status.code());
|
||||
|
||||
if !stdout.trim().is_empty() {
|
||||
error!("Make stdout:");
|
||||
for line in stdout.lines() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
if !stderr.trim().is_empty() {
|
||||
error!("Make stderr:");
|
||||
for line in stderr.lines() {
|
||||
error!(" {}", line);
|
||||
}
|
||||
}
|
||||
|
||||
return Err(eyre!(
|
||||
"reth-bench compilation failed with exit code: {:?}",
|
||||
output.status.code()
|
||||
));
|
||||
}
|
||||
|
||||
info!("Reth-bench compilation completed");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
328
bin/reth-bench-compare/src/git.rs
Normal file
328
bin/reth-bench-compare/src/git.rs
Normal file
@@ -0,0 +1,328 @@
|
||||
//! Git operations for branch management.
|
||||
|
||||
use eyre::{eyre, Result, WrapErr};
|
||||
use std::process::Command;
|
||||
use tracing::{info, warn};
|
||||
|
||||
/// Manages git operations for branch switching
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct GitManager {
|
||||
repo_root: String,
|
||||
}
|
||||
|
||||
impl GitManager {
|
||||
/// Create a new `GitManager`, detecting the repository root
|
||||
pub(crate) fn new() -> Result<Self> {
|
||||
let output = Command::new("git")
|
||||
.args(["rev-parse", "--show-toplevel"])
|
||||
.output()
|
||||
.wrap_err("Failed to execute git command - is git installed?")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(eyre!("Not in a git repository or git command failed"));
|
||||
}
|
||||
|
||||
let repo_root = String::from_utf8(output.stdout)
|
||||
.wrap_err("Git output is not valid UTF-8")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
let manager = Self { repo_root };
|
||||
info!(
|
||||
"Detected git repository at: {}, current reference: {}",
|
||||
manager.repo_root(),
|
||||
manager.get_current_ref()?
|
||||
);
|
||||
|
||||
Ok(manager)
|
||||
}
|
||||
|
||||
/// Get the current git branch name
|
||||
pub(crate) fn get_current_branch(&self) -> Result<String> {
|
||||
let output = Command::new("git")
|
||||
.args(["branch", "--show-current"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to get current branch")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(eyre!("Failed to determine current branch"));
|
||||
}
|
||||
|
||||
let branch = String::from_utf8(output.stdout)
|
||||
.wrap_err("Branch name is not valid UTF-8")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
if branch.is_empty() {
|
||||
return Err(eyre!("Not on a named branch (detached HEAD?)"));
|
||||
}
|
||||
|
||||
Ok(branch)
|
||||
}
|
||||
|
||||
/// Get the current git reference (branch name, tag, or commit hash)
|
||||
pub(crate) fn get_current_ref(&self) -> Result<String> {
|
||||
// First try to get branch name
|
||||
if let Ok(branch) = self.get_current_branch() {
|
||||
return Ok(branch);
|
||||
}
|
||||
|
||||
// If not on a branch, check if we're on a tag
|
||||
let tag_output = Command::new("git")
|
||||
.args(["describe", "--exact-match", "--tags", "HEAD"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to check for tag")?;
|
||||
|
||||
if tag_output.status.success() {
|
||||
let tag = String::from_utf8(tag_output.stdout)
|
||||
.wrap_err("Tag name is not valid UTF-8")?
|
||||
.trim()
|
||||
.to_string();
|
||||
return Ok(tag);
|
||||
}
|
||||
|
||||
// If not on a branch or tag, return the commit hash
|
||||
let commit_output = Command::new("git")
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to get current commit")?;
|
||||
|
||||
if !commit_output.status.success() {
|
||||
return Err(eyre!("Failed to get current commit hash"));
|
||||
}
|
||||
|
||||
let commit_hash = String::from_utf8(commit_output.stdout)
|
||||
.wrap_err("Commit hash is not valid UTF-8")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
Ok(commit_hash)
|
||||
}
|
||||
|
||||
/// Check if the git working directory has uncommitted changes to tracked files
|
||||
pub(crate) fn validate_clean_state(&self) -> Result<()> {
|
||||
let output = Command::new("git")
|
||||
.args(["status", "--porcelain"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to check git status")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(eyre!("Git status command failed"));
|
||||
}
|
||||
|
||||
let status_output =
|
||||
String::from_utf8(output.stdout).wrap_err("Git status output is not valid UTF-8")?;
|
||||
|
||||
// Check for uncommitted changes to tracked files
|
||||
// Status codes: M = modified, A = added, D = deleted, R = renamed, C = copied, U = updated
|
||||
// ?? = untracked files (we want to ignore these)
|
||||
let has_uncommitted_changes = status_output.lines().any(|line| {
|
||||
if line.len() >= 2 {
|
||||
let status = &line[0..2];
|
||||
// Ignore untracked files (??) and ignored files (!!)
|
||||
!matches!(status, "??" | "!!")
|
||||
} else {
|
||||
false
|
||||
}
|
||||
});
|
||||
|
||||
if has_uncommitted_changes {
|
||||
warn!("Git working directory has uncommitted changes to tracked files:");
|
||||
for line in status_output.lines() {
|
||||
if line.len() >= 2 && !matches!(&line[0..2], "??" | "!!") {
|
||||
warn!(" {}", line);
|
||||
}
|
||||
}
|
||||
return Err(eyre!(
|
||||
"Git working directory has uncommitted changes to tracked files. Please commit or stash changes before running benchmark comparison."
|
||||
));
|
||||
}
|
||||
|
||||
// Check if there are untracked files and log them as info
|
||||
let untracked_files: Vec<&str> =
|
||||
status_output.lines().filter(|line| line.starts_with("??")).collect();
|
||||
|
||||
if !untracked_files.is_empty() {
|
||||
info!(
|
||||
"Git working directory has {} untracked files (this is OK)",
|
||||
untracked_files.len()
|
||||
);
|
||||
}
|
||||
|
||||
info!("Git working directory is clean (no uncommitted changes to tracked files)");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Fetch all refs from remote to ensure we have latest branches and tags
|
||||
pub(crate) fn fetch_all(&self) -> Result<()> {
|
||||
let output = Command::new("git")
|
||||
.args(["fetch", "--all", "--tags", "--quiet", "--force"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to fetch latest refs")?;
|
||||
|
||||
if output.status.success() {
|
||||
info!("Fetched latest refs");
|
||||
} else {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
// Only warn if there's actual error content, not just fetch progress
|
||||
if !stderr.trim().is_empty() && !stderr.contains("-> origin/") {
|
||||
warn!("Git fetch encountered issues (continuing anyway): {}", stderr);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate that the specified git references exist (branches, tags, or commits)
|
||||
pub(crate) fn validate_refs(&self, refs: &[&str]) -> Result<()> {
|
||||
for &git_ref in refs {
|
||||
// Try to resolve the ref similar to `git checkout` by peeling to a commit.
|
||||
// First try the ref as-is with ^{commit}, then fall back to origin/{ref}^{commit}.
|
||||
let as_is = format!("{git_ref}^{{commit}}");
|
||||
let ref_check = Command::new("git")
|
||||
.args(["rev-parse", "--verify", &as_is])
|
||||
.current_dir(&self.repo_root)
|
||||
.output();
|
||||
|
||||
let found = if let Ok(output) = ref_check &&
|
||||
output.status.success()
|
||||
{
|
||||
info!("Validated reference exists: {}", git_ref);
|
||||
true
|
||||
} else {
|
||||
// Try remote-only branches via origin/{ref}
|
||||
let origin_ref = format!("origin/{git_ref}^{{commit}}");
|
||||
let origin_check = Command::new("git")
|
||||
.args(["rev-parse", "--verify", &origin_ref])
|
||||
.current_dir(&self.repo_root)
|
||||
.output();
|
||||
|
||||
if let Ok(output) = origin_check &&
|
||||
output.status.success()
|
||||
{
|
||||
info!("Validated remote reference exists: origin/{}", git_ref);
|
||||
true
|
||||
} else {
|
||||
false
|
||||
}
|
||||
};
|
||||
|
||||
if !found {
|
||||
return Err(eyre!(
|
||||
"Git reference '{}' does not exist as branch, tag, or commit (tried '{}' and 'origin/{}^{{commit}}')",
|
||||
git_ref,
|
||||
format!("{git_ref}^{{commit}}"),
|
||||
git_ref,
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Switch to the specified git reference (branch, tag, or commit)
|
||||
pub(crate) fn switch_ref(&self, git_ref: &str) -> Result<()> {
|
||||
// First checkout the reference
|
||||
let output = Command::new("git")
|
||||
.args(["checkout", git_ref])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err_with(|| format!("Failed to switch to reference '{git_ref}'"))?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
return Err(eyre!("Failed to switch to reference '{}': {}", git_ref, stderr));
|
||||
}
|
||||
|
||||
// Check if this is a branch that tracks a remote and pull latest changes
|
||||
let is_branch = Command::new("git")
|
||||
.args(["show-ref", "--verify", "--quiet", &format!("refs/heads/{git_ref}")])
|
||||
.current_dir(&self.repo_root)
|
||||
.status()
|
||||
.map(|s| s.success())
|
||||
.unwrap_or(false);
|
||||
|
||||
if is_branch {
|
||||
// Check if the branch tracks a remote
|
||||
let tracking_output = Command::new("git")
|
||||
.args([
|
||||
"rev-parse",
|
||||
"--abbrev-ref",
|
||||
"--symbolic-full-name",
|
||||
&format!("{git_ref}@{{upstream}}"),
|
||||
])
|
||||
.current_dir(&self.repo_root)
|
||||
.output();
|
||||
|
||||
if let Ok(output) = tracking_output &&
|
||||
output.status.success()
|
||||
{
|
||||
let upstream = String::from_utf8_lossy(&output.stdout).trim().to_string();
|
||||
if !upstream.is_empty() && upstream != format!("{git_ref}@{{upstream}}") {
|
||||
// Branch tracks a remote, pull latest changes
|
||||
info!("Pulling latest changes for branch: {}", git_ref);
|
||||
|
||||
let pull_output = Command::new("git")
|
||||
.args(["pull", "--ff-only"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err_with(|| {
|
||||
format!("Failed to pull latest changes for branch '{git_ref}'")
|
||||
})?;
|
||||
|
||||
if pull_output.status.success() {
|
||||
info!("Successfully pulled latest changes for branch: {}", git_ref);
|
||||
} else {
|
||||
let stderr = String::from_utf8_lossy(&pull_output.stderr);
|
||||
warn!("Failed to pull latest changes for branch '{}': {}", git_ref, stderr);
|
||||
// Continue anyway, we'll use whatever version we have
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Verify the checkout succeeded by checking the current commit
|
||||
let current_commit_output = Command::new("git")
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to get current commit")?;
|
||||
|
||||
if !current_commit_output.status.success() {
|
||||
return Err(eyre!("Failed to verify git checkout"));
|
||||
}
|
||||
|
||||
info!("Switched to reference: {}", git_ref);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get the current commit hash
|
||||
pub(crate) fn get_current_commit(&self) -> Result<String> {
|
||||
let output = Command::new("git")
|
||||
.args(["rev-parse", "HEAD"])
|
||||
.current_dir(&self.repo_root)
|
||||
.output()
|
||||
.wrap_err("Failed to get current commit")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(eyre!("Failed to get current commit hash"));
|
||||
}
|
||||
|
||||
let commit_hash = String::from_utf8(output.stdout)
|
||||
.wrap_err("Commit hash is not valid UTF-8")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
Ok(commit_hash)
|
||||
}
|
||||
|
||||
/// Get the repository root path
|
||||
pub(crate) fn repo_root(&self) -> &str {
|
||||
&self.repo_root
|
||||
}
|
||||
}
|
||||
47
bin/reth-bench-compare/src/main.rs
Normal file
47
bin/reth-bench-compare/src/main.rs
Normal file
@@ -0,0 +1,47 @@
|
||||
//! # reth-bench-compare
|
||||
//!
|
||||
//! Automated tool for comparing reth performance between two git branches.
|
||||
//! This tool automates the complete workflow of compiling, running, and benchmarking
|
||||
//! reth on different branches to provide meaningful performance comparisons.
|
||||
|
||||
#![doc(
|
||||
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
|
||||
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
|
||||
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
|
||||
)]
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
|
||||
#[global_allocator]
|
||||
static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
|
||||
|
||||
use alloy_primitives as _;
|
||||
|
||||
mod benchmark;
|
||||
mod cli;
|
||||
mod comparison;
|
||||
mod compilation;
|
||||
mod git;
|
||||
mod node;
|
||||
|
||||
use clap::Parser;
|
||||
use cli::{run_comparison, Args};
|
||||
use eyre::Result;
|
||||
use reth_cli_runner::CliRunner;
|
||||
|
||||
fn main() -> Result<()> {
|
||||
// Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
|
||||
if std::env::var_os("RUST_BACKTRACE").is_none() {
|
||||
unsafe {
|
||||
std::env::set_var("RUST_BACKTRACE", "1");
|
||||
}
|
||||
}
|
||||
|
||||
let args = Args::parse();
|
||||
|
||||
// Initialize tracing
|
||||
let _guard = args.init_tracing()?;
|
||||
|
||||
// Run until either exit or sigint or sigterm
|
||||
let runner = CliRunner::try_default_runtime()?;
|
||||
runner.run_command_until_exit(|ctx| run_comparison(args, ctx))
|
||||
}
|
||||
695
bin/reth-bench-compare/src/node.rs
Normal file
695
bin/reth-bench-compare/src/node.rs
Normal file
@@ -0,0 +1,695 @@
|
||||
//! Node management for starting, stopping, and controlling reth instances.
|
||||
|
||||
use crate::cli::Args;
|
||||
use alloy_provider::{Provider, ProviderBuilder};
|
||||
use alloy_rpc_client::RpcClient;
|
||||
use alloy_rpc_types_eth::SyncStatus;
|
||||
use alloy_transport_ws::WsConnect;
|
||||
use eyre::{eyre, OptionExt, Result, WrapErr};
|
||||
#[cfg(unix)]
|
||||
use nix::sys::signal::{killpg, Signal};
|
||||
#[cfg(unix)]
|
||||
use nix::unistd::Pid;
|
||||
use reth_chainspec::Chain;
|
||||
use std::{fs, path::PathBuf, time::Duration};
|
||||
use tokio::{
|
||||
fs::File as AsyncFile,
|
||||
io::{AsyncBufReadExt, AsyncWriteExt, BufReader as AsyncBufReader},
|
||||
process::Command,
|
||||
time::{sleep, timeout},
|
||||
};
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
/// Default websocket RPC port used by reth
|
||||
const DEFAULT_WS_RPC_PORT: u16 = 8546;
|
||||
|
||||
/// Manages reth node lifecycle and operations
|
||||
pub(crate) struct NodeManager {
|
||||
datadir: Option<String>,
|
||||
metrics_port: u16,
|
||||
chain: Chain,
|
||||
use_sudo: bool,
|
||||
binary_path: Option<std::path::PathBuf>,
|
||||
enable_profiling: bool,
|
||||
output_dir: PathBuf,
|
||||
additional_reth_args: Vec<String>,
|
||||
comparison_dir: Option<PathBuf>,
|
||||
tracing_endpoint: Option<String>,
|
||||
otlp_max_queue_size: usize,
|
||||
}
|
||||
|
||||
impl NodeManager {
|
||||
/// Create a new `NodeManager` with configuration from CLI args
|
||||
pub(crate) fn new(args: &Args) -> Self {
|
||||
Self {
|
||||
datadir: Some(args.datadir_path().to_string_lossy().to_string()),
|
||||
metrics_port: args.metrics_port,
|
||||
chain: args.chain,
|
||||
use_sudo: args.sudo,
|
||||
binary_path: None,
|
||||
enable_profiling: args.profile,
|
||||
output_dir: args.output_dir_path(),
|
||||
// Filter out empty strings to prevent invalid arguments being passed to reth node
|
||||
additional_reth_args: args
|
||||
.reth_args
|
||||
.iter()
|
||||
.filter(|s| !s.is_empty())
|
||||
.cloned()
|
||||
.collect(),
|
||||
comparison_dir: None,
|
||||
tracing_endpoint: args.traces.otlp.as_ref().map(|u| u.to_string()),
|
||||
otlp_max_queue_size: args.otlp_max_queue_size,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the comparison directory path for logging
|
||||
pub(crate) fn set_comparison_dir(&mut self, dir: PathBuf) {
|
||||
self.comparison_dir = Some(dir);
|
||||
}
|
||||
|
||||
/// Get the log file path for a given reference type
|
||||
fn get_log_file_path(&self, ref_type: &str) -> Result<PathBuf> {
|
||||
let comparison_dir = self
|
||||
.comparison_dir
|
||||
.as_ref()
|
||||
.ok_or_eyre("Comparison directory not set. Call set_comparison_dir first.")?;
|
||||
|
||||
// The comparison directory already contains the full path to results/<timestamp>
|
||||
let log_dir = comparison_dir.join(ref_type);
|
||||
|
||||
// Create the directory if it doesn't exist
|
||||
fs::create_dir_all(&log_dir)
|
||||
.wrap_err(format!("Failed to create log directory: {:?}", log_dir))?;
|
||||
|
||||
let log_file = log_dir.join("reth_node.log");
|
||||
Ok(log_file)
|
||||
}
|
||||
|
||||
/// Get the perf event max sample rate from the system, capped at 10000
|
||||
fn get_perf_sample_rate(&self) -> Option<String> {
|
||||
let perf_rate_file = "/proc/sys/kernel/perf_event_max_sample_rate";
|
||||
if let Ok(content) = fs::read_to_string(perf_rate_file) {
|
||||
let rate_str = content.trim();
|
||||
if !rate_str.is_empty() {
|
||||
if let Ok(system_rate) = rate_str.parse::<u32>() {
|
||||
let capped_rate = std::cmp::min(system_rate, 10000);
|
||||
info!(
|
||||
"Detected perf_event_max_sample_rate: {}, using: {}",
|
||||
system_rate, capped_rate
|
||||
);
|
||||
return Some(capped_rate.to_string());
|
||||
}
|
||||
warn!("Failed to parse perf_event_max_sample_rate: {}", rate_str);
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
/// Get the absolute path to samply using 'which' command
|
||||
async fn get_samply_path(&self) -> Result<String> {
|
||||
let output = Command::new("which")
|
||||
.arg("samply")
|
||||
.output()
|
||||
.await
|
||||
.wrap_err("Failed to execute 'which samply' command")?;
|
||||
|
||||
if !output.status.success() {
|
||||
return Err(eyre!("samply not found in PATH"));
|
||||
}
|
||||
|
||||
let samply_path = String::from_utf8(output.stdout)
|
||||
.wrap_err("samply path is not valid UTF-8")?
|
||||
.trim()
|
||||
.to_string();
|
||||
|
||||
if samply_path.is_empty() {
|
||||
return Err(eyre!("which samply returned empty path"));
|
||||
}
|
||||
|
||||
Ok(samply_path)
|
||||
}
|
||||
|
||||
/// Build reth arguments as a vector of strings
|
||||
fn build_reth_args(
|
||||
&self,
|
||||
binary_path_str: &str,
|
||||
additional_args: &[String],
|
||||
ref_type: &str,
|
||||
) -> (Vec<String>, String) {
|
||||
let mut reth_args = vec![binary_path_str.to_string(), "node".to_string()];
|
||||
|
||||
// Add chain argument (skip for mainnet as it's the default)
|
||||
let chain_str = self.chain.to_string();
|
||||
if chain_str != "mainnet" {
|
||||
reth_args.extend_from_slice(&["--chain".to_string(), chain_str.clone()]);
|
||||
}
|
||||
|
||||
// Add datadir if specified
|
||||
if let Some(ref datadir) = self.datadir {
|
||||
reth_args.extend_from_slice(&["--datadir".to_string(), datadir.clone()]);
|
||||
}
|
||||
|
||||
// Add reth-specific arguments
|
||||
let metrics_arg = format!("0.0.0.0:{}", self.metrics_port);
|
||||
reth_args.extend_from_slice(&[
|
||||
"--engine.accept-execution-requests-hash".to_string(),
|
||||
"--metrics".to_string(),
|
||||
metrics_arg,
|
||||
"--http".to_string(),
|
||||
"--http.api".to_string(),
|
||||
"eth,reth".to_string(),
|
||||
"--ws".to_string(),
|
||||
"--ws.api".to_string(),
|
||||
"eth,reth".to_string(),
|
||||
"--disable-discovery".to_string(),
|
||||
"--trusted-only".to_string(),
|
||||
"--disable-tx-gossip".to_string(),
|
||||
]);
|
||||
|
||||
// Add tracing arguments if OTLP endpoint is configured
|
||||
if let Some(ref endpoint) = self.tracing_endpoint {
|
||||
info!("Enabling OTLP tracing export to: {} (service: reth-{})", endpoint, ref_type);
|
||||
// Endpoint requires equals per clap settings in reth
|
||||
reth_args.push(format!("--tracing-otlp={}", endpoint));
|
||||
}
|
||||
|
||||
// Add any additional arguments passed via command line (common to both baseline and
|
||||
// feature)
|
||||
reth_args.extend_from_slice(&self.additional_reth_args);
|
||||
|
||||
// Add reference-specific additional arguments
|
||||
reth_args.extend_from_slice(additional_args);
|
||||
|
||||
(reth_args, chain_str)
|
||||
}
|
||||
|
||||
/// Create a command for profiling mode
|
||||
async fn create_profiling_command(
|
||||
&self,
|
||||
ref_type: &str,
|
||||
reth_args: &[String],
|
||||
) -> Result<Command> {
|
||||
// Create profiles directory if it doesn't exist
|
||||
let profile_dir = self.output_dir.join("profiles");
|
||||
fs::create_dir_all(&profile_dir).wrap_err("Failed to create profiles directory")?;
|
||||
|
||||
let profile_path = profile_dir.join(format!("{}.json.gz", ref_type));
|
||||
info!("Starting reth node with samply profiling...");
|
||||
info!("Profile output: {:?}", profile_path);
|
||||
|
||||
// Get absolute path to samply
|
||||
let samply_path = self.get_samply_path().await?;
|
||||
|
||||
let mut cmd = if self.use_sudo {
|
||||
let mut sudo_cmd = Command::new("sudo");
|
||||
sudo_cmd.arg(&samply_path);
|
||||
sudo_cmd
|
||||
} else {
|
||||
Command::new(&samply_path)
|
||||
};
|
||||
|
||||
// Add samply arguments
|
||||
cmd.args(["record", "--save-only", "-o", &profile_path.to_string_lossy()]);
|
||||
|
||||
// Add rate argument if available
|
||||
if let Some(rate) = self.get_perf_sample_rate() {
|
||||
cmd.args(["--rate", &rate]);
|
||||
}
|
||||
|
||||
// Add separator and complete reth command
|
||||
cmd.arg("--");
|
||||
cmd.args(reth_args);
|
||||
|
||||
// Enable tracing-samply
|
||||
if supports_samply_flags(&reth_args[0]) {
|
||||
cmd.arg("--log.samply");
|
||||
}
|
||||
|
||||
// Set environment variable to disable log styling
|
||||
cmd.env("RUST_LOG_STYLE", "never");
|
||||
|
||||
Ok(cmd)
|
||||
}
|
||||
|
||||
/// Create a command for direct reth execution
|
||||
fn create_direct_command(&self, reth_args: &[String]) -> Command {
|
||||
let binary_path = &reth_args[0];
|
||||
|
||||
let mut cmd = if self.use_sudo {
|
||||
info!("Starting reth node with sudo...");
|
||||
let mut sudo_cmd = Command::new("sudo");
|
||||
sudo_cmd.args(reth_args);
|
||||
sudo_cmd
|
||||
} else {
|
||||
info!("Starting reth node...");
|
||||
let mut reth_cmd = Command::new(binary_path);
|
||||
reth_cmd.args(&reth_args[1..]); // Skip the binary path since it's the command
|
||||
reth_cmd
|
||||
};
|
||||
|
||||
// Set environment variable to disable log styling
|
||||
cmd.env("RUST_LOG_STYLE", "never");
|
||||
|
||||
cmd
|
||||
}
|
||||
|
||||
/// Start a reth node using the specified binary path and return the process handle
|
||||
/// along with the formatted reth command string for reporting.
|
||||
pub(crate) async fn start_node(
|
||||
&mut self,
|
||||
binary_path: &std::path::Path,
|
||||
_git_ref: &str,
|
||||
ref_type: &str,
|
||||
additional_args: &[String],
|
||||
) -> Result<(tokio::process::Child, String)> {
|
||||
// Store the binary path for later use (e.g., in unwind_to_block)
|
||||
self.binary_path = Some(binary_path.to_path_buf());
|
||||
|
||||
let binary_path_str = binary_path.to_string_lossy();
|
||||
let (reth_args, _) = self.build_reth_args(&binary_path_str, additional_args, ref_type);
|
||||
|
||||
// Format the reth command string for reporting
|
||||
let reth_command = shlex::try_join(reth_args.iter().map(|s| s.as_str()))
|
||||
.wrap_err("Failed to format reth command string")?;
|
||||
|
||||
// Log additional arguments if any
|
||||
if !self.additional_reth_args.is_empty() {
|
||||
info!("Using common additional reth arguments: {:?}", self.additional_reth_args);
|
||||
}
|
||||
if !additional_args.is_empty() {
|
||||
info!("Using reference-specific additional reth arguments: {:?}", additional_args);
|
||||
}
|
||||
|
||||
let mut cmd = if self.enable_profiling {
|
||||
self.create_profiling_command(ref_type, &reth_args).await?
|
||||
} else {
|
||||
self.create_direct_command(&reth_args)
|
||||
};
|
||||
|
||||
// Set process group for better signal handling
|
||||
#[cfg(unix)]
|
||||
{
|
||||
cmd.process_group(0);
|
||||
}
|
||||
|
||||
// Set high queue size to prevent trace dropping during benchmarks
|
||||
if self.tracing_endpoint.is_some() {
|
||||
cmd.env("OTEL_BSP_MAX_QUEUE_SIZE", self.otlp_max_queue_size.to_string()); // Traces
|
||||
cmd.env("OTEL_BLRP_MAX_QUEUE_SIZE", "10000"); // Logs
|
||||
|
||||
// Set service name to differentiate baseline vs feature runs in Jaeger
|
||||
cmd.env("OTEL_SERVICE_NAME", format!("reth-{}", ref_type));
|
||||
}
|
||||
|
||||
debug!("Executing reth command: {cmd:?}");
|
||||
|
||||
let mut child = cmd
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.kill_on_drop(true) // Kill on drop so that on Ctrl-C for parent process we stop all child processes
|
||||
.spawn()
|
||||
.wrap_err("Failed to start reth node")?;
|
||||
|
||||
info!(
|
||||
"Reth node started with PID: {:?} (binary: {})",
|
||||
child.id().ok_or_eyre("Reth node is not running")?,
|
||||
binary_path_str
|
||||
);
|
||||
|
||||
// Prepare log file path
|
||||
let log_file_path = self.get_log_file_path(ref_type)?;
|
||||
info!("Reth node logs will be saved to: {:?}", log_file_path);
|
||||
|
||||
// Stream stdout and stderr with prefixes at debug level and to log file
|
||||
if let Some(stdout) = child.stdout.take() {
|
||||
let log_file = AsyncFile::create(&log_file_path)
|
||||
.await
|
||||
.wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
|
||||
tokio::spawn(async move {
|
||||
let reader = AsyncBufReader::new(stdout);
|
||||
let mut lines = reader.lines();
|
||||
let mut log_file = log_file;
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[RETH] {}", line);
|
||||
// Write to log file (reth already includes timestamps)
|
||||
let log_line = format!("{}\n", line);
|
||||
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
|
||||
debug!("Failed to write to log file: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(stderr) = child.stderr.take() {
|
||||
let log_file = AsyncFile::options()
|
||||
.create(true)
|
||||
.append(true)
|
||||
.open(&log_file_path)
|
||||
.await
|
||||
.wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
|
||||
tokio::spawn(async move {
|
||||
let reader = AsyncBufReader::new(stderr);
|
||||
let mut lines = reader.lines();
|
||||
let mut log_file = log_file;
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[RETH] {}", line);
|
||||
// Write to log file (reth already includes timestamps)
|
||||
let log_line = format!("{}\n", line);
|
||||
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
|
||||
debug!("Failed to write to log file: {}", e);
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Give the node a moment to start up
|
||||
sleep(Duration::from_secs(5)).await;
|
||||
|
||||
Ok((child, reth_command))
|
||||
}
|
||||
|
||||
/// Wait for the node to be ready and return its current tip.
|
||||
///
|
||||
/// Fails early if the node process exits before becoming ready.
|
||||
pub(crate) async fn wait_for_node_ready_and_get_tip(
|
||||
&self,
|
||||
child: &mut tokio::process::Child,
|
||||
) -> Result<u64> {
|
||||
info!("Waiting for node to be ready and synced...");
|
||||
|
||||
let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync
|
||||
let check_interval = Duration::from_secs(2);
|
||||
let rpc_url = "http://localhost:8545";
|
||||
|
||||
// Create Alloy provider
|
||||
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
|
||||
let provider = ProviderBuilder::new().connect_http(url);
|
||||
|
||||
let start_time = tokio::time::Instant::now();
|
||||
let mut iteration = 0;
|
||||
|
||||
timeout(max_wait, async {
|
||||
loop {
|
||||
iteration += 1;
|
||||
debug!(
|
||||
"Readiness check iteration {} (elapsed: {:?})",
|
||||
iteration,
|
||||
start_time.elapsed()
|
||||
);
|
||||
|
||||
// Check if the node process has exited.
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Err(eyre!("Node process exited unexpectedly with {status}"));
|
||||
}
|
||||
|
||||
// First check if RPC is up and node is not syncing
|
||||
match provider.syncing().await {
|
||||
Ok(sync_result) => {
|
||||
match sync_result {
|
||||
SyncStatus::Info(sync_info) => {
|
||||
debug!("Node is still syncing {sync_info:?}, waiting...");
|
||||
}
|
||||
_ => {
|
||||
debug!("HTTP RPC is up and node is not syncing, checking block number...");
|
||||
// Node is not syncing, now get the tip
|
||||
match provider.get_block_number().await {
|
||||
Ok(tip) => {
|
||||
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
|
||||
|
||||
// Verify WebSocket RPC is ready (public endpoint, no JWT required)
|
||||
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
|
||||
debug!("Attempting WebSocket connection to {} (public endpoint)", ws_url);
|
||||
let ws_connect = WsConnect::new(&ws_url);
|
||||
|
||||
match RpcClient::connect_pubsub(ws_connect).await
|
||||
{
|
||||
Ok(_) => {
|
||||
info!(
|
||||
"Node is ready (HTTP and WebSocket) at block: {} (took {:?}, {} iterations)",
|
||||
tip, start_time.elapsed(), iteration
|
||||
);
|
||||
return Ok(tip);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
|
||||
iteration, e
|
||||
);
|
||||
debug!("WebSocket error details: {}", e);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Failed to get block number (iteration {}): {:?}", iteration, e);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("Node RPC not ready yet or failed to check sync status (iteration {}): {:?}", iteration, e);
|
||||
}
|
||||
}
|
||||
|
||||
debug!("Sleeping for {:?} before next check", check_interval);
|
||||
sleep(check_interval).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.wrap_err("Timed out waiting for node to be ready and synced")?
|
||||
}
|
||||
|
||||
/// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
|
||||
///
|
||||
/// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
|
||||
/// the node is still syncing.
|
||||
pub(crate) async fn wait_for_rpc_and_get_tip(
|
||||
&self,
|
||||
child: &mut tokio::process::Child,
|
||||
) -> Result<u64> {
|
||||
info!("Waiting for node RPC to be ready (skipping sync wait)...");
|
||||
|
||||
let max_wait = Duration::from_secs(60);
|
||||
let check_interval = Duration::from_secs(2);
|
||||
let rpc_url = "http://localhost:8545";
|
||||
|
||||
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
|
||||
let provider = ProviderBuilder::new().connect_http(url);
|
||||
|
||||
let start_time = tokio::time::Instant::now();
|
||||
let mut iteration = 0;
|
||||
|
||||
timeout(max_wait, async {
|
||||
loop {
|
||||
iteration += 1;
|
||||
debug!(
|
||||
"RPC readiness check iteration {} (elapsed: {:?})",
|
||||
iteration,
|
||||
start_time.elapsed()
|
||||
);
|
||||
|
||||
if let Some(status) = child.try_wait()? {
|
||||
return Err(eyre!("Node process exited unexpectedly with {status}"));
|
||||
}
|
||||
|
||||
match provider.get_block_number().await {
|
||||
Ok(tip) => {
|
||||
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
|
||||
|
||||
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
|
||||
let ws_connect = WsConnect::new(&ws_url);
|
||||
|
||||
match RpcClient::connect_pubsub(ws_connect).await {
|
||||
Ok(_) => {
|
||||
info!(
|
||||
"Node RPC is ready at block: {} (took {:?}, {} iterations)",
|
||||
tip,
|
||||
start_time.elapsed(),
|
||||
iteration
|
||||
);
|
||||
return Ok(tip);
|
||||
}
|
||||
Err(e) => {
|
||||
debug!(
|
||||
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
|
||||
iteration, e
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(e) => {
|
||||
debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
|
||||
}
|
||||
}
|
||||
|
||||
sleep(check_interval).await;
|
||||
}
|
||||
})
|
||||
.await
|
||||
.wrap_err("Timed out waiting for node RPC to be ready")?
|
||||
}
|
||||
|
||||
/// Stop the reth node gracefully
|
||||
pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
|
||||
let pid = child.id().ok_or_eyre("Child process ID should be available")?;
|
||||
|
||||
// Check if the process has already exited
|
||||
match child.try_wait() {
|
||||
Ok(Some(status)) => {
|
||||
info!("Reth node (PID: {}) has already exited with status: {:?}", pid, status);
|
||||
return Ok(());
|
||||
}
|
||||
Ok(None) => {
|
||||
// Process is still running, proceed to stop it
|
||||
info!("Stopping process gracefully with SIGINT (PID: {})...", pid);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(eyre!("Failed to check process status: {}", e));
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(unix)]
|
||||
{
|
||||
// Send SIGINT to process group to mimic Ctrl-C behavior
|
||||
let nix_pgid = Pid::from_raw(pid as i32);
|
||||
|
||||
match killpg(nix_pgid, Signal::SIGINT) {
|
||||
Ok(()) => {}
|
||||
Err(nix::errno::Errno::ESRCH) => {
|
||||
info!("Process group {} has already exited", pid);
|
||||
}
|
||||
Err(e) => {
|
||||
return Err(eyre!("Failed to send SIGINT to process group {}: {}", pid, e));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(not(unix))]
|
||||
{
|
||||
// On non-Unix systems, fall back to using external kill command
|
||||
let output = Command::new("taskkill")
|
||||
.args(["/PID", &pid.to_string(), "/F"])
|
||||
.output()
|
||||
.await
|
||||
.wrap_err("Failed to execute taskkill command")?;
|
||||
|
||||
if !output.status.success() {
|
||||
let stderr = String::from_utf8_lossy(&output.stderr);
|
||||
// Check if the error is because the process doesn't exist
|
||||
if stderr.contains("not found") || stderr.contains("not exist") {
|
||||
info!("Process {} has already exited", pid);
|
||||
} else {
|
||||
return Err(eyre!("Failed to kill process {}: {}", pid, stderr));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Wait for the process to exit
|
||||
match child.wait().await {
|
||||
Ok(status) => {
|
||||
info!("Reth node (PID: {}) exited with status: {:?}", pid, status);
|
||||
}
|
||||
Err(e) => {
|
||||
// If we get an error here, it might be because the process already exited
|
||||
debug!("Error waiting for process exit (may have already exited): {}", e);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Unwind the node to a specific block
|
||||
pub(crate) async fn unwind_to_block(&self, block_number: u64) -> Result<()> {
|
||||
if self.use_sudo {
|
||||
info!("Unwinding node to block: {} (with sudo)", block_number);
|
||||
} else {
|
||||
info!("Unwinding node to block: {}", block_number);
|
||||
}
|
||||
|
||||
// Use the binary path from the last start_node call, or fallback to default
|
||||
let binary_path = self
|
||||
.binary_path
|
||||
.as_ref()
|
||||
.map(|p| p.to_string_lossy().to_string())
|
||||
.unwrap_or_else(|| "./target/profiling/reth".to_string());
|
||||
|
||||
let mut cmd = if self.use_sudo {
|
||||
let mut sudo_cmd = Command::new("sudo");
|
||||
sudo_cmd.args([&binary_path, "stage", "unwind"]);
|
||||
sudo_cmd
|
||||
} else {
|
||||
let mut reth_cmd = Command::new(&binary_path);
|
||||
reth_cmd.args(["stage", "unwind"]);
|
||||
reth_cmd
|
||||
};
|
||||
|
||||
// Add chain argument (skip for mainnet as it's the default)
|
||||
let chain_str = self.chain.to_string();
|
||||
if chain_str != "mainnet" {
|
||||
cmd.args(["--chain", &chain_str]);
|
||||
}
|
||||
|
||||
// Add datadir if specified
|
||||
if let Some(ref datadir) = self.datadir {
|
||||
cmd.args(["--datadir", datadir]);
|
||||
}
|
||||
|
||||
cmd.args(["to-block", &block_number.to_string()]);
|
||||
|
||||
// Set environment variable to disable log styling
|
||||
cmd.env("RUST_LOG_STYLE", "never");
|
||||
|
||||
// Debug log the command
|
||||
debug!("Executing reth unwind command: {:?}", cmd);
|
||||
|
||||
let mut child = cmd
|
||||
.stdout(std::process::Stdio::piped())
|
||||
.stderr(std::process::Stdio::piped())
|
||||
.spawn()
|
||||
.wrap_err("Failed to start unwind command")?;
|
||||
|
||||
// Stream stdout and stderr with prefixes in real-time
|
||||
if let Some(stdout) = child.stdout.take() {
|
||||
tokio::spawn(async move {
|
||||
let reader = AsyncBufReader::new(stdout);
|
||||
let mut lines = reader.lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[RETH-UNWIND] {}", line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
if let Some(stderr) = child.stderr.take() {
|
||||
tokio::spawn(async move {
|
||||
let reader = AsyncBufReader::new(stderr);
|
||||
let mut lines = reader.lines();
|
||||
while let Ok(Some(line)) = lines.next_line().await {
|
||||
debug!("[RETH-UNWIND] {}", line);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
// Wait for the command to complete
|
||||
let status = child.wait().await.wrap_err("Failed to wait for unwind command")?;
|
||||
|
||||
if !status.success() {
|
||||
return Err(eyre!("Unwind command failed with exit code: {:?}", status.code()));
|
||||
}
|
||||
|
||||
info!("Unwound to block: {}", block_number);
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
fn supports_samply_flags(bin: &str) -> bool {
|
||||
let mut cmd = std::process::Command::new(bin);
|
||||
// NOTE: The flag to check must come before --help.
|
||||
// We pass --help as a shortcut to not execute any command.
|
||||
cmd.args(["--log.samply", "--help"]);
|
||||
debug!(?cmd, "Checking samply flags support");
|
||||
let Ok(output) = cmd.output() else {
|
||||
return false;
|
||||
};
|
||||
debug!(?output, "Samply flags support check");
|
||||
output.status.success()
|
||||
}
|
||||
@@ -17,6 +17,7 @@ workspace = true
|
||||
reth-cli-runner.workspace = true
|
||||
reth-cli-util.workspace = true
|
||||
reth-engine-primitives.workspace = true
|
||||
reth-ethereum-primitives.workspace = true
|
||||
reth-fs-util.workspace = true
|
||||
reth-node-api.workspace = true
|
||||
reth-node-core.workspace = true
|
||||
@@ -24,11 +25,13 @@ reth-primitives-traits.workspace = true
|
||||
reth-rpc-api.workspace = true
|
||||
|
||||
reth-tracing.workspace = true
|
||||
reth-chainspec.workspace = true
|
||||
|
||||
# alloy
|
||||
alloy-eips.workspace = true
|
||||
alloy-json-rpc.workspace = true
|
||||
|
||||
alloy-consensus.workspace = true
|
||||
alloy-network.workspace = true
|
||||
alloy-primitives = { workspace = true, features = ["rand"] }
|
||||
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
|
||||
alloy-pubsub.workspace = true
|
||||
|
||||
@@ -35,10 +35,6 @@ The `new-payload-fcu` command supports two optional waiting modes that can be us
|
||||
- `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms` or `--wait-time 400` for 400ms)
|
||||
- `--wait-for-persistence`: Waits for blocks to be persisted using the `reth_subscribePersistedBlock` subscription
|
||||
|
||||
Both `new-payload-fcu` and `new-payload-only` support `--rpc-block-fetch-retries <RETRIES>`
|
||||
to control how many times block fetches are retried after an RPC failure. The default is `10`.
|
||||
Use `--rpc-block-fetch-retries forever` to keep retrying indefinitely.
|
||||
|
||||
When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
|
||||
|
||||
By default, the WebSocket URL for persistence subscriptions is derived from `--engine-rpc-url` (converting to ws:// on port 8546). Use `--ws-rpc-url` to override this.
|
||||
|
||||
@@ -33,10 +33,6 @@ pub(crate) struct BenchContext {
|
||||
pub(crate) use_reth_namespace: bool,
|
||||
/// Whether to fetch and replay RLP-encoded blocks.
|
||||
pub(crate) rlp_blocks: bool,
|
||||
/// Whether to skip waiting for persistence (pass `wait_for_persistence: false`).
|
||||
pub(crate) no_wait_for_persistence: bool,
|
||||
/// Whether to skip waiting for caches (pass `wait_for_caches: false`).
|
||||
pub(crate) no_wait_for_caches: bool,
|
||||
}
|
||||
|
||||
impl BenchContext {
|
||||
@@ -64,9 +60,8 @@ impl BenchContext {
|
||||
.and_then(|t| t.as_http_error())
|
||||
.is_some_and(|e| e.status == 502)
|
||||
});
|
||||
let max_retries = bench_args.rpc_block_fetch_retries.as_max_retries();
|
||||
let client = ClientBuilder::default()
|
||||
.layer(RetryBackoffLayer::new_with_policy(max_retries, 800, u64::MAX, retry_policy))
|
||||
.layer(RetryBackoffLayer::new_with_policy(10, 800, u64::MAX, retry_policy))
|
||||
.http(rpc_url.parse()?);
|
||||
let block_provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
@@ -167,8 +162,6 @@ impl BenchContext {
|
||||
let next_block = first_block.header.number + 1;
|
||||
let rlp_blocks = bench_args.rlp_blocks;
|
||||
let use_reth_namespace = bench_args.reth_new_payload || rlp_blocks;
|
||||
let no_wait_for_persistence = bench_args.no_wait_for_persistence;
|
||||
let no_wait_for_caches = bench_args.no_wait_for_caches;
|
||||
Ok(Self {
|
||||
auth_provider,
|
||||
block_provider,
|
||||
@@ -177,8 +170,6 @@ impl BenchContext {
|
||||
is_optimism,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
no_wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
251
bin/reth-bench/src/bench/gas_limit_ramp.rs
Normal file
251
bin/reth-bench/src/bench/gas_limit_ramp.rs
Normal file
@@ -0,0 +1,251 @@
|
||||
//! Benchmarks empty block processing by ramping the block gas limit.
|
||||
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
bench::{
|
||||
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
|
||||
output::GasRampPayloadFile,
|
||||
},
|
||||
valid_payload::{
|
||||
call_forkchoice_updated_with_reth, call_new_payload_with_reth, payload_to_new_payload,
|
||||
},
|
||||
};
|
||||
use alloy_eips::BlockNumberOrTag;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
|
||||
|
||||
use clap::Parser;
|
||||
use reqwest::Url;
|
||||
use reth_chainspec::ChainSpec;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_ethereum_primitives::TransactionSigned;
|
||||
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
|
||||
use reth_rpc_api::RethNewPayloadInput;
|
||||
use std::{path::PathBuf, time::Instant};
|
||||
use tracing::info;
|
||||
|
||||
/// `reth benchmark gas-limit-ramp` command.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command {
|
||||
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
|
||||
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
|
||||
blocks: Option<u64>,
|
||||
|
||||
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
|
||||
/// reaches or exceeds this value. Mutually exclusive with --blocks.
|
||||
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
|
||||
/// billion).
|
||||
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
|
||||
target_gas_limit: Option<u64>,
|
||||
|
||||
/// The Engine API RPC URL.
|
||||
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
|
||||
engine_rpc_url: String,
|
||||
|
||||
/// Path to the JWT secret for Engine API authentication.
|
||||
#[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
|
||||
jwt_secret: PathBuf,
|
||||
|
||||
/// Output directory for benchmark results and generated payloads.
|
||||
#[arg(long, value_name = "OUTPUT")]
|
||||
output: PathBuf,
|
||||
|
||||
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
///
|
||||
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
|
||||
/// directly, waits for persistence and cache updates to complete before processing,
|
||||
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
}
|
||||
|
||||
/// Mode for determining when to stop ramping.
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
enum RampMode {
|
||||
/// Ramp for a fixed number of blocks.
|
||||
Blocks(u64),
|
||||
/// Ramp until reaching or exceeding target gas limit.
|
||||
TargetGasLimit(u64),
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute `benchmark gas-limit-ramp` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
let mode = match (self.blocks, self.target_gas_limit) {
|
||||
(Some(blocks), None) => {
|
||||
if blocks == 0 {
|
||||
return Err(eyre::eyre!("--blocks must be greater than 0"));
|
||||
}
|
||||
RampMode::Blocks(blocks)
|
||||
}
|
||||
(None, Some(target)) => {
|
||||
if target == 0 {
|
||||
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
|
||||
}
|
||||
RampMode::TargetGasLimit(target)
|
||||
}
|
||||
_ => {
|
||||
return Err(eyre::eyre!(
|
||||
"Exactly one of --blocks or --target-gas-limit must be specified"
|
||||
));
|
||||
}
|
||||
};
|
||||
|
||||
// Ensure output directory exists
|
||||
if self.output.is_file() {
|
||||
return Err(eyre::eyre!("Output path must be a directory"));
|
||||
}
|
||||
if !self.output.exists() {
|
||||
std::fs::create_dir_all(&self.output)?;
|
||||
info!(target: "reth-bench", "Created output directory: {:?}", self.output);
|
||||
}
|
||||
|
||||
// Set up authenticated provider (used for both Engine API and eth_ methods)
|
||||
let jwt = std::fs::read_to_string(&self.jwt_secret)?;
|
||||
let jwt = JwtSecret::from_hex(jwt)?;
|
||||
let auth_url = Url::parse(&self.engine_rpc_url)?;
|
||||
|
||||
info!(target: "reth-bench", "Connecting to Engine RPC at {}", auth_url);
|
||||
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
|
||||
let client = ClientBuilder::default().connect_with(auth_transport).await?;
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
|
||||
// Get chain spec - required for fork detection
|
||||
let chain_id = provider.get_chain_id().await?;
|
||||
let chain_spec = ChainSpec::from_chain_id(chain_id)
|
||||
.ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
|
||||
|
||||
// Fetch the current head block as parent
|
||||
let parent_block = provider
|
||||
.get_block_by_number(BlockNumberOrTag::Latest)
|
||||
.full()
|
||||
.await?
|
||||
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
|
||||
|
||||
let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
|
||||
|
||||
let canonical_parent = parent_header.number;
|
||||
let start_block = canonical_parent + 1;
|
||||
|
||||
match mode {
|
||||
RampMode::Blocks(blocks) => {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
canonical_parent,
|
||||
start_block,
|
||||
end_block = start_block + blocks - 1,
|
||||
"Starting gas limit ramp benchmark (block count mode)"
|
||||
);
|
||||
}
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
canonical_parent,
|
||||
start_block,
|
||||
current_gas_limit = parent_header.gas_limit,
|
||||
target_gas_limit = target,
|
||||
"Starting gas limit ramp benchmark (target gas limit mode)"
|
||||
);
|
||||
}
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
|
||||
}
|
||||
|
||||
let mut blocks_processed = 0u64;
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
|
||||
let timestamp = parent_header.timestamp.saturating_add(1);
|
||||
|
||||
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
|
||||
let new_payload_version = request.new_payload_version;
|
||||
|
||||
let (payload, sidecar) = build_payload(&provider, request).await?;
|
||||
|
||||
let mut block =
|
||||
payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
|
||||
|
||||
let max_increase = max_gas_limit_increase(parent_header.gas_limit);
|
||||
let gas_limit =
|
||||
parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
|
||||
|
||||
block.header.gas_limit = gas_limit;
|
||||
|
||||
let block_hash = block.header.hash_slow();
|
||||
// Regenerate the payload from the modified block, but keep the original sidecar
|
||||
// which contains the actual execution requests data (not just the hash)
|
||||
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
|
||||
let (version, params, execution_data) = payload_to_new_payload(
|
||||
payload,
|
||||
sidecar,
|
||||
false,
|
||||
block.header.withdrawals_root,
|
||||
Some(new_payload_version),
|
||||
)?;
|
||||
|
||||
let (version, params) = if self.reth_new_payload {
|
||||
(None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?)
|
||||
} else {
|
||||
(Some(version), params)
|
||||
};
|
||||
|
||||
// Save payload to file with version info for replay
|
||||
let payload_path =
|
||||
self.output.join(format!("payload_block_{}.json", block.header.number));
|
||||
let file = GasRampPayloadFile {
|
||||
version: version.map(|v| v as u8),
|
||||
block_hash,
|
||||
params: params.clone(),
|
||||
};
|
||||
let payload_json = serde_json::to_string_pretty(&file)?;
|
||||
std::fs::write(&payload_path, &payload_json)?;
|
||||
info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
|
||||
|
||||
let _ = call_new_payload_with_reth(&provider, version, params).await?;
|
||||
|
||||
let forkchoice_state = ForkchoiceState {
|
||||
head_block_hash: block_hash,
|
||||
safe_block_hash: block_hash,
|
||||
finalized_block_hash: block_hash,
|
||||
};
|
||||
call_forkchoice_updated_with_reth(&provider, version, forkchoice_state).await?;
|
||||
|
||||
parent_header = block.header;
|
||||
parent_hash = block_hash;
|
||||
blocks_processed += 1;
|
||||
|
||||
let progress = match mode {
|
||||
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
|
||||
RampMode::TargetGasLimit(target) => {
|
||||
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
|
||||
format!("{pct:.1}%")
|
||||
}
|
||||
};
|
||||
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
|
||||
}
|
||||
|
||||
let final_gas_limit = parent_header.gas_limit;
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
total_duration=?total_benchmark_duration.elapsed(),
|
||||
blocks_processed,
|
||||
final_gas_limit,
|
||||
"Benchmark complete"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
|
||||
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
|
||||
}
|
||||
|
||||
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
|
||||
match mode {
|
||||
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
|
||||
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
|
||||
}
|
||||
}
|
||||
@@ -773,7 +773,6 @@ impl Command {
|
||||
suggested_fee_recipient: alloy_primitives::Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
},
|
||||
transactions: transactions.to_vec(),
|
||||
extra_data: None,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use crate::valid_payload::call_forkchoice_updated;
|
||||
use eyre::Result;
|
||||
use std::{
|
||||
io::{BufReader, Read},
|
||||
@@ -69,6 +70,180 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
use alloy_consensus::Header;
|
||||
use alloy_eips::eip4844::kzg_to_versioned_hash;
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
|
||||
PayloadAttributes, PayloadId,
|
||||
};
|
||||
use eyre::OptionExt;
|
||||
use reth_chainspec::{ChainSpec, EthereumHardforks};
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use tracing::debug;
|
||||
|
||||
/// Prepared payload request data for triggering block building.
|
||||
pub(crate) struct PayloadRequest {
|
||||
/// The payload attributes for the new block.
|
||||
pub(crate) attributes: PayloadAttributes,
|
||||
/// The forkchoice state pointing to the parent block.
|
||||
pub(crate) forkchoice_state: ForkchoiceState,
|
||||
/// The engine API version for FCU calls.
|
||||
pub(crate) fcu_version: EngineApiMessageVersion,
|
||||
/// The getPayload version to use (1-5).
|
||||
pub(crate) get_payload_version: u8,
|
||||
/// The newPayload version to use.
|
||||
pub(crate) new_payload_version: EngineApiMessageVersion,
|
||||
}
|
||||
|
||||
/// Prepare payload attributes and forkchoice state for a new block.
|
||||
pub(crate) fn prepare_payload_request(
|
||||
chain_spec: &ChainSpec,
|
||||
timestamp: u64,
|
||||
parent_hash: B256,
|
||||
) -> PayloadRequest {
|
||||
let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
|
||||
let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
|
||||
let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
|
||||
let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
|
||||
|
||||
// FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
|
||||
let fcu_version = if cancun_active {
|
||||
EngineApiMessageVersion::V3
|
||||
} else if shanghai_active {
|
||||
EngineApiMessageVersion::V2
|
||||
} else {
|
||||
EngineApiMessageVersion::V1
|
||||
};
|
||||
|
||||
// getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
|
||||
// newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
|
||||
let (get_payload_version, new_payload_version) = if osaka_active {
|
||||
(5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
|
||||
} else if prague_active {
|
||||
(4, EngineApiMessageVersion::V4)
|
||||
} else if cancun_active {
|
||||
(3, EngineApiMessageVersion::V3)
|
||||
} else if shanghai_active {
|
||||
(2, EngineApiMessageVersion::V2)
|
||||
} else {
|
||||
(1, EngineApiMessageVersion::V1)
|
||||
};
|
||||
|
||||
PayloadRequest {
|
||||
attributes: PayloadAttributes {
|
||||
timestamp,
|
||||
prev_randao: B256::ZERO,
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: shanghai_active.then(Vec::new),
|
||||
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
|
||||
},
|
||||
forkchoice_state: ForkchoiceState {
|
||||
head_block_hash: parent_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
},
|
||||
fcu_version,
|
||||
get_payload_version,
|
||||
new_payload_version,
|
||||
}
|
||||
}
|
||||
|
||||
/// Trigger payload building via FCU and retrieve the built payload.
|
||||
///
|
||||
/// This sends a forkchoiceUpdated with payload attributes to start building,
|
||||
/// then calls getPayload to retrieve the result.
|
||||
pub(crate) async fn build_payload(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
request: PayloadRequest,
|
||||
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
|
||||
let fcu_result = call_forkchoice_updated(
|
||||
provider,
|
||||
request.fcu_version,
|
||||
request.forkchoice_state,
|
||||
Some(request.attributes.clone()),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let payload_id =
|
||||
fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
|
||||
|
||||
get_payload_with_sidecar(
|
||||
provider,
|
||||
request.get_payload_version,
|
||||
payload_id,
|
||||
request.attributes.parent_beacon_block_root,
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
/// Convert an RPC block to a consensus header and block hash.
|
||||
pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
|
||||
let block_hash = block.header.hash;
|
||||
let header = block.header.inner.clone().into_header_with_defaults();
|
||||
(header, block_hash)
|
||||
}
|
||||
|
||||
/// Compute versioned hashes from KZG commitments.
|
||||
fn versioned_hashes_from_commitments(
|
||||
commitments: &[alloy_primitives::FixedBytes<48>],
|
||||
) -> Vec<B256> {
|
||||
commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
|
||||
}
|
||||
|
||||
/// Fetch an execution payload using the appropriate engine API version.
|
||||
pub(crate) async fn get_payload_with_sidecar(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
version: u8,
|
||||
payload_id: PayloadId,
|
||||
parent_beacon_block_root: Option<B256>,
|
||||
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
|
||||
debug!(target: "reth-bench", get_payload_version = ?version, ?payload_id, "Sending getPayload");
|
||||
|
||||
match version {
|
||||
1 => {
|
||||
let payload = provider.get_payload_v1(payload_id).await?;
|
||||
Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
|
||||
}
|
||||
2 => {
|
||||
let envelope = provider.get_payload_v2(payload_id).await?;
|
||||
let payload = match envelope.execution_payload {
|
||||
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
|
||||
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
|
||||
};
|
||||
Ok((payload, ExecutionPayloadSidecar::none()))
|
||||
}
|
||||
3 => {
|
||||
let envelope = provider.get_payload_v3(payload_id).await?;
|
||||
let versioned_hashes =
|
||||
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
|
||||
let cancun_fields = CancunPayloadFields {
|
||||
parent_beacon_block_root: parent_beacon_block_root
|
||||
.ok_or_eyre("parent_beacon_block_root required for V3")?,
|
||||
versioned_hashes,
|
||||
};
|
||||
Ok((
|
||||
ExecutionPayload::V3(envelope.execution_payload),
|
||||
ExecutionPayloadSidecar::v3(cancun_fields),
|
||||
))
|
||||
}
|
||||
4 => {
|
||||
let envelope = provider.get_payload_v4(payload_id).await?;
|
||||
Ok(envelope.into_payload_and_sidecar(
|
||||
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V4")?,
|
||||
))
|
||||
}
|
||||
5 => {
|
||||
let envelope = provider.get_payload_v5(payload_id).await?;
|
||||
Ok(envelope.into_payload_and_sidecar(
|
||||
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V5")?,
|
||||
))
|
||||
}
|
||||
_ => panic!("This tool does not support getPayload versions past v5"),
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -6,6 +6,7 @@ use reth_node_core::args::LogArgs;
|
||||
use reth_tracing::FileWorkerGuard;
|
||||
|
||||
mod context;
|
||||
mod gas_limit_ramp;
|
||||
mod generate_big_block;
|
||||
pub(crate) mod helpers;
|
||||
pub use generate_big_block::{
|
||||
@@ -15,6 +16,7 @@ pub(crate) mod metrics_scraper;
|
||||
mod new_payload_fcu;
|
||||
mod new_payload_only;
|
||||
mod output;
|
||||
mod persistence_waiter;
|
||||
mod replay_payloads;
|
||||
mod send_invalid_payload;
|
||||
mod send_payload;
|
||||
@@ -35,6 +37,9 @@ pub enum Subcommands {
|
||||
/// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
|
||||
NewPayloadFcu(new_payload_fcu::Command),
|
||||
|
||||
/// Benchmark which builds empty blocks with a ramped gas limit.
|
||||
GasLimitRamp(gas_limit_ramp::Command),
|
||||
|
||||
/// Benchmark which only calls subsequent `newPayload` calls.
|
||||
NewPayloadOnly(new_payload_only::Command),
|
||||
|
||||
@@ -94,6 +99,7 @@ impl BenchmarkCommand {
|
||||
|
||||
match self.command {
|
||||
Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
|
||||
Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
|
||||
Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
|
||||
Subcommands::SendPayload(command) => command.execute(ctx).await,
|
||||
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
|
||||
|
||||
@@ -1,5 +1,13 @@
|
||||
//! Runs the `reth bench` command, calling first newPayload for each block, then calling
|
||||
//! forkchoiceUpdated.
|
||||
//!
|
||||
//! Supports configurable waiting behavior:
|
||||
//! - **`--wait-time`**: Fixed sleep interval between blocks.
|
||||
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
|
||||
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
|
||||
//! threshold. This ensures the benchmark doesn't outpace persistence.
|
||||
//!
|
||||
//! Both options can be used together or independently.
|
||||
|
||||
use crate::{
|
||||
bench::{
|
||||
@@ -9,6 +17,9 @@ use crate::{
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
persistence_waiter::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
},
|
||||
},
|
||||
valid_payload::{
|
||||
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
|
||||
@@ -18,7 +29,6 @@ use alloy_provider::{ext::DebugApi, Provider};
|
||||
use alloy_rpc_types_engine::ForkchoiceState;
|
||||
use clap::Parser;
|
||||
use eyre::{Context, OptionExt};
|
||||
use futures::{stream, StreamExt, TryStreamExt};
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_core::args::BenchmarkArgs;
|
||||
@@ -39,6 +49,16 @@ pub struct Command {
|
||||
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
|
||||
wait_time: Option<Duration>,
|
||||
|
||||
/// Wait for blocks to be persisted before sending the next batch.
|
||||
///
|
||||
/// When enabled, waits for every Nth block to be persisted using the
|
||||
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
|
||||
/// doesn't outpace persistence.
|
||||
///
|
||||
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
wait_for_persistence: bool,
|
||||
|
||||
/// Engine persistence threshold used for deciding when to wait for persistence.
|
||||
///
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
|
||||
@@ -86,17 +106,55 @@ impl Command {
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
|
||||
}
|
||||
if self.wait_for_persistence {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
|
||||
self.persistence_threshold + 1,
|
||||
self.persistence_threshold
|
||||
);
|
||||
}
|
||||
|
||||
// Set up waiter based on configured options
|
||||
// When both are set: wait at least wait_time, and also wait for persistence if needed
|
||||
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
|
||||
(Some(duration), true) => {
|
||||
let ws_url = derive_ws_rpc_url(
|
||||
self.benchmark.ws_rpc_url.as_deref(),
|
||||
&self.benchmark.engine_rpc_url,
|
||||
)?;
|
||||
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
|
||||
Some(PersistenceWaiter::with_duration_and_subscription(
|
||||
duration,
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
self.persistence_timeout,
|
||||
))
|
||||
}
|
||||
(Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
|
||||
(None, true) => {
|
||||
let ws_url = derive_ws_rpc_url(
|
||||
self.benchmark.ws_rpc_url.as_deref(),
|
||||
&self.benchmark.engine_rpc_url,
|
||||
)?;
|
||||
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
|
||||
Some(PersistenceWaiter::with_subscription(
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
self.persistence_timeout,
|
||||
))
|
||||
}
|
||||
(None, false) => None,
|
||||
};
|
||||
|
||||
let BenchContext {
|
||||
benchmark_mode,
|
||||
block_provider,
|
||||
auth_provider,
|
||||
next_block,
|
||||
mut next_block,
|
||||
is_optimism,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
no_wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
@@ -109,71 +167,70 @@ impl Command {
|
||||
|
||||
let buffer_size = self.rpc_block_buffer_size;
|
||||
|
||||
let mut blocks = Box::pin(
|
||||
stream::iter((next_block..)
|
||||
.take_while(|next_block| {
|
||||
benchmark_mode.contains(*next_block)
|
||||
}))
|
||||
.map(|next_block| {
|
||||
let block_provider = block_provider.clone();
|
||||
async move {
|
||||
let block_res = block_provider
|
||||
.get_block_by_number(next_block.into())
|
||||
.full()
|
||||
.await
|
||||
.wrap_err_with(|| {
|
||||
format!("Failed to fetch block by number {next_block}")
|
||||
});
|
||||
let block =
|
||||
match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
|
||||
Ok(block) => block,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
|
||||
return Err(e)
|
||||
}
|
||||
};
|
||||
// Use a oneshot channel to propagate errors from the spawned task
|
||||
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
|
||||
|
||||
let rlp = if rlp_blocks {
|
||||
let rlp = match block_provider
|
||||
.debug_get_raw_block(next_block.into())
|
||||
.await
|
||||
{
|
||||
Ok(rlp) => rlp,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
|
||||
return Err(e.into())
|
||||
}
|
||||
};
|
||||
Some(rlp)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let head_block_hash = block.header.hash;
|
||||
let safe_block_hash = block_provider
|
||||
.get_block_by_number(block.header.number.saturating_sub(32).into());
|
||||
|
||||
let finalized_block_hash = block_provider
|
||||
.get_block_by_number(block.header.number.saturating_sub(64).into());
|
||||
|
||||
let (safe, finalized) =
|
||||
tokio::join!(safe_block_hash, finalized_block_hash);
|
||||
|
||||
let safe_block_hash = match safe {
|
||||
Ok(Some(block)) => block.header.hash,
|
||||
Ok(None) | Err(_) => head_block_hash,
|
||||
};
|
||||
|
||||
let finalized_block_hash = match finalized {
|
||||
Ok(Some(block)) => block.header.hash,
|
||||
Ok(None) | Err(_) => head_block_hash,
|
||||
};
|
||||
|
||||
Ok((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
|
||||
tokio::task::spawn(async move {
|
||||
while benchmark_mode.contains(next_block) {
|
||||
let block_res = block_provider
|
||||
.get_block_by_number(next_block.into())
|
||||
.full()
|
||||
.await
|
||||
.wrap_err_with(|| format!("Failed to fetch block by number {next_block}"));
|
||||
let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
|
||||
Ok(block) => block,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
|
||||
let _ = error_sender.send(e);
|
||||
break;
|
||||
}
|
||||
})
|
||||
.buffered(buffer_size),
|
||||
);
|
||||
};
|
||||
|
||||
let rlp = if rlp_blocks {
|
||||
let rlp = match block_provider.debug_get_raw_block(next_block.into()).await {
|
||||
Ok(rlp) => rlp,
|
||||
Err(e) => {
|
||||
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
|
||||
let _ = error_sender
|
||||
.send(eyre::eyre!("Failed to fetch raw block {next_block}: {e}"));
|
||||
break;
|
||||
}
|
||||
};
|
||||
Some(rlp)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let head_block_hash = block.header.hash;
|
||||
let safe_block_hash = block_provider
|
||||
.get_block_by_number(block.header.number.saturating_sub(32).into());
|
||||
|
||||
let finalized_block_hash = block_provider
|
||||
.get_block_by_number(block.header.number.saturating_sub(64).into());
|
||||
|
||||
let (safe, finalized) = tokio::join!(safe_block_hash, finalized_block_hash,);
|
||||
|
||||
let safe_block_hash = match safe {
|
||||
Ok(Some(block)) => block.header.hash,
|
||||
Ok(None) | Err(_) => head_block_hash,
|
||||
};
|
||||
|
||||
let finalized_block_hash = match finalized {
|
||||
Ok(Some(block)) => block.header.hash,
|
||||
Ok(None) | Err(_) => head_block_hash,
|
||||
};
|
||||
|
||||
next_block += 1;
|
||||
if let Err(e) = sender
|
||||
.send((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
|
||||
.await
|
||||
{
|
||||
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
|
||||
break;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
let mut results = Vec::new();
|
||||
let mut blocks_processed = 0u64;
|
||||
@@ -182,7 +239,7 @@ impl Command {
|
||||
|
||||
while let Some((block, head, safe, finalized, rlp)) = {
|
||||
let wait_start = Instant::now();
|
||||
let result = blocks.try_next().await?;
|
||||
let result = receiver.recv().await;
|
||||
total_wait_time += wait_start.elapsed();
|
||||
result
|
||||
} {
|
||||
@@ -199,14 +256,8 @@ impl Command {
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
is_optimism,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
no_wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
)?;
|
||||
let (version, params) =
|
||||
block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
call_new_payload_with_reth(&auth_provider, version, params).await?;
|
||||
@@ -264,8 +315,8 @@ impl Command {
|
||||
warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
|
||||
}
|
||||
|
||||
if let Some(wait_time) = self.wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
}
|
||||
|
||||
let gas_row =
|
||||
@@ -273,6 +324,15 @@ impl Command {
|
||||
results.push((gas_row, combined_result));
|
||||
}
|
||||
|
||||
// Check if the spawned task encountered an error
|
||||
if let Ok(error) = error_receiver.try_recv() {
|
||||
return Err(error);
|
||||
}
|
||||
|
||||
// Drop waiter - we don't need to wait for final blocks to persist
|
||||
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
|
||||
drop(waiter);
|
||||
|
||||
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
|
||||
results.into_iter().unzip();
|
||||
|
||||
|
||||
@@ -52,8 +52,6 @@ impl Command {
|
||||
is_optimism,
|
||||
use_reth_namespace,
|
||||
rlp_blocks,
|
||||
no_wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
|
||||
|
||||
let total_blocks = benchmark_mode.total_blocks();
|
||||
@@ -124,14 +122,8 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
is_optimism,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
no_wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
)?;
|
||||
let (version, params) =
|
||||
block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
|
||||
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
//! Contains various benchmark output formats, either for logging or for
|
||||
//! serialization to / from files.
|
||||
|
||||
use alloy_primitives::B256;
|
||||
use csv::Writer;
|
||||
use eyre::OptionExt;
|
||||
use reth_primitives_traits::constants::GIGAGAS;
|
||||
use serde::{ser::SerializeStruct, Serialize};
|
||||
use serde::{ser::SerializeStruct, Deserialize, Serialize};
|
||||
use std::{fs, path::Path, time::Duration};
|
||||
use tracing::info;
|
||||
|
||||
@@ -17,6 +18,20 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
|
||||
/// This is the suffix for new payload output csv files.
|
||||
pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
|
||||
|
||||
/// Serialized format for gas ramp payloads on disk.
|
||||
#[derive(Debug, Serialize, Deserialize)]
|
||||
pub(crate) struct GasRampPayloadFile {
|
||||
/// Engine API version (1-5).
|
||||
///
|
||||
/// `None` indicates that `reth_newPayload` should be used.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub(crate) version: Option<u8>,
|
||||
/// The block hash for FCU.
|
||||
pub(crate) block_hash: B256,
|
||||
/// The params to pass to newPayload.
|
||||
pub(crate) params: serde_json::Value,
|
||||
}
|
||||
|
||||
/// This represents the results of a single `newPayload` call in the benchmark, containing the gas
|
||||
/// used and the `newPayload` latency.
|
||||
#[derive(Debug)]
|
||||
|
||||
334
bin/reth-bench/src/bench/persistence_waiter.rs
Normal file
334
bin/reth-bench/src/bench/persistence_waiter.rs
Normal file
@@ -0,0 +1,334 @@
|
||||
//! Persistence waiting utilities for benchmarks.
|
||||
//!
|
||||
//! Provides waiting behavior to control benchmark pacing:
|
||||
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
|
||||
//! - **Persistence-based waits**: Wait for blocks to be persisted using
|
||||
//! `reth_subscribePersistedBlock` subscription
|
||||
//! - **Combined mode**: Wait at least the fixed duration, and also wait for persistence if the
|
||||
//! block hasn't been persisted yet (whichever takes longer)
|
||||
|
||||
use alloy_eips::BlockNumHash;
|
||||
use alloy_network::Ethereum;
|
||||
use alloy_provider::{Provider, RootProvider};
|
||||
use alloy_pubsub::SubscriptionStream;
|
||||
use alloy_rpc_client::RpcClient;
|
||||
use alloy_transport_ws::WsConnect;
|
||||
use eyre::Context;
|
||||
use futures::StreamExt;
|
||||
use std::time::Duration;
|
||||
use tracing::{debug, info};
|
||||
|
||||
/// Default `WebSocket` RPC port for reth.
|
||||
const DEFAULT_WS_RPC_PORT: u16 = 8546;
|
||||
use url::Url;
|
||||
|
||||
/// Returns the websocket RPC URL used for the persistence subscription.
|
||||
///
|
||||
/// Preference:
|
||||
/// - If `ws_rpc_url` is provided, use it directly.
|
||||
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
|
||||
///
|
||||
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
|
||||
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
|
||||
/// Since we may only have the engine URL by default, we convert the scheme
|
||||
/// (http→ws, https→wss) and force the port to 8546.
|
||||
pub(crate) fn derive_ws_rpc_url(
|
||||
ws_rpc_url: Option<&str>,
|
||||
engine_rpc_url: &str,
|
||||
) -> eyre::Result<Url> {
|
||||
if let Some(ws_url) = ws_rpc_url {
|
||||
let parsed: Url = ws_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
|
||||
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
|
||||
Ok(parsed)
|
||||
} else {
|
||||
let derived = engine_url_to_ws_url(engine_rpc_url)?;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
engine_url = %engine_rpc_url,
|
||||
%derived,
|
||||
"Derived WebSocket RPC URL from engine RPC URL"
|
||||
);
|
||||
Ok(derived)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an engine API URL to the default RPC websocket URL.
|
||||
///
|
||||
/// Transformations:
|
||||
/// - `http` → `ws`
|
||||
/// - `https` → `wss`
|
||||
/// - `ws` / `wss` keep their scheme
|
||||
/// - Port is always set to `8546`, reth's default RPC websocket port.
|
||||
///
|
||||
/// This is used when we only know the engine API URL (typically `:8551`) but
|
||||
/// need to connect to the node's WS RPC endpoint for persistence events.
|
||||
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
|
||||
let url: Url = engine_url
|
||||
.parse()
|
||||
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
|
||||
|
||||
let mut ws_url = url.clone();
|
||||
|
||||
match ws_url.scheme() {
|
||||
"http" => ws_url
|
||||
.set_scheme("ws")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
|
||||
"https" => ws_url
|
||||
.set_scheme("wss")
|
||||
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
|
||||
"ws" | "wss" => {}
|
||||
scheme => {
|
||||
return Err(eyre::eyre!(
|
||||
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
ws_url
|
||||
.set_port(Some(DEFAULT_WS_RPC_PORT))
|
||||
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
|
||||
|
||||
Ok(ws_url)
|
||||
}
|
||||
|
||||
/// Waits until the persistence subscription reports that `target` has been persisted.
|
||||
///
|
||||
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
|
||||
/// - the subscription stream ends unexpectedly, or
|
||||
/// - `timeout` elapses before `target` is observed.
|
||||
async fn wait_for_persistence(
|
||||
stream: &mut SubscriptionStream<BlockNumHash>,
|
||||
target: u64,
|
||||
last_persisted: &mut u64,
|
||||
timeout: Duration,
|
||||
) -> eyre::Result<()> {
|
||||
tokio::time::timeout(timeout, async {
|
||||
while *last_persisted < target {
|
||||
match stream.next().await {
|
||||
Some(persisted) => {
|
||||
*last_persisted = persisted.number;
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted_block = ?last_persisted,
|
||||
"Received persistence notification"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
})
|
||||
.await
|
||||
.map_err(|_| {
|
||||
eyre::eyre!(
|
||||
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
|
||||
target,
|
||||
timeout,
|
||||
last_persisted
|
||||
)
|
||||
})?
|
||||
}
|
||||
|
||||
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
|
||||
/// The provider must be kept alive for the subscription to continue receiving events.
|
||||
pub(crate) struct PersistenceSubscription {
|
||||
_provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
}
|
||||
|
||||
impl PersistenceSubscription {
|
||||
const fn new(
|
||||
provider: RootProvider<Ethereum>,
|
||||
stream: SubscriptionStream<BlockNumHash>,
|
||||
) -> Self {
|
||||
Self { _provider: provider, stream }
|
||||
}
|
||||
|
||||
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
|
||||
&mut self.stream
|
||||
}
|
||||
}
|
||||
|
||||
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
|
||||
///
|
||||
/// The `keepalive_interval` is set to match `persistence_timeout` so that the `WebSocket`
|
||||
/// connection is not dropped during long MDBX commits that block the server from responding
|
||||
/// to pings.
|
||||
pub(crate) async fn setup_persistence_subscription(
|
||||
ws_url: Url,
|
||||
persistence_timeout: Duration,
|
||||
) -> eyre::Result<PersistenceSubscription> {
|
||||
info!(target: "reth-bench", "Connecting to WebSocket at {} for persistence subscription", ws_url);
|
||||
|
||||
let ws_connect =
|
||||
WsConnect::new(ws_url.to_string()).with_keepalive_interval(persistence_timeout);
|
||||
let client = RpcClient::connect_pubsub(ws_connect)
|
||||
.await
|
||||
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
|
||||
let provider: RootProvider<Ethereum> = RootProvider::new(client);
|
||||
|
||||
let subscription = provider
|
||||
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
|
||||
.await
|
||||
.wrap_err("Failed to subscribe to persistence notifications")?;
|
||||
|
||||
info!(target: "reth-bench", "Subscribed to persistence notifications");
|
||||
|
||||
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
|
||||
}
|
||||
|
||||
/// Encapsulates the block waiting logic.
|
||||
///
|
||||
/// Provides a simple `on_block()` interface that handles both:
|
||||
/// - Fixed duration waits (when `wait_time` is set)
|
||||
/// - Persistence-based waits (when `subscription` is set)
|
||||
///
|
||||
/// For persistence mode, waits after every `(threshold + 1)` blocks.
|
||||
pub(crate) struct PersistenceWaiter {
|
||||
wait_time: Option<Duration>,
|
||||
subscription: Option<PersistenceSubscription>,
|
||||
blocks_sent: u64,
|
||||
last_persisted: u64,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
}
|
||||
|
||||
impl PersistenceWaiter {
|
||||
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
|
||||
Self {
|
||||
wait_time: Some(wait_time),
|
||||
subscription: None,
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold: 0,
|
||||
timeout: Duration::ZERO,
|
||||
}
|
||||
}
|
||||
|
||||
pub(crate) const fn with_subscription(
|
||||
subscription: PersistenceSubscription,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
wait_time: None,
|
||||
subscription: Some(subscription),
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold,
|
||||
timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Creates a waiter that combines both duration and persistence waiting.
|
||||
///
|
||||
/// Waits at least `wait_time` between blocks, and also waits for persistence
|
||||
/// if the block hasn't been persisted yet (whichever takes longer).
|
||||
pub(crate) const fn with_duration_and_subscription(
|
||||
wait_time: Duration,
|
||||
subscription: PersistenceSubscription,
|
||||
threshold: u64,
|
||||
timeout: Duration,
|
||||
) -> Self {
|
||||
Self {
|
||||
wait_time: Some(wait_time),
|
||||
subscription: Some(subscription),
|
||||
blocks_sent: 0,
|
||||
last_persisted: 0,
|
||||
threshold,
|
||||
timeout,
|
||||
}
|
||||
}
|
||||
|
||||
/// Called once per block. Waits based on the configured mode.
|
||||
///
|
||||
/// When both `wait_time` and `subscription` are set (combined mode):
|
||||
/// - Always waits at least `wait_time`
|
||||
/// - Additionally waits for persistence if we're at a persistence checkpoint
|
||||
#[allow(clippy::manual_is_multiple_of)]
|
||||
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
|
||||
// Always wait for the fixed duration if configured
|
||||
if let Some(wait_time) = self.wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
}
|
||||
|
||||
// Check persistence if subscription is configured
|
||||
let Some(ref mut subscription) = self.subscription else {
|
||||
return Ok(());
|
||||
};
|
||||
|
||||
self.blocks_sent += 1;
|
||||
|
||||
if self.blocks_sent % (self.threshold + 1) == 0 {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
target_block = ?block_number,
|
||||
last_persisted = self.last_persisted,
|
||||
blocks_sent = self.blocks_sent,
|
||||
"Waiting for persistence"
|
||||
);
|
||||
|
||||
wait_for_persistence(
|
||||
subscription.stream_mut(),
|
||||
block_number,
|
||||
&mut self.last_persisted,
|
||||
self.timeout,
|
||||
)
|
||||
.await?;
|
||||
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
persisted = self.last_persisted,
|
||||
"Persistence caught up"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::time::Instant;
|
||||
|
||||
#[test]
|
||||
fn test_engine_url_to_ws_url() {
|
||||
// http -> ws, always uses port 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "ws://localhost:8546/");
|
||||
|
||||
// https -> wss
|
||||
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
|
||||
assert_eq!(result.as_str(), "wss://localhost:8546/");
|
||||
|
||||
// Custom engine port still maps to 8546
|
||||
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
|
||||
assert_eq!(result.port(), Some(8546));
|
||||
|
||||
// Already ws passthrough
|
||||
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
|
||||
assert_eq!(result.scheme(), "ws");
|
||||
|
||||
// Invalid inputs
|
||||
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
|
||||
assert!(engine_url_to_ws_url("not a valid url").is_err());
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn test_waiter_with_duration() {
|
||||
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
|
||||
|
||||
let start = Instant::now();
|
||||
waiter.on_block(1).await.unwrap();
|
||||
waiter.on_block(2).await.unwrap();
|
||||
waiter.on_block(3).await.unwrap();
|
||||
|
||||
// Should have waited ~3ms total
|
||||
assert!(start.elapsed() >= Duration::from_millis(3));
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,15 @@
|
||||
//! Command for replaying pre-generated payloads from disk.
|
||||
//!
|
||||
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
|
||||
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
|
||||
//!
|
||||
//! Supports configurable waiting behavior:
|
||||
//! - **`--wait-time`**: Fixed sleep interval between blocks.
|
||||
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
|
||||
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
|
||||
//! threshold. This ensures the benchmark doesn't outpace persistence.
|
||||
//!
|
||||
//! Both options can be used together or independently.
|
||||
|
||||
use crate::{
|
||||
authenticated_transport::AuthenticatedTransportConnect,
|
||||
@@ -6,7 +17,11 @@ use crate::{
|
||||
helpers::parse_duration,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
|
||||
TotalGasOutput, TotalGasRow,
|
||||
},
|
||||
persistence_waiter::{
|
||||
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
|
||||
},
|
||||
},
|
||||
valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
|
||||
@@ -21,13 +36,14 @@ use alloy_rpc_types_engine::{
|
||||
use clap::Parser;
|
||||
use eyre::Context;
|
||||
use reth_cli_runner::CliContext;
|
||||
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
|
||||
use reth_node_api::EngineApiMessageVersion;
|
||||
use reth_rpc_api::RethNewPayloadInput;
|
||||
use std::{
|
||||
path::PathBuf,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::{debug, info, warn};
|
||||
use tracing::{debug, info};
|
||||
use url::Url;
|
||||
|
||||
/// `reth bench replay-payloads` command
|
||||
@@ -57,9 +73,9 @@ pub struct Command {
|
||||
#[arg(long, value_name = "SKIP", default_value = "0")]
|
||||
skip: usize,
|
||||
|
||||
/// Deprecated: gas ramp is no longer needed. Use `--testing.skip-gas-limit-ramp-check`
|
||||
/// and `--testing.gas-limit` on the reth node instead. This flag is accepted but ignored.
|
||||
#[arg(long, value_name = "GAS_RAMP_DIR", hide = true)]
|
||||
/// Optional directory containing gas ramp payloads to replay first.
|
||||
/// These are replayed before the main payloads to warm up the gas limit.
|
||||
#[arg(long, value_name = "GAS_RAMP_DIR")]
|
||||
gas_ramp_dir: Option<PathBuf>,
|
||||
|
||||
/// Optional output directory for benchmark results (CSV files).
|
||||
@@ -73,6 +89,47 @@ pub struct Command {
|
||||
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
|
||||
wait_time: Option<Duration>,
|
||||
|
||||
/// Wait for blocks to be persisted before sending the next batch.
|
||||
///
|
||||
/// When enabled, waits for every Nth block to be persisted using the
|
||||
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
|
||||
/// doesn't outpace persistence.
|
||||
///
|
||||
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
wait_for_persistence: bool,
|
||||
|
||||
/// Engine persistence threshold used for deciding when to wait for persistence.
|
||||
///
|
||||
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
|
||||
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
|
||||
/// at blocks 3, 6, 9, etc.
|
||||
#[arg(
|
||||
long = "persistence-threshold",
|
||||
value_name = "PERSISTENCE_THRESHOLD",
|
||||
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
persistence_threshold: u64,
|
||||
|
||||
/// Timeout for waiting on persistence at each checkpoint.
|
||||
///
|
||||
/// Must be long enough to account for the persistence thread being blocked
|
||||
/// by pruning after the previous save.
|
||||
#[arg(
|
||||
long = "persistence-timeout",
|
||||
value_name = "PERSISTENCE_TIMEOUT",
|
||||
value_parser = parse_duration,
|
||||
default_value = "120s",
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
persistence_timeout: Duration,
|
||||
|
||||
/// Optional `WebSocket` RPC URL for persistence subscription.
|
||||
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
|
||||
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
|
||||
ws_rpc_url: Option<String>,
|
||||
|
||||
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
|
||||
///
|
||||
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
|
||||
@@ -81,20 +138,6 @@ pub struct Command {
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment)]
|
||||
reth_new_payload: bool,
|
||||
|
||||
/// Skip waiting for in-flight persistence before processing.
|
||||
///
|
||||
/// Only works with `--reth-new-payload`. When set, passes `wait_for_persistence: false`
|
||||
/// to the `reth_newPayload` endpoint.
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
|
||||
no_wait_for_persistence: bool,
|
||||
|
||||
/// Skip waiting for execution cache and sparse trie locks before processing.
|
||||
///
|
||||
/// Only works with `--reth-new-payload`. When set, passes `wait_for_caches: false`
|
||||
/// to the `reth_newPayload` endpoint.
|
||||
#[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
|
||||
no_wait_for_caches: bool,
|
||||
|
||||
/// Optional Prometheus metrics endpoint to scrape after each block.
|
||||
///
|
||||
/// When provided, reth-bench will fetch metrics from this URL after each
|
||||
@@ -114,6 +157,18 @@ struct LoadedPayload {
|
||||
block_hash: B256,
|
||||
}
|
||||
|
||||
/// A gas ramp payload loaded from disk.
|
||||
struct GasRampPayload {
|
||||
/// Block number from filename.
|
||||
block_number: u64,
|
||||
/// Engine API version for newPayload.
|
||||
///
|
||||
/// `None` indicates that `reth_newPayload` should be used.
|
||||
version: Option<EngineApiMessageVersion>,
|
||||
/// The file contents.
|
||||
file: GasRampPayloadFile,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
/// Execute the `replay-payloads` command.
|
||||
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
|
||||
@@ -123,10 +178,44 @@ impl Command {
|
||||
if let Some(duration) = self.wait_time {
|
||||
info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
|
||||
}
|
||||
if self.wait_for_persistence {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
|
||||
self.persistence_threshold + 1,
|
||||
self.persistence_threshold
|
||||
);
|
||||
}
|
||||
if self.reth_new_payload {
|
||||
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
|
||||
}
|
||||
|
||||
// Set up waiter based on configured options
|
||||
// When both are set: wait at least wait_time, and also wait for persistence if needed
|
||||
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
|
||||
(Some(duration), true) => {
|
||||
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
|
||||
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
|
||||
Some(PersistenceWaiter::with_duration_and_subscription(
|
||||
duration,
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
self.persistence_timeout,
|
||||
))
|
||||
}
|
||||
(Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
|
||||
(None, true) => {
|
||||
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
|
||||
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
|
||||
Some(PersistenceWaiter::with_subscription(
|
||||
sub,
|
||||
self.persistence_threshold,
|
||||
self.persistence_timeout,
|
||||
))
|
||||
}
|
||||
(None, false) => None,
|
||||
};
|
||||
|
||||
let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
|
||||
|
||||
// Set up authenticated engine provider
|
||||
@@ -156,16 +245,18 @@ impl Command {
|
||||
"Using initial parent block"
|
||||
);
|
||||
|
||||
// Warn if deprecated --gas-ramp-dir is passed
|
||||
if self.gas_ramp_dir.is_some() {
|
||||
warn!(
|
||||
target: "reth-bench",
|
||||
"--gas-ramp-dir is deprecated and ignored. Use --testing.skip-gas-limit-ramp-check \
|
||||
and --testing.gas-limit on the reth node instead."
|
||||
);
|
||||
}
|
||||
|
||||
// Load all payloads upfront to avoid I/O delays between phases
|
||||
let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
|
||||
let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
|
||||
if payloads.is_empty() {
|
||||
return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
|
||||
}
|
||||
info!(target: "reth-bench", count = payloads.len(), "Loaded gas ramp payloads from disk");
|
||||
payloads
|
||||
} else {
|
||||
Vec::new()
|
||||
};
|
||||
|
||||
let payloads = self.load_payloads()?;
|
||||
if payloads.is_empty() {
|
||||
return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
|
||||
@@ -174,6 +265,40 @@ impl Command {
|
||||
|
||||
let mut parent_hash = initial_parent_hash;
|
||||
|
||||
// Replay gas ramp payloads first
|
||||
for (i, payload) in gas_ramp_payloads.iter().enumerate() {
|
||||
info!(
|
||||
target: "reth-bench",
|
||||
gas_ramp_payload = i + 1,
|
||||
total = gas_ramp_payloads.len(),
|
||||
block_number = payload.block_number,
|
||||
block_hash = %payload.file.block_hash,
|
||||
"Executing gas ramp payload (newPayload + FCU)"
|
||||
);
|
||||
|
||||
let _ = call_new_payload_with_reth(
|
||||
&auth_provider,
|
||||
payload.version,
|
||||
payload.file.params.clone(),
|
||||
)
|
||||
.await?;
|
||||
|
||||
let fcu_state = ForkchoiceState {
|
||||
head_block_hash: payload.file.block_hash,
|
||||
safe_block_hash: parent_hash,
|
||||
finalized_block_hash: parent_hash,
|
||||
};
|
||||
call_forkchoice_updated_with_reth(&auth_provider, payload.version, fcu_state).await?;
|
||||
|
||||
info!(target: "reth-bench", gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
|
||||
|
||||
parent_hash = payload.file.block_hash;
|
||||
}
|
||||
|
||||
if !gas_ramp_payloads.is_empty() {
|
||||
info!(target: "reth-bench", count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
|
||||
}
|
||||
|
||||
let mut results = Vec::new();
|
||||
let total_benchmark_duration = Instant::now();
|
||||
|
||||
@@ -220,14 +345,7 @@ impl Command {
|
||||
},
|
||||
),
|
||||
};
|
||||
(
|
||||
None,
|
||||
serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(reth_data),
|
||||
self.no_wait_for_persistence.then_some(false),
|
||||
self.no_wait_for_caches.then_some(false),
|
||||
))?,
|
||||
)
|
||||
(None, serde_json::to_value((RethNewPayloadInput::ExecutionData(reth_data),))?)
|
||||
} else {
|
||||
(
|
||||
Some(EngineApiMessageVersion::V4),
|
||||
@@ -291,6 +409,10 @@ impl Command {
|
||||
tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
|
||||
}
|
||||
|
||||
if let Some(w) = &mut waiter {
|
||||
w.on_block(block_number).await?;
|
||||
}
|
||||
|
||||
let gas_row =
|
||||
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
|
||||
results.push((gas_row, combined_result));
|
||||
@@ -298,6 +420,10 @@ impl Command {
|
||||
parent_hash = block_hash;
|
||||
}
|
||||
|
||||
// Drop waiter - we don't need to wait for final blocks to persist
|
||||
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
|
||||
drop(waiter);
|
||||
|
||||
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
|
||||
results.into_iter().unzip();
|
||||
|
||||
@@ -384,4 +510,65 @@ impl Command {
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
|
||||
/// Load and parse gas ramp payload files from a directory.
|
||||
fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
|
||||
let mut payloads = Vec::new();
|
||||
|
||||
let entries: Vec<_> = std::fs::read_dir(dir)
|
||||
.wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
|
||||
.filter_map(|e| e.ok())
|
||||
.filter(|e| {
|
||||
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
|
||||
e.file_name().to_string_lossy().starts_with("payload_block_")
|
||||
})
|
||||
.collect();
|
||||
|
||||
// Parse filenames to get block numbers and sort
|
||||
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
|
||||
.into_iter()
|
||||
.filter_map(|e| {
|
||||
let name = e.file_name();
|
||||
let name_str = name.to_string_lossy();
|
||||
// Extract block number from "payload_block_NNN.json"
|
||||
let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
|
||||
let block_number: u64 = block_str.parse().ok()?;
|
||||
Some((block_number, e.path()))
|
||||
})
|
||||
.collect();
|
||||
|
||||
indexed_paths.sort_by_key(|(num, _)| *num);
|
||||
|
||||
for (block_number, path) in indexed_paths {
|
||||
let content = std::fs::read_to_string(&path)
|
||||
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
|
||||
let file: GasRampPayloadFile = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
|
||||
let version = if let Some(version) = file.version {
|
||||
match version {
|
||||
1 => EngineApiMessageVersion::V1,
|
||||
2 => EngineApiMessageVersion::V2,
|
||||
3 => EngineApiMessageVersion::V3,
|
||||
4 => EngineApiMessageVersion::V4,
|
||||
5 => EngineApiMessageVersion::V5,
|
||||
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
|
||||
}
|
||||
.into()
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
info!(
|
||||
block_number,
|
||||
block_hash = %file.block_hash,
|
||||
path = %path.display(),
|
||||
"Loaded gas ramp payload"
|
||||
);
|
||||
|
||||
payloads.push(GasRampPayload { block_number, version, file });
|
||||
}
|
||||
|
||||
Ok(payloads)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -240,7 +240,6 @@ impl Command {
|
||||
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
|
||||
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
|
||||
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
|
||||
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
|
||||
};
|
||||
|
||||
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
|
||||
@@ -255,9 +254,6 @@ impl Command {
|
||||
ExecutionPayload::V1(p) => p.block_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
|
||||
ExecutionPayload::V4(p) => {
|
||||
p.payload_inner.payload_inner.payload_inner.block_hash
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -266,9 +262,6 @@ impl Command {
|
||||
ExecutionPayload::V1(p) => p.block_hash = new_hash,
|
||||
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
|
||||
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
|
||||
ExecutionPayload::V4(p) => {
|
||||
p.payload_inner.payload_inner.payload_inner.block_hash = new_hash
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -167,25 +167,16 @@ where
|
||||
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
|
||||
///
|
||||
/// Returns `(version, versioned_params, execution_data)`.
|
||||
///
|
||||
/// When `no_wait_for_persistence` or `no_wait_for_caches` is `true` and using `reth_newPayload`,
|
||||
/// passes the corresponding `wait_for_*: false` to skip that wait.
|
||||
pub(crate) fn block_to_new_payload(
|
||||
block: AnyRpcBlock,
|
||||
is_optimism: bool,
|
||||
rlp: Option<Bytes>,
|
||||
reth_new_payload: bool,
|
||||
no_wait_for_persistence: bool,
|
||||
no_wait_for_caches: bool,
|
||||
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
|
||||
if let Some(rlp) = rlp {
|
||||
return Ok((
|
||||
None,
|
||||
serde_json::to_value((
|
||||
RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),
|
||||
no_wait_for_persistence.then_some(false),
|
||||
no_wait_for_caches.then_some(false),
|
||||
))?,
|
||||
serde_json::to_value((RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),))?,
|
||||
));
|
||||
}
|
||||
let block = block
|
||||
@@ -203,14 +194,7 @@ pub(crate) fn block_to_new_payload(
|
||||
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)?;
|
||||
|
||||
if reth_new_payload {
|
||||
Ok((
|
||||
None,
|
||||
serde_json::to_value((
|
||||
RethNewPayloadInput::ExecutionData(execution_data),
|
||||
no_wait_for_persistence.then_some(false),
|
||||
no_wait_for_caches.then_some(false),
|
||||
))?,
|
||||
))
|
||||
Ok((None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?))
|
||||
} else {
|
||||
Ok((Some(version), params))
|
||||
}
|
||||
@@ -230,20 +214,6 @@ pub(crate) fn payload_to_new_payload(
|
||||
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
|
||||
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V4(payload) => {
|
||||
let cancun = sidecar.cancun().unwrap();
|
||||
let prague = sidecar.prague().unwrap();
|
||||
let requests = prague.requests.requests_hash();
|
||||
(
|
||||
EngineApiMessageVersion::V5,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar.cancun().unwrap();
|
||||
|
||||
@@ -401,12 +371,9 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
|
||||
forkchoice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
// FCU V3 is used for Cancun, Prague, and Amsterdam (there is no FCU V4-V6)
|
||||
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
|
||||
match message_version {
|
||||
EngineApiMessageVersion::V3 |
|
||||
EngineApiMessageVersion::V4 |
|
||||
EngineApiMessageVersion::V5 |
|
||||
EngineApiMessageVersion::V6 => {
|
||||
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
|
||||
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
EngineApiMessageVersion::V2 => {
|
||||
|
||||
@@ -70,7 +70,7 @@ aquamarine.workspace = true
|
||||
clap = { workspace = true, features = ["derive", "env"] }
|
||||
|
||||
[dev-dependencies]
|
||||
alloy-node-bindings = "1.5.2"
|
||||
alloy-node-bindings = "1.6.3"
|
||||
alloy-provider = { workspace = true, features = ["reqwest"] }
|
||||
alloy-rpc-types-eth.workspace = true
|
||||
backon.workspace = true
|
||||
@@ -124,7 +124,6 @@ jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
"reth-node-metrics/jemalloc",
|
||||
"reth-ethereum-cli/jemalloc",
|
||||
"reth-provider/jemalloc",
|
||||
]
|
||||
jemalloc-prof = [
|
||||
"reth-cli-util/jemalloc",
|
||||
|
||||
@@ -36,6 +36,15 @@ pub use spec::{
|
||||
DepositContract, ForkBaseFeeParams, DEV, HOLESKY, HOODI, MAINNET, SEPOLIA,
|
||||
};
|
||||
|
||||
use reth_primitives_traits::sync::OnceLock;
|
||||
|
||||
/// Simple utility to create a thread-safe sync cell with a value set.
|
||||
pub fn once_cell_set<T>(value: T) -> OnceLock<T> {
|
||||
let once = OnceLock::new();
|
||||
let _ = once.set(value);
|
||||
once
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -28,7 +28,7 @@ use alloy_consensus::{
|
||||
};
|
||||
use alloy_eips::{
|
||||
eip1559::INITIAL_BASE_FEE, eip7685::EMPTY_REQUESTS_HASH, eip7840::BlobParams,
|
||||
eip7892::BlobScheduleBlobParams, eip7928::EMPTY_BLOCK_ACCESS_LIST_HASH,
|
||||
eip7892::BlobScheduleBlobParams,
|
||||
};
|
||||
use alloy_genesis::{ChainConfig, Genesis};
|
||||
use alloy_primitives::{address, b256, Address, BlockNumber, B256, U256};
|
||||
@@ -76,18 +76,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
|
||||
.active_at_timestamp(genesis.timestamp)
|
||||
.then_some(EMPTY_REQUESTS_HASH);
|
||||
|
||||
// If Amsterdam is activated at genesis we set block access list hash to empty hash.
|
||||
let block_access_list_hash = hardforks
|
||||
.fork(EthereumHardfork::Amsterdam)
|
||||
.active_at_timestamp(genesis.timestamp)
|
||||
.then_some(EMPTY_BLOCK_ACCESS_LIST_HASH);
|
||||
|
||||
// If Amsterdam is activated at genesis we set slot number to 0.
|
||||
let slot_number = hardforks
|
||||
.fork(EthereumHardfork::Amsterdam)
|
||||
.active_at_timestamp(genesis.timestamp)
|
||||
.then_some(0);
|
||||
|
||||
Header {
|
||||
number: genesis.number.unwrap_or_default(),
|
||||
parent_hash: genesis.parent_hash.unwrap_or_default(),
|
||||
@@ -105,8 +93,6 @@ pub fn make_genesis_header(genesis: &Genesis, hardforks: &ChainHardforks) -> Hea
|
||||
blob_gas_used,
|
||||
excess_blob_gas,
|
||||
requests_hash,
|
||||
block_access_list_hash,
|
||||
slot_number,
|
||||
..Default::default()
|
||||
}
|
||||
}
|
||||
@@ -312,7 +298,6 @@ pub fn create_chain_config(
|
||||
cancun_time: timestamp(EthereumHardfork::Cancun),
|
||||
prague_time: timestamp(EthereumHardfork::Prague),
|
||||
osaka_time: timestamp(EthereumHardfork::Osaka),
|
||||
amsterdam_time: timestamp(EthereumHardfork::Amsterdam),
|
||||
bpo1_time: timestamp(EthereumHardfork::Bpo1),
|
||||
bpo2_time: timestamp(EthereumHardfork::Bpo2),
|
||||
bpo3_time: timestamp(EthereumHardfork::Bpo3),
|
||||
@@ -895,7 +880,6 @@ impl From<Genesis> for ChainSpec {
|
||||
(EthereumHardfork::Cancun.boxed(), genesis.config.cancun_time),
|
||||
(EthereumHardfork::Prague.boxed(), genesis.config.prague_time),
|
||||
(EthereumHardfork::Osaka.boxed(), genesis.config.osaka_time),
|
||||
(EthereumHardfork::Amsterdam.boxed(), genesis.config.amsterdam_time),
|
||||
(EthereumHardfork::Bpo1.boxed(), genesis.config.bpo1_time),
|
||||
(EthereumHardfork::Bpo2.boxed(), genesis.config.bpo2_time),
|
||||
(EthereumHardfork::Bpo3.boxed(), genesis.config.bpo3_time),
|
||||
@@ -1207,19 +1191,6 @@ impl ChainSpecBuilder {
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable Amsterdam at genesis.
|
||||
pub fn amsterdam_activated(mut self) -> Self {
|
||||
self = self.osaka_activated();
|
||||
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(0));
|
||||
self
|
||||
}
|
||||
|
||||
/// Enable Amsterdam at the given timestamp.
|
||||
pub fn with_amsterdam_at(mut self, timestamp: u64) -> Self {
|
||||
self.hardforks.insert(EthereumHardfork::Amsterdam, ForkCondition::Timestamp(timestamp));
|
||||
self
|
||||
}
|
||||
|
||||
/// Build the resulting [`ChainSpec`].
|
||||
///
|
||||
/// # Panics
|
||||
|
||||
@@ -187,6 +187,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
where
|
||||
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
|
||||
{
|
||||
let prune_modes = config.prune.segments.clone();
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
|
||||
db,
|
||||
self.chain.clone(),
|
||||
@@ -194,8 +195,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
rocksdb_provider,
|
||||
runtime,
|
||||
)?
|
||||
.with_prune_modes(config.prune.segments.clone())
|
||||
.with_minimum_pruning_distance(config.prune.minimum_pruning_distance);
|
||||
.with_prune_modes(prune_modes.clone());
|
||||
|
||||
// Check for consistency between database and static files.
|
||||
if !access.is_read_only_inconsistent() &&
|
||||
@@ -229,13 +229,10 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
NoopBodiesDownloader::default(),
|
||||
NoopEvmConfig::<N::Evm>::default(),
|
||||
config.stages.clone(),
|
||||
config.prune.segments.clone(),
|
||||
prune_modes.clone(),
|
||||
None,
|
||||
))
|
||||
.build(
|
||||
factory.clone(),
|
||||
StaticFileProducer::new(factory.clone(), config.prune.segments.clone()),
|
||||
);
|
||||
.build(factory.clone(), StaticFileProducer::new(factory.clone(), prune_modes));
|
||||
|
||||
// Move all applicable data from database to static files.
|
||||
pipeline.move_to_static_files()?;
|
||||
|
||||
@@ -199,7 +199,7 @@ pub(crate) fn config_for_selections(
|
||||
}
|
||||
|
||||
return Config {
|
||||
prune: PruneConfig { segments, ..Default::default() },
|
||||
prune: PruneConfig { block_interval: PruneConfig::default().block_interval, segments },
|
||||
static_files,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -201,7 +201,6 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
|
||||
let net =
|
||||
NetworkConfigBuilder::<N::NetworkPrimitives>::new(p2p_secret_key, Runtime::test())
|
||||
.peer_config(config.peers_config_with_basic_nodes_from_file(None))
|
||||
.sessions_config(config.sessions)
|
||||
.external_ip_resolver(self.network.nat.clone())
|
||||
.network_id(self.network.network_id)
|
||||
.boot_nodes(boot_nodes.clone())
|
||||
|
||||
@@ -179,10 +179,8 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
}
|
||||
};
|
||||
|
||||
let bal= executor.take_bal();
|
||||
|
||||
if let Err(err) = consensus
|
||||
.validate_block_post_execution(&block, &result, None, bal, true)
|
||||
.validate_block_post_execution(&block, &result, None)
|
||||
.wrap_err_with(|| {
|
||||
format!(
|
||||
"Failed to validate block {} {}",
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Configuration files.
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_prune_types::{PruneModes, MINIMUM_UNWIND_SAFE_DISTANCE};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_stages_types::ExecutionStageThresholds;
|
||||
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
|
||||
use std::{
|
||||
@@ -540,24 +540,11 @@ pub struct PruneConfig {
|
||||
/// Pruning configuration for every part of the data that can be pruned.
|
||||
#[cfg_attr(feature = "serde", serde(alias = "parts"))]
|
||||
pub segments: PruneModes,
|
||||
/// Minimum distance from the tip required for pruning. Controls the safety margin for
|
||||
/// reorgs and manual unwinds. Defaults to [`MINIMUM_UNWIND_SAFE_DISTANCE`].
|
||||
#[cfg_attr(feature = "serde", serde(default = "default_minimum_pruning_distance"))]
|
||||
pub minimum_pruning_distance: u64,
|
||||
}
|
||||
|
||||
/// Returns the default minimum pruning distance.
|
||||
const fn default_minimum_pruning_distance() -> u64 {
|
||||
MINIMUM_UNWIND_SAFE_DISTANCE
|
||||
}
|
||||
|
||||
impl Default for PruneConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
block_interval: DEFAULT_BLOCK_INTERVAL,
|
||||
segments: PruneModes::default(),
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
}
|
||||
Self { block_interval: DEFAULT_BLOCK_INTERVAL, segments: PruneModes::default() }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -590,7 +577,6 @@ impl PruneConfig {
|
||||
bodies_history,
|
||||
receipts_log_filter,
|
||||
},
|
||||
minimum_pruning_distance,
|
||||
} = other;
|
||||
|
||||
// Merge block_interval, only update if it's the default interval
|
||||
@@ -598,11 +584,6 @@ impl PruneConfig {
|
||||
self.block_interval = block_interval;
|
||||
}
|
||||
|
||||
// Merge minimum_pruning_distance, only update if it's the default
|
||||
if self.minimum_pruning_distance == MINIMUM_UNWIND_SAFE_DISTANCE {
|
||||
self.minimum_pruning_distance = minimum_pruning_distance;
|
||||
}
|
||||
|
||||
// Merge the various segment prune modes
|
||||
self.segments.sender_recovery = self.segments.sender_recovery.or(sender_recovery);
|
||||
self.segments.transaction_lookup = self.segments.transaction_lookup.or(transaction_lookup);
|
||||
@@ -642,9 +623,7 @@ mod tests {
|
||||
use crate::PruneConfig;
|
||||
use alloy_primitives::Address;
|
||||
use reth_network_peers::TrustedPeer;
|
||||
use reth_prune_types::{
|
||||
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
};
|
||||
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig};
|
||||
use std::{collections::BTreeMap, path::Path, str::FromStr, time::Duration};
|
||||
|
||||
fn with_tempdir(filename: &str, proc: fn(&std::path::Path)) {
|
||||
@@ -1114,7 +1093,6 @@ receipts = { distance = 16384 }
|
||||
fn test_prune_config_merge() {
|
||||
let mut config1 = PruneConfig {
|
||||
block_interval: 5,
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: None,
|
||||
@@ -1131,7 +1109,6 @@ receipts = { distance = 16384 }
|
||||
|
||||
let config2 = PruneConfig {
|
||||
block_interval: 10,
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Distance(500)),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
|
||||
@@ -87,27 +87,6 @@ pub fn validate_cancun_gas<B: Block>(block: &SealedBlock<B>) -> Result<(), Conse
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Validate that Amsterdam header fields are present in the block.
|
||||
///
|
||||
/// This checks that the `block_access_list_hash` and `slot_number` are set in the header,
|
||||
/// as required post-Amsterdam.
|
||||
///
|
||||
/// See [EIP-7928]: Block-level Access Lists
|
||||
/// See [EIP-7778]: Slot Number in Block Header
|
||||
///
|
||||
/// [EIP-7928]: https://eips.ethereum.org/EIPS/eip-7928
|
||||
/// [EIP-7778]: https://eips.ethereum.org/EIPS/eip-7778
|
||||
#[inline]
|
||||
pub fn validate_amsterdam_header_fields<H: BlockHeader>(header: &H) -> Result<(), ConsensusError> {
|
||||
if header.block_access_list_hash().is_none() {
|
||||
return Err(ConsensusError::BlockAccessListHashMissing);
|
||||
}
|
||||
if header.slot_number().is_none() {
|
||||
return Err(ConsensusError::SlotNumberMissing);
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Ensures the block response data matches the header.
|
||||
///
|
||||
/// This ensures the body response items match the header's hashes:
|
||||
|
||||
@@ -18,7 +18,6 @@ reth-primitives-traits.workspace = true
|
||||
# ethereum
|
||||
alloy-primitives.workspace = true
|
||||
alloy-consensus.workspace = true
|
||||
alloy-eip7928.workspace = true
|
||||
|
||||
# misc
|
||||
auto_impl.workspace = true
|
||||
@@ -30,8 +29,10 @@ std = [
|
||||
"reth-primitives-traits/std",
|
||||
"alloy-primitives/std",
|
||||
"alloy-consensus/std",
|
||||
"alloy-eip7928/std",
|
||||
"reth-primitives-traits/std",
|
||||
"reth-execution-types/std",
|
||||
"thiserror/std",
|
||||
]
|
||||
test-utils = ["reth-primitives-traits/test-utils"]
|
||||
test-utils = [
|
||||
"reth-primitives-traits/test-utils",
|
||||
]
|
||||
|
||||
@@ -13,7 +13,6 @@ extern crate alloc;
|
||||
|
||||
use alloc::{boxed::Box, fmt::Debug, string::String, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::Header;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use alloy_primitives::{BlockHash, BlockNumber, Bloom, B256};
|
||||
use core::error::Error;
|
||||
|
||||
@@ -55,17 +54,12 @@ pub trait FullConsensus<N: NodePrimitives>: Consensus<N::Block> {
|
||||
/// If `receipt_root_bloom` is provided, the implementation should use the pre-computed
|
||||
/// receipt root and logs bloom instead of computing them from the receipts.
|
||||
///
|
||||
/// If `allow_bal_check` is enabled, we calculate the bal hash and match with the header bal
|
||||
/// hash. We don't do by default because for payload validation, we do the same bal check
|
||||
///
|
||||
/// Note: validating blocks does not include other validations of the Consensus
|
||||
fn validate_block_post_execution(
|
||||
&self,
|
||||
block: &RecoveredBlock<N::Block>,
|
||||
result: &BlockExecutionResult<N::Receipt>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
block_access_list: Option<BlockAccessList>,
|
||||
allow_bal_check: bool,
|
||||
) -> Result<(), ConsensusError>;
|
||||
}
|
||||
|
||||
@@ -337,30 +331,6 @@ pub enum ConsensusError {
|
||||
#[error("unexpected parent beacon block root")]
|
||||
ParentBeaconBlockRootUnexpected,
|
||||
|
||||
/// Error when the block access list hash is missing.
|
||||
#[error("missing block access list hash")]
|
||||
BlockAccessListHashMissing,
|
||||
|
||||
/// Error when an unexpected block access list hash is encountered.
|
||||
#[error("unexpected block access list hash")]
|
||||
BlockAccessListHashUnexpected,
|
||||
|
||||
/// Error when an unexpected block access list cost is encountered.
|
||||
#[error("block access list cost exceeds gas limit")]
|
||||
BlockAccessListCostMoreThanGasLimit,
|
||||
|
||||
/// Error when the block access list hash doesn't match the expected value.
|
||||
#[error("block access list hash mismatch: {0}")]
|
||||
BlockAccessListHashMismatch(GotExpectedBoxed<B256>),
|
||||
|
||||
/// Error when the slot number is missing.
|
||||
#[error("missing slot number")]
|
||||
SlotNumberMissing,
|
||||
|
||||
/// Error when an unexpected slot number is encountered.
|
||||
#[error("unexpected slot number")]
|
||||
SlotNumberUnexpected,
|
||||
|
||||
/// Error when blob gas used exceeds the maximum allowed.
|
||||
#[error("blob gas used {blob_gas_used} exceeds maximum allowance {max_blob_gas_per_block}")]
|
||||
BlobGasUsedExceedsMaxBlobGasPerBlock {
|
||||
|
||||
@@ -20,7 +20,6 @@
|
||||
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use alloc::sync::Arc;
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
|
||||
@@ -78,8 +77,6 @@ impl<N: NodePrimitives> FullConsensus<N> for NoopConsensus {
|
||||
_block: &RecoveredBlock<N::Block>,
|
||||
_result: &BlockExecutionResult<N::Receipt>,
|
||||
_receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
_block_access_list: Option<BlockAccessList>,
|
||||
_allow_bal_check: bool,
|
||||
) -> Result<(), ConsensusError> {
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
use crate::{Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom};
|
||||
use alloy_eip7928::BlockAccessList;
|
||||
use core::sync::atomic::{AtomicBool, Ordering};
|
||||
use reth_execution_types::BlockExecutionResult;
|
||||
use reth_primitives_traits::{Block, NodePrimitives, RecoveredBlock, SealedBlock, SealedHeader};
|
||||
@@ -53,8 +52,6 @@ impl<N: NodePrimitives> FullConsensus<N> for TestConsensus {
|
||||
_block: &RecoveredBlock<N::Block>,
|
||||
_result: &BlockExecutionResult<N::Receipt>,
|
||||
_receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
_block_access_list: Option<BlockAccessList>,
|
||||
_allow_bal_check: bool,
|
||||
) -> Result<(), ConsensusError> {
|
||||
if self.fail_validation() {
|
||||
Err(ConsensusError::BaseFeeMissing)
|
||||
|
||||
@@ -96,11 +96,6 @@ where
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the pruning arguments for the test nodes.
|
||||
pub fn with_pruning(self, pruning: reth_node_core::args::PruningArgs) -> Self {
|
||||
self.with_node_config_modifier(move |config| config.with_pruning(pruning.clone()))
|
||||
}
|
||||
|
||||
/// Enables v2 storage defaults (`--storage.v2`), routing tx hashes, history
|
||||
/// indices, etc. to `RocksDB` and changesets/senders to static files.
|
||||
pub fn with_storage_v2(self) -> Self {
|
||||
|
||||
@@ -55,8 +55,6 @@ pub fn generate_test_blocks(chain_spec: &ChainSpec, count: u64) -> Vec<SealedBlo
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
block_access_list_hash: None,
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
// Set required fields based on chain spec
|
||||
|
||||
@@ -227,7 +227,6 @@ where
|
||||
suggested_fee_recipient: alloy_primitives::Address::random(),
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
env.active_node_state_mut()?
|
||||
@@ -300,7 +299,6 @@ where
|
||||
suggested_fee_recipient: alloy_primitives::Address::random(),
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
let fresh_fcu_result = EngineApiClient::<Engine>::fork_choice_updated_v3(
|
||||
|
||||
@@ -271,7 +271,6 @@ where
|
||||
suggested_fee_recipient: alloy_primitives::Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
};
|
||||
@@ -302,7 +301,6 @@ where
|
||||
suggested_fee_recipient: alloy_primitives::Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
<<N as NodeTypes>::Payload as PayloadTypes>::PayloadBuilderAttributes::from(
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes),
|
||||
|
||||
@@ -161,7 +161,6 @@ async fn test_testsuite_assert_mine_block() -> Result<()> {
|
||||
suggested_fee_recipient: Address::random(),
|
||||
withdrawals: None,
|
||||
parent_beacon_block_root: None,
|
||||
slot_number: None,
|
||||
},
|
||||
));
|
||||
|
||||
|
||||
@@ -1,9 +1,8 @@
|
||||
//! E2E tests for `RocksDB` provider functionality.
|
||||
|
||||
use alloy_consensus::BlockHeader;
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use alloy_primitives::{Address, Bytes, TxKind, B256, U256};
|
||||
use alloy_rpc_types_eth::{Transaction, TransactionInput, TransactionReceipt, TransactionRequest};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rpc_types_eth::{Transaction, TransactionReceipt};
|
||||
use eyre::Result;
|
||||
use jsonrpsee::core::client::ClientT;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
@@ -90,7 +89,6 @@ fn test_attributes_generator(timestamp: u64) -> EthPayloadBuilderAttributes {
|
||||
suggested_fee_recipient: alloy_primitives::Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
}
|
||||
@@ -575,426 +573,3 @@ async fn test_rocksdb_reorg_unwind() -> Result<()> {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Historical account queries: verifies that `eth_getBalance` and `eth_getTransactionCount`
|
||||
/// return correct values at past block numbers after the account state has changed.
|
||||
///
|
||||
/// This test exercises the database-backed historical state lookup path. After mining the
|
||||
/// blocks we care about (1-3), we mine additional blocks to advance the canonical head
|
||||
/// far enough that the engine tree's persistence + in-memory eviction cycle guarantees
|
||||
/// blocks 1-3 are no longer in the in-memory overlay. Historical queries for those blocks
|
||||
/// must then be served from `RocksDB` changesets.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_historical_account_queries() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
let (mut nodes, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let sender: Address = signer.address();
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
|
||||
// Query the sender's balance and nonce at genesis (block 0)
|
||||
let genesis_balance: U256 = client.request("eth_getBalance", (sender, "0x0")).await?;
|
||||
let genesis_nonce: U256 = client.request("eth_getTransactionCount", (sender, "0x0")).await?;
|
||||
assert!(genesis_balance > U256::ZERO, "Sender should have genesis balance");
|
||||
assert_eq!(genesis_nonce, U256::ZERO, "Sender nonce should be 0 at genesis");
|
||||
|
||||
// Mine block 1 with a transfer (nonce 0)
|
||||
let raw_tx1 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 0).await;
|
||||
let tx_hash1 = nodes[0].rpc.inject_tx(raw_tx1).await?;
|
||||
wait_for_pending_tx(&client, tx_hash1).await;
|
||||
|
||||
let payload1 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash1).await;
|
||||
|
||||
// Record state after block 1
|
||||
let balance_at_1: U256 = client.request("eth_getBalance", (sender, "0x1")).await?;
|
||||
let nonce_at_1: U256 = client.request("eth_getTransactionCount", (sender, "0x1")).await?;
|
||||
assert!(balance_at_1 < genesis_balance, "Balance should decrease after transfer + gas");
|
||||
assert_eq!(nonce_at_1, U256::from(1), "Nonce should be 1 after first tx");
|
||||
|
||||
// Mine block 2 with another transfer (nonce 1)
|
||||
let raw_tx2 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 1).await;
|
||||
let tx_hash2 = nodes[0].rpc.inject_tx(raw_tx2).await?;
|
||||
wait_for_pending_tx(&client, tx_hash2).await;
|
||||
|
||||
let payload2 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload2.block().number(), 2);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash2).await;
|
||||
|
||||
let balance_at_2: U256 = client.request("eth_getBalance", (sender, "0x2")).await?;
|
||||
let nonce_at_2: U256 = client.request("eth_getTransactionCount", (sender, "0x2")).await?;
|
||||
assert!(balance_at_2 < balance_at_1, "Balance should decrease further after second tx");
|
||||
assert_eq!(nonce_at_2, U256::from(2), "Nonce should be 2 after second tx");
|
||||
|
||||
// Mine block 3 with a third transfer (nonce 2)
|
||||
let raw_tx3 =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), 2).await;
|
||||
let tx_hash3 = nodes[0].rpc.inject_tx(raw_tx3).await?;
|
||||
wait_for_pending_tx(&client, tx_hash3).await;
|
||||
|
||||
let payload3 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload3.block().number(), 3);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, tx_hash3).await;
|
||||
|
||||
let balance_at_3: U256 = client.request("eth_getBalance", (sender, "0x3")).await?;
|
||||
let nonce_at_3: U256 = client.request("eth_getTransactionCount", (sender, "0x3")).await?;
|
||||
assert!(balance_at_3 < balance_at_2, "Balance should decrease further after third tx");
|
||||
assert_eq!(nonce_at_3, U256::from(3), "Nonce should be 3 after third tx");
|
||||
|
||||
// Mine additional blocks to push blocks 1-3 out of the in-memory overlay.
|
||||
// With persistence_threshold=0 and memory_block_buffer_target=0, each new block
|
||||
// triggers persistence up to `head` followed by in-memory eviction. Mining several
|
||||
// more blocks ensures the engine loop has completed at least one full
|
||||
// persist-then-evict cycle covering blocks 1-3.
|
||||
// Each block needs a transaction because the payload builder requires non-empty payloads.
|
||||
for nonce in 3..8u64 {
|
||||
let raw_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce)
|
||||
.await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
nodes[0].advance_block().await?;
|
||||
}
|
||||
// Allow the engine loop to process the persistence completions
|
||||
tokio::time::sleep(Duration::from_millis(500)).await;
|
||||
|
||||
// Historical queries — blocks 1-3 should now be served from the database.
|
||||
let hist_balance_0: U256 = client.request("eth_getBalance", (sender, "0x0")).await?;
|
||||
let hist_nonce_0: U256 = client.request("eth_getTransactionCount", (sender, "0x0")).await?;
|
||||
assert_eq!(
|
||||
hist_balance_0, genesis_balance,
|
||||
"Historical balance at block 0 should match genesis"
|
||||
);
|
||||
assert_eq!(hist_nonce_0, U256::ZERO, "Historical nonce at block 0 should be 0");
|
||||
|
||||
let hist_balance_1: U256 = client.request("eth_getBalance", (sender, "0x1")).await?;
|
||||
let hist_nonce_1: U256 = client.request("eth_getTransactionCount", (sender, "0x1")).await?;
|
||||
assert_eq!(hist_balance_1, balance_at_1, "Historical balance at block 1 should match");
|
||||
assert_eq!(hist_nonce_1, U256::from(1), "Historical nonce at block 1 should be 1");
|
||||
|
||||
let hist_balance_2: U256 = client.request("eth_getBalance", (sender, "0x2")).await?;
|
||||
let hist_nonce_2: U256 = client.request("eth_getTransactionCount", (sender, "0x2")).await?;
|
||||
assert_eq!(hist_balance_2, balance_at_2, "Historical balance at block 2 should match");
|
||||
assert_eq!(hist_nonce_2, U256::from(2), "Historical nonce at block 2 should be 2");
|
||||
|
||||
let hist_balance_3: U256 = client.request("eth_getBalance", (sender, "0x3")).await?;
|
||||
let hist_nonce_3: U256 = client.request("eth_getTransactionCount", (sender, "0x3")).await?;
|
||||
assert_eq!(hist_balance_3, balance_at_3, "Historical balance at block 3 should match");
|
||||
assert_eq!(hist_nonce_3, U256::from(3), "Historical nonce at block 3 should be 3");
|
||||
|
||||
// "latest" should still match head
|
||||
let latest_balance: U256 = client.request("eth_getBalance", (sender, "latest")).await?;
|
||||
let latest_nonce: U256 = client.request("eth_getTransactionCount", (sender, "latest")).await?;
|
||||
assert_eq!(
|
||||
latest_nonce,
|
||||
U256::from(8),
|
||||
"Latest nonce should be 8 (3 original + 5 extra blocks)"
|
||||
);
|
||||
assert!(latest_balance < balance_at_3, "Latest balance should be less than block 3 balance");
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Reproduces the race condition between `save_blocks` and `RocksDB` pruning described in
|
||||
/// <https://github.com/paradigmxyz/reth/pull/23081>.
|
||||
///
|
||||
/// Both `save_blocks` and the pruner push to `pending_rocksdb_batches` before a single
|
||||
/// `commit()`. The pruner reads committed (stale) state that doesn't include `save_blocks`'
|
||||
/// new entry, filters it, and pushes its own batch. On commit the pruner's batch overwrites
|
||||
/// `save_blocks`' batch for the same `ShardedKey(addr, u64::MAX)`. Every cycle the new
|
||||
/// block's history entry is silently lost. After enough cycles the shard is completely empty.
|
||||
///
|
||||
/// This test mines blocks with account-history pruning (`block_interval=1`,
|
||||
/// `minimum_distance=5`), waits for persistence, then reads the `AccountsHistory` shard
|
||||
/// directly from `RocksDB`. Without the fix the shard is empty — all entries were
|
||||
/// overwritten by the pruner's stale batches. With the fix, entries for the most recent
|
||||
/// blocks survive.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_account_history_pruning() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
const PRUNE_DISTANCE: u64 = 5;
|
||||
const TOTAL_BLOCKS: u64 = 20;
|
||||
|
||||
let (mut nodes, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.with_node_config_modifier(|mut config| {
|
||||
config.pruning.account_history_distance = Some(PRUNE_DISTANCE);
|
||||
config.pruning.minimum_distance = Some(PRUNE_DISTANCE);
|
||||
config.pruning.block_interval = Some(1);
|
||||
config
|
||||
})
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let sender: Address = signer.address();
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
|
||||
// Mine blocks one at a time with a delay so each save_blocks + pruner cycle
|
||||
// completes independently. The race fires every cycle (the pruner reads stale
|
||||
// committed state that doesn't include save_blocks' pending batch), but processing
|
||||
// one block at a time makes the outcome deterministic.
|
||||
let mut last_tx_hash = B256::ZERO;
|
||||
for nonce in 0..TOTAL_BLOCKS {
|
||||
let raw_tx =
|
||||
TransactionTestContext::transfer_tx_bytes_with_nonce(chain_id, signer.clone(), nonce)
|
||||
.await;
|
||||
let tx_hash = nodes[0].rpc.inject_tx(raw_tx).await?;
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), nonce + 1);
|
||||
last_tx_hash = tx_hash;
|
||||
|
||||
// Let the persistence cycle (save_blocks → pruner → commit) complete before
|
||||
// producing the next block, so each cycle processes exactly one block.
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
}
|
||||
|
||||
// Wait for the last block to be fully persisted to RocksDB.
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, last_tx_hash).await;
|
||||
|
||||
// Read the AccountsHistory shard for `sender` directly from RocksDB.
|
||||
// This is the data structure corrupted by the race.
|
||||
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
|
||||
let shards = rocksdb.account_history_shards(sender).unwrap();
|
||||
let all_entries: Vec<u64> = shards.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
|
||||
// The sender has a transfer in every block, so the shard should contain an entry
|
||||
// for every block in the retention window: (TOTAL_BLOCKS - PRUNE_DISTANCE, TOTAL_BLOCKS].
|
||||
//
|
||||
// Without the fix: the pruner reads stale committed state each cycle, overwrites
|
||||
// save_blocks' entry, and only the very last block survives (no subsequent pruner
|
||||
// cycle to overwrite it). The shard ends up as just [TOTAL_BLOCKS].
|
||||
//
|
||||
// With the fix: save_blocks' batch is committed before the pruner reads, so the
|
||||
// pruner sees the new entry and preserves it. All retained blocks are present.
|
||||
let expected: Vec<u64> = ((TOTAL_BLOCKS - PRUNE_DISTANCE + 1)..=TOTAL_BLOCKS).collect();
|
||||
assert_eq!(
|
||||
all_entries, expected,
|
||||
"AccountsHistory shard for sender doesn't match expected retention window. \
|
||||
Expected {expected:?}, got {all_entries:?}. \
|
||||
The pruner's stale batch overwrote save_blocks' entries \
|
||||
(save_blocks/pruner race, see PR #23081)."
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Mirrors [`test_rocksdb_account_history_pruning`] for the `StoragesHistory` table.
|
||||
///
|
||||
/// The same race condition between `save_blocks` and the pruner that affects
|
||||
/// `AccountsHistory` also affects `StoragesHistory`:
|
||||
/// - `write_storage_history` reads committed `RocksDB` state and pushes a batch.
|
||||
/// - `prune_storage_history_batch` also reads stale committed state and pushes its own batch.
|
||||
/// - On a single `commit()`, the pruner's batch overwrites `save_blocks`' batch for the same
|
||||
/// `StorageShardedKey(addr, slot, u64::MAX)`.
|
||||
///
|
||||
/// This test deploys a minimal contract that writes to storage slot 0 each block,
|
||||
/// then verifies that `StoragesHistory` contains entries for the full retention
|
||||
/// window. Without the fix the shard would be truncated — new entries silently
|
||||
/// lost every cycle.
|
||||
#[tokio::test]
|
||||
async fn test_rocksdb_storage_history_pruning() -> Result<()> {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let chain_spec = test_chain_spec();
|
||||
let chain_id = chain_spec.chain().id();
|
||||
|
||||
const PRUNE_DISTANCE: u64 = 5;
|
||||
const TOTAL_BLOCKS: u64 = 20;
|
||||
|
||||
let (mut nodes, _) = E2ETestSetupBuilder::<EthereumNode, _>::new(
|
||||
1,
|
||||
chain_spec.clone(),
|
||||
test_attributes_generator,
|
||||
)
|
||||
.with_storage_v2()
|
||||
.with_tree_config_modifier(|config| config.with_persistence_threshold(0))
|
||||
.with_node_config_modifier(|mut config| {
|
||||
config.pruning.storage_history_distance = Some(PRUNE_DISTANCE);
|
||||
config.pruning.minimum_distance = Some(PRUNE_DISTANCE);
|
||||
config.pruning.block_interval = Some(1);
|
||||
config
|
||||
})
|
||||
.build()
|
||||
.await?;
|
||||
|
||||
assert_eq!(nodes.len(), 1);
|
||||
|
||||
let wallets = wallet::Wallet::new(1).with_chain_id(chain_id).wallet_gen();
|
||||
let signer = wallets[0].clone();
|
||||
let client = nodes[0].rpc_client().expect("RPC client");
|
||||
|
||||
// Deploy a minimal contract that stores CALLDATA[0..32] into slot 0:
|
||||
// PUSH0 ; [0]
|
||||
// CALLDATALOAD ; [calldata[0..32]]
|
||||
// PUSH0 ; [0, calldata[0..32]]
|
||||
// SSTORE ; sstore(0, calldata[0..32])
|
||||
// STOP
|
||||
// Bytecode: 5f355f5500
|
||||
//
|
||||
// Init code that deploys this runtime:
|
||||
// PUSH5 5f355f5500 ; push 5-byte runtime
|
||||
// PUSH0 ; offset 0 in memory
|
||||
// MSTORE ; store at mem[0..32] (right-padded in 32 bytes)
|
||||
// PUSH1 0x05 ; size = 5
|
||||
// PUSH1 0x1b ; offset = 27 (32 - 5)
|
||||
// PUSH0 ; destOffset = 0
|
||||
// CODECOPY ; copy runtime to mem[0..5]
|
||||
// PUSH1 0x05 ; size = 5
|
||||
// PUSH0 ; offset = 0
|
||||
// RETURN ; return mem[0..5]
|
||||
//
|
||||
// We can simplify: just use PUSH + MSTORE + RETURN pattern.
|
||||
// Init code (hex):
|
||||
// 645f355f5500 PUSH5 runtime_bytecode
|
||||
// 5f PUSH0 (memory offset for MSTORE, stores at 27..32)
|
||||
// 52 MSTORE
|
||||
// 6005 PUSH1 5 (size)
|
||||
// 601b PUSH1 27 (offset = 32-5)
|
||||
// f3 RETURN
|
||||
let init_code = Bytes::from_static(&[
|
||||
0x64, 0x5f, 0x35, 0x5f, 0x55, 0x00, // PUSH5 runtime
|
||||
0x5f, // PUSH0
|
||||
0x52, // MSTORE
|
||||
0x60, 0x05, // PUSH1 5
|
||||
0x60, 0x1b, // PUSH1 27
|
||||
0xf3, // RETURN
|
||||
]);
|
||||
|
||||
// Deploy in block 1 (nonce 0)
|
||||
let deploy_tx = TransactionRequest {
|
||||
nonce: Some(0),
|
||||
value: Some(U256::ZERO),
|
||||
to: Some(TxKind::Create),
|
||||
gas: Some(100_000),
|
||||
max_fee_per_gas: Some(1000e9 as u128),
|
||||
max_priority_fee_per_gas: Some(20e9 as u128),
|
||||
chain_id: Some(chain_id),
|
||||
input: TransactionInput { input: None, data: Some(init_code) },
|
||||
..Default::default()
|
||||
};
|
||||
let signed_deploy = TransactionTestContext::sign_tx(signer.clone(), deploy_tx).await;
|
||||
let deploy_bytes: Bytes = signed_deploy.encoded_2718().into();
|
||||
let deploy_hash = nodes[0].rpc.inject_tx(deploy_bytes).await?;
|
||||
wait_for_pending_tx(&client, deploy_hash).await;
|
||||
|
||||
let payload1 = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload1.block().number(), 1);
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, deploy_hash).await;
|
||||
|
||||
// Let the persistence cycle complete before the next block (same cadence as the loop below)
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
|
||||
// Get the deployed contract address from the receipt
|
||||
let receipt: Option<TransactionReceipt> =
|
||||
client.request("eth_getTransactionReceipt", [deploy_hash]).await?;
|
||||
let contract_address = receipt
|
||||
.expect("deploy receipt should exist")
|
||||
.contract_address
|
||||
.expect("deploy should create a contract");
|
||||
|
||||
// Sanity check: verify the runtime bytecode is what we expect
|
||||
let code: Bytes = client.request("eth_getCode", (contract_address, "latest")).await?;
|
||||
assert_eq!(
|
||||
code,
|
||||
Bytes::from_static(&[0x5f, 0x35, 0x5f, 0x55, 0x00]),
|
||||
"Deployed runtime should be PUSH0 CALLDATALOAD PUSH0 SSTORE STOP"
|
||||
);
|
||||
|
||||
// The storage slot we track: slot 0, encoded as B256
|
||||
let storage_slot = B256::ZERO;
|
||||
|
||||
// Mine TOTAL_BLOCKS - 1 more blocks (block 2..=TOTAL_BLOCKS), each calling the
|
||||
// contract to write a new value to slot 0. This creates a storage changeset entry
|
||||
// per block for (contract_address, slot 0).
|
||||
let mut last_tx_hash = deploy_hash;
|
||||
for nonce in 1..TOTAL_BLOCKS {
|
||||
// calldata = abi encode the block number so each write is unique
|
||||
let block_num = nonce + 1;
|
||||
let calldata = B256::from(U256::from(block_num));
|
||||
|
||||
let call_tx = TransactionRequest {
|
||||
nonce: Some(nonce),
|
||||
value: Some(U256::ZERO),
|
||||
to: Some(TxKind::Call(contract_address)),
|
||||
gas: Some(50_000),
|
||||
max_fee_per_gas: Some(1000e9 as u128),
|
||||
max_priority_fee_per_gas: Some(20e9 as u128),
|
||||
chain_id: Some(chain_id),
|
||||
input: TransactionInput { input: None, data: Some(Bytes::from(calldata.0)) },
|
||||
..Default::default()
|
||||
};
|
||||
let signed_call = TransactionTestContext::sign_tx(signer.clone(), call_tx).await;
|
||||
let call_bytes: Bytes = signed_call.encoded_2718().into();
|
||||
let tx_hash = nodes[0].rpc.inject_tx(call_bytes).await?;
|
||||
wait_for_pending_tx(&client, tx_hash).await;
|
||||
|
||||
let payload = nodes[0].advance_block().await?;
|
||||
assert_eq!(payload.block().number(), block_num);
|
||||
last_tx_hash = tx_hash;
|
||||
|
||||
// Let the persistence cycle complete before the next block
|
||||
tokio::time::sleep(Duration::from_millis(300)).await;
|
||||
}
|
||||
|
||||
// Wait for the last block to be fully persisted to RocksDB
|
||||
poll_tx_in_rocksdb(&nodes[0].inner.provider, last_tx_hash).await;
|
||||
|
||||
// Read StoragesHistory shard for (contract_address, slot 0) directly from RocksDB
|
||||
let rocksdb = nodes[0].inner.provider.rocksdb_provider();
|
||||
let shards = rocksdb.storage_history_shards(contract_address, storage_slot).unwrap();
|
||||
let all_entries: Vec<u64> = shards.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
|
||||
// The contract has a storage write in blocks 2..=TOTAL_BLOCKS (the deploy in block 1
|
||||
// only executes init code — no SSTORE — so block 1 has no StoragesHistory entry for
|
||||
// slot 0). With pruning distance=5, the retention window should be
|
||||
// (TOTAL_BLOCKS - PRUNE_DISTANCE, TOTAL_BLOCKS] = blocks 16..=20.
|
||||
//
|
||||
// Without the fix: the pruner overwrites save_blocks' entries each cycle,
|
||||
// leaving only the very last block (or empty).
|
||||
//
|
||||
// With the fix: all blocks in the retention window are present.
|
||||
let expected: Vec<u64> = ((TOTAL_BLOCKS - PRUNE_DISTANCE + 1)..=TOTAL_BLOCKS).collect();
|
||||
assert_eq!(
|
||||
all_entries, expected,
|
||||
"StoragesHistory shard for contract slot 0 doesn't match expected retention window. \
|
||||
Expected {expected:?}, got {all_entries:?}. \
|
||||
The pruner's stale batch overwrote save_blocks' entries \
|
||||
(save_blocks/pruner race for StorageHistory, see PR #23081)."
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
@@ -57,7 +57,6 @@ where
|
||||
.chain_spec
|
||||
.is_cancun_active_at_timestamp(timestamp)
|
||||
.then(B256::random),
|
||||
slot_number: None,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -146,22 +146,13 @@ pub struct TreeConfig {
|
||||
slow_block_threshold: Option<Duration>,
|
||||
/// Whether to fully disable sparse trie cache pruning between blocks.
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Whether to use the arena-based sparse trie implementation.
|
||||
enable_arena_sparse_trie: bool,
|
||||
/// Timeout for the state root task before spawning a sequential fallback computation.
|
||||
/// If `Some`, after waiting this duration for the state root task, a sequential state root
|
||||
/// computation is spawned in parallel and whichever finishes first is used.
|
||||
/// If `None`, the timeout fallback is disabled.
|
||||
state_root_task_timeout: Option<Duration>,
|
||||
/// Whether to disable BAL (Block Access List, EIP-7928) based parallel execution.
|
||||
/// When disabled, falls back to transaction-based prewarming even when a BAL is available.
|
||||
disable_bal_parallel_execution: bool,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
/// When disabled, the BAL hashed post state is not sent to the multiproof task for
|
||||
/// early parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
/// Whether to disable BAL (Block Access List) batched IO during prewarming.
|
||||
/// When disabled, falls back to individual per-slot storage reads instead of
|
||||
/// batched cursor reads via `storage_range`.
|
||||
disable_bal_batch_io: bool,
|
||||
/// Maximum random jitter applied before each proof computation (trie-debug only).
|
||||
/// When set, each proof worker sleeps for a random duration up to this value
|
||||
/// before starting a proof calculation.
|
||||
@@ -196,10 +187,8 @@ impl Default for TreeConfig {
|
||||
sparse_trie_max_hot_accounts: DEFAULT_SPARSE_TRIE_MAX_HOT_ACCOUNTS,
|
||||
slow_block_threshold: None,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
enable_arena_sparse_trie: false,
|
||||
state_root_task_timeout: Some(DEFAULT_STATE_ROOT_TASK_TIMEOUT),
|
||||
disable_bal_parallel_execution: false,
|
||||
disable_bal_parallel_state_root: false,
|
||||
disable_bal_batch_io: false,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
proof_jitter: None,
|
||||
}
|
||||
@@ -260,10 +249,8 @@ impl TreeConfig {
|
||||
sparse_trie_max_hot_accounts,
|
||||
slow_block_threshold,
|
||||
disable_sparse_trie_cache_pruning: false,
|
||||
enable_arena_sparse_trie: false,
|
||||
state_root_task_timeout,
|
||||
disable_bal_parallel_execution: false,
|
||||
disable_bal_parallel_state_root: false,
|
||||
disable_bal_batch_io: false,
|
||||
#[cfg(feature = "trie-debug")]
|
||||
proof_jitter: None,
|
||||
}
|
||||
@@ -565,6 +552,17 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether the arena-based sparse trie is enabled.
|
||||
pub const fn enable_arena_sparse_trie(&self) -> bool {
|
||||
self.enable_arena_sparse_trie
|
||||
}
|
||||
|
||||
/// Setter for whether to enable the arena-based sparse trie.
|
||||
pub const fn with_enable_arena_sparse_trie(mut self, value: bool) -> Self {
|
||||
self.enable_arena_sparse_trie = value;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the state root task timeout.
|
||||
pub const fn state_root_task_timeout(&self) -> Option<Duration> {
|
||||
self.state_root_task_timeout
|
||||
@@ -576,34 +574,6 @@ impl TreeConfig {
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL-based parallel execution is disabled.
|
||||
pub const fn disable_bal_parallel_execution(&self) -> bool {
|
||||
self.disable_bal_parallel_execution
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL-based parallel execution.
|
||||
pub const fn without_bal_parallel_execution(
|
||||
mut self,
|
||||
disable_bal_parallel_execution: bool,
|
||||
) -> Self {
|
||||
self.disable_bal_parallel_execution = disable_bal_parallel_execution;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL-driven parallel state root computation is disabled.
|
||||
pub const fn disable_bal_parallel_state_root(&self) -> bool {
|
||||
self.disable_bal_parallel_state_root
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL-driven parallel state root computation.
|
||||
pub const fn without_bal_parallel_state_root(
|
||||
mut self,
|
||||
disable_bal_parallel_state_root: bool,
|
||||
) -> Self {
|
||||
self.disable_bal_parallel_state_root = disable_bal_parallel_state_root;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the proof jitter duration, if configured (trie-debug only).
|
||||
#[cfg(feature = "trie-debug")]
|
||||
pub const fn proof_jitter(&self) -> Option<Duration> {
|
||||
@@ -616,15 +586,4 @@ impl TreeConfig {
|
||||
self.proof_jitter = proof_jitter;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns whether BAL batched IO is disabled.
|
||||
pub const fn disable_bal_batch_io(&self) -> bool {
|
||||
self.disable_bal_batch_io
|
||||
}
|
||||
|
||||
/// Setter for whether to disable BAL batched IO.
|
||||
pub const fn without_bal_batch_io(mut self, disable_bal_batch_io: bool) -> Self {
|
||||
self.disable_bal_batch_io = disable_bal_batch_io;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
@@ -149,17 +149,12 @@ pub struct NewPayloadTimings {
|
||||
/// Server-side execution latency.
|
||||
pub latency: Duration,
|
||||
/// Time spent waiting for persistence to complete.
|
||||
///
|
||||
/// `None` when wasn't asked to wait for persistence.
|
||||
/// `None` when no persistence was in-flight.
|
||||
pub persistence_wait: Option<Duration>,
|
||||
/// Time spent waiting for the execution cache lock.
|
||||
///
|
||||
/// `None` when wasn't asked to wait for execution cache.
|
||||
pub execution_cache_wait: Option<Duration>,
|
||||
/// Time spent waiting for the sparse trie cache lock.
|
||||
///
|
||||
/// `None` when wasn't asked to wait for sparse trie cache.
|
||||
pub sparse_trie_wait: Option<Duration>,
|
||||
pub execution_cache_wait: Duration,
|
||||
/// Time spent waiting for the sparse trie lock.
|
||||
pub sparse_trie_wait: Duration,
|
||||
}
|
||||
|
||||
/// A message for the beacon engine from other components of the node (engine RPC API invoked by the
|
||||
@@ -175,17 +170,11 @@ pub enum BeaconEngineMessage<Payload: PayloadTypes> {
|
||||
},
|
||||
/// Message with new payload used by `reth_newPayload` endpoint.
|
||||
///
|
||||
/// Supports independent control over waiting for persistence and cache locks before
|
||||
/// processing, providing unbiased timing measurements when enabled.
|
||||
///
|
||||
/// Returns detailed timing breakdown alongside the payload status.
|
||||
/// Waits for persistence, execution cache, and sparse trie locks before processing,
|
||||
/// and returns detailed timing breakdown alongside the payload status.
|
||||
RethNewPayload {
|
||||
/// The execution payload received by Engine API.
|
||||
payload: Payload::ExecutionData,
|
||||
/// Whether to wait for in-flight persistence to complete before processing.
|
||||
wait_for_persistence: bool,
|
||||
/// Whether to wait for execution cache and sparse trie locks before processing.
|
||||
wait_for_caches: bool,
|
||||
/// The sender for returning payload status result and timing breakdown.
|
||||
tx: oneshot::Sender<Result<(PayloadStatus, NewPayloadTimings), BeaconOnNewPayloadError>>,
|
||||
},
|
||||
@@ -270,23 +259,14 @@ where
|
||||
|
||||
/// Sends a new payload message used by `reth_newPayload` endpoint.
|
||||
///
|
||||
/// `wait_for_persistence`: waits for in-flight persistence to complete.
|
||||
/// `wait_for_caches`: waits for execution cache and sparse trie locks.
|
||||
///
|
||||
/// Returns detailed timing breakdown alongside the payload status.
|
||||
/// Waits for persistence, execution cache, and sparse trie locks before processing,
|
||||
/// and returns detailed timing breakdown alongside the payload status.
|
||||
pub async fn reth_new_payload(
|
||||
&self,
|
||||
payload: Payload::ExecutionData,
|
||||
wait_for_persistence: bool,
|
||||
wait_for_caches: bool,
|
||||
) -> Result<(PayloadStatus, NewPayloadTimings), BeaconOnNewPayloadError> {
|
||||
let (tx, rx) = oneshot::channel();
|
||||
let _ = self.to_engine.send(BeaconEngineMessage::RethNewPayload {
|
||||
payload,
|
||||
wait_for_persistence,
|
||||
wait_for_caches,
|
||||
tx,
|
||||
});
|
||||
let _ = self.to_engine.send(BeaconEngineMessage::RethNewPayload { payload, tx });
|
||||
rx.await.map_err(|_| BeaconOnNewPayloadError::EngineUnavailable)?
|
||||
}
|
||||
|
||||
|
||||
@@ -60,7 +60,6 @@ metrics.workspace = true
|
||||
reth-metrics = { workspace = true, features = ["common"] }
|
||||
|
||||
# misc
|
||||
itertools.workspace = true
|
||||
schnellru.workspace = true
|
||||
rayon.workspace = true
|
||||
tracing.workspace = true
|
||||
|
||||
@@ -177,25 +177,18 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
// Run the pruner in a separate provider so it reads committed RocksDB state
|
||||
// that includes the history entries written by save_blocks above.
|
||||
//
|
||||
// The pruner reads the indices from rocksdb, filters it, and writes to indices, so it
|
||||
// must be able to read anything written by save_blocks.
|
||||
if self.pruner.is_pruning_needed(last.number) {
|
||||
debug!(target: "engine::persistence", block_num=?last.number, "Running pruner");
|
||||
let prune_start = Instant::now();
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
let _ = self.pruner.run_with_provider(&provider_rw, last.number)?;
|
||||
provider_rw.commit()?;
|
||||
debug!(target: "engine::persistence", tip=?last.number, "Finished pruning after saving blocks");
|
||||
self.metrics.prune_before_duration_seconds.record(prune_start.elapsed());
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
self.metrics.save_blocks_batch_size.record(block_count as f64);
|
||||
self.metrics.save_blocks_duration_seconds.record(elapsed);
|
||||
@@ -461,52 +454,4 @@ mod tests {
|
||||
assert_eq!(last_hash, result.last_block.unwrap().hash);
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies that committing `save_blocks` history before running the pruner
|
||||
/// prevents the pruner from overwriting new entries.
|
||||
///
|
||||
/// Previously, both `save_blocks` and the pruner pushed `RocksDB` batches before
|
||||
/// a single commit. Both read committed state, so the pruner didn't see the
|
||||
/// new entries and its batch overwrote them. The fix commits `save_blocks`
|
||||
/// first, then runs the pruner against committed state in a separate provider.
|
||||
#[test]
|
||||
fn test_save_blocks_then_prune_preserves_new_history() {
|
||||
use reth_db::{models::ShardedKey, tables, BlockNumberList};
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let tracked_addr = alloy_primitives::Address::from([0xBE; 20]);
|
||||
|
||||
// Phase 1: Establish baseline history for blocks 0..20.
|
||||
let rocksdb = provider_factory.rocksdb_provider();
|
||||
{
|
||||
let mut batch = rocksdb.batch();
|
||||
let initial_blocks: Vec<u64> = (0..20).collect();
|
||||
let shard = BlockNumberList::new_pre_sorted(initial_blocks.iter().copied());
|
||||
batch
|
||||
.put::<tables::AccountsHistory>(ShardedKey::new(tracked_addr, u64::MAX), &shard)
|
||||
.unwrap();
|
||||
batch.commit().unwrap();
|
||||
}
|
||||
|
||||
// Phase 2: Simulate the fixed on_save_blocks flow.
|
||||
// Step 1: save_blocks appends new entries 20..25 and commits immediately.
|
||||
let mut batch1 = rocksdb.batch();
|
||||
batch1.append_account_history_shard(tracked_addr, 20..25u64).unwrap();
|
||||
batch1.commit().unwrap();
|
||||
|
||||
// Step 2: Pruner runs AFTER commit, so it reads state that includes 20..25.
|
||||
// Prunes entries ≤ 14, leaving [15..25).
|
||||
let mut batch2 = rocksdb.batch();
|
||||
batch2.prune_account_history_to(tracked_addr, 14).unwrap();
|
||||
batch2.commit().unwrap();
|
||||
|
||||
// Verify new entries survived pruning.
|
||||
let shards = rocksdb.account_history_shards(tracked_addr).unwrap();
|
||||
let entries: Vec<u64> = shards.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
let expected: Vec<u64> = (15..25).collect();
|
||||
assert_eq!(entries, expected, "new entries 20..25 must survive pruning");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -6,7 +6,7 @@ use alloy_primitives::{
|
||||
use fixed_cache::{AnyRef, CacheConfig, Stats, StatsHandler};
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use parking_lot::Once;
|
||||
use reth_errors::{ProviderError, ProviderResult};
|
||||
use reth_errors::ProviderResult;
|
||||
use reth_metrics::Metrics;
|
||||
use reth_primitives_traits::{Account, Bytecode};
|
||||
use reth_provider::{
|
||||
@@ -466,55 +466,6 @@ impl<S: StateProvider, const PREWARM: bool> StateProvider for CachedStateProvide
|
||||
self.state_provider.storage(account, storage_key)
|
||||
}
|
||||
}
|
||||
|
||||
fn storage_range(
|
||||
&self,
|
||||
account: Address,
|
||||
keys: &[StorageKey],
|
||||
) -> ProviderResult<Vec<(StorageKey, StorageValue)>> {
|
||||
let mut uncached_keys = Vec::new();
|
||||
let mut result = Vec::with_capacity(keys.len());
|
||||
|
||||
for &key in keys {
|
||||
if let Some(value) = self.caches.get_storage(account, key) {
|
||||
if !value.is_zero() {
|
||||
result.push((key, value));
|
||||
}
|
||||
} else {
|
||||
uncached_keys.push(key);
|
||||
}
|
||||
}
|
||||
|
||||
// Batch-fetch all uncached keys from the inner provider
|
||||
if !uncached_keys.is_empty() {
|
||||
let mut fetched = self.state_provider.storage_range(account, &uncached_keys)?;
|
||||
// Sort by raw key to align with uncached_keys for the merge-join below.
|
||||
// The inner provider may return results in a different order (e.g. hashed state
|
||||
// iterates by hashed slot).
|
||||
fetched.sort_unstable_by_key(|(k, _)| *k);
|
||||
// Merge-join to find zero slots without allocating a HashSet.
|
||||
let mut fetched_iter = fetched.iter();
|
||||
let mut next = fetched_iter.next();
|
||||
for &key in &uncached_keys {
|
||||
if let Some(&(fk, fv)) = next &&
|
||||
fk == key
|
||||
{
|
||||
let _ = self.caches.get_or_try_insert_storage_with(account, key, || {
|
||||
Ok::<_, ProviderError>(fv)
|
||||
});
|
||||
result.push((key, fv));
|
||||
next = fetched_iter.next();
|
||||
continue;
|
||||
}
|
||||
// key not returned by inner provider → zero slot
|
||||
let _ = self.caches.get_or_try_insert_storage_with(account, key, || {
|
||||
Ok::<_, ProviderError>(StorageValue::ZERO)
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
Ok(result)
|
||||
}
|
||||
}
|
||||
|
||||
impl<S: BytecodeReader, const PREWARM: bool> BytecodeReader for CachedStateProvider<S, PREWARM> {
|
||||
@@ -805,11 +756,6 @@ impl ExecutionCache {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a cached storage value if present, without inserting.
|
||||
pub fn get_storage(&self, address: Address, key: StorageKey) -> Option<StorageValue> {
|
||||
self.0.storage_cache.get(&(address, key))
|
||||
}
|
||||
|
||||
/// Insert storage value into cache.
|
||||
pub fn insert_storage(&self, address: Address, key: StorageKey, value: Option<StorageValue>) {
|
||||
self.0.storage_cache.insert((address, key), value.unwrap_or_default());
|
||||
|
||||
@@ -791,10 +791,15 @@ where
|
||||
// If the canonical chain is ahead of the new chain,
|
||||
// gather all blocks until new head number.
|
||||
while current_canonical_number > current_number {
|
||||
let block = self.canonical_block_by_hash(old_hash)?;
|
||||
old_hash = block.recovered_block().parent_hash();
|
||||
old_chain.push(block);
|
||||
current_canonical_number -= 1;
|
||||
if let Some(block) = self.canonical_block_by_hash(old_hash)? {
|
||||
old_hash = block.recovered_block().parent_hash();
|
||||
old_chain.push(block);
|
||||
current_canonical_number -= 1;
|
||||
} else {
|
||||
// This shouldn't happen as we're walking back the canonical chain
|
||||
warn!(target: "engine::tree", current_hash=?old_hash, "Canonical block not found in TreeState");
|
||||
return Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
// Both new and old chain pointers are now at the same height.
|
||||
@@ -803,9 +808,14 @@ where
|
||||
// Walk both chains from specified hashes at same height until
|
||||
// a common ancestor (fork block) is reached.
|
||||
while old_hash != current_hash {
|
||||
let block = self.canonical_block_by_hash(old_hash)?;
|
||||
old_hash = block.recovered_block().parent_hash();
|
||||
old_chain.push(block);
|
||||
if let Some(block) = self.canonical_block_by_hash(old_hash)? {
|
||||
old_hash = block.recovered_block().parent_hash();
|
||||
old_chain.push(block);
|
||||
} else {
|
||||
// This shouldn't happen as we're walking back the canonical chain
|
||||
warn!(target: "engine::tree", current_hash=?old_hash, "Canonical block not found in TreeState");
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
if let Some(block) = self.state.tree_state.executed_block_by_hash(current_hash).cloned()
|
||||
{
|
||||
@@ -932,22 +942,36 @@ where
|
||||
let new_head_hash = canonical_header.hash();
|
||||
let new_head_number = canonical_header.number();
|
||||
|
||||
// Load the canonical ancestor's block
|
||||
let executed_block = self.canonical_block_by_hash(new_head_hash)?;
|
||||
// Perform the reorg to properly handle the unwind
|
||||
self.canonical_in_memory_state
|
||||
.update_chain(NewCanonicalChain::Reorg { new: vec![executed_block], old: old_blocks });
|
||||
// Try to load the canonical ancestor's block
|
||||
match self.canonical_block_by_hash(new_head_hash)? {
|
||||
Some(executed_block) => {
|
||||
// Perform the reorg to properly handle the unwind
|
||||
self.canonical_in_memory_state.update_chain(NewCanonicalChain::Reorg {
|
||||
new: vec![executed_block],
|
||||
old: old_blocks,
|
||||
});
|
||||
|
||||
// CRITICAL: Update the canonical head after the reorg
|
||||
// This ensures get_canonical_head() returns the correct block
|
||||
self.canonical_in_memory_state.set_canonical_head(canonical_header.clone());
|
||||
// CRITICAL: Update the canonical head after the reorg
|
||||
// This ensures get_canonical_head() returns the correct block
|
||||
self.canonical_in_memory_state.set_canonical_head(canonical_header.clone());
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
block_number = new_head_number,
|
||||
block_hash = ?new_head_hash,
|
||||
"Successfully loaded canonical ancestor into memory via reorg"
|
||||
);
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
block_number = new_head_number,
|
||||
block_hash = ?new_head_hash,
|
||||
"Successfully loaded canonical ancestor into memory via reorg"
|
||||
);
|
||||
}
|
||||
None => {
|
||||
// Fallback: update header only if block cannot be found
|
||||
warn!(
|
||||
target: "engine::tree",
|
||||
block_hash = ?new_head_hash,
|
||||
"Could not find canonical ancestor block, updating header only"
|
||||
);
|
||||
self.canonical_in_memory_state.set_canonical_head(canonical_header.clone());
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -973,17 +997,18 @@ where
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
// Load the block from storage
|
||||
let executed_block = self.canonical_block_by_hash(block_hash)?;
|
||||
self.canonical_in_memory_state
|
||||
.update_chain(NewCanonicalChain::Commit { new: vec![executed_block] });
|
||||
// Try to load the block from storage
|
||||
if let Some(executed_block) = self.canonical_block_by_hash(block_hash)? {
|
||||
self.canonical_in_memory_state
|
||||
.update_chain(NewCanonicalChain::Commit { new: vec![executed_block] });
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
block_number,
|
||||
block_hash = ?block_hash,
|
||||
"Added canonical block to in-memory state"
|
||||
);
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
block_number,
|
||||
block_hash = ?block_hash,
|
||||
"Added canonical block to in-memory state"
|
||||
);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
@@ -1544,52 +1569,58 @@ where
|
||||
// handle the event if any
|
||||
self.on_maybe_tree_event(maybe_event)?;
|
||||
}
|
||||
BeaconEngineMessage::RethNewPayload {
|
||||
payload,
|
||||
wait_for_persistence,
|
||||
wait_for_caches,
|
||||
tx,
|
||||
} => {
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
wait_for_persistence,
|
||||
wait_for_caches,
|
||||
"Processing reth_newPayload"
|
||||
);
|
||||
BeaconEngineMessage::RethNewPayload { payload, tx } => {
|
||||
// Before processing the new payload, we wait for persistence and
|
||||
// cache updates to complete. We do it in parallel, spawning
|
||||
// persistence and cache update wait tasks with Tokio, so that we
|
||||
// can get an unbiased breakdown on how long did every step take.
|
||||
//
|
||||
// If we first wait for persistence, and only then for cache
|
||||
// updates, we will offset the cache update waits by the duration of
|
||||
// persistence, which is incorrect.
|
||||
debug!(target: "engine::tree", "Waiting for persistence and caches in parallel before processing reth_newPayload");
|
||||
|
||||
let persistence_wait = if wait_for_persistence {
|
||||
let pending_persistence = self.persistence_state.rx.take();
|
||||
if let Some((rx, start_time, _action)) = pending_persistence {
|
||||
let (persistence_tx, persistence_rx) =
|
||||
std::sync::mpsc::channel();
|
||||
self.runtime.spawn_blocking_named(
|
||||
"wait-persist",
|
||||
move || {
|
||||
let start = Instant::now();
|
||||
let result = rx
|
||||
.recv()
|
||||
.expect("persistence state channel closed");
|
||||
let _ = persistence_tx.send((
|
||||
result,
|
||||
start_time,
|
||||
start.elapsed(),
|
||||
));
|
||||
},
|
||||
);
|
||||
let (result, start_time, wait_duration) = persistence_rx
|
||||
.recv()
|
||||
.expect("persistence result channel closed");
|
||||
let _ = self.on_persistence_complete(result, start_time);
|
||||
Some(wait_duration)
|
||||
} else {
|
||||
Some(Duration::ZERO)
|
||||
}
|
||||
let pending_persistence = self.persistence_state.rx.take();
|
||||
let persistence_rx = if let Some((rx, start_time, _action)) =
|
||||
pending_persistence
|
||||
{
|
||||
let (persistence_tx, persistence_rx) =
|
||||
std::sync::mpsc::channel();
|
||||
self.runtime.spawn_blocking_named("wait-persist", move || {
|
||||
let start = Instant::now();
|
||||
let result =
|
||||
rx.recv().expect("persistence state channel closed");
|
||||
let _ = persistence_tx.send((
|
||||
result,
|
||||
start_time,
|
||||
start.elapsed(),
|
||||
));
|
||||
});
|
||||
Some(persistence_rx)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let cache_wait = wait_for_caches
|
||||
.then(|| self.payload_validator.wait_for_caches());
|
||||
let cache_wait = self.payload_validator.wait_for_caches();
|
||||
|
||||
let persistence_wait = if let Some(persistence_rx) = persistence_rx
|
||||
{
|
||||
let (result, start_time, wait_duration) = persistence_rx
|
||||
.recv()
|
||||
.expect("persistence result channel closed");
|
||||
let _ = self.on_persistence_complete(result, start_time);
|
||||
Some(wait_duration)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
debug!(
|
||||
target: "engine::tree",
|
||||
?persistence_wait,
|
||||
execution_cache_wait = ?cache_wait.execution_cache,
|
||||
sparse_trie_wait = ?cache_wait.sparse_trie,
|
||||
"Persistence finished and caches updated for reth_newPayload"
|
||||
);
|
||||
|
||||
let start = Instant::now();
|
||||
let gas_used = payload.gas_used();
|
||||
@@ -1609,9 +1640,8 @@ where
|
||||
let timings = NewPayloadTimings {
|
||||
latency,
|
||||
persistence_wait,
|
||||
execution_cache_wait: cache_wait
|
||||
.map(|wait| wait.execution_cache),
|
||||
sparse_trie_wait: cache_wait.map(|wait| wait.sparse_trie),
|
||||
execution_cache_wait: cache_wait.execution_cache,
|
||||
sparse_trie_wait: cache_wait.sparse_trie,
|
||||
};
|
||||
if let Err(err) =
|
||||
tx.send(output.map(|o| (o.outcome, timings)).map_err(|e| {
|
||||
@@ -1999,11 +2029,11 @@ where
|
||||
/// pruned for a given block, this operation will return an error. On archive nodes, it
|
||||
/// can retrieve any block.
|
||||
#[instrument(level = "debug", target = "engine::tree", skip(self))]
|
||||
fn canonical_block_by_hash(&self, hash: B256) -> ProviderResult<ExecutedBlock<N>> {
|
||||
fn canonical_block_by_hash(&self, hash: B256) -> ProviderResult<Option<ExecutedBlock<N>>> {
|
||||
trace!(target: "engine::tree", ?hash, "Fetching executed block by hash");
|
||||
// check memory first
|
||||
if let Some(block) = self.state.tree_state.executed_block_by_hash(hash) {
|
||||
return Ok(block.clone())
|
||||
return Ok(Some(block.clone()))
|
||||
}
|
||||
|
||||
let (block, senders) = self
|
||||
@@ -2045,22 +2075,11 @@ where
|
||||
},
|
||||
});
|
||||
|
||||
Ok(ExecutedBlock::new(
|
||||
Ok(Some(ExecutedBlock::new(
|
||||
Arc::new(RecoveredBlock::new_sealed(block, senders)),
|
||||
execution_output,
|
||||
trie_data,
|
||||
))
|
||||
}
|
||||
|
||||
/// Returns `true` if a block with the given hash is known, either in memory or in the
|
||||
/// database. This is a lightweight existence check that avoids constructing a full
|
||||
/// [`SealedHeader`].
|
||||
fn has_block_by_hash(&self, hash: B256) -> ProviderResult<bool> {
|
||||
if self.state.tree_state.contains_hash(&hash) {
|
||||
Ok(true)
|
||||
} else {
|
||||
self.provider.is_known(hash)
|
||||
}
|
||||
)))
|
||||
}
|
||||
|
||||
/// Return sealed block header from in-memory state or database by hash.
|
||||
@@ -2107,7 +2126,7 @@ where
|
||||
parent_hash: B256,
|
||||
) -> ProviderResult<Option<B256>> {
|
||||
// Check if parent exists in side chain or in canonical chain.
|
||||
if self.has_block_by_hash(parent_hash)? {
|
||||
if self.sealed_header_by_hash(parent_hash)?.is_some() {
|
||||
return Ok(Some(parent_hash))
|
||||
}
|
||||
|
||||
@@ -2121,7 +2140,7 @@ where
|
||||
|
||||
// If current_header is None, then the current_hash does not have an invalid
|
||||
// ancestor in the cache, check its presence in blockchain tree
|
||||
if current_block.is_none() && self.has_block_by_hash(current_hash)? {
|
||||
if current_block.is_none() && self.sealed_header_by_hash(current_hash)?.is_some() {
|
||||
return Ok(Some(current_hash))
|
||||
}
|
||||
}
|
||||
@@ -2788,7 +2807,7 @@ where
|
||||
debug!(target: "engine::tree", block=?block_num_hash, parent = ?block_id.parent, "Inserting new block into tree");
|
||||
|
||||
// Check if block already exists - first in memory, then DB only if it could be persisted
|
||||
if self.state.tree_state.contains_hash(&block_num_hash.hash) {
|
||||
if self.state.tree_state.sealed_header_by_hash(&block_num_hash.hash).is_some() {
|
||||
convert_to_block(self, input)?;
|
||||
return Ok(InsertPayloadOk::AlreadySeen(BlockStatus::Valid));
|
||||
}
|
||||
|
||||
@@ -38,9 +38,7 @@ use reth_trie_parallel::{
|
||||
proof_task::{ProofTaskCtx, ProofWorkerHandle},
|
||||
root::ParallelStateRootError,
|
||||
};
|
||||
use reth_trie_sparse::{
|
||||
ArenaParallelSparseTrie, ConfigurableSparseTrie, RevealableSparseTrie, SparseStateTrie,
|
||||
};
|
||||
use reth_trie_sparse::ParallelismThresholds;
|
||||
use std::{
|
||||
ops::Not,
|
||||
sync::{
|
||||
@@ -59,7 +57,18 @@ pub mod prewarm;
|
||||
pub mod receipt_root_task;
|
||||
pub mod sparse_trie;
|
||||
|
||||
use preserved_sparse_trie::{PreservedSparseTrie, SharedPreservedSparseTrie};
|
||||
pub use preserved_sparse_trie::{
|
||||
PayloadSparseTrieCache, PayloadSparseTrieKind, PayloadSparseTrieStoreOutcome,
|
||||
SparseTrieCheckout,
|
||||
};
|
||||
|
||||
/// Default parallelism thresholds to use with the [`ParallelSparseTrie`].
|
||||
///
|
||||
/// These values were determined by performing benchmarks using gradually increasing values to judge
|
||||
/// the effects. Below 100 throughput would generally be equal or slightly less, while above 150 it
|
||||
/// would deteriorate to the point where PST might as well not be used.
|
||||
const PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS: ParallelismThresholds =
|
||||
ParallelismThresholds { min_revealed_nodes: 100, min_updated_nodes: 100 };
|
||||
|
||||
/// Default node capacity for shrinking the sparse trie. This is used to limit the number of trie
|
||||
/// nodes in allocated sparse tries.
|
||||
@@ -95,6 +104,52 @@ type IteratorPayloadHandle<Evm, I, N> = PayloadHandle<
|
||||
<N as NodePrimitives>::Receipt,
|
||||
>;
|
||||
|
||||
/// Shared cache handles that can be exported to engine consumers and downstream payload builders.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct EngineSharedCaches<Evm: ConfigureEvm> {
|
||||
execution_cache: PayloadExecutionCache,
|
||||
sparse_trie_cache: PayloadSparseTrieCache,
|
||||
precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
}
|
||||
|
||||
impl<Evm> Default for EngineSharedCaches<Evm>
|
||||
where
|
||||
Evm: ConfigureEvm,
|
||||
{
|
||||
fn default() -> Self {
|
||||
Self::with_sparse_trie_kind(PayloadSparseTrieKind::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl<Evm> EngineSharedCaches<Evm>
|
||||
where
|
||||
Evm: ConfigureEvm,
|
||||
{
|
||||
/// Creates shared caches backed by the requested sparse trie implementation.
|
||||
pub fn with_sparse_trie_kind(sparse_trie_kind: PayloadSparseTrieKind) -> Self {
|
||||
Self {
|
||||
execution_cache: Default::default(),
|
||||
sparse_trie_cache: PayloadSparseTrieCache::new(sparse_trie_kind),
|
||||
precompile_cache_map: Default::default(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the shared execution cache handle for engine-internal use.
|
||||
pub(crate) fn execution_cache(&self) -> PayloadExecutionCache {
|
||||
self.execution_cache.clone()
|
||||
}
|
||||
|
||||
/// Returns the shared sparse trie cache handle.
|
||||
pub fn sparse_trie_cache(&self) -> PayloadSparseTrieCache {
|
||||
self.sparse_trie_cache.clone()
|
||||
}
|
||||
|
||||
/// Returns the shared precompile cache map.
|
||||
pub fn precompile_cache_map(&self) -> PrecompileCacheMap<SpecFor<Evm>> {
|
||||
self.precompile_cache_map.clone()
|
||||
}
|
||||
}
|
||||
|
||||
/// Entrypoint for executing the payload.
|
||||
#[derive(Debug)]
|
||||
pub struct PayloadProcessor<Evm>
|
||||
@@ -103,8 +158,8 @@ where
|
||||
{
|
||||
/// The executor used by to spawn tasks.
|
||||
executor: Runtime,
|
||||
/// The most recent cache used for execution.
|
||||
execution_cache: PayloadExecutionCache,
|
||||
/// Shared caches reused across payload processing.
|
||||
shared_caches: EngineSharedCaches<Evm>,
|
||||
/// Metrics for trie operations
|
||||
trie_metrics: MultiProofTaskMetrics,
|
||||
/// Cross-block cache size in bytes.
|
||||
@@ -117,12 +172,6 @@ where
|
||||
evm_config: Evm,
|
||||
/// Whether precompile cache should be disabled.
|
||||
precompile_cache_disabled: bool,
|
||||
/// Precompile cache map.
|
||||
precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// A pruned `SparseStateTrie`, kept around as a cache of already revealed trie nodes and to
|
||||
/// re-use allocated memory. Stored with the block hash it was computed for to enable trie
|
||||
/// preservation across sequential payload validations.
|
||||
sparse_state_trie: SharedPreservedSparseTrie,
|
||||
/// LFU hot-slot capacity: max storage slots retained across prune cycles.
|
||||
sparse_trie_max_hot_slots: usize,
|
||||
/// LFU hot-account capacity: max account addresses retained across prune cycles.
|
||||
@@ -131,12 +180,6 @@ where
|
||||
disable_sparse_trie_cache_pruning: bool,
|
||||
/// Whether to disable cache metrics recording.
|
||||
disable_cache_metrics: bool,
|
||||
/// Whether to disable BAL-based parallel execution (falls back to tx-based prewarming).
|
||||
disable_bal_parallel_execution: bool,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
/// Whether BAL batched IO is disabled.
|
||||
disable_bal_batch_io: bool,
|
||||
}
|
||||
|
||||
impl<N, Evm> PayloadProcessor<Evm>
|
||||
@@ -154,26 +197,21 @@ where
|
||||
executor: Runtime,
|
||||
evm_config: Evm,
|
||||
config: &TreeConfig,
|
||||
precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
shared_caches: EngineSharedCaches<Evm>,
|
||||
) -> Self {
|
||||
Self {
|
||||
executor,
|
||||
execution_cache: Default::default(),
|
||||
shared_caches,
|
||||
trie_metrics: Default::default(),
|
||||
cross_block_cache_size: config.cross_block_cache_size(),
|
||||
disable_transaction_prewarming: config.disable_prewarming(),
|
||||
evm_config,
|
||||
disable_state_cache: config.disable_state_cache(),
|
||||
precompile_cache_disabled: config.precompile_cache_disabled(),
|
||||
precompile_cache_map,
|
||||
sparse_state_trie: SharedPreservedSparseTrie::default(),
|
||||
sparse_trie_max_hot_slots: config.sparse_trie_max_hot_slots(),
|
||||
sparse_trie_max_hot_accounts: config.sparse_trie_max_hot_accounts(),
|
||||
disable_sparse_trie_cache_pruning: config.disable_sparse_trie_cache_pruning(),
|
||||
disable_cache_metrics: config.disable_cache_metrics(),
|
||||
disable_bal_parallel_execution: config.disable_bal_parallel_execution(),
|
||||
disable_bal_parallel_state_root: config.disable_bal_parallel_state_root(),
|
||||
disable_bal_batch_io: config.disable_bal_batch_io(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -186,8 +224,8 @@ where
|
||||
debug!(target: "engine::tree::payload_processor", "Waiting for execution cache and sparse trie locks");
|
||||
|
||||
// Wait for both caches in parallel using std threads
|
||||
let execution_cache = self.execution_cache.clone();
|
||||
let sparse_trie = self.sparse_state_trie.clone();
|
||||
let execution_cache = self.shared_caches.execution_cache();
|
||||
let sparse_trie = self.shared_caches.sparse_trie_cache();
|
||||
|
||||
// Use channels and spawn_blocking instead of std::thread::spawn
|
||||
let (execution_tx, execution_rx) = std::sync::mpsc::channel();
|
||||
@@ -501,25 +539,22 @@ where
|
||||
terminate_execution: Arc::new(AtomicBool::new(false)),
|
||||
executed_tx_index: Arc::clone(&executed_tx_index),
|
||||
precompile_cache_disabled: self.precompile_cache_disabled,
|
||||
precompile_cache_map: self.precompile_cache_map.clone(),
|
||||
disable_bal_batch_io: self.disable_bal_batch_io,
|
||||
precompile_cache_map: self.shared_caches.precompile_cache_map(),
|
||||
};
|
||||
|
||||
let (prewarm_task, to_prewarm_task) = PrewarmCacheTask::new(
|
||||
self.executor.clone(),
|
||||
self.execution_cache.clone(),
|
||||
self.shared_caches.execution_cache(),
|
||||
prewarm_ctx,
|
||||
to_multi_proof,
|
||||
self.disable_bal_parallel_state_root,
|
||||
);
|
||||
|
||||
{
|
||||
let to_prewarm_task = to_prewarm_task.clone();
|
||||
let disable_bal_parallel_execution = self.disable_bal_parallel_execution;
|
||||
self.executor.spawn_blocking_named("prewarm", move || {
|
||||
let mode = if skip_prewarm {
|
||||
PrewarmMode::Skipped
|
||||
} else if let Some(bal) = bal.filter(|_| !disable_bal_parallel_execution) {
|
||||
} else if let Some(bal) = bal {
|
||||
PrewarmMode::BlockAccessList(bal)
|
||||
} else {
|
||||
PrewarmMode::Transactions(transactions)
|
||||
@@ -537,7 +572,7 @@ where
|
||||
/// instance.
|
||||
#[instrument(level = "debug", target = "engine::caching", skip(self))]
|
||||
fn cache_for(&self, parent_hash: B256) -> SavedCache {
|
||||
if let Some(cache) = self.execution_cache.get_cache_for(parent_hash) {
|
||||
if let Some(cache) = self.shared_caches.execution_cache().get_cache_for(parent_hash) {
|
||||
debug!("reusing execution cache");
|
||||
cache
|
||||
} else {
|
||||
@@ -562,7 +597,7 @@ where
|
||||
parent_state_root: B256,
|
||||
chunk_size: usize,
|
||||
) {
|
||||
let preserved_sparse_trie = self.sparse_state_trie.clone();
|
||||
let sparse_trie_cache = self.shared_caches.sparse_trie_cache();
|
||||
let trie_metrics = self.trie_metrics.clone();
|
||||
let max_hot_slots = self.sparse_trie_max_hot_slots;
|
||||
let max_hot_accounts = self.sparse_trie_max_hot_accounts;
|
||||
@@ -576,39 +611,19 @@ where
|
||||
let _enter = debug_span!(target: "engine::tree::payload_processor", parent: parent_span, "sparse_trie_task")
|
||||
.entered();
|
||||
|
||||
// Reuse a stored SparseStateTrie if available, applying continuation logic.
|
||||
// If this payload's parent state root matches the preserved trie's anchor,
|
||||
// we can reuse the pruned trie structure. Otherwise, we clear the trie but
|
||||
// keep allocations.
|
||||
let start = Instant::now();
|
||||
let preserved = preserved_sparse_trie.take();
|
||||
let mut checkout = sparse_trie_cache.take_or_create_for(parent_state_root);
|
||||
trie_metrics
|
||||
.sparse_trie_cache_wait_duration_histogram
|
||||
.record(start.elapsed().as_secs_f64());
|
||||
checkout.set_hot_cache_capacities(max_hot_slots, max_hot_accounts);
|
||||
|
||||
let mut sparse_state_trie = preserved
|
||||
.map(|preserved| preserved.into_trie_for(parent_state_root))
|
||||
.unwrap_or_else(|| {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
"Creating new sparse trie - no preserved trie available"
|
||||
);
|
||||
let default_trie = RevealableSparseTrie::blind_from(
|
||||
ConfigurableSparseTrie::Arena(ArenaParallelSparseTrie::default()),
|
||||
);
|
||||
SparseStateTrie::default()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true)
|
||||
});
|
||||
sparse_state_trie.set_hot_cache_capacities(max_hot_slots, max_hot_accounts);
|
||||
|
||||
let mut task = SparseTrieCacheTask::new_with_trie(
|
||||
let mut task = SparseTrieCacheTask::new_with_checkout(
|
||||
&executor,
|
||||
from_multi_proof,
|
||||
proof_worker_handle,
|
||||
trie_metrics.clone(),
|
||||
sparse_state_trie,
|
||||
checkout,
|
||||
chunk_size,
|
||||
);
|
||||
|
||||
@@ -619,7 +634,7 @@ where
|
||||
// causing take() to return None and forcing it to create a new empty trie
|
||||
// instead of reusing the preserved one. Holding the guard ensures the next
|
||||
// block's take() blocks until we've stored the trie for reuse.
|
||||
let mut guard = preserved_sparse_trie.lock();
|
||||
let mut guard = sparse_trie_cache.lock();
|
||||
|
||||
let task_result = result.as_ref().ok().cloned();
|
||||
// Send state root computation result - next block may start but will block on take()
|
||||
@@ -634,7 +649,7 @@ where
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
);
|
||||
guard.store(PreservedSparseTrie::cleared(trie));
|
||||
trie.store_prepared_cleared_with_guard(&mut guard);
|
||||
drop(guard);
|
||||
executor.spawn_drop(deferred);
|
||||
return;
|
||||
@@ -663,7 +678,7 @@ where
|
||||
trie_metrics
|
||||
.sparse_trie_retained_storage_tries
|
||||
.set(trie.retained_storage_tries_count() as f64);
|
||||
guard.store(PreservedSparseTrie::anchored(trie, result.state_root));
|
||||
trie.store_anchored_with_guard(&mut guard, result.state_root);
|
||||
deferred
|
||||
} else {
|
||||
debug!(
|
||||
@@ -674,7 +689,7 @@ where
|
||||
SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
);
|
||||
guard.store(PreservedSparseTrie::cleared(trie));
|
||||
trie.store_prepared_cleared_with_guard(&mut guard);
|
||||
deferred
|
||||
};
|
||||
drop(guard);
|
||||
@@ -695,7 +710,7 @@ where
|
||||
bundle_state: &BundleState,
|
||||
) {
|
||||
let disable_cache_metrics = self.disable_cache_metrics;
|
||||
self.execution_cache.update_with_guard(|cached| {
|
||||
self.shared_caches.execution_cache().update_with_guard(|cached| {
|
||||
if cached.as_ref().is_some_and(|c| c.executed_block_hash() != block_with_parent.parent) {
|
||||
debug!(
|
||||
target: "engine::caching",
|
||||
@@ -999,7 +1014,7 @@ impl<R> Drop for CacheTaskHandle<R> {
|
||||
/// - Prepares data for state root proof computation
|
||||
/// - Runs concurrently but must not interfere with cache saves
|
||||
#[derive(Clone, Debug, Default)]
|
||||
pub struct PayloadExecutionCache {
|
||||
pub(crate) struct PayloadExecutionCache {
|
||||
/// Guarded cloneable cache identified by a block hash.
|
||||
inner: Arc<RwLock<Option<SavedCache>>>,
|
||||
/// Metrics for cache operations.
|
||||
@@ -1007,11 +1022,11 @@ pub struct PayloadExecutionCache {
|
||||
}
|
||||
|
||||
impl PayloadExecutionCache {
|
||||
/// Returns the cache for `parent_hash` if it's available for use.
|
||||
/// Returns the cache backing store for `parent_hash` if it's available for reuse.
|
||||
///
|
||||
/// A cache is considered available when:
|
||||
/// - It exists and matches the requested parent hash
|
||||
/// - No other tasks are currently using it (checked via Arc reference count)
|
||||
/// If the tracked cache is available but keyed to a different parent hash, the cache is
|
||||
/// cleared and returned so callers can reuse the underlying allocations without carrying over
|
||||
/// stale state.
|
||||
#[instrument(level = "debug", target = "engine::tree::payload_processor", skip(self))]
|
||||
pub(crate) fn get_cache_for(&self, parent_hash: B256) -> Option<SavedCache> {
|
||||
let start = Instant::now();
|
||||
@@ -1050,7 +1065,7 @@ impl PayloadExecutionCache {
|
||||
// and picking up polluted data if the fork block fails.
|
||||
c.clear_with_hash(parent_hash);
|
||||
}
|
||||
return Some(c.clone())
|
||||
return Some(c.clone());
|
||||
} else if hash_matches {
|
||||
self.metrics.execution_cache_in_use.increment(1);
|
||||
}
|
||||
@@ -1067,7 +1082,7 @@ impl PayloadExecutionCache {
|
||||
/// This is useful for synchronization before starting payload processing.
|
||||
///
|
||||
/// Returns the time spent waiting for the lock.
|
||||
pub fn wait_for_availability(&self) -> Duration {
|
||||
pub(crate) fn wait_for_availability(&self) -> Duration {
|
||||
let start = Instant::now();
|
||||
// Acquire write lock to wait for any current holders to finish
|
||||
let _guard = self.inner.write();
|
||||
@@ -1095,7 +1110,7 @@ impl PayloadExecutionCache {
|
||||
///
|
||||
/// Violating this requirement can result in cache corruption, incorrect state data,
|
||||
/// and potential consensus failures.
|
||||
pub fn update_with_guard<F>(&self, update_fn: F)
|
||||
pub(crate) fn update_with_guard<F>(&self, update_fn: F)
|
||||
where
|
||||
F: FnOnce(&mut Option<SavedCache>),
|
||||
{
|
||||
@@ -1164,8 +1179,9 @@ mod tests {
|
||||
use super::PayloadExecutionCache;
|
||||
use crate::tree::{
|
||||
cached_state::{CachedStateMetrics, ExecutionCache, SavedCache},
|
||||
payload_processor::{evm_state_to_hashed_post_state, ExecutionEnv, PayloadProcessor},
|
||||
precompile_cache::PrecompileCacheMap,
|
||||
payload_processor::{
|
||||
evm_state_to_hashed_post_state, EngineSharedCaches, ExecutionEnv, PayloadProcessor,
|
||||
},
|
||||
StateProviderBuilder, TreeConfig,
|
||||
};
|
||||
use alloy_eips::eip1898::{BlockNumHash, BlockWithParent};
|
||||
@@ -1277,7 +1293,7 @@ mod tests {
|
||||
reth_tasks::Runtime::test(),
|
||||
EthEvmConfig::new(Arc::new(ChainSpec::default())),
|
||||
&TreeConfig::default(),
|
||||
PrecompileCacheMap::default(),
|
||||
EngineSharedCaches::default(),
|
||||
);
|
||||
|
||||
let parent_hash = B256::from([1u8; 32]);
|
||||
@@ -1289,13 +1305,17 @@ mod tests {
|
||||
let bundle_state = BundleState::default();
|
||||
|
||||
// Cache should be empty initially
|
||||
assert!(payload_processor.execution_cache.get_cache_for(block_hash).is_none());
|
||||
assert!(payload_processor
|
||||
.shared_caches
|
||||
.execution_cache()
|
||||
.get_cache_for(block_hash)
|
||||
.is_none());
|
||||
|
||||
// Update cache with inserted block
|
||||
payload_processor.on_inserted_executed_block(block_with_parent, &bundle_state);
|
||||
|
||||
// Cache should now exist for the block hash
|
||||
let cached = payload_processor.execution_cache.get_cache_for(block_hash);
|
||||
let cached = payload_processor.shared_caches.execution_cache().get_cache_for(block_hash);
|
||||
assert!(cached.is_some());
|
||||
assert_eq!(cached.unwrap().executed_block_hash(), block_hash);
|
||||
}
|
||||
@@ -1306,13 +1326,14 @@ mod tests {
|
||||
reth_tasks::Runtime::test(),
|
||||
EthEvmConfig::new(Arc::new(ChainSpec::default())),
|
||||
&TreeConfig::default(),
|
||||
PrecompileCacheMap::default(),
|
||||
EngineSharedCaches::default(),
|
||||
);
|
||||
|
||||
// Setup: populate cache with block 1
|
||||
let block1_hash = B256::from([1u8; 32]);
|
||||
payload_processor
|
||||
.execution_cache
|
||||
.shared_caches
|
||||
.execution_cache()
|
||||
.update_with_guard(|slot| *slot = Some(make_saved_cache(block1_hash)));
|
||||
|
||||
// Try to insert block 3 with wrong parent (should skip and keep block 1's cache)
|
||||
@@ -1327,11 +1348,11 @@ mod tests {
|
||||
payload_processor.on_inserted_executed_block(block_with_parent, &bundle_state);
|
||||
|
||||
// Cache should still be for block 1 (unchanged)
|
||||
let cached = payload_processor.execution_cache.get_cache_for(block1_hash);
|
||||
let cached = payload_processor.shared_caches.execution_cache().get_cache_for(block1_hash);
|
||||
assert!(cached.is_some(), "Original cache should be preserved");
|
||||
|
||||
// Cache for block 3 should not exist
|
||||
let cached3 = payload_processor.execution_cache.get_cache_for(block3_hash);
|
||||
let cached3 = payload_processor.shared_caches.execution_cache().get_cache_for(block3_hash);
|
||||
assert!(cached3.is_none(), "New block cache should not be created on mismatch");
|
||||
}
|
||||
|
||||
@@ -1441,7 +1462,7 @@ mod tests {
|
||||
reth_tasks::Runtime::test(),
|
||||
EthEvmConfig::new(factory.chain_spec()),
|
||||
&TreeConfig::default(),
|
||||
PrecompileCacheMap::default(),
|
||||
EngineSharedCaches::default(),
|
||||
);
|
||||
|
||||
let provider_factory = BlockchainProvider::new(factory).unwrap();
|
||||
|
||||
@@ -1,44 +1,128 @@
|
||||
//! Preserved sparse trie for reuse across payload validations.
|
||||
|
||||
use super::{
|
||||
PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS, SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY,
|
||||
SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY,
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use parking_lot::Mutex;
|
||||
use reth_trie_sparse::{ConfigurableSparseTrie, SparseStateTrie};
|
||||
use std::{sync::Arc, time::Instant};
|
||||
use reth_trie_sparse::{
|
||||
ArenaParallelSparseTrie, ConfigurableSparseTrie, ParallelSparseTrie, RevealableSparseTrie,
|
||||
SparseStateTrie,
|
||||
};
|
||||
use std::{
|
||||
ops::{Deref, DerefMut},
|
||||
sync::Arc,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::debug;
|
||||
|
||||
/// Type alias for the sparse trie type used in preservation.
|
||||
pub(super) type SparseTrie = SparseStateTrie<ConfigurableSparseTrie, ConfigurableSparseTrie>;
|
||||
type SparseTrie = SparseStateTrie<ConfigurableSparseTrie, ConfigurableSparseTrie>;
|
||||
|
||||
/// Shared handle to a preserved sparse trie that can be reused across payload validations.
|
||||
/// Sparse trie implementation used by [`PayloadSparseTrieCache`].
|
||||
#[derive(Debug, Default, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PayloadSparseTrieKind {
|
||||
/// Back sparse trie storage with hash maps.
|
||||
#[default]
|
||||
HashMap,
|
||||
/// Back sparse trie storage with arena allocations.
|
||||
Arena,
|
||||
}
|
||||
|
||||
impl From<bool> for PayloadSparseTrieKind {
|
||||
fn from(enable_arena_sparse_trie: bool) -> Self {
|
||||
if enable_arena_sparse_trie {
|
||||
Self::Arena
|
||||
} else {
|
||||
Self::HashMap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Default)]
|
||||
struct PayloadSparseTrieState {
|
||||
latest_checkout_id: u64,
|
||||
preserved: Option<PreservedSparseTrie>,
|
||||
}
|
||||
|
||||
/// Outcome of storing a checked-out sparse trie back into the shared cache.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum PayloadSparseTrieStoreOutcome {
|
||||
/// The checkout was the most recent lease and the trie was stored.
|
||||
Stored,
|
||||
/// A newer checkout had already been issued, so this stale lease was ignored.
|
||||
IgnoredStaleCheckout,
|
||||
}
|
||||
|
||||
/// Shared sparse trie cache that can be reused across payload validations.
|
||||
///
|
||||
/// This is stored in [`PayloadProcessor`](super::PayloadProcessor) and cloned to pass to
|
||||
/// [`SparseTrieCacheTask`](super::sparse_trie::SparseTrieCacheTask) for trie reuse.
|
||||
#[derive(Debug, Default, Clone)]
|
||||
pub(super) struct SharedPreservedSparseTrie(Arc<Mutex<Option<PreservedSparseTrie>>>);
|
||||
/// This is the public sparse-trie SDK surface exposed through
|
||||
/// [`EngineSharedCaches`](super::EngineSharedCaches). Callers take or create a trie, use it for
|
||||
/// payload work, then store it back either anchored to the resulting state root or cleared for
|
||||
/// allocation reuse.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct PayloadSparseTrieCache {
|
||||
kind: PayloadSparseTrieKind,
|
||||
state: Arc<Mutex<PayloadSparseTrieState>>,
|
||||
}
|
||||
|
||||
impl SharedPreservedSparseTrie {
|
||||
/// Takes the preserved trie if present, leaving `None` in its place.
|
||||
pub(super) fn take(&self) -> Option<PreservedSparseTrie> {
|
||||
self.0.lock().take()
|
||||
impl Default for PayloadSparseTrieCache {
|
||||
fn default() -> Self {
|
||||
Self::new(PayloadSparseTrieKind::default())
|
||||
}
|
||||
}
|
||||
|
||||
impl PayloadSparseTrieCache {
|
||||
/// Creates a sparse trie cache backed by the requested trie implementation.
|
||||
pub fn new(kind: PayloadSparseTrieKind) -> Self {
|
||||
Self { kind, state: Arc::new(Mutex::new(PayloadSparseTrieState::default())) }
|
||||
}
|
||||
|
||||
/// Acquires a guard that blocks `take()` until dropped.
|
||||
/// Use this before sending the state root result to ensure the next block
|
||||
/// waits for the trie to be stored.
|
||||
pub(super) fn lock(&self) -> PreservedTrieGuard<'_> {
|
||||
PreservedTrieGuard(self.0.lock())
|
||||
/// Returns the sparse trie implementation used when the cache needs to create a new trie.
|
||||
pub const fn kind(&self) -> PayloadSparseTrieKind {
|
||||
self.kind
|
||||
}
|
||||
|
||||
/// Takes a preserved trie for `parent_state_root` or creates a new trie if the cache is empty.
|
||||
pub fn take_or_create_for(&self, parent_state_root: B256) -> SparseTrieCheckout {
|
||||
let start = Instant::now();
|
||||
let mut state = self.state.lock();
|
||||
state.latest_checkout_id += 1;
|
||||
let checkout_id = state.latest_checkout_id;
|
||||
let trie = state
|
||||
.preserved
|
||||
.take()
|
||||
.map(|preserved| preserved.into_trie_for(parent_state_root))
|
||||
.unwrap_or_else(|| {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
%parent_state_root,
|
||||
kind = ?self.kind,
|
||||
"Creating new sparse trie - no preserved trie available"
|
||||
);
|
||||
new_sparse_trie(self.kind)
|
||||
});
|
||||
drop(state);
|
||||
|
||||
let elapsed = start.elapsed();
|
||||
if elapsed.as_millis() > 5 {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
blocked_for=?elapsed,
|
||||
"Waited for preserved sparse trie checkout"
|
||||
);
|
||||
}
|
||||
|
||||
SparseTrieCheckout { trie: Some(trie), cache: self.clone(), checkout_id }
|
||||
}
|
||||
|
||||
/// Waits until the sparse trie lock becomes available.
|
||||
///
|
||||
/// This acquires and immediately releases the lock, ensuring that any
|
||||
/// ongoing operations complete before returning. Useful for synchronization
|
||||
/// before starting payload processing.
|
||||
///
|
||||
/// Returns the time spent waiting for the lock.
|
||||
pub(super) fn wait_for_availability(&self) -> std::time::Duration {
|
||||
pub fn wait_for_availability(&self) -> Duration {
|
||||
let start = Instant::now();
|
||||
let _guard = self.0.lock();
|
||||
let _guard = self.state.lock();
|
||||
let elapsed = start.elapsed();
|
||||
if elapsed.as_millis() > 5 {
|
||||
debug!(
|
||||
@@ -49,27 +133,142 @@ impl SharedPreservedSparseTrie {
|
||||
}
|
||||
elapsed
|
||||
}
|
||||
|
||||
/// Acquires a guard that blocks cache mutation until dropped.
|
||||
///
|
||||
/// Engine-internal code uses this before making the state-root result visible so the next
|
||||
/// payload cannot observe an empty cache between send and store.
|
||||
pub(super) fn lock(&self) -> PreservedTrieGuard<'_> {
|
||||
PreservedTrieGuard { state: self.state.lock() }
|
||||
}
|
||||
}
|
||||
|
||||
/// A checked-out sparse trie lease.
|
||||
///
|
||||
/// This dereferences to [`SparseStateTrie`] so callers can reuse the trie directly. If the lease is
|
||||
/// dropped without being stored back, a cleared trie is returned to the shared cache unless a newer
|
||||
/// checkout has already superseded it.
|
||||
#[derive(Debug)]
|
||||
pub struct SparseTrieCheckout {
|
||||
trie: Option<SparseTrie>,
|
||||
cache: PayloadSparseTrieCache,
|
||||
checkout_id: u64,
|
||||
}
|
||||
|
||||
impl SparseTrieCheckout {
|
||||
/// Stores the trie back into the shared cache anchored to the given state root.
|
||||
pub fn store_anchored(self, state_root: B256) -> PayloadSparseTrieStoreOutcome {
|
||||
let cache = self.cache.clone();
|
||||
let mut guard = cache.lock();
|
||||
self.store_anchored_with_guard(&mut guard, state_root)
|
||||
}
|
||||
|
||||
/// Stores the trie back into the shared cache in a cleared state.
|
||||
pub fn store_cleared(mut self) -> PayloadSparseTrieStoreOutcome {
|
||||
let cache = self.cache.clone();
|
||||
let mut trie = self.take_trie();
|
||||
prepare_cleared_trie(&mut trie);
|
||||
let deferred = trie.take_deferred_drops();
|
||||
let mut guard = cache.lock();
|
||||
let outcome = guard.store(self.checkout_id, PreservedSparseTrie::cleared(trie));
|
||||
drop(guard);
|
||||
drop(deferred);
|
||||
outcome
|
||||
}
|
||||
|
||||
/// Stores the trie back into the shared cache anchored to the given state root while the
|
||||
/// caller is already holding the preservation lock.
|
||||
pub(super) fn store_anchored_with_guard(
|
||||
mut self,
|
||||
guard: &mut PreservedTrieGuard<'_>,
|
||||
state_root: B256,
|
||||
) -> PayloadSparseTrieStoreOutcome {
|
||||
guard.store(self.checkout_id, PreservedSparseTrie::anchored(self.take_trie(), state_root))
|
||||
}
|
||||
|
||||
/// Stores an already-cleared trie back into the shared cache while the caller is already
|
||||
/// holding the preservation lock.
|
||||
pub(super) fn store_prepared_cleared_with_guard(
|
||||
mut self,
|
||||
guard: &mut PreservedTrieGuard<'_>,
|
||||
) -> PayloadSparseTrieStoreOutcome {
|
||||
guard.store(self.checkout_id, PreservedSparseTrie::cleared(self.take_trie()))
|
||||
}
|
||||
|
||||
fn take_trie(&mut self) -> SparseTrie {
|
||||
self.trie.take().expect("sparse trie checkout must hold a trie until it is stored")
|
||||
}
|
||||
}
|
||||
|
||||
impl Deref for SparseTrieCheckout {
|
||||
type Target = SparseTrie;
|
||||
|
||||
fn deref(&self) -> &Self::Target {
|
||||
self.trie.as_ref().expect("sparse trie checkout must hold a trie until it is stored")
|
||||
}
|
||||
}
|
||||
|
||||
impl DerefMut for SparseTrieCheckout {
|
||||
fn deref_mut(&mut self) -> &mut Self::Target {
|
||||
self.trie.as_mut().expect("sparse trie checkout must hold a trie until it is stored")
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for SparseTrieCheckout {
|
||||
fn drop(&mut self) {
|
||||
let Some(mut trie) = self.trie.take() else { return };
|
||||
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
checkout_id = self.checkout_id,
|
||||
"Sparse trie checkout dropped before store, returning cleared trie to cache"
|
||||
);
|
||||
|
||||
prepare_cleared_trie(&mut trie);
|
||||
let deferred = trie.take_deferred_drops();
|
||||
let mut guard = self.cache.lock();
|
||||
let _ = guard.store(self.checkout_id, PreservedSparseTrie::cleared(trie));
|
||||
drop(guard);
|
||||
drop(deferred);
|
||||
}
|
||||
}
|
||||
|
||||
/// Guard that holds the lock on the preserved trie.
|
||||
/// While held, `take()` will block. Call `store()` to save the trie before dropping.
|
||||
pub(super) struct PreservedTrieGuard<'a>(parking_lot::MutexGuard<'a, Option<PreservedSparseTrie>>);
|
||||
/// While held, take-or-create calls will block. Call `store()` to save the trie before dropping.
|
||||
pub(super) struct PreservedTrieGuard<'a> {
|
||||
state: parking_lot::MutexGuard<'a, PayloadSparseTrieState>,
|
||||
}
|
||||
|
||||
impl PreservedTrieGuard<'_> {
|
||||
/// Stores a preserved trie for later reuse.
|
||||
pub(super) fn store(&mut self, trie: PreservedSparseTrie) {
|
||||
self.0.replace(trie);
|
||||
/// Stores a preserved trie for later reuse if the checkout is still current.
|
||||
fn store(
|
||||
&mut self,
|
||||
checkout_id: u64,
|
||||
trie: PreservedSparseTrie,
|
||||
) -> PayloadSparseTrieStoreOutcome {
|
||||
if checkout_id != self.state.latest_checkout_id {
|
||||
debug!(
|
||||
target: "engine::tree::payload_processor",
|
||||
checkout_id,
|
||||
latest_checkout_id = self.state.latest_checkout_id,
|
||||
"Ignoring stale sparse trie checkout"
|
||||
);
|
||||
return PayloadSparseTrieStoreOutcome::IgnoredStaleCheckout;
|
||||
}
|
||||
|
||||
self.state.preserved.replace(trie);
|
||||
PayloadSparseTrieStoreOutcome::Stored
|
||||
}
|
||||
}
|
||||
|
||||
/// A preserved sparse trie that can be reused across payload validations.
|
||||
///
|
||||
/// The trie exists in one of two states:
|
||||
/// - **Anchored**: Has a computed state root and can be reused for payloads whose parent state root
|
||||
/// matches the anchor.
|
||||
/// - **Anchored**: Has a computed state root and can be reused for payloads whose parent state
|
||||
/// root matches the anchor.
|
||||
/// - **Cleared**: Trie data has been cleared but allocations are preserved for reuse.
|
||||
#[derive(Debug)]
|
||||
pub(super) enum PreservedSparseTrie {
|
||||
enum PreservedSparseTrie {
|
||||
/// Trie with a computed state root that can be reused for continuation payloads.
|
||||
Anchored {
|
||||
/// The sparse state trie (pruned after root computation).
|
||||
@@ -87,24 +286,17 @@ pub(super) enum PreservedSparseTrie {
|
||||
|
||||
impl PreservedSparseTrie {
|
||||
/// Creates a new anchored preserved trie.
|
||||
///
|
||||
/// The `state_root` is the computed state root from the trie, which becomes the
|
||||
/// anchor for determining if subsequent payloads can reuse this trie.
|
||||
pub(super) const fn anchored(trie: SparseTrie, state_root: B256) -> Self {
|
||||
const fn anchored(trie: SparseTrie, state_root: B256) -> Self {
|
||||
Self::Anchored { trie, state_root }
|
||||
}
|
||||
|
||||
/// Creates a cleared preserved trie (allocations preserved, data cleared).
|
||||
pub(super) const fn cleared(trie: SparseTrie) -> Self {
|
||||
const fn cleared(trie: SparseTrie) -> Self {
|
||||
Self::Cleared { trie }
|
||||
}
|
||||
|
||||
/// Consumes self and returns the trie for reuse.
|
||||
///
|
||||
/// If the preserved trie is anchored and the parent state root matches, the pruned
|
||||
/// trie structure is reused directly. Otherwise, the trie is cleared but allocations
|
||||
/// are preserved to reduce memory overhead.
|
||||
pub(super) fn into_trie_for(self, parent_state_root: B256) -> SparseTrie {
|
||||
fn into_trie_for(self, parent_state_root: B256) -> SparseTrie {
|
||||
match self {
|
||||
Self::Anchored { trie, state_root } if state_root == parent_state_root => {
|
||||
debug!(
|
||||
@@ -135,3 +327,111 @@ impl PreservedSparseTrie {
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn new_sparse_trie(kind: PayloadSparseTrieKind) -> SparseTrie {
|
||||
let default_trie = match kind {
|
||||
PayloadSparseTrieKind::HashMap => {
|
||||
RevealableSparseTrie::blind_from(ConfigurableSparseTrie::HashMap(
|
||||
ParallelSparseTrie::default()
|
||||
.with_parallelism_thresholds(PARALLEL_SPARSE_TRIE_PARALLELISM_THRESHOLDS),
|
||||
))
|
||||
}
|
||||
PayloadSparseTrieKind::Arena => RevealableSparseTrie::blind_from(
|
||||
ConfigurableSparseTrie::Arena(ArenaParallelSparseTrie::default()),
|
||||
),
|
||||
};
|
||||
|
||||
SparseStateTrie::default()
|
||||
.with_accounts_trie(default_trie.clone())
|
||||
.with_default_storage_trie(default_trie)
|
||||
.with_updates(true)
|
||||
}
|
||||
|
||||
fn prepare_cleared_trie(trie: &mut SparseTrie) {
|
||||
trie.clear();
|
||||
trie.shrink_to(SPARSE_TRIE_MAX_NODES_SHRINK_CAPACITY, SPARSE_TRIE_MAX_VALUES_SHRINK_CAPACITY);
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn take_or_create_reuses_matching_anchor() {
|
||||
let cache = PayloadSparseTrieCache::default();
|
||||
let state_root = B256::with_last_byte(1);
|
||||
|
||||
assert_eq!(
|
||||
cache.take_or_create_for(state_root).store_anchored(state_root),
|
||||
PayloadSparseTrieStoreOutcome::Stored
|
||||
);
|
||||
|
||||
match cache.state.lock().preserved.as_ref() {
|
||||
Some(PreservedSparseTrie::Anchored { state_root: anchored, .. }) => {
|
||||
assert_eq!(*anchored, state_root);
|
||||
}
|
||||
other => panic!("expected anchored trie, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn drop_restores_cleared_trie() {
|
||||
let cache = PayloadSparseTrieCache::default();
|
||||
let state_root = B256::with_last_byte(2);
|
||||
|
||||
let mut checkout = cache.take_or_create_for(state_root);
|
||||
checkout.set_updates(true);
|
||||
drop(checkout);
|
||||
|
||||
match cache.state.lock().preserved.as_ref() {
|
||||
Some(PreservedSparseTrie::Cleared { .. }) => {}
|
||||
other => panic!("expected cleared trie, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_checkout_does_not_overwrite_newer_store() {
|
||||
let cache = PayloadSparseTrieCache::default();
|
||||
let parent_state_root = B256::with_last_byte(3);
|
||||
let anchored_state_root = B256::with_last_byte(4);
|
||||
|
||||
let stale = cache.take_or_create_for(parent_state_root);
|
||||
let fresh = cache.take_or_create_for(parent_state_root);
|
||||
|
||||
assert_eq!(
|
||||
fresh.store_anchored(anchored_state_root),
|
||||
PayloadSparseTrieStoreOutcome::Stored
|
||||
);
|
||||
assert_eq!(stale.store_cleared(), PayloadSparseTrieStoreOutcome::IgnoredStaleCheckout);
|
||||
|
||||
match cache.state.lock().preserved.as_ref() {
|
||||
Some(PreservedSparseTrie::Anchored { state_root, .. }) => {
|
||||
assert_eq!(*state_root, anchored_state_root);
|
||||
}
|
||||
other => panic!("expected anchored trie to survive stale checkout, got {other:?}"),
|
||||
}
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn stale_checkout_drop_does_not_overwrite_newer_store() {
|
||||
let cache = PayloadSparseTrieCache::default();
|
||||
let parent_state_root = B256::with_last_byte(5);
|
||||
let anchored_state_root = B256::with_last_byte(6);
|
||||
|
||||
let stale = cache.take_or_create_for(parent_state_root);
|
||||
let fresh = cache.take_or_create_for(parent_state_root);
|
||||
|
||||
assert_eq!(
|
||||
fresh.store_anchored(anchored_state_root),
|
||||
PayloadSparseTrieStoreOutcome::Stored
|
||||
);
|
||||
drop(stale);
|
||||
|
||||
match cache.state.lock().preserved.as_ref() {
|
||||
Some(PreservedSparseTrie::Anchored { state_root, .. }) => {
|
||||
assert_eq!(*state_root, anchored_state_root);
|
||||
}
|
||||
other => panic!("expected anchored trie to survive stale checkout drop, got {other:?}"),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -22,7 +22,6 @@ use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{keccak256, StorageKey, B256};
|
||||
use crossbeam_channel::Sender as CrossbeamSender;
|
||||
use itertools::{EitherOrBoth, Itertools};
|
||||
use metrics::{Counter, Gauge, Histogram};
|
||||
use rayon::prelude::*;
|
||||
use reth_evm::{execute::ExecutableTxFor, ConfigureEvm, Evm, EvmFor, RecoveredTx, SpecFor};
|
||||
@@ -40,7 +39,7 @@ use std::sync::{
|
||||
mpsc::{self, channel, Receiver, Sender},
|
||||
Arc,
|
||||
};
|
||||
use tracing::{debug, debug_span, instrument, trace, trace_span, warn, Span};
|
||||
use tracing::{debug, debug_span, instrument, trace, warn, Span};
|
||||
|
||||
/// Determines the prewarming mode: transaction-based, BAL-based, or skipped.
|
||||
#[derive(Debug)]
|
||||
@@ -76,8 +75,6 @@ where
|
||||
actions_rx: Receiver<PrewarmTaskEvent<N::Receipt>>,
|
||||
/// Parent span for tracing
|
||||
parent_span: Span,
|
||||
/// Whether to disable BAL-driven parallel state root computation.
|
||||
disable_bal_parallel_state_root: bool,
|
||||
}
|
||||
|
||||
impl<N, P, Evm> PrewarmCacheTask<N, P, Evm>
|
||||
@@ -87,12 +84,11 @@ where
|
||||
Evm: ConfigureEvm<Primitives = N> + 'static,
|
||||
{
|
||||
/// Initializes the task with the given transactions pending execution
|
||||
pub fn new(
|
||||
pub(crate) fn new(
|
||||
executor: Runtime,
|
||||
execution_cache: PayloadExecutionCache,
|
||||
ctx: PrewarmContext<N, P, Evm>,
|
||||
to_multi_proof: Option<CrossbeamSender<MultiProofMessage>>,
|
||||
disable_bal_parallel_state_root: bool,
|
||||
) -> (Self, Sender<PrewarmTaskEvent<N::Receipt>>) {
|
||||
let (actions_tx, actions_rx) = channel();
|
||||
|
||||
@@ -111,7 +107,6 @@ where
|
||||
to_multi_proof,
|
||||
actions_rx,
|
||||
parent_span: Span::current(),
|
||||
disable_bal_parallel_state_root,
|
||||
},
|
||||
actions_tx,
|
||||
)
|
||||
@@ -170,7 +165,7 @@ where
|
||||
tx_count += 1;
|
||||
let parent_span = Span::current();
|
||||
s.spawn(move |_| {
|
||||
let _enter = trace_span!(
|
||||
let _enter = debug_span!(
|
||||
target: "engine::tree::payload_processor::prewarm",
|
||||
parent: parent_span,
|
||||
"prewarm_tx",
|
||||
@@ -295,19 +290,24 @@ where
|
||||
let new_cache = SavedCache::new(hash, caches, cache_metrics)
|
||||
.with_disable_cache_metrics(disable_cache_metrics);
|
||||
|
||||
// Insert state into cache while holding the lock
|
||||
// Access the BundleState through the shared ExecutionOutcome
|
||||
if new_cache.cache().insert_state(&execution_outcome.state).is_err() {
|
||||
// Clear the cache on error to prevent having a polluted cache
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
return;
|
||||
}
|
||||
|
||||
new_cache.update_metrics();
|
||||
|
||||
// Defer `insert_state` until after validation: FixedCache's non-blocking
|
||||
// inserts silently drop writes under concurrent reader contention.
|
||||
if valid_block_rx.recv().is_ok() {
|
||||
if new_cache.cache().insert_state(&execution_outcome.state).is_err() {
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on update error");
|
||||
return;
|
||||
}
|
||||
|
||||
// Replace the shared cache with the new one; the previous cache (if any) is
|
||||
// dropped.
|
||||
*cached = Some(new_cache);
|
||||
} else {
|
||||
// Block was invalid; caches were already mutated by insert_state above,
|
||||
// so we must clear to prevent using polluted state
|
||||
*cached = None;
|
||||
debug!(target: "engine::caching", "cleared execution cache on invalid block");
|
||||
}
|
||||
@@ -356,12 +356,7 @@ where
|
||||
let ctx = self.ctx.clone();
|
||||
self.executor.prewarming_pool().install_fn(|| {
|
||||
bal.par_iter().for_each_init(
|
||||
|| {
|
||||
(
|
||||
ctx.clone(),
|
||||
None::<CachedStateProvider<reth_provider::StateProviderBox, true>>,
|
||||
)
|
||||
},
|
||||
|| (ctx.clone(), None::<CachedStateProvider<reth_provider::StateProviderBox>>),
|
||||
|(ctx, provider), account| {
|
||||
if ctx.should_stop() {
|
||||
return;
|
||||
@@ -386,9 +381,6 @@ where
|
||||
/// Converts the BAL to [`HashedPostState`](reth_trie::HashedPostState) and sends it to the
|
||||
/// multiproof task.
|
||||
fn send_bal_hashed_state(&self, bal: &BlockAccessList) {
|
||||
if self.disable_bal_parallel_state_root {
|
||||
return;
|
||||
}
|
||||
let Some(to_multi_proof) = &self.to_multi_proof else { return };
|
||||
|
||||
let provider = match self.ctx.provider.build() {
|
||||
@@ -523,8 +515,6 @@ where
|
||||
pub precompile_cache_disabled: bool,
|
||||
/// The precompile cache map.
|
||||
pub precompile_cache_map: PrecompileCacheMap<SpecFor<Evm>>,
|
||||
/// Whether BAL batched IO is disabled.
|
||||
pub disable_bal_batch_io: bool,
|
||||
}
|
||||
|
||||
/// Per-thread EVM state initialised by [`PrewarmContext::evm_for_ctx`] and stored in
|
||||
@@ -610,7 +600,7 @@ where
|
||||
/// thread.
|
||||
fn prefetch_bal_account(
|
||||
&self,
|
||||
provider: &mut Option<CachedStateProvider<reth_provider::StateProviderBox, true>>,
|
||||
provider: &mut Option<CachedStateProvider<reth_provider::StateProviderBox>>,
|
||||
account: &alloy_eip7928::AccountChanges,
|
||||
) {
|
||||
let state_provider = match provider {
|
||||
@@ -631,7 +621,7 @@ where
|
||||
self.saved_cache.as_ref().expect("BAL prewarm should only run with cache");
|
||||
let caches = saved_cache.cache().clone();
|
||||
let cache_metrics = saved_cache.metrics().clone();
|
||||
slot.insert(CachedStateProvider::new_prewarm(built, caches, cache_metrics))
|
||||
slot.insert(CachedStateProvider::new(built, caches, cache_metrics))
|
||||
}
|
||||
};
|
||||
|
||||
@@ -639,24 +629,11 @@ where
|
||||
|
||||
let _ = state_provider.basic_account(&account.address);
|
||||
|
||||
if self.disable_bal_batch_io {
|
||||
for slot in &account.storage_changes {
|
||||
let _ = state_provider.storage(account.address, StorageKey::from(slot.slot));
|
||||
}
|
||||
for &slot in &account.storage_reads {
|
||||
let _ = state_provider.storage(account.address, StorageKey::from(slot));
|
||||
}
|
||||
} else {
|
||||
let slots: Vec<StorageKey> = account
|
||||
.storage_changes
|
||||
.iter()
|
||||
.map(|s| StorageKey::from(s.slot))
|
||||
.merge_join_by(account.storage_reads.iter().map(|&s| StorageKey::from(s)), Ord::cmp)
|
||||
.map(|either| match either {
|
||||
EitherOrBoth::Left(k) | EitherOrBoth::Right(k) | EitherOrBoth::Both(k, _) => k,
|
||||
})
|
||||
.collect();
|
||||
let _ = state_provider.storage_range(account.address, &slots);
|
||||
for slot in &account.storage_changes {
|
||||
let _ = state_provider.storage(account.address, StorageKey::from(slot.slot));
|
||||
}
|
||||
for &slot in &account.storage_reads {
|
||||
let _ = state_provider.storage(account.address, StorageKey::from(slot));
|
||||
}
|
||||
|
||||
self.metrics.bal_slot_iteration_duration.record(start.elapsed().as_secs_f64());
|
||||
|
||||
@@ -7,7 +7,7 @@ use crate::tree::{
|
||||
dispatch_with_chunking, evm_state_to_hashed_post_state, MultiProofMessage,
|
||||
DEFAULT_MAX_TARGETS_FOR_CHUNKING,
|
||||
},
|
||||
payload_processor::multiproof::MultiProofTaskMetrics,
|
||||
payload_processor::{multiproof::MultiProofTaskMetrics, SparseTrieCheckout},
|
||||
};
|
||||
use alloy_primitives::B256;
|
||||
use alloy_rlp::{Decodable, Encodable};
|
||||
@@ -30,7 +30,7 @@ use reth_trie_parallel::{
|
||||
use reth_trie_sparse::debug_recorder::TrieDebugRecorder;
|
||||
use reth_trie_sparse::{
|
||||
errors::SparseTrieResult, ConfigurableSparseTrie, DeferredDrops, LeafUpdate,
|
||||
RevealableSparseTrie, SparseStateTrie, SparseTrie,
|
||||
RevealableSparseTrie,
|
||||
};
|
||||
use revm_primitives::{hash_map::Entry, B256Map};
|
||||
use tracing::{debug, debug_span, error, instrument, trace_span};
|
||||
@@ -39,7 +39,7 @@ use tracing::{debug, debug_span, error, instrument, trace_span};
|
||||
const MAX_PENDING_UPDATES: usize = 100;
|
||||
|
||||
/// Sparse trie task implementation that uses in-memory sparse trie data to schedule proof fetching.
|
||||
pub(super) struct SparseTrieCacheTask<A = ConfigurableSparseTrie, S = ConfigurableSparseTrie> {
|
||||
pub(super) struct SparseTrieCacheTask {
|
||||
/// Sender for proof results.
|
||||
proof_result_tx: CrossbeamSender<ProofResultMessage>,
|
||||
/// Receiver for proof results directly from workers.
|
||||
@@ -47,7 +47,7 @@ pub(super) struct SparseTrieCacheTask<A = ConfigurableSparseTrie, S = Configurab
|
||||
/// Receives updates from execution and prewarming.
|
||||
updates: CrossbeamReceiver<SparseTrieTaskMessage>,
|
||||
/// `SparseStateTrie` used for computing the state root.
|
||||
trie: SparseStateTrie<A, S>,
|
||||
trie: SparseTrieCheckout,
|
||||
/// Handle to the proof worker pools (storage and account).
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
|
||||
@@ -110,18 +110,14 @@ pub(super) struct SparseTrieCacheTask<A = ConfigurableSparseTrie, S = Configurab
|
||||
metrics: MultiProofTaskMetrics,
|
||||
}
|
||||
|
||||
impl<A, S> SparseTrieCacheTask<A, S>
|
||||
where
|
||||
A: SparseTrie + Default,
|
||||
S: SparseTrie + Default + Clone,
|
||||
{
|
||||
impl SparseTrieCacheTask {
|
||||
/// Creates a new sparse trie, pre-populating with an existing [`SparseStateTrie`].
|
||||
pub(super) fn new_with_trie(
|
||||
pub(super) fn new_with_checkout(
|
||||
executor: &Runtime,
|
||||
updates: CrossbeamReceiver<MultiProofMessage>,
|
||||
proof_worker_handle: ProofWorkerHandle,
|
||||
metrics: MultiProofTaskMetrics,
|
||||
trie: SparseStateTrie<A, S>,
|
||||
trie: SparseTrieCheckout,
|
||||
chunk_size: usize,
|
||||
) -> Self {
|
||||
let (proof_result_tx, proof_result_rx) = crossbeam_channel::unbounded();
|
||||
@@ -205,7 +201,7 @@ where
|
||||
max_values_capacity: usize,
|
||||
disable_pruning: bool,
|
||||
updates: &TrieUpdates,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
) -> (SparseTrieCheckout, DeferredDrops) {
|
||||
let Self { mut trie, .. } = self;
|
||||
trie.commit_updates(updates);
|
||||
if !disable_pruning {
|
||||
@@ -224,7 +220,7 @@ where
|
||||
self,
|
||||
max_nodes_capacity: usize,
|
||||
max_values_capacity: usize,
|
||||
) -> (SparseStateTrie<A, S>, DeferredDrops) {
|
||||
) -> (SparseTrieCheckout, DeferredDrops) {
|
||||
let Self { mut trie, .. } = self;
|
||||
trie.clear();
|
||||
trie.shrink_to(max_nodes_capacity, max_values_capacity);
|
||||
@@ -306,20 +302,14 @@ where
|
||||
self.promote_pending_account_updates()?;
|
||||
self.metrics.sparse_trie_process_updates_duration_histogram.record(t.elapsed());
|
||||
|
||||
if self.finished_state_updates &&
|
||||
self.account_updates.is_empty() &&
|
||||
self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
|
||||
if self.finished_state_updates
|
||||
&& self.account_updates.is_empty()
|
||||
&& self.storage_updates.iter().all(|(_, updates)| updates.is_empty())
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
self.dispatch_pending_targets();
|
||||
|
||||
// If there's still no pending updates spend some time pre-computing the account
|
||||
// trie upper hashes
|
||||
if self.proof_result_rx.is_empty() {
|
||||
self.trie.calculate_subtries();
|
||||
}
|
||||
} else if self.updates.is_empty() || self.pending_updates > MAX_PENDING_UPDATES {
|
||||
// If we don't have any pending updates OR we've accumulated a lot already, apply
|
||||
// them to the trie,
|
||||
@@ -606,7 +596,6 @@ where
|
||||
|
||||
Ok(updates_len_after < updates_len_before)
|
||||
}
|
||||
|
||||
/// Computes storage roots for accounts whose storage updates are fully drained.
|
||||
///
|
||||
/// For each storage trie T that:
|
||||
@@ -627,16 +616,16 @@ where
|
||||
.filter_map(|(address, updates)| updates.is_empty().then_some(*address))
|
||||
.collect();
|
||||
|
||||
struct SendStorageTriePtr<S>(*mut RevealableSparseTrie<S>);
|
||||
struct SendStorageTriePtr(*mut RevealableSparseTrie<ConfigurableSparseTrie>);
|
||||
// SAFETY: this wrapper only forwards the pointer across rayon; deref invariants are
|
||||
// documented at the use site below.
|
||||
unsafe impl<S: Send> Send for SendStorageTriePtr<S> {}
|
||||
unsafe impl Send for SendStorageTriePtr {}
|
||||
|
||||
let mut tries_to_compute_roots: Vec<(B256, SendStorageTriePtr<S>)> =
|
||||
let mut tries_to_compute_roots: Vec<(B256, SendStorageTriePtr)> =
|
||||
Vec::with_capacity(addresses_to_compute_roots.len());
|
||||
for address in addresses_to_compute_roots {
|
||||
if let Some(trie) = self.trie.storage_tries_mut().get_mut(&address) &&
|
||||
!trie.is_root_cached()
|
||||
if let Some(trie) = self.trie.storage_tries_mut().get_mut(&address)
|
||||
&& !trie.is_root_cached()
|
||||
{
|
||||
tries_to_compute_roots.push((address, SendStorageTriePtr(trie)));
|
||||
}
|
||||
@@ -735,7 +724,7 @@ where
|
||||
// We need to keep iterating if any updates are being drained because that might
|
||||
// indicate that more pending account updates can be promoted.
|
||||
if num_promoted == 0 || !self.process_account_leaf_updates(false)? {
|
||||
break
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -856,7 +845,6 @@ pub struct StateRootComputeOutcome {
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{keccak256, Address, B256, U256};
|
||||
use reth_trie_sparse::ArenaParallelSparseTrie;
|
||||
|
||||
#[test]
|
||||
fn test_run_hashing_task_hashed_state_update_forwards() {
|
||||
@@ -879,10 +867,7 @@ mod tests {
|
||||
let expected_state = hashed_state.clone();
|
||||
|
||||
let handle = std::thread::spawn(move || {
|
||||
SparseTrieCacheTask::<ArenaParallelSparseTrie, ArenaParallelSparseTrie>::run_hashing_task(
|
||||
updates_rx,
|
||||
hashed_state_tx,
|
||||
);
|
||||
SparseTrieCacheTask::run_hashing_task(updates_rx, hashed_state_tx);
|
||||
});
|
||||
|
||||
updates_tx.send(MultiProofMessage::HashedStateUpdate(hashed_state)).unwrap();
|
||||
|
||||
@@ -4,7 +4,7 @@ use crate::tree::{
|
||||
cached_state::{CacheStats, CachedStateProvider},
|
||||
error::{InsertBlockError, InsertBlockErrorKind, InsertPayloadError},
|
||||
instrumented_state::{InstrumentedStateProvider, StateProviderStats},
|
||||
payload_processor::PayloadProcessor,
|
||||
payload_processor::{EngineSharedCaches, PayloadProcessor},
|
||||
precompile_cache::{CachedPrecompile, CachedPrecompileMetrics, PrecompileCacheMap},
|
||||
sparse_trie::StateRootComputeOutcome,
|
||||
CacheWaitDurations, EngineApiMetrics, EngineApiTreeState, ExecutionEnv, PayloadHandle,
|
||||
@@ -15,7 +15,6 @@ use alloy_eip7928::BlockAccessList;
|
||||
use alloy_eips::{eip1898::BlockWithParent, eip4895::Withdrawal, NumHash};
|
||||
use alloy_evm::Evm;
|
||||
use alloy_primitives::{map::B256Set, B256};
|
||||
use alloy_rlp::Decodable;
|
||||
#[cfg(feature = "trie-debug")]
|
||||
use reth_trie_sparse::debug_recorder::TrieDebugRecorder;
|
||||
|
||||
@@ -191,16 +190,13 @@ where
|
||||
validator: V,
|
||||
config: TreeConfig,
|
||||
invalid_block_hook: Box<dyn InvalidBlockHook<N>>,
|
||||
shared_caches: EngineSharedCaches<Evm>,
|
||||
changeset_cache: ChangesetCache,
|
||||
runtime: reth_tasks::Runtime,
|
||||
) -> Self {
|
||||
let precompile_cache_map = PrecompileCacheMap::default();
|
||||
let payload_processor = PayloadProcessor::new(
|
||||
runtime.clone(),
|
||||
evm_config.clone(),
|
||||
&config,
|
||||
precompile_cache_map.clone(),
|
||||
);
|
||||
let precompile_cache_map = shared_caches.precompile_cache_map();
|
||||
let payload_processor =
|
||||
PayloadProcessor::new(runtime.clone(), evm_config.clone(), &config, shared_caches);
|
||||
Self {
|
||||
provider,
|
||||
consensus,
|
||||
@@ -314,7 +310,7 @@ where
|
||||
// Validate block consensus rules which includes header validation
|
||||
if let Err(consensus_err) = self.validate_block_inner(&block, None) {
|
||||
// Header validation error takes precedence over execution error
|
||||
return Err(InsertBlockError::new(block, consensus_err.into()).into())
|
||||
return Err(InsertBlockError::new(block, consensus_err.into()).into());
|
||||
}
|
||||
|
||||
// Also validate against the parent
|
||||
@@ -322,7 +318,7 @@ where
|
||||
self.consensus.validate_header_against_parent(block.sealed_header(), parent_block)
|
||||
{
|
||||
// Parent validation error takes precedence over execution error
|
||||
return Err(InsertBlockError::new(block, consensus_err.into()).into())
|
||||
return Err(InsertBlockError::new(block, consensus_err.into()).into());
|
||||
}
|
||||
|
||||
// No header validation errors, return the original execution error
|
||||
@@ -397,7 +393,7 @@ where
|
||||
Ok(val) => val,
|
||||
Err(e) => {
|
||||
let block = convert_to_block(input)?;
|
||||
return Err(InsertBlockError::new(block, e.into()).into())
|
||||
return Err(InsertBlockError::new(block, e.into()).into());
|
||||
}
|
||||
}
|
||||
};
|
||||
@@ -430,7 +426,7 @@ where
|
||||
convert_to_block(input)?,
|
||||
ProviderError::HeaderNotFound(parent_hash.into()).into(),
|
||||
)
|
||||
.into())
|
||||
.into());
|
||||
};
|
||||
let mut state_provider = ensure_ok!(provider_builder.build());
|
||||
drop(_enter);
|
||||
@@ -443,7 +439,7 @@ where
|
||||
convert_to_block(input)?,
|
||||
ProviderError::HeaderNotFound(parent_hash.into()).into(),
|
||||
)
|
||||
.into())
|
||||
.into());
|
||||
};
|
||||
|
||||
let evm_env = debug_span!(target: "engine::tree::payload_validator", "evm_env")
|
||||
@@ -763,7 +759,7 @@ where
|
||||
)
|
||||
.into(),
|
||||
)
|
||||
.into())
|
||||
.into());
|
||||
}
|
||||
|
||||
let timing_stats = state_provider_stats.map(|stats| {
|
||||
@@ -825,14 +821,14 @@ where
|
||||
) -> Result<(), ConsensusError> {
|
||||
if let Err(e) = self.consensus.validate_header(block.sealed_header()) {
|
||||
error!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {}: {e}", block.hash());
|
||||
return Err(e)
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
if let Err(e) =
|
||||
self.consensus.validate_block_pre_execution_with_tx_root(block, transaction_root)
|
||||
{
|
||||
error!(target: "engine::tree::payload_validator", ?block, "Failed to validate block {}: {e}", block.hash());
|
||||
return Err(e)
|
||||
return Err(e);
|
||||
}
|
||||
|
||||
Ok(())
|
||||
@@ -865,21 +861,15 @@ where
|
||||
S: StateProvider + Send,
|
||||
Err: core::error::Error + Send + Sync + 'static,
|
||||
V: PayloadValidator<T, Block = N::Block>,
|
||||
T: PayloadTypes<
|
||||
BuiltPayload: BuiltPayload<Primitives = N>,
|
||||
ExecutionData: ExecutionPayload,
|
||||
>,
|
||||
T: PayloadTypes<BuiltPayload: BuiltPayload<Primitives = N>>,
|
||||
Evm: ConfigureEngineEvm<T::ExecutionData, Primitives = N>,
|
||||
{
|
||||
debug!(target: "engine::tree::payload_validator", "Executing block");
|
||||
|
||||
let has_bal = input.block_access_list().is_some();
|
||||
|
||||
let mut db = debug_span!(target: "engine::tree", "build_state_db").in_scope(|| {
|
||||
State::builder()
|
||||
.with_database(StateProviderDatabase::new(state_provider))
|
||||
.with_bundle_update()
|
||||
.with_bal_builder_if(has_bal)
|
||||
.build()
|
||||
});
|
||||
|
||||
@@ -938,7 +928,6 @@ where
|
||||
handle.iter_transactions(),
|
||||
&receipt_tx,
|
||||
&executed_tx_index,
|
||||
has_bal,
|
||||
)?;
|
||||
drop(receipt_tx);
|
||||
|
||||
@@ -953,32 +942,6 @@ where
|
||||
debug_span!(target: "engine::tree", "merge_transitions")
|
||||
.in_scope(|| db.merge_transitions(BundleRetention::Reverts));
|
||||
|
||||
// Validate BAL hash if we executed with BAL tracking
|
||||
if has_bal {
|
||||
// Get the expected BAL from input and the built BAL from execution
|
||||
let expected_bal =
|
||||
input.block_access_list().transpose().map_err(BlockExecutionError::other)?;
|
||||
|
||||
let built_bal = db.take_built_alloy_bal();
|
||||
|
||||
// Compute hashes and compare
|
||||
let expected_hash = expected_bal
|
||||
.as_ref()
|
||||
.map(|bal| alloy_eips::eip7928::compute_block_access_list_hash(bal));
|
||||
|
||||
let built_hash = built_bal
|
||||
.as_ref()
|
||||
.map(|bal| alloy_eips::eip7928::compute_block_access_list_hash(bal));
|
||||
|
||||
if let (Some(expected), Some(got)) = (expected_hash, built_hash) &&
|
||||
expected != got
|
||||
{
|
||||
return Err(InsertBlockErrorKind::Consensus(
|
||||
ConsensusError::BlockAccessListHashMismatch((got, expected).into()),
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
let output = BlockExecutionOutput { result, state: db.take_bundle() };
|
||||
|
||||
let execution_duration = execution_start.elapsed();
|
||||
@@ -996,21 +959,18 @@ where
|
||||
/// - Executing each transaction with timing metrics
|
||||
/// - Streaming receipts to the receipt root computation task
|
||||
/// - Collecting transaction senders for later use
|
||||
/// - Bumping BAL index after each transaction when BAL tracking is enabled
|
||||
///
|
||||
/// Returns the executor (for finalization) and the collected senders.
|
||||
fn execute_transactions<'a, E, Tx, InnerTx, Err, DB>(
|
||||
fn execute_transactions<E, Tx, InnerTx, Err>(
|
||||
&self,
|
||||
mut executor: E,
|
||||
transaction_count: usize,
|
||||
transactions: impl Iterator<Item = Result<Tx, Err>>,
|
||||
receipt_tx: &crossbeam_channel::Sender<IndexedReceipt<N::Receipt>>,
|
||||
executed_tx_index: &AtomicUsize,
|
||||
has_bal: bool,
|
||||
) -> Result<(E, Vec<Address>), BlockExecutionError>
|
||||
where
|
||||
E: BlockExecutor<Receipt = N::Receipt, Evm: alloy_evm::Evm<DB = &'a mut State<DB>>>,
|
||||
DB: revm::Database + 'a,
|
||||
E: BlockExecutor<Receipt = N::Receipt>,
|
||||
Tx: alloy_evm::block::ExecutableTx<E> + alloy_evm::RecoveredTx<InnerTx>,
|
||||
InnerTx: TxHashRef,
|
||||
Err: core::error::Error + Send + Sync + 'static,
|
||||
@@ -1023,11 +983,6 @@ where
|
||||
.in_scope(|| executor.apply_pre_execution_changes())?;
|
||||
self.metrics.record_pre_execution(pre_exec_start.elapsed());
|
||||
|
||||
// Bump BAL index after pre-execution changes (EIP-7928: index 0 is pre-execution)
|
||||
if has_bal {
|
||||
executor.evm_mut().db_mut().bump_bal_index();
|
||||
}
|
||||
|
||||
// Execute transactions
|
||||
let exec_span = debug_span!(target: "engine::tree", "execution").entered();
|
||||
let mut transactions = transactions.into_iter();
|
||||
@@ -1072,10 +1027,6 @@ where
|
||||
let _ = receipt_tx.send(IndexedReceipt::new(tx_index, receipt.clone()));
|
||||
}
|
||||
}
|
||||
// Bump BAL index after each transaction (EIP-7928)
|
||||
if has_bal {
|
||||
executor.evm_mut().db_mut().bump_bal_index();
|
||||
}
|
||||
}
|
||||
drop(exec_span);
|
||||
|
||||
@@ -1369,7 +1320,7 @@ where
|
||||
trace!(target: "engine::tree::payload_validator", block=?block.num_hash(), "Validating block consensus");
|
||||
// validate block consensus rules
|
||||
if let Err(e) = self.validate_block_inner(block, transaction_root) {
|
||||
return Err(e.into())
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
// now validate against the parent
|
||||
@@ -1378,7 +1329,7 @@ where
|
||||
self.consensus.validate_header_against_parent(block.sealed_header(), parent_block)
|
||||
{
|
||||
warn!(target: "engine::tree::payload_validator", ?block, "Failed to validate header {} against parent: {e}", block.hash());
|
||||
return Err(e.into())
|
||||
return Err(e.into());
|
||||
}
|
||||
drop(_enter);
|
||||
|
||||
@@ -1386,16 +1337,12 @@ where
|
||||
let _enter =
|
||||
debug_span!(target: "engine::tree::payload_validator", "validate_block_post_execution")
|
||||
.entered();
|
||||
if let Err(err) = self.consensus.validate_block_post_execution(
|
||||
block,
|
||||
output,
|
||||
receipt_root_bloom,
|
||||
None,
|
||||
false,
|
||||
) {
|
||||
if let Err(err) =
|
||||
self.consensus.validate_block_post_execution(block, output, receipt_root_bloom)
|
||||
{
|
||||
// call post-block hook
|
||||
self.on_invalid_block(parent_block, block, output, None, ctx.state_mut());
|
||||
return Err(err.into())
|
||||
return Err(err.into());
|
||||
}
|
||||
drop(_enter);
|
||||
|
||||
@@ -1411,7 +1358,7 @@ where
|
||||
{
|
||||
// call post-block hook
|
||||
self.on_invalid_block(parent_block, block, output, None, ctx.state_mut());
|
||||
return Err(err.into())
|
||||
return Err(err.into());
|
||||
}
|
||||
|
||||
// record post-execution validation duration
|
||||
@@ -1519,7 +1466,7 @@ where
|
||||
self.provider.clone(),
|
||||
historical,
|
||||
Some(blocks),
|
||||
)))
|
||||
)));
|
||||
}
|
||||
|
||||
// Check if the block is persisted
|
||||
@@ -1527,7 +1474,7 @@ where
|
||||
debug!(target: "engine::tree::payload_validator", %hash, number = %header.number(), "found canonical state for block in database, creating provider builder");
|
||||
// For persisted blocks, we create a builder that will fetch state directly from the
|
||||
// database
|
||||
return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None)))
|
||||
return Ok(Some(StateProviderBuilder::new(self.provider.clone(), hash, None)));
|
||||
}
|
||||
|
||||
debug!(target: "engine::tree::payload_validator", %hash, "no canonical state found for block");
|
||||
@@ -1559,7 +1506,7 @@ where
|
||||
) {
|
||||
if state.invalid_headers.get(&block.hash()).is_some() {
|
||||
// we already marked this block as invalid
|
||||
return
|
||||
return;
|
||||
}
|
||||
self.invalid_block_hook.on_invalid_block(parent_header, block, output, trie_updates);
|
||||
}
|
||||
@@ -2078,16 +2025,9 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
}
|
||||
|
||||
/// Returns the block access list if available.
|
||||
pub fn block_access_list(&self) -> Option<Result<BlockAccessList, alloy_rlp::Error>>
|
||||
where
|
||||
T::ExecutionData: ExecutionPayload,
|
||||
{
|
||||
match self {
|
||||
Self::Payload(payload) => payload
|
||||
.block_access_list()
|
||||
.map(|bytes| BlockAccessList::decode(&mut bytes.as_ref())),
|
||||
Self::Block(_) => None,
|
||||
}
|
||||
pub const fn block_access_list(&self) -> Option<Result<BlockAccessList, alloy_rlp::Error>> {
|
||||
// TODO decode and return `BlockAccessList`
|
||||
None
|
||||
}
|
||||
|
||||
/// Returns the number of transactions in the payload or block.
|
||||
@@ -2122,15 +2062,4 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
Self::Block(block) => block.gas_used(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the gas limit used by the block.
|
||||
pub fn gas_limit(&self) -> u64
|
||||
where
|
||||
T::ExecutionData: ExecutionPayload,
|
||||
{
|
||||
match self {
|
||||
Self::Payload(payload) => payload.gas_limit(),
|
||||
Self::Block(block) => block.gas_limit(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -74,11 +74,6 @@ impl<N: NodePrimitives> TreeState<N> {
|
||||
self.blocks_by_hash.get(&hash)
|
||||
}
|
||||
|
||||
/// Returns `true` if a block with the given hash exists in memory.
|
||||
pub fn contains_hash(&self, hash: &B256) -> bool {
|
||||
self.blocks_by_hash.contains_key(hash)
|
||||
}
|
||||
|
||||
/// Returns the sealed block header by hash.
|
||||
pub fn sealed_header_by_hash(&self, hash: &B256) -> Option<SealedHeader<N::BlockHeader>> {
|
||||
self.blocks_by_hash.get(hash).map(|b| b.sealed_block().sealed_header().clone())
|
||||
|
||||
@@ -203,6 +203,7 @@ impl TestHarness {
|
||||
payload_validator,
|
||||
TreeConfig::default(),
|
||||
Box::new(NoopInvalidBlockHook::default()),
|
||||
EngineSharedCaches::default(),
|
||||
changeset_cache.clone(),
|
||||
reth_tasks::Runtime::test(),
|
||||
);
|
||||
@@ -407,6 +408,7 @@ impl ValidatorTestHarness {
|
||||
payload_validator,
|
||||
TreeConfig::default(),
|
||||
Box::new(NoopInvalidBlockHook::default()),
|
||||
EngineSharedCaches::default(),
|
||||
changeset_cache,
|
||||
reth_tasks::Runtime::test(),
|
||||
);
|
||||
|
||||
27
crates/engine/tree/tests/shared_caches_sdk.rs
Normal file
27
crates/engine/tree/tests/shared_caches_sdk.rs
Normal file
@@ -0,0 +1,27 @@
|
||||
//! SDK smoke tests for `EngineSharedCaches`.
|
||||
|
||||
use alloy_primitives::B256;
|
||||
use reth_engine_tree::tree::{
|
||||
EngineSharedCaches, PayloadSparseTrieKind, PayloadSparseTrieStoreOutcome,
|
||||
};
|
||||
use reth_evm_ethereum::EthEvmConfig;
|
||||
|
||||
#[test]
|
||||
fn engine_shared_caches_exposes_public_sparse_trie_sdk() {
|
||||
let caches =
|
||||
EngineSharedCaches::<EthEvmConfig>::with_sparse_trie_kind(PayloadSparseTrieKind::Arena);
|
||||
|
||||
let _precompile_cache_map = caches.precompile_cache_map();
|
||||
|
||||
let sparse_trie_cache = caches.sparse_trie_cache();
|
||||
assert_eq!(sparse_trie_cache.kind(), PayloadSparseTrieKind::Arena);
|
||||
let state_root = B256::with_last_byte(1);
|
||||
|
||||
assert_eq!(
|
||||
sparse_trie_cache.take_or_create_for(state_root).store_anchored(state_root),
|
||||
PayloadSparseTrieStoreOutcome::Stored
|
||||
);
|
||||
|
||||
let checkout = sparse_trie_cache.take_or_create_for(state_root);
|
||||
assert!(checkout.memory_size() > 0 || checkout.retained_storage_tries_count() == 0);
|
||||
}
|
||||
@@ -4,6 +4,7 @@ use eyre::{eyre, OptionExt};
|
||||
use futures_util::{stream::StreamExt, Stream, TryStreamExt};
|
||||
use reqwest::{Client, IntoUrl, Url};
|
||||
use reth_era::common::file_ops::EraFileType;
|
||||
use reth_fs_util::FsPathError;
|
||||
use sha2::{Digest, Sha256};
|
||||
use std::{future::Future, path::Path, str::FromStr};
|
||||
use tokio::{
|
||||
@@ -136,7 +137,7 @@ impl<Http: HttpClient + Clone> EraClient<Http> {
|
||||
let Some(number) = self.file_name_to_number(name) &&
|
||||
(number < index || number >= last)
|
||||
{
|
||||
reth_fs_util::remove_file_if_exists(entry.path())?;
|
||||
remove_file_ignore_not_found(entry.path())?;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -321,6 +322,16 @@ impl<Http: HttpClient + Clone> EraClient<Http> {
|
||||
}
|
||||
}
|
||||
|
||||
fn remove_file_ignore_not_found(path: impl AsRef<Path>) -> eyre::Result<()> {
|
||||
match reth_fs_util::remove_file(path) {
|
||||
Ok(()) => Ok(()),
|
||||
Err(FsPathError::RemoveFile { source, .. }) if source.kind() == io::ErrorKind::NotFound => {
|
||||
Ok(())
|
||||
}
|
||||
Err(err) => Err(err.into()),
|
||||
}
|
||||
}
|
||||
|
||||
async fn checksum(mut reader: impl AsyncRead + Unpin) -> eyre::Result<Vec<u8>> {
|
||||
let mut hasher = Sha256::new();
|
||||
|
||||
@@ -367,4 +378,25 @@ mod tests {
|
||||
|
||||
assert_eq!(actual_number, expected_number);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_file_ignore_not_found() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let path = temp_dir.path().join("missing.era1");
|
||||
|
||||
assert!(remove_file_ignore_not_found(&path).is_ok());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_remove_file_ignore_not_found_preserves_other_errors() {
|
||||
let temp_dir = tempfile::tempdir().unwrap();
|
||||
let path = temp_dir.path().join("dir");
|
||||
std::fs::create_dir_all(&path).unwrap();
|
||||
|
||||
let err = remove_file_ignore_not_found(&path).unwrap_err();
|
||||
assert!(matches!(
|
||||
err.downcast_ref::<FsPathError>(),
|
||||
Some(FsPathError::RemoveFile { source, .. }) if source.kind() != io::ErrorKind::NotFound
|
||||
));
|
||||
}
|
||||
}
|
||||
|
||||
@@ -34,8 +34,6 @@ pub(crate) fn create_header() -> Header {
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
block_access_list_hash: None,
|
||||
slot_number: None,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -140,8 +138,6 @@ pub(crate) fn create_test_block_with_compressed_data(number: BlockNumber) -> Blo
|
||||
excess_blob_gas: None,
|
||||
parent_beacon_block_root: None,
|
||||
requests_hash: None,
|
||||
block_access_list_hash: None,
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
// Create test body
|
||||
|
||||
@@ -13,7 +13,7 @@ extern crate alloc;
|
||||
|
||||
use alloc::{fmt::Debug, sync::Arc};
|
||||
use alloy_consensus::{constants::MAXIMUM_EXTRA_DATA_SIZE, EMPTY_OMMER_ROOT_HASH};
|
||||
use alloy_eips::{eip7840::BlobParams, eip7928::BlockAccessList};
|
||||
use alloy_eips::eip7840::BlobParams;
|
||||
use reth_chainspec::{EthChainSpec, EthereumHardforks};
|
||||
use reth_consensus::{
|
||||
Consensus, ConsensusError, FullConsensus, HeaderValidator, ReceiptRootBloom, TransactionRoot,
|
||||
@@ -43,18 +43,12 @@ pub struct EthBeaconConsensus<ChainSpec> {
|
||||
chain_spec: Arc<ChainSpec>,
|
||||
/// Maximum allowed extra data size in bytes
|
||||
max_extra_data_size: usize,
|
||||
/// When true, skips the gas limit change validation between parent and child blocks.
|
||||
skip_gas_limit_ramp_check: bool,
|
||||
}
|
||||
|
||||
impl<ChainSpec: EthChainSpec + EthereumHardforks> EthBeaconConsensus<ChainSpec> {
|
||||
/// Create a new instance of [`EthBeaconConsensus`]
|
||||
pub const fn new(chain_spec: Arc<ChainSpec>) -> Self {
|
||||
Self {
|
||||
chain_spec,
|
||||
max_extra_data_size: MAXIMUM_EXTRA_DATA_SIZE,
|
||||
skip_gas_limit_ramp_check: false,
|
||||
}
|
||||
Self { chain_spec, max_extra_data_size: MAXIMUM_EXTRA_DATA_SIZE }
|
||||
}
|
||||
|
||||
/// Returns the maximum allowed extra data size.
|
||||
@@ -68,12 +62,6 @@ impl<ChainSpec: EthChainSpec + EthereumHardforks> EthBeaconConsensus<ChainSpec>
|
||||
self
|
||||
}
|
||||
|
||||
/// Disables the gas limit change validation between parent and child blocks.
|
||||
pub const fn with_skip_gas_limit_ramp_check(mut self, skip: bool) -> Self {
|
||||
self.skip_gas_limit_ramp_check = skip;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns the chain spec associated with this consensus engine.
|
||||
pub const fn chain_spec(&self) -> &Arc<ChainSpec> {
|
||||
&self.chain_spec
|
||||
@@ -90,8 +78,6 @@ where
|
||||
block: &RecoveredBlock<N::Block>,
|
||||
result: &BlockExecutionResult<N::Receipt>,
|
||||
receipt_root_bloom: Option<ReceiptRootBloom>,
|
||||
block_access_list: Option<BlockAccessList>,
|
||||
allow_bal_check: bool,
|
||||
) -> Result<(), ConsensusError> {
|
||||
validate_block_post_execution(
|
||||
block,
|
||||
@@ -99,9 +85,6 @@ where
|
||||
&result.receipts,
|
||||
&result.requests,
|
||||
receipt_root_bloom,
|
||||
&block_access_list,
|
||||
allow_bal_check,
|
||||
Some(result.gas_used),
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -222,9 +205,7 @@ where
|
||||
|
||||
validate_against_parent_timestamp(header.header(), parent.header())?;
|
||||
|
||||
if !self.skip_gas_limit_ramp_check {
|
||||
validate_against_parent_gas_limit(header, parent, &self.chain_spec)?;
|
||||
}
|
||||
validate_against_parent_gas_limit(header, parent, &self.chain_spec)?;
|
||||
|
||||
validate_against_parent_eip1559_base_fee(
|
||||
header.header(),
|
||||
|
||||
@@ -1,10 +1,6 @@
|
||||
use alloc::vec::Vec;
|
||||
use alloy_consensus::{proofs::calculate_receipt_root, BlockHeader, TxReceipt};
|
||||
use alloy_eips::{
|
||||
eip7685::Requests,
|
||||
eip7928::{compute_block_access_list_hash, BlockAccessList},
|
||||
Encodable2718,
|
||||
};
|
||||
use alloy_eips::{eip7685::Requests, Encodable2718};
|
||||
use alloy_primitives::{Bloom, Bytes, B256};
|
||||
use reth_chainspec::EthereumHardforks;
|
||||
use reth_consensus::ConsensusError;
|
||||
@@ -19,56 +15,27 @@ use reth_primitives_traits::{
|
||||
///
|
||||
/// If `receipt_root_bloom` is provided, the pre-computed receipt root and logs bloom are used
|
||||
/// instead of computing them from the receipts.
|
||||
///
|
||||
/// If `allow_bal_check` is true, we compute the bal hash and match with the header bal hash
|
||||
///
|
||||
/// `gas_spent` is the `gas_used` value from the block execution result. When EIP-7778
|
||||
/// (Amsterdam) is active, block header `gas_used` tracks gas before refunds while receipt
|
||||
/// `cumulative_gas_used` tracks gas after refunds. In that case, the header must be validated
|
||||
/// against the execution result's `gas_used` rather than the receipt value.
|
||||
#[allow(clippy::too_many_arguments)]
|
||||
pub fn validate_block_post_execution<B, R, ChainSpec>(
|
||||
block: &RecoveredBlock<B>,
|
||||
chain_spec: &ChainSpec,
|
||||
receipts: &[R],
|
||||
requests: &Requests,
|
||||
receipt_root_bloom: Option<(B256, Bloom)>,
|
||||
block_access_list: &Option<BlockAccessList>,
|
||||
allow_bal_check: bool,
|
||||
gas_spent: Option<u64>,
|
||||
) -> Result<(), ConsensusError>
|
||||
where
|
||||
B: Block,
|
||||
R: Receipt,
|
||||
ChainSpec: EthereumHardforks,
|
||||
{
|
||||
// EIP-7778: When Amsterdam is active, block header gas_used tracks gas before refunds,
|
||||
// but receipt cumulative_gas_used still tracks gas after refunds. Use the execution
|
||||
// result's gas_used which always matches the header semantics.
|
||||
// Check if gas used matches the value set in header.
|
||||
let cumulative_gas_used =
|
||||
if chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()) {
|
||||
gas_spent.unwrap_or_default()
|
||||
} else {
|
||||
receipts.last().map(|receipt| receipt.cumulative_gas_used()).unwrap_or(0)
|
||||
};
|
||||
receipts.last().map(|receipt| receipt.cumulative_gas_used()).unwrap_or(0);
|
||||
if block.header().gas_used() != cumulative_gas_used {
|
||||
return Err(ConsensusError::BlockGasUsed {
|
||||
gas: GotExpected { got: cumulative_gas_used, expected: block.header().gas_used() },
|
||||
gas_spent_by_tx: gas_spent_by_transactions(receipts),
|
||||
})
|
||||
}
|
||||
// Validate that the block access list hash matches the calculated block access list hash
|
||||
if chain_spec.is_amsterdam_active_at_timestamp(block.header().timestamp()) && allow_bal_check {
|
||||
let block_bal_hash = block.header().block_access_list_hash().unwrap_or_default();
|
||||
let default_bal = BlockAccessList::default();
|
||||
let block_access_list_hash =
|
||||
compute_block_access_list_hash(block_access_list.as_ref().unwrap_or(&default_bal));
|
||||
if block_access_list_hash != block_bal_hash {
|
||||
return Err(ConsensusError::BlockAccessListHashMismatch(
|
||||
(block_access_list_hash, block_bal_hash).into(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
// Before Byzantium, receipts contained state root that would mean that expensive
|
||||
// operation as hashing that is required for state root got calculated in every
|
||||
|
||||
@@ -17,12 +17,11 @@ pub use payload::{payload_id, BlobSidecars, EthBuiltPayload, EthPayloadBuilderAt
|
||||
mod error;
|
||||
pub use error::*;
|
||||
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
|
||||
};
|
||||
use alloy_rpc_types_engine::{ExecutionData, ExecutionPayload};
|
||||
pub use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3, ExecutionPayloadEnvelopeV4,
|
||||
ExecutionPayloadV1, PayloadAttributes as EthPayloadAttributes,
|
||||
ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6, ExecutionPayloadV1,
|
||||
PayloadAttributes as EthPayloadAttributes,
|
||||
};
|
||||
use reth_engine_primitives::EngineTypes;
|
||||
use reth_payload_primitives::{BuiltPayload, PayloadTypes};
|
||||
|
||||
@@ -6,15 +6,13 @@ use alloy_eips::{
|
||||
eip4895::Withdrawals,
|
||||
eip7594::{BlobTransactionSidecarEip7594, BlobTransactionSidecarVariant},
|
||||
eip7685::Requests,
|
||||
eip7928::BlockAccessList,
|
||||
};
|
||||
use alloy_primitives::{Address, B256, U256};
|
||||
use alloy_rlp::Encodable;
|
||||
use alloy_rpc_types_engine::{
|
||||
BlobsBundleV1, BlobsBundleV2, ExecutionPayloadEnvelopeV2, ExecutionPayloadEnvelopeV3,
|
||||
ExecutionPayloadEnvelopeV4, ExecutionPayloadEnvelopeV5, ExecutionPayloadEnvelopeV6,
|
||||
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, ExecutionPayloadV4,
|
||||
PayloadAttributes, PayloadId,
|
||||
ExecutionPayloadFieldV2, ExecutionPayloadV1, ExecutionPayloadV3, PayloadAttributes, PayloadId,
|
||||
};
|
||||
use core::convert::Infallible;
|
||||
use reth_ethereum_primitives::EthPrimitives;
|
||||
@@ -43,8 +41,6 @@ pub struct EthBuiltPayload<N: NodePrimitives = EthPrimitives> {
|
||||
pub(crate) sidecars: BlobSidecars,
|
||||
/// The requests of the payload
|
||||
pub(crate) requests: Option<Requests>,
|
||||
/// The block access list of the payload
|
||||
pub(crate) block_access_list: Option<BlockAccessList>,
|
||||
}
|
||||
|
||||
// === impl BuiltPayload ===
|
||||
@@ -58,9 +54,8 @@ impl<N: NodePrimitives> EthBuiltPayload<N> {
|
||||
block: Arc<SealedBlock<N::Block>>,
|
||||
fees: U256,
|
||||
requests: Option<Requests>,
|
||||
block_access_list: Option<BlockAccessList>,
|
||||
) -> Self {
|
||||
Self { id, block, fees, requests, sidecars: BlobSidecars::Empty, block_access_list }
|
||||
Self { id, block, fees, requests, sidecars: BlobSidecars::Empty }
|
||||
}
|
||||
|
||||
/// Returns the identifier of the payload.
|
||||
@@ -167,35 +162,10 @@ impl EthBuiltPayload {
|
||||
}
|
||||
|
||||
/// Try converting built payload into [`ExecutionPayloadEnvelopeV6`].
|
||||
///
|
||||
/// Note: Amsterdam fork is not yet implemented, so this conversion is not yet supported.
|
||||
pub fn try_into_v6(self) -> Result<ExecutionPayloadEnvelopeV6, BuiltPayloadConversionError> {
|
||||
let Self { block, fees, sidecars, requests, block_access_list, .. } = self;
|
||||
|
||||
let blobs_bundle = match sidecars {
|
||||
BlobSidecars::Empty => BlobsBundleV2::empty(),
|
||||
BlobSidecars::Eip7594(sidecars) => BlobsBundleV2::from(sidecars),
|
||||
BlobSidecars::Eip4844(_) => {
|
||||
return Err(BuiltPayloadConversionError::UnexpectedEip4844Sidecars)
|
||||
}
|
||||
};
|
||||
Ok(ExecutionPayloadEnvelopeV6 {
|
||||
execution_payload: ExecutionPayloadV4::from_block_unchecked_with_bal(
|
||||
block.hash(),
|
||||
&Arc::unwrap_or_clone(block).into_block(),
|
||||
alloy_rlp::encode(block_access_list.unwrap_or_default()).into(),
|
||||
),
|
||||
block_value: fees,
|
||||
// From the engine API spec:
|
||||
//
|
||||
// > Client software **MAY** use any heuristics to decide whether to set
|
||||
// `shouldOverrideBuilder` flag or not. If client software does not implement any
|
||||
// heuristic this flag **SHOULD** be set to `false`.
|
||||
//
|
||||
// Spec:
|
||||
// <https://github.com/ethereum/execution-apis/blob/fe8e13c288c592ec154ce25c534e26cb7ce0530d/src/engine/cancun.md#specification-2>
|
||||
should_override_builder: false,
|
||||
blobs_bundle,
|
||||
execution_requests: requests.unwrap_or_default(),
|
||||
})
|
||||
unimplemented!("ExecutionPayloadEnvelopeV6 not yet supported")
|
||||
}
|
||||
}
|
||||
|
||||
@@ -378,8 +348,6 @@ pub struct EthPayloadBuilderAttributes {
|
||||
pub withdrawals: Withdrawals,
|
||||
/// Root of the parent beacon block
|
||||
pub parent_beacon_block_root: Option<B256>,
|
||||
/// Slot number (EIP-7928, Amsterdam).
|
||||
pub slot_number: Option<u64>,
|
||||
}
|
||||
|
||||
// === impl EthPayloadBuilderAttributes ===
|
||||
@@ -404,7 +372,6 @@ impl EthPayloadBuilderAttributes {
|
||||
prev_randao: attributes.prev_randao,
|
||||
withdrawals: attributes.withdrawals.unwrap_or_default().into(),
|
||||
parent_beacon_block_root: attributes.parent_beacon_block_root,
|
||||
slot_number: attributes.slot_number,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -451,10 +418,6 @@ impl PayloadBuilderAttributes for EthPayloadBuilderAttributes {
|
||||
fn withdrawals(&self) -> &Withdrawals {
|
||||
&self.withdrawals
|
||||
}
|
||||
|
||||
fn slot_number(&self) -> Option<u64> {
|
||||
self.slot_number
|
||||
}
|
||||
}
|
||||
|
||||
/// Generates the payload id for the configured payload from the [`PayloadAttributes`].
|
||||
@@ -477,10 +440,6 @@ pub fn payload_id(parent: &B256, attributes: &PayloadAttributes) -> PayloadId {
|
||||
hasher.update(parent_beacon_block);
|
||||
}
|
||||
|
||||
if let Some(slot_number) = attributes.slot_number {
|
||||
hasher.update(&slot_number.to_be_bytes()[..]);
|
||||
}
|
||||
|
||||
let out = hasher.finalize();
|
||||
|
||||
#[allow(deprecated)] // generic-array 0.14 deprecated
|
||||
@@ -518,7 +477,6 @@ mod tests {
|
||||
.unwrap(),
|
||||
withdrawals: None,
|
||||
parent_beacon_block_root: None,
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
// Verify that the generated payload ID matches the expected value
|
||||
@@ -556,7 +514,6 @@ mod tests {
|
||||
},
|
||||
]),
|
||||
parent_beacon_block_root: None,
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
// Verify that the generated payload ID matches the expected value
|
||||
@@ -589,7 +546,6 @@ mod tests {
|
||||
)
|
||||
.unwrap(),
|
||||
),
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
// Verify that the generated payload ID matches the expected value
|
||||
|
||||
@@ -45,11 +45,11 @@ where
|
||||
execution_ctx: ctx,
|
||||
parent,
|
||||
transactions,
|
||||
output: BlockExecutionResult { receipts, requests, gas_used, blob_gas_used, .. },
|
||||
output: BlockExecutionResult { receipts, requests, gas_used, blob_gas_used },
|
||||
state_root,
|
||||
block_access_list_hash,
|
||||
..
|
||||
} = input;
|
||||
|
||||
let timestamp = evm_env.block_env.timestamp().saturating_to();
|
||||
|
||||
let transactions_root = proofs::calculate_transaction_root(&transactions);
|
||||
@@ -90,12 +90,6 @@ where
|
||||
};
|
||||
}
|
||||
|
||||
let bal_hash = if self.chain_spec.is_amsterdam_active_at_timestamp(timestamp) {
|
||||
block_access_list_hash
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let header = Header {
|
||||
parent_hash: ctx.parent_hash,
|
||||
ommers_hash: EMPTY_OMMER_ROOT_HASH,
|
||||
@@ -118,8 +112,6 @@ where
|
||||
blob_gas_used: block_blob_gas_used,
|
||||
excess_blob_gas,
|
||||
requests_hash,
|
||||
block_access_list_hash: bal_hash,
|
||||
slot_number: ctx.slot_number,
|
||||
};
|
||||
|
||||
Ok(Block {
|
||||
|
||||
@@ -175,7 +175,6 @@ where
|
||||
suggested_fee_recipient: attributes.suggested_fee_recipient,
|
||||
prev_randao: attributes.prev_randao,
|
||||
gas_limit: attributes.gas_limit,
|
||||
slot_number: attributes.slot_number,
|
||||
},
|
||||
self.chain_spec().next_block_base_fee(parent, attributes.timestamp).unwrap_or_default(),
|
||||
self.chain_spec(),
|
||||
@@ -195,7 +194,6 @@ where
|
||||
ommers: &block.body().ommers,
|
||||
withdrawals: block.body().withdrawals.as_ref().map(|w| Cow::Borrowed(w.as_slice())),
|
||||
extra_data: block.header().extra_data.clone(),
|
||||
slot_number: block.header().slot_number,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -211,7 +209,6 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: attributes.withdrawals.map(|w| Cow::Owned(w.into_inner())),
|
||||
extra_data: attributes.extra_data,
|
||||
slot_number: attributes.slot_number,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -276,11 +273,7 @@ where
|
||||
gas_limit: payload.payload.gas_limit(),
|
||||
basefee: payload.payload.saturated_base_fee_per_gas(),
|
||||
blob_excess_gas_and_price,
|
||||
slot_num: if spec >= SpecId::AMSTERDAM {
|
||||
payload.payload.as_v4().unwrap().slot_number
|
||||
} else {
|
||||
Default::default()
|
||||
},
|
||||
slot_num: 0,
|
||||
};
|
||||
|
||||
Ok(EvmEnv { cfg_env, block_env })
|
||||
@@ -297,7 +290,6 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: payload.payload.withdrawals().map(|w| Cow::Borrowed(w.as_slice())),
|
||||
extra_data: payload.payload.as_v1().extra_data.clone(),
|
||||
slot_number: payload.payload.as_v4().map(|v4| v4.slot_number),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -14,17 +14,13 @@ impl ReceiptBuilder for RethReceiptBuilder {
|
||||
type Receipt = Receipt;
|
||||
|
||||
fn build_receipt<E: Evm>(&self, ctx: ReceiptBuilderCtx<'_, TxType, E>) -> Self::Receipt {
|
||||
let ReceiptBuilderCtx { tx_type, result, cumulative_gas_used, gas_spent, .. } = ctx;
|
||||
// EIP-7778: when active, `cumulative_gas_used` tracks gas before refunds (for block
|
||||
// accounting), but receipts must use gas after refunds (unchanged). `gas_spent` holds the
|
||||
// after-refund cumulative gas when EIP-7778 is active.
|
||||
let receipt_gas = gas_spent.unwrap_or(cumulative_gas_used);
|
||||
let ReceiptBuilderCtx { tx_type, result, cumulative_gas_used, .. } = ctx;
|
||||
Receipt {
|
||||
tx_type,
|
||||
// Success flag was added in `EIP-658: Embedding transaction status code in
|
||||
// receipts`.
|
||||
success: result.is_success(),
|
||||
cumulative_gas_used: receipt_gas,
|
||||
cumulative_gas_used,
|
||||
logs: result.into_logs(),
|
||||
}
|
||||
}
|
||||
|
||||
@@ -293,7 +293,6 @@ where
|
||||
EthConfigHandler::new(ctx.node.provider().clone(), ctx.node.evm_config().clone());
|
||||
|
||||
let testing_skip_invalid_transactions = ctx.config.rpc.testing_skip_invalid_transactions;
|
||||
let testing_gas_limit_override = ctx.config.rpc.testing_gas_limit;
|
||||
|
||||
self.inner
|
||||
.launch_add_ons_with(ctx, move |container| {
|
||||
@@ -315,9 +314,6 @@ where
|
||||
if testing_skip_invalid_transactions {
|
||||
testing_api = testing_api.with_skip_invalid_transactions();
|
||||
}
|
||||
if let Some(gas_limit) = testing_gas_limit_override {
|
||||
testing_api = testing_api.with_gas_limit_override(gas_limit);
|
||||
}
|
||||
container
|
||||
.modules
|
||||
.merge_if_module_configured(RethRpcModule::Testing, testing_api.into_rpc())?;
|
||||
@@ -567,10 +563,7 @@ where
|
||||
type Consensus = Arc<EthBeaconConsensus<<Node::Types as NodeTypes>::ChainSpec>>;
|
||||
|
||||
async fn build_consensus(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Consensus> {
|
||||
Ok(Arc::new(
|
||||
EthBeaconConsensus::new(ctx.chain_spec())
|
||||
.with_skip_gas_limit_ramp_check(ctx.config().rpc.testing_skip_gas_limit_ramp_check),
|
||||
))
|
||||
Ok(Arc::new(EthBeaconConsensus::new(ctx.chain_spec())))
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -221,7 +221,6 @@ async fn test_testing_build_block_v1_osaka() -> eyre::Result<()> {
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
let request = TestingBuildBlockRequestV1 {
|
||||
|
||||
@@ -25,7 +25,6 @@ pub(crate) fn eth_payload_attributes(timestamp: u64) -> EthPayloadBuilderAttribu
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: Some(B256::ZERO),
|
||||
slot_number: None,
|
||||
};
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
}
|
||||
@@ -39,7 +38,6 @@ pub(crate) fn eth_payload_attributes_shanghai(timestamp: u64) -> EthPayloadBuild
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: Some(vec![]),
|
||||
parent_beacon_block_root: None,
|
||||
slot_number: None,
|
||||
};
|
||||
EthPayloadBuilderAttributes::new(B256::ZERO, attributes)
|
||||
}
|
||||
@@ -85,9 +83,7 @@ where
|
||||
tx = tx.into_create().with_input(dummy_bytecode.clone());
|
||||
} else {
|
||||
tx = tx.with_to(*call_destinations.choose(rng).unwrap()).with_input(
|
||||
(0..rng.random_range(0..10000))
|
||||
.map(|_| rng.random::<u8>())
|
||||
.collect::<Vec<u8>>(),
|
||||
(0..rng.random_range(0..10000)).map(|_| rng.random()).collect::<Vec<u8>>(),
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -56,7 +56,6 @@ async fn testing_rpc_build_block_works() -> eyre::Result<()> {
|
||||
suggested_fee_recipient: Address::ZERO,
|
||||
withdrawals: None,
|
||||
parent_beacon_block_root: None,
|
||||
slot_number: None,
|
||||
};
|
||||
|
||||
let request = TestingBuildBlockRequestV1 {
|
||||
|
||||
@@ -154,16 +154,8 @@ where
|
||||
|
||||
let state_provider = client.state_by_block_hash(parent_header.hash())?;
|
||||
let state = StateProviderDatabase::new(state_provider.as_ref());
|
||||
let chain_spec = client.chain_spec();
|
||||
|
||||
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(attributes.timestamp());
|
||||
|
||||
// Build state with BAL builder enabled when Amsterdam is active
|
||||
let mut db = State::builder()
|
||||
.with_database(cached_reads.as_db_mut(state))
|
||||
.with_bundle_update()
|
||||
.with_bal_builder_if(is_amsterdam)
|
||||
.build();
|
||||
let mut db =
|
||||
State::builder().with_database(cached_reads.as_db_mut(state)).with_bundle_update().build();
|
||||
|
||||
let mut builder = evm_config
|
||||
.builder_for_next_block(
|
||||
@@ -177,11 +169,12 @@ where
|
||||
parent_beacon_block_root: attributes.parent_beacon_block_root(),
|
||||
withdrawals: Some(attributes.withdrawals().clone()),
|
||||
extra_data: builder_config.extra_data,
|
||||
slot_number: attributes.slot_number,
|
||||
},
|
||||
)
|
||||
.map_err(PayloadBuilderError::other)?;
|
||||
|
||||
let chain_spec = client.chain_spec();
|
||||
|
||||
debug!(target: "payload_builder", id=%attributes.id, parent_header = ?parent_header.hash(), parent_number = parent_header.number, "building new payload");
|
||||
let mut cumulative_gas_used = 0;
|
||||
let block_gas_limit: u64 = builder.evm_mut().block().gas_limit();
|
||||
@@ -331,25 +324,6 @@ where
|
||||
}
|
||||
continue
|
||||
}
|
||||
// EIP-7778: the executor tracks gas_before_refund while the payload builder's
|
||||
// pre-check uses gas_after_refund. Near-full blocks can pass the pre-check but
|
||||
// fail the executor's check. Skip the tx and continue building.
|
||||
Err(BlockExecutionError::Validation(
|
||||
BlockValidationError::TransactionGasLimitMoreThanAvailableBlockGas {
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
},
|
||||
)) => {
|
||||
trace!(target: "payload_builder", %transaction_gas_limit, %block_available_gas, ?tx, "skipping transaction exceeding block gas limit");
|
||||
best_txs.mark_invalid(
|
||||
&pool_tx,
|
||||
&InvalidPoolTransactionError::ExceedsGasLimit(
|
||||
transaction_gas_limit,
|
||||
block_available_gas,
|
||||
),
|
||||
);
|
||||
continue
|
||||
}
|
||||
// this is an error that we should treat as fatal for this attempt
|
||||
Err(err) => return Err(PayloadBuilderError::evm(err)),
|
||||
};
|
||||
@@ -386,7 +360,7 @@ where
|
||||
return Ok(BuildOutcome::Aborted { fees: total_fees, cached_reads })
|
||||
}
|
||||
|
||||
let BlockBuilderOutcome { execution_result, block, block_access_list, .. } =
|
||||
let BlockBuilderOutcome { execution_result, block, .. } =
|
||||
builder.finish(state_provider.as_ref())?;
|
||||
|
||||
let requests = chain_spec
|
||||
@@ -403,10 +377,9 @@ where
|
||||
}));
|
||||
}
|
||||
|
||||
let payload =
|
||||
EthBuiltPayload::new(attributes.id, sealed_block, total_fees, requests, block_access_list)
|
||||
// add blob sidecars from the executed txs
|
||||
.with_sidecars(blob_sidecars);
|
||||
let payload = EthBuiltPayload::new(attributes.id, sealed_block, total_fees, requests)
|
||||
// add blob sidecars from the executed txs
|
||||
.with_sidecars(blob_sidecars);
|
||||
|
||||
Ok(BuildOutcome::Better { payload, cached_reads })
|
||||
}
|
||||
|
||||
@@ -152,7 +152,6 @@ jemalloc = [
|
||||
"reth-cli-util?/jemalloc",
|
||||
"reth-ethereum-cli?/jemalloc",
|
||||
"reth-node-core?/jemalloc",
|
||||
"reth-provider?/jemalloc",
|
||||
]
|
||||
jemalloc-prof = [
|
||||
"jemalloc",
|
||||
|
||||
@@ -72,12 +72,6 @@ where
|
||||
Self::Right(b) => b.into_state(),
|
||||
}
|
||||
}
|
||||
fn take_bal(&mut self) -> Option<alloy_eips::eip7928::BlockAccessList> {
|
||||
match self {
|
||||
Self::Left(a) => a.take_bal(),
|
||||
Self::Right(b) => b.take_bal(),
|
||||
}
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> usize {
|
||||
match self {
|
||||
|
||||
@@ -3,10 +3,7 @@
|
||||
use crate::{ConfigureEvm, Database, OnStateHook, TxEnvFor};
|
||||
use alloc::{boxed::Box, sync::Arc, vec::Vec};
|
||||
use alloy_consensus::{BlockHeader, Header};
|
||||
use alloy_eips::{
|
||||
eip2718::WithEncoded,
|
||||
eip7928::{compute_block_access_list_hash, BlockAccessList},
|
||||
};
|
||||
use alloy_eips::eip2718::WithEncoded;
|
||||
pub use alloy_evm::block::{BlockExecutor, BlockExecutorFactory};
|
||||
use alloy_evm::{
|
||||
block::{CommitChanges, ExecutableTxParts},
|
||||
@@ -27,7 +24,6 @@ use reth_trie_common::{updates::TrieUpdates, HashedPostState};
|
||||
use revm::{
|
||||
context::result::ExecutionResult,
|
||||
database::{states::bundle_state::BundleRetention, BundleState, State},
|
||||
state::bal::Bal,
|
||||
};
|
||||
|
||||
/// A type that knows how to execute a block. It is assumed to operate on a
|
||||
@@ -148,9 +144,6 @@ pub trait Executor<DB: Database>: Sized {
|
||||
/// Consumes the executor and returns the [`State`] containing all state changes.
|
||||
fn into_state(self) -> State<DB>;
|
||||
|
||||
/// Take built [`BlockAccessList`] from executor
|
||||
fn take_bal(&mut self) -> Option<BlockAccessList>;
|
||||
|
||||
/// The size hint of the batch's tracked state size.
|
||||
///
|
||||
/// This is used to optimize DB commits depending on the size of the state.
|
||||
@@ -172,7 +165,6 @@ pub trait Executor<DB: Database>: Sized {
|
||||
/// - `bundle_state`: Accumulated state changes from all transactions
|
||||
/// - `state_provider`: Access to the current state for additional lookups
|
||||
/// - `state_root`: The calculated state root after all changes
|
||||
/// - `block_access_list_hash`: Block access list hash (EIP-7928, Amsterdam)
|
||||
///
|
||||
/// # Usage
|
||||
///
|
||||
@@ -216,8 +208,6 @@ pub struct BlockAssemblerInput<'a, 'b, F: BlockExecutorFactory, H = Header> {
|
||||
pub state_provider: &'b dyn StateProvider,
|
||||
/// State root for this block.
|
||||
pub state_root: B256,
|
||||
/// Block access list hash (EIP-7928, Amsterdam).
|
||||
pub block_access_list_hash: Option<B256>,
|
||||
}
|
||||
|
||||
impl<'a, 'b, F: BlockExecutorFactory, H> BlockAssemblerInput<'a, 'b, F, H> {
|
||||
@@ -235,7 +225,6 @@ impl<'a, 'b, F: BlockExecutorFactory, H> BlockAssemblerInput<'a, 'b, F, H> {
|
||||
bundle_state: &'a BundleState,
|
||||
state_provider: &'b dyn StateProvider,
|
||||
state_root: B256,
|
||||
block_access_list_hash: Option<B256>,
|
||||
) -> Self {
|
||||
Self {
|
||||
evm_env,
|
||||
@@ -246,7 +235,6 @@ impl<'a, 'b, F: BlockExecutorFactory, H> BlockAssemblerInput<'a, 'b, F, H> {
|
||||
bundle_state,
|
||||
state_provider,
|
||||
state_root,
|
||||
block_access_list_hash,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -316,8 +304,6 @@ pub struct BlockBuilderOutcome<N: NodePrimitives> {
|
||||
pub trie_updates: TrieUpdates,
|
||||
/// The built block.
|
||||
pub block: RecoveredBlock<N::Block>,
|
||||
/// Block access list built during execution (EIP-7928, Amsterdam).
|
||||
pub block_access_list: Option<BlockAccessList>,
|
||||
}
|
||||
|
||||
/// A type that knows how to execute and build a block.
|
||||
@@ -467,11 +453,7 @@ where
|
||||
type Executor = Executor;
|
||||
|
||||
fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
|
||||
self.executor.apply_pre_execution_changes()?;
|
||||
// Bump BAL index after pre-execution changes (EIP-7928: index 0 is pre-execution)
|
||||
self.executor.evm_mut().db_mut().bump_bal_index();
|
||||
|
||||
Ok(())
|
||||
self.executor.apply_pre_execution_changes()
|
||||
}
|
||||
|
||||
fn execute_transaction_with_commit_condition(
|
||||
@@ -486,9 +468,6 @@ where
|
||||
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
|
||||
{
|
||||
self.transactions.push(tx);
|
||||
// Bump BAL index after each committed transaction (EIP-7928)
|
||||
self.executor.evm_mut().db_mut().bump_bal_index();
|
||||
|
||||
Ok(Some(gas_used))
|
||||
} else {
|
||||
Ok(None)
|
||||
@@ -505,11 +484,6 @@ where
|
||||
// merge all transitions into bundle state
|
||||
db.merge_transitions(BundleRetention::Reverts);
|
||||
|
||||
// extract the built block access list (EIP-7928, Amsterdam) and compute its hash
|
||||
let block_access_list = db.take_built_alloy_bal();
|
||||
let block_access_list_hash =
|
||||
block_access_list.as_ref().map(|bal| compute_block_access_list_hash(bal));
|
||||
|
||||
// calculate the state root
|
||||
let hashed_state = state.hashed_post_state(&db.bundle_state);
|
||||
let (state_root, trie_updates) = state
|
||||
@@ -528,18 +502,11 @@ where
|
||||
bundle_state: &db.bundle_state,
|
||||
state_provider: &state,
|
||||
state_root,
|
||||
block_access_list_hash,
|
||||
})?;
|
||||
|
||||
let block = RecoveredBlock::new_unhashed(block, senders);
|
||||
|
||||
Ok(BlockBuilderOutcome {
|
||||
execution_result: result,
|
||||
hashed_state,
|
||||
trie_updates,
|
||||
block,
|
||||
block_access_list,
|
||||
})
|
||||
Ok(BlockBuilderOutcome { execution_result: result, hashed_state, trie_updates, block })
|
||||
}
|
||||
|
||||
fn executor_mut(&mut self) -> &mut Self::Executor {
|
||||
@@ -568,11 +535,7 @@ pub struct BasicBlockExecutor<F, DB> {
|
||||
impl<F, DB: Database> BasicBlockExecutor<F, DB> {
|
||||
/// Creates a new `BasicBlockExecutor` with the given strategy.
|
||||
pub fn new(strategy_factory: F, db: DB) -> Self {
|
||||
let db = State::builder()
|
||||
.with_database(db)
|
||||
.with_bundle_update()
|
||||
.with_bal_builder_if(true)
|
||||
.build();
|
||||
let db = State::builder().with_database(db).with_bundle_update().build();
|
||||
Self { strategy_factory, db }
|
||||
}
|
||||
}
|
||||
@@ -590,21 +553,11 @@ where
|
||||
block: &RecoveredBlock<<Self::Primitives as NodePrimitives>::Block>,
|
||||
) -> Result<BlockExecutionResult<<Self::Primitives as NodePrimitives>::Receipt>, Self::Error>
|
||||
{
|
||||
let mut executor = self
|
||||
let result = self
|
||||
.strategy_factory
|
||||
.executor_for_block(&mut self.db, block)
|
||||
.map_err(BlockExecutionError::other)?;
|
||||
|
||||
executor.evm_mut().db_mut().bal_state.bal_builder = Some(Bal::new());
|
||||
executor.apply_pre_execution_changes()?;
|
||||
executor.evm_mut().db_mut().bump_bal_index();
|
||||
|
||||
for tx in block.transactions_recovered() {
|
||||
executor.execute_transaction(tx)?;
|
||||
executor.evm_mut().db_mut().bump_bal_index();
|
||||
}
|
||||
|
||||
let result = executor.apply_post_execution_changes()?;
|
||||
.map_err(BlockExecutionError::other)?
|
||||
.execute_block(block.transactions_recovered())?;
|
||||
|
||||
self.db.merge_transitions(BundleRetention::Reverts);
|
||||
|
||||
@@ -635,10 +588,6 @@ where
|
||||
self.db
|
||||
}
|
||||
|
||||
fn take_bal(&mut self) -> Option<BlockAccessList> {
|
||||
self.db.take_built_alloy_bal()
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> usize {
|
||||
self.db.bundle_state.size_hint()
|
||||
}
|
||||
@@ -744,10 +693,6 @@ mod tests {
|
||||
unreachable!()
|
||||
}
|
||||
|
||||
fn take_bal(&mut self) -> Option<BlockAccessList> {
|
||||
None
|
||||
}
|
||||
|
||||
fn size_hint(&self) -> usize {
|
||||
0
|
||||
}
|
||||
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user