Compare commits

..

1 Commits

Author SHA1 Message Date
Arsenii Kulikov
3a18325e0e flush 2026-03-17 21:39:49 +01:00
62 changed files with 1279 additions and 2083 deletions

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: patch
---
Fixed another branch collapse edge case where `check_subtrie_collapse_needs_proof` incorrectly compared removal count against total update count (including `Touched` entries), causing it to skip proof requests for blinded siblings and panic when the subtrie emptied. Added a regression test covering the removals + `Touched` + blinded sibling scenario.

View File

@@ -1,132 +0,0 @@
#!/usr/bin/env bash
#
# Resolves baseline and feature refs for nightly regression benchmark runs.
#
# Queries the latest successful scheduled docker.yml run via GitHub API
# to find the commit that built the nightly Docker image. Compares with
# the last successful feature ref (from GH Actions cache) to determine
# baseline, detect staleness, and decide whether to skip.
#
# Usage: bench-nightly-refs.sh [--force]
#
# Outputs (via GITHUB_OUTPUT):
# baseline-ref — commit SHA for baseline
# feature-ref — commit SHA for feature (current nightly)
# should-skip — "true" if no new nightly since last run
# is-stale — "true" if latest nightly build is >24h old
# stale-age-hours — age of the nightly build in hours (only if stale)
# nightly-created — ISO timestamp of the nightly build
#
# Reads:
# .nightly-state/last-feature-ref (from GH Actions cache, may not exist)
#
# Requires: gh (GitHub CLI), jq, date
set -euo pipefail
FORCE="${1:-false}"
REPO="${GITHUB_REPOSITORY:-paradigmxyz/reth}"
# --- Step 1: Query latest successful scheduled docker.yml run ---
echo "::group::Querying latest nightly docker build"
RUNS_JSON=$(gh run list \
-R "$REPO" \
--workflow=docker.yml \
--event=schedule \
--status=completed \
--limit 5 \
--json headSha,createdAt,conclusion)
# Find the most recent successful run
LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
if [ -z "$LATEST" ]; then
echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
echo "Runs found: $RUNS_JSON"
exit 1
fi
FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt')
echo "Latest nightly commit: $FEATURE_REF"
echo "Built at: $CREATED_AT"
echo "::endgroup::"
# --- Step 2: Staleness check ---
echo "::group::Checking staleness"
NOW_EPOCH=$(date +%s)
# Handle both GNU date (-d) and BSD date (-j -f) for cross-platform compat
CREATED_EPOCH=$(date -d "$CREATED_AT" +%s 2>/dev/null || \
date -j -f "%Y-%m-%dT%H:%M:%SZ" "$CREATED_AT" +%s 2>/dev/null || \
date -j -f "%Y-%m-%dT%T%z" "$CREATED_AT" +%s 2>/dev/null || \
{ echo "::error::Cannot parse date: $CREATED_AT"; exit 1; })
AGE_SECONDS=$(( NOW_EPOCH - CREATED_EPOCH ))
AGE_HOURS=$(( AGE_SECONDS / 3600 ))
IS_STALE="false"
if [ "$AGE_HOURS" -gt 24 ]; then
IS_STALE="true"
echo "::warning::STALE NIGHTLY: Build is ${AGE_HOURS}h old (>24h threshold)"
echo "This indicates the nightly docker build failed — no new image was produced"
else
echo "Nightly build age: ${AGE_HOURS}h (within 24h threshold)"
fi
echo "::endgroup::"
# --- Step 3: Read last successful feature ref from cache ---
echo "::group::Reading cached state"
LAST_FEATURE_REF=""
STATE_FILE=".nightly-state/last-feature-ref"
if [ -f "$STATE_FILE" ]; then
LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
echo "Previous feature ref: $LAST_FEATURE_REF"
else
echo "No cached state found (first run)"
fi
echo "::endgroup::"
# --- Step 4: Determine baseline and skip logic ---
echo "::group::Resolving refs"
SHOULD_SKIP="false"
BASELINE_REF="$FEATURE_REF" # default for first run
if [ "$IS_STALE" = "true" ]; then
# Stale = error path, don't skip (will alert and fail downstream)
SHOULD_SKIP="false"
BASELINE_REF="${LAST_FEATURE_REF:-$FEATURE_REF}"
echo "Stale nightly detected — will alert and fail"
elif [ -z "$LAST_FEATURE_REF" ]; then
# First run: baseline = feature (self-comparison to establish baseline)
BASELINE_REF="$FEATURE_REF"
echo "First run — will benchmark nightly against itself to establish baseline"
elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
# No new nightly since last successful run
if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
echo "No new nightly, but force=true — running anyway"
BASELINE_REF="$LAST_FEATURE_REF"
else
SHOULD_SKIP="true"
echo "No new nightly since last run — will skip"
fi
else
# Normal case: new nightly available
BASELINE_REF="$LAST_FEATURE_REF"
echo "New nightly detected"
fi
echo "Baseline: $BASELINE_REF"
echo "Feature: $FEATURE_REF"
echo "Skip: $SHOULD_SKIP"
echo "Stale: $IS_STALE"
echo "::endgroup::"
# --- Step 5: Write outputs ---
{
echo "baseline-ref=$BASELINE_REF"
echo "feature-ref=$FEATURE_REF"
echo "should-skip=$SHOULD_SKIP"
echo "is-stale=$IS_STALE"
echo "stale-age-hours=$AGE_HOURS"
echo "nightly-created=$CREATED_AT"
} >> "$GITHUB_OUTPUT"

View File

@@ -1,865 +0,0 @@
# Nightly regression benchmark.
#
# Compares the previous nightly build against the current nightly build to
# detect performance regressions. Runs daily after docker.yml produces a new
# nightly image at 01:00 UTC.
#
# State is persisted between runs via GitHub Actions cache: each successful
# run saves the feature commit SHA so the next run knows what to compare against.
on:
schedule:
- cron: "30 5 * * *" # 06:30 UTC daily
workflow_dispatch:
inputs:
force:
description: "Force run even if no new nightly (bypass skip logic)"
required: false
default: false
type: boolean
no_slack:
description: "Suppress Slack notifications"
required: false
default: true
type: boolean
env:
CARGO_TERM_COLOR: always
RUSTC_WRAPPER: "sccache"
name: bench-scheduled
permissions:
contents: read
actions: read
jobs:
# ---------------------------------------------------------------------------
# Job 1: Resolve nightly refs, check staleness, manage state
# ---------------------------------------------------------------------------
resolve-refs:
name: resolve-refs
runs-on: ubuntu-latest
outputs:
baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
feature-ref: ${{ steps.refs.outputs.feature-ref }}
should-skip: ${{ steps.refs.outputs.should-skip }}
is-stale: ${{ steps.refs.outputs.is-stale }}
stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
nightly-created: ${{ steps.refs.outputs.nightly-created }}
steps:
- uses: actions/checkout@v6
with:
sparse-checkout: .github/scripts
sparse-checkout-cone-mode: true
- name: Restore nightly state
id: state-cache
uses: actions/cache/restore@v4
with:
path: .nightly-state
key: bench-scheduled-state-dummy
restore-keys: |
bench-scheduled-state-
- name: Resolve nightly refs
id: refs
env:
GH_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
run: |
FORCE="${{ inputs.force || 'false' }}"
.github/scripts/bench-scheduled-refs.sh "$FORCE"
- name: Alert on stale nightly
if: steps.refs.outputs.is-stale == 'true'
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const token = process.env.SLACK_BENCH_BOT_TOKEN;
const channel = process.env.SLACK_BENCH_CHANNEL;
if (!token || !channel) {
core.warning('Slack credentials not set, skipping stale nightly alert');
return;
}
const ageHours = '${{ steps.refs.outputs.stale-age-hours }}';
const created = '${{ steps.refs.outputs.nightly-created }}';
const featureRef = '${{ steps.refs.outputs.feature-ref }}';
const shortSha = featureRef.slice(0, 8);
const repo = '${{ github.repository }}';
const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: ':rotating_light: Nightly Regression: nightly build is stale', emoji: true },
},
{
type: 'section',
text: {
type: 'mrkdwn',
text: [
'*Nightly regression did not run* — nightly build is stale',
'',
`The latest nightly image was built from a commit that is *${ageHours}h old* (threshold: 24h).`,
`This means today's nightly docker build likely failed and no new image was produced.`,
'',
`Stale commit: \`${shortSha}\` (built at ${created})`,
'',
'*Action required:* Check the <https://github.com/' + repo + '/actions/workflows/docker.yml|docker.yml> workflow for failures.',
].join('\n'),
},
},
{
type: 'actions',
elements: [
{
type: 'button',
text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
url: runUrl,
action_id: 'ci_button',
},
],
},
];
const resp = await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
channel,
blocks,
text: 'Nightly regression: nightly build is stale',
unfurl_links: false,
}),
});
const data = await resp.json();
if (!data.ok) {
core.warning(`Slack API error: ${JSON.stringify(data)}`);
}
- name: Fail on stale nightly
if: steps.refs.outputs.is-stale == 'true'
run: |
echo "::error::Nightly build is stale (>24h old). Aborting."
exit 1
# ---------------------------------------------------------------------------
# Job 2: Run the benchmark
# ---------------------------------------------------------------------------
bench-scheduled:
needs: resolve-refs
if: |
needs.resolve-refs.outputs.should-skip != 'true' &&
needs.resolve-refs.outputs.is-stale != 'true'
name: bench-scheduled
runs-on: [self-hosted, Linux, X64, available]
timeout-minutes: 120
env:
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
SCHELK_MOUNT: /reth-bench
BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
BENCH_PR: ""
BENCH_ACTOR: "nightly-regression"
BENCH_BLOCKS: "2000"
BENCH_WARMUP_BLOCKS: "500"
BENCH_SAMPLY: "false"
BENCH_CORES: "0"
BENCH_BIG_BLOCKS: "false"
BENCH_RETH_NEW_PAYLOAD: "true"
BENCH_WAIT_TIME: ""
BENCH_BASELINE_ARGS: ""
BENCH_FEATURE_ARGS: ""
BENCH_ABBA: "true"
BENCH_COMMENT_ID: ""
BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
BENCH_METRICS_ADDR: "127.0.0.1:9100"
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }}
steps:
- name: Clean up previous bench-work
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
- uses: actions/checkout@v6
with:
submodules: true
fetch-depth: 0
ref: ${{ needs.resolve-refs.outputs.feature-ref }}
- name: Resolve job URL
id: job-url
uses: actions/github-script@v8
with:
script: |
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
});
const job = jobs.jobs.find(j => j.name === 'bench-scheduled');
const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
core.exportVariable('BENCH_JOB_URL', jobUrl);
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
continue-on-error: true
- name: Install dependencies
env:
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
run: |
mkdir -p "$HOME/.local/bin"
# apt packages
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends \
python3 make jq zstd curl dmsetup \
linux-tools-"$(uname -r)" || \
sudo apt-get install -y --no-install-recommends linux-tools-generic
# mc (MinIO client)
if ! command -v mc &>/dev/null; then
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
chmod +x "$HOME/.local/bin/mc"
fi
# uv (Python package manager)
if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
fi
# Configure git auth for private repos
git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
# thin-provisioning-tools (era_invalidate, required by schelk)
if ! command -v era_invalidate &>/dev/null; then
git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
sudo make -C /tmp/tpt install
rm -rf /tmp/tpt
fi
# schelk (snapshot rollback tool, invoked via sudo)
if ! sudo sh -c 'command -v schelk' &>/dev/null; then
cargo install --git https://github.com/tempoxyz/schelk --locked
sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
fi
- name: Check dependencies
run: |
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
missing=()
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
echo "::error::Missing required tools: ${missing[*]}"
exit 1
fi
echo "All dependencies found"
- name: Resolve display names
id: refs
run: |
BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
echo "baseline-name=nightly-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
echo "feature-name=nightly-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
- name: Check if snapshot needs update
id: snapshot-check
run: |
if .github/scripts/bench-reth-snapshot.sh --check; then
echo "needed=false" >> "$GITHUB_OUTPUT"
else
echo "needed=true" >> "$GITHUB_OUTPUT"
fi
- name: Prepare source dirs
run: |
if [ -d ../reth-baseline ]; then
git -C ../reth-baseline fetch origin "$BASELINE_REF"
else
git clone . ../reth-baseline
fi
git -C ../reth-baseline checkout "$BASELINE_REF"
if [ -d ../reth-feature ]; then
git -C ../reth-feature fetch origin "$FEATURE_REF"
else
git clone . ../reth-feature
fi
git -C ../reth-feature checkout "$FEATURE_REF"
- name: Build binaries and download snapshot in parallel
id: build
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BENCH_REPO: ${{ github.repository }}
SNAPSHOT_NEEDED: ${{ steps.snapshot-check.outputs.needed }}
run: |
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
FEATURE_DIR="$(cd ../reth-feature && pwd)"
.github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "$BASELINE_REF" &
PID_BASELINE=$!
.github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "$FEATURE_REF" &
PID_FEATURE=$!
PID_SNAPSHOT=
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
.github/scripts/bench-reth-snapshot.sh &
PID_SNAPSHOT=$!
fi
FAIL=0
wait $PID_BASELINE || FAIL=1
wait $PID_FEATURE || FAIL=1
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
if [ $FAIL -ne 0 ]; then
echo "::error::One or more parallel tasks failed (builds / snapshot download)"
exit 1
fi
# System tuning for reproducible benchmarks
- name: System setup
run: |
sudo cpupower frequency-set -g performance || true
# Disable turbo boost (Intel and AMD paths)
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
sudo swapoff -a || true
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
# Disable SMT (hyperthreading)
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
if [ "$current" != "$first" ]; then
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
fi
done
echo "Online CPUs: $(nproc)"
# Disable transparent huge pages
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
done || true
# Prevent deep C-states
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
# Move all IRQs to core 0
for irq in /proc/irq/*/smp_affinity_list; do
echo 0 | sudo tee "$irq" 2>/dev/null || true
done
# Stop noisy background services
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
echo "=== Benchmark environment ==="
uname -r
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
free -h
- name: Pre-flight cleanup
run: |
sudo pkill -9 reth || true
sleep 1
if mountpoint -q "$SCHELK_MOUNT"; then
sudo umount -l "$SCHELK_MOUNT" || true
sudo schelk recover -y || true
fi
rm -rf "$BENCH_WORK_DIR"
mkdir -p "$BENCH_WORK_DIR"
- name: Start metrics proxy
run: |
BENCH_ID="nightly-${{ github.run_id }}"
BENCH_REFERENCE_EPOCH=$(date +%s)
echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
LABELS_FILE="/tmp/bench-metrics-labels.json"
echo '{}' > "$LABELS_FILE"
echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
python3 .github/scripts/bench-metrics-proxy.py \
--labels "$LABELS_FILE" \
--upstream "http://${BENCH_METRICS_ADDR}/" \
--subnet 10.10.0.0/24 \
--port 9090 &
PROXY_PID=$!
echo "BENCH_METRICS_PROXY_PID=${PROXY_PID}" >> "$GITHUB_ENV"
echo "Metrics proxy started (PID $PROXY_PID)"
# Interleaved run order (B-F-F-B) to reduce systematic bias
- name: "Run benchmark: baseline (1/2)"
id: run-baseline-1
run: |
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
- name: "Run benchmark: feature (1/2)"
id: run-feature-1
run: |
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
- name: "Run benchmark: feature (2/2)"
id: run-feature-2
run: |
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
- name: "Run benchmark: baseline (2/2)"
id: run-baseline-2
run: |
LAST_RUN_START=$(date +%s)
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
- name: Stop metrics proxy & generate Grafana URL
id: metrics
if: "!cancelled()"
run: |
kill "$BENCH_METRICS_PROXY_PID" 2>/dev/null || true
LAST_RUN_DURATION=$(( $(date +%s) - BENCH_LAST_RUN_START ))
FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=ef57fux92e9z4e&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
echo "Grafana URL: ${GRAFANA_URL}"
- name: Scan logs for errors
if: "!cancelled()"
run: |
ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
found=false
for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
LOG="$BENCH_WORK_DIR/$run_dir/node.log"
if [ ! -f "$LOG" ]; then continue; fi
panics=$(grep -c -E 'panicked at' "$LOG" || true)
errors=$(grep -c ' ERROR ' "$LOG" || true)
if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
if [ "$found" = false ]; then
printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
found=true
fi
printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
if [ "$panics" -gt 0 ]; then
printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
if [ "$errors" -gt 0 ]; then
printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
printf '\n</details>\n\n' >> "$ERRORS_FILE"
fi
done
- name: Parse results
id: results
if: success()
env:
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
run: |
SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
if [ -n "$GRAFANA_URL" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
fi
# shellcheck disable=SC2086
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
- name: Generate charts
if: success()
env:
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
run: |
CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
# shellcheck disable=SC2086
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
- name: Upload results
if: "!cancelled()"
uses: actions/upload-artifact@v7
with:
name: bench-scheduled-results
path: ${{ env.BENCH_WORK_DIR }}
- name: Push charts
id: push-charts
if: success()
run: |
RUN_ID=${{ github.run_id }}
CHART_DIR="nightly/${RUN_ID}"
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
TMP_DIR=$(mktemp -d)
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
true
else
git init "${TMP_DIR}"
git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
fi
mkdir -p "${TMP_DIR}/${CHART_DIR}"
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
git -C "${TMP_DIR}" add "${CHART_DIR}"
git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
commit -m "nightly bench charts for run ${RUN_ID}"
git -C "${TMP_DIR}" push origin HEAD:main
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
rm -rf "${TMP_DIR}"
- name: Write job summary
if: success()
uses: actions/github-script@v8
with:
script: |
const fs = require('fs');
const { verdict, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const commitUrl = `https://github.com/${repo}/commit`;
const { emoji, label } = verdict(summary.changes);
const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
let md = `# ${emoji} Nightly Regression: ${label}\n\n`;
md += `**Baseline:** ${baselineLink}\n`;
md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
const rows = metricRows(summary);
md += `| Metric | Baseline | Feature | Change |\n`;
md += `|--------|----------|---------|--------|\n`;
for (const r of rows) {
md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
}
md += '\n';
const wtRows = waitTimeRows(summary);
if (wtRows.length > 0) {
md += `### Wait Time Breakdown\n\n`;
md += `| Metric | Baseline | Feature |\n`;
md += `|--------|----------|--------|\n`;
for (const r of wtRows) {
md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
}
md += '\n';
}
// Charts
const chartSha = '${{ steps.push-charts.outputs.sha }}';
if (chartSha) {
const runId = '${{ github.run_id }}';
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`;
const charts = [
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
];
md += `### Charts\n\n`;
for (const chart of charts) {
md += `<details><summary>${chart.label}</summary>\n\n`;
md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
md += `</details>\n\n`;
}
}
const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}';
if (grafanaUrl) {
md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
}
try {
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
if (errors.trim()) md += '\n' + errors + '\n';
} catch {}
await core.summary.addRaw(md).write();
- name: Send Slack notification (success)
if: success() && env.BENCH_NO_SLACK != 'true'
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const fs = require('fs');
const { verdict, fmtChange, fmtMs, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
const token = process.env.SLACK_BENCH_BOT_TOKEN;
const channel = process.env.SLACK_BENCH_CHANNEL;
if (!token || !channel) {
core.info('Slack credentials not set, skipping notification');
return;
}
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
core.warning('Could not read summary.json for Slack notification');
return;
}
// Only notify on significant changes (regression OR improvement)
const changes = summary.changes || {};
const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
if (!hasSignificant) {
core.info('No significant changes detected, skipping nightly Slack notification');
return;
}
const SLACK_VERDICT = {
'⚠️': ':warning:',
'❌': ':x:',
'✅': ':white_check_mark:',
'⚪': ':white_circle:',
};
const repo = `${context.repo.owner}/${context.repo.repo}`;
const { emoji, label } = verdict(changes);
const headerEmoji = SLACK_VERDICT[emoji] || emoji;
const commitUrl = `https://github.com/${repo}/commit`;
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
const sectionText = [
'*Nightly Regression*',
'',
`*Baseline:* ${baselineLink}`,
`*Feature:* ${featureLink}`,
blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '),
].join('\n');
const rows = metricRows(summary);
const tableRows = [
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
];
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: `${headerEmoji} Nightly: ${label}`, emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: sectionText },
},
{
type: 'table',
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }, { align: 'right' }],
rows: tableRows,
},
{
type: 'actions',
elements: [
{
type: 'button',
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
},
{
type: 'button',
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
url: diffUrl,
action_id: 'diff_button',
},
],
},
];
const text = `Nightly Regression: ${summary.baseline.name} vs ${summary.feature.name}`;
const resp = await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ channel, blocks, text, unfurl_links: false }),
});
const data = await resp.json();
if (!data.ok) {
core.warning(`Slack API error: ${JSON.stringify(data)}`);
return;
}
// Post wait time breakdown as threaded reply
const wtRows = waitTimeRows(summary);
if (data.ts && wtRows.length > 0) {
const waitTableRows = [
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
];
await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
channel,
thread_ts: data.ts,
blocks: [{
type: 'table',
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }],
rows: waitTableRows,
}],
text: 'Wait time breakdown',
unfurl_links: false,
}),
});
}
- name: Send Slack notification (failure)
if: failure()
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const token = process.env.SLACK_BENCH_BOT_TOKEN;
const channel = process.env.SLACK_BENCH_CHANNEL;
if (!token || !channel) return;
const steps_status = [
['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
];
const failed = steps_status.find(([, o]) => o === 'failure');
const failedStep = failed ? failed[0] : 'unknown step';
const repo = `${context.repo.owner}/${context.repo.repo}`;
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: ':rotating_light: Nightly Bench Failed', emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: `*Nightly regression* failed while *${failedStep}*` },
},
{
type: 'actions',
elements: [{
type: 'button',
text: { type: 'plain_text', text: 'View Logs :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
}],
},
];
await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
channel,
blocks,
text: `Nightly bench failed while ${failedStep}`,
unfurl_links: false,
}),
});
- name: Restore system settings
if: always()
run: |
sudo systemctl start irqbalance cron atd 2>/dev/null || true
# ---------------------------------------------------------------------------
# Job 3: Save state on success
# ---------------------------------------------------------------------------
save-state:
needs: [resolve-refs, bench-scheduled]
if: success()
name: save-state
runs-on: ubuntu-latest
steps:
- name: Write state file
run: |
mkdir -p .nightly-state
echo "${{ needs.resolve-refs.outputs.feature-ref }}" > .nightly-state/last-feature-ref
- name: Save nightly state
uses: actions/cache/save@v4
with:
path: .nightly-state
key: bench-scheduled-state-${{ needs.resolve-refs.outputs.feature-ref }}

View File

@@ -71,11 +71,6 @@ on:
required: false
default: "true"
type: boolean
otlp:
description: "Export OTLP traces and logs"
required: false
default: "true"
type: boolean
env:
CARGO_TERM_COLOR: always
@@ -115,7 +110,6 @@ jobs:
baseline-args: ${{ steps.args.outputs.baseline-args }}
feature-args: ${{ steps.args.outputs.feature-args }}
abba: ${{ steps.args.outputs.abba }}
otlp: ${{ steps.args.outputs.otlp }}
comment-id: ${{ steps.ack.outputs.comment-id }}
steps:
- name: Check org membership
@@ -157,7 +151,6 @@ jobs:
bigBlocks = blocks === 'big' ? 'true' : 'false';
var rethNewPayload = '${{ github.event.inputs.reth_newPayload }}' !== 'false' ? 'true' : 'false';
var abba = '${{ github.event.inputs.abba }}' !== 'false' ? 'true' : 'false';
var otlp = '${{ github.event.inputs.otlp }}' !== 'false' ? 'true' : 'false';
var waitTime = '${{ github.event.inputs.wait_time }}' || '';
var baselineNodeArgs = '${{ github.event.inputs.baseline_args }}' || '';
var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || '';
@@ -184,10 +177,10 @@ jobs:
const intOrKeywordArgs = new Map([['blocks', new Set(['big'])]]);
const refArgs = new Set(['baseline', 'feature']);
const boolArgs = new Set(['samply', 'no-slack']);
const boolDefaultTrue = new Set(['reth_newPayload', 'abba', 'otlp']);
const boolDefaultTrue = new Set(['reth_newPayload', 'abba']);
const durationArgs = new Set(['wait-time']);
const stringArgs = new Set(['baseline-args', 'feature-args']);
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const unknown = [];
const invalid = [];
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -257,7 +250,7 @@ jobs:
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
if (errors.length) {
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [otlp=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
await github.rest.issues.createComment({
owner: context.repo.owner,
repo: context.repo.repo,
@@ -277,7 +270,6 @@ jobs:
bigBlocks = blocks === 'big' ? 'true' : 'false';
var rethNewPayload = defaults.reth_newPayload;
var abba = defaults.abba;
var otlp = defaults.otlp;
var waitTime = defaults['wait-time'];
var baselineNodeArgs = defaults['baseline-args'];
var featureNodeArgs = defaults['feature-args'];
@@ -316,7 +308,6 @@ jobs:
core.setOutput('baseline-args', baselineNodeArgs);
core.setOutput('feature-args', featureNodeArgs);
core.setOutput('abba', abba);
core.setOutput('otlp', otlp);
- name: Acknowledge request
id: ack
@@ -384,8 +375,6 @@ jobs:
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
@@ -393,7 +382,7 @@ jobs:
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
const { data: comment } = await github.rest.issues.createComment({
owner: context.repo.owner,
@@ -428,8 +417,6 @@ jobs:
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
@@ -437,7 +424,7 @@ jobs:
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
@@ -507,12 +494,11 @@ jobs:
BENCH_BASELINE_ARGS: ${{ needs.reth-bench-ack.outputs.baseline-args }}
BENCH_FEATURE_ARGS: ${{ needs.reth-bench-ack.outputs.feature-args }}
BENCH_ABBA: ${{ needs.reth-bench-ack.outputs.abba }}
BENCH_OTLP: ${{ needs.reth-bench-ack.outputs.otlp }}
BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
BENCH_NO_SLACK: ${{ needs.reth-bench-ack.outputs.no-slack }}
BENCH_METRICS_ADDR: "127.0.0.1:9100"
BENCH_OTLP_TRACES_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_TRACES_ENDPOINT || '' }}
BENCH_OTLP_LOGS_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_LOGS_ENDPOINT || '' }}
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
steps:
- name: Clean up previous bench-work
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
@@ -573,8 +559,6 @@ jobs:
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
const abbaEnabled = (process.env.BENCH_ABBA || 'true') !== 'false';
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
const otlpEnabled = (process.env.BENCH_OTLP || 'true') !== 'false';
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
const waitTimeVal = process.env.BENCH_WAIT_TIME || '';
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
const baselineArgsVal = process.env.BENCH_BASELINE_ARGS || '';
@@ -582,7 +566,7 @@ jobs:
const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
const { buildBody } = require('./.github/scripts/bench-update-status.js');
await github.rest.issues.updateComment({

150
Cargo.lock generated
View File

@@ -290,17 +290,21 @@ dependencies = [
[[package]]
name = "alloy-evm"
version = "0.29.2"
source = "git+https://github.com/alloy-rs/evm?rev=b0eb7e6#b0eb7e617f964f7090c504f21a5977cc440117f7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3cb6ba2dafd6327f78f2b59ae539bd5c39c57a01dc76763e92942619d934a7bb"
dependencies = [
"alloy-consensus",
"alloy-eips",
"alloy-hardforks 0.4.7",
"alloy-op-hardforks",
"alloy-primitives",
"alloy-rpc-types-engine",
"alloy-rpc-types-eth",
"alloy-sol-types",
"auto_impl",
"derive_more",
"op-alloy",
"op-revm",
"revm",
"thiserror 2.0.18",
"tracing",
@@ -436,6 +440,18 @@ dependencies = [
"url",
]
[[package]]
name = "alloy-op-hardforks"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6472c610150c4c4c15be9e1b964c9b78068f933bda25fb9cdf09b9ac2bb66f36"
dependencies = [
"alloy-chains",
"alloy-hardforks 0.4.7",
"alloy-primitives",
"auto_impl",
]
[[package]]
name = "alloy-primitives"
version = "1.5.7"
@@ -1021,7 +1037,7 @@ version = "1.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
dependencies = [
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -1032,7 +1048,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
dependencies = [
"anstyle",
"once_cell_polyfill",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -3456,7 +3472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
dependencies = [
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -4270,8 +4286,8 @@ dependencies = [
"libc",
"log",
"rustversion",
"windows-link 0.1.3",
"windows-result 0.3.4",
"windows-link 0.2.1",
"windows-result 0.4.1",
]
[[package]]
@@ -4827,7 +4843,7 @@ dependencies = [
"js-sys",
"log",
"wasm-bindgen",
"windows-core 0.61.2",
"windows-core 0.62.2",
]
[[package]]
@@ -5179,7 +5195,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
dependencies = [
"hermit-abi",
"libc",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6134,7 +6150,7 @@ version = "0.50.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
dependencies = [
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -6339,6 +6355,19 @@ version = "11.1.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
[[package]]
name = "op-alloy"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "a95dd0974d5e60ffe9342a70cc0033d299244fab01cb16a958eb7352ddba1fa7"
dependencies = [
"op-alloy-consensus",
"op-alloy-network",
"op-alloy-provider",
"op-alloy-rpc-types",
"op-alloy-rpc-types-engine",
]
[[package]]
name = "op-alloy-consensus"
version = "0.24.0"
@@ -6359,6 +6388,37 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "op-alloy-network"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c8ea44162d493219cc678aaca1253d46c3aa73aa361326dfa9d406f086dfa135"
dependencies = [
"alloy-consensus",
"alloy-network",
"alloy-primitives",
"alloy-provider",
"alloy-rpc-types-eth",
"alloy-signer",
"op-alloy-consensus",
"op-alloy-rpc-types",
]
[[package]]
name = "op-alloy-provider"
version = "0.24.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "83aa8dc34bdf077c8e6d48ff75beff4ac14b428d982c9722483ccd7473c0e114"
dependencies = [
"alloy-network",
"alloy-primitives",
"alloy-provider",
"alloy-rpc-types-engine",
"alloy-transport",
"async-trait",
"op-alloy-rpc-types-engine",
]
[[package]]
name = "op-alloy-rpc-types"
version = "0.24.0"
@@ -6400,6 +6460,17 @@ dependencies = [
"thiserror 2.0.18",
]
[[package]]
name = "op-revm"
version = "17.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "57a98f3a512a7e02a1dcf1242b57302d83657b265a665d50ad98d0b158efaf2c"
dependencies = [
"auto_impl",
"revm",
"serde",
]
[[package]]
name = "opaque-debug"
version = "0.3.1"
@@ -10592,7 +10663,8 @@ dependencies = [
[[package]]
name = "revm"
version = "36.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b0abc15d09cd211e9e73410ada10134069c794d4bcdb787dfc16a1bf0939849c"
dependencies = [
"revm-bytecode",
"revm-context",
@@ -10610,7 +10682,8 @@ dependencies = [
[[package]]
name = "revm-bytecode"
version = "9.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e86e468df3cf5cf59fa7ef71a3e9ccabb76bb336401ea2c0674f563104cf3c5e"
dependencies = [
"bitvec",
"phf",
@@ -10621,7 +10694,8 @@ dependencies = [
[[package]]
name = "revm-context"
version = "15.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9eb1f0a76b14d684a444fc52f7bf6b7564bf882599d91ee62e76d602e7a187c7"
dependencies = [
"bitvec",
"cfg-if",
@@ -10637,7 +10711,8 @@ dependencies = [
[[package]]
name = "revm-context-interface"
version = "16.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc256b27743e2912ca16899568e6652a372eb5d1d573e6edb16c7836b16cf487"
dependencies = [
"alloy-eip2930",
"alloy-eip7702",
@@ -10652,7 +10727,8 @@ dependencies = [
[[package]]
name = "revm-database"
version = "12.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2c0a7d6da41061f2c50f99a2632571026b23684b5449ff319914151f4449b6c8"
dependencies = [
"alloy-eips",
"revm-bytecode",
@@ -10665,7 +10741,8 @@ dependencies = [
[[package]]
name = "revm-database-interface"
version = "10.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bd497a38a79057b94a049552cb1f925ad15078bc1a479c132aeeebd1d2ccc768"
dependencies = [
"auto_impl",
"either",
@@ -10678,7 +10755,8 @@ dependencies = [
[[package]]
name = "revm-handler"
version = "17.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9f1eed729ca9b228ae98688f352235871e9b8be3d568d488e4070f64c56e9d3d"
dependencies = [
"auto_impl",
"derive-where",
@@ -10696,7 +10774,8 @@ dependencies = [
[[package]]
name = "revm-inspector"
version = "17.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cbf5102391706513689f91cb3cb3d97b5f13a02e8647e6e9cb7620877ef84847"
dependencies = [
"auto_impl",
"either",
@@ -10712,8 +10791,9 @@ dependencies = [
[[package]]
name = "revm-inspectors"
version = "0.36.1"
source = "git+https://github.com/paradigmxyz/revm-inspectors?rev=24becc3#24becc35973c6c1d4e1c1475fa51a83d36d50d48"
version = "0.36.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cfb0f462c8a3d9989d3dbc62d7cca4dfecd7072cfa5d563ab90ced60590ed1da"
dependencies = [
"alloy-primitives",
"alloy-rpc-types-eth",
@@ -10732,7 +10812,8 @@ dependencies = [
[[package]]
name = "revm-interpreter"
version = "34.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "cf22f80612bb8f58fd1f578750281f2afadb6c93835b14ae6a4d6b75ca26f445"
dependencies = [
"revm-bytecode",
"revm-context-interface",
@@ -10744,7 +10825,8 @@ dependencies = [
[[package]]
name = "revm-precompile"
version = "32.1.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e2ec11f45deec71e4945e1809736bb20d454285f9167ab53c5159dae1deb603f"
dependencies = [
"ark-bls12-381",
"ark-bn254",
@@ -10758,7 +10840,6 @@ dependencies = [
"cfg-if",
"k256",
"p256",
"revm-context-interface",
"revm-primitives",
"ripemd",
"secp256k1 0.31.1",
@@ -10768,7 +10849,8 @@ dependencies = [
[[package]]
name = "revm-primitives"
version = "22.1.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4bcfb5ce6cf18b118932bcdb7da05cd9c250f2cb9f64131396b55f3fe3537c35"
dependencies = [
"alloy-primitives",
"num_enum",
@@ -10779,7 +10861,8 @@ dependencies = [
[[package]]
name = "revm-state"
version = "10.0.0"
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d29404707763da607e5d6e4771cb203998c28159279c2f64cc32de08d2814651"
dependencies = [
"alloy-eip7928",
"bitflags 2.11.0",
@@ -11022,7 +11105,7 @@ dependencies = [
"errno",
"libc",
"linux-raw-sys",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -11613,8 +11696,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
[[package]]
name = "slotmap"
version = "1.1.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"
source = "git+https://github.com/DaniPopes/slotmap.git?branch=dani%2Fshrink-methods#09fbd360968f9ecef19fc9559037cf869d33858b"
dependencies = [
"version_check",
]
@@ -11680,7 +11762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
dependencies = [
"libc",
"windows-sys 0.60.2",
"windows-sys 0.61.2",
]
[[package]]
@@ -11880,7 +11962,7 @@ dependencies = [
"getrandom 0.4.2",
"once_cell",
"rustix",
"windows-sys 0.59.0",
"windows-sys 0.61.2",
]
[[package]]
@@ -12221,11 +12303,7 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857"
dependencies = [
"futures-util",
"log",
"rustls",
"rustls-native-certs",
"rustls-pki-types",
"tokio",
"tokio-rustls",
"tungstenite 0.28.0",
]
@@ -12605,7 +12683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c5f7c95348f20c1c913d72157b3c6dee6ea3e30b3d19502c5a7f6d3f160dacbf"
dependencies = [
"cc",
"windows-targets 0.48.5",
"windows-targets 0.52.6",
]
[[package]]
@@ -12680,8 +12758,6 @@ dependencies = [
"httparse",
"log",
"rand 0.9.2",
"rustls",
"rustls-pki-types",
"sha1",
"thiserror 2.0.18",
"utf-8",
@@ -13184,7 +13260,7 @@ version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
dependencies = [
"windows-sys 0.48.0",
"windows-sys 0.61.2",
]
[[package]]

View File

@@ -399,9 +399,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
reth-payload-primitives = { path = "crates/payload/primitives" }
reth-payload-validator = { path = "crates/payload/validator" }
reth-payload-util = { path = "crates/payload/util" }
reth-primitives = { path = "crates/primitives", default-features = false, features = [
"__internal",
] }
reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
reth-provider = { path = "crates/storage/provider" }
reth-prune = { path = "crates/prune/prune" }
@@ -446,22 +444,18 @@ revm-state = { version = "10.0.0", default-features = false }
revm-primitives = { version = "22.1.0", default-features = false }
revm-interpreter = { version = "34.0.0", default-features = false }
revm-database-interface = { version = "10.0.0", default-features = false }
revm-inspectors = "0.36.1"
revm-inspectors = "0.36.0"
# eth
alloy-dyn-abi = "1.5.6"
alloy-primitives = { version = "1.5.6", default-features = false, features = [
"map-foldhash",
] }
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.6", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.29.2", default-features = false }
alloy-rlp = { version = "0.3.13", default-features = false, features = [
"core-net",
] }
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-hardforks = "0.4.5"
@@ -473,15 +467,10 @@ alloy-genesis = { version = "1.7.3", default-features = false }
alloy-json-rpc = { version = "1.7.3", default-features = false }
alloy-network = { version = "1.7.3", default-features = false }
alloy-network-primitives = { version = "1.7.3", default-features = false }
alloy-provider = { version = "1.7.3", features = [
"reqwest",
"debug-api",
], default-features = false }
alloy-provider = { version = "1.7.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.7.3", default-features = false }
alloy-rpc-client = { version = "1.7.3", default-features = false }
alloy-rpc-types = { version = "1.7.3", features = [
"eth",
], default-features = false }
alloy-rpc-types = { version = "1.7.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.7.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.7.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.7.3", default-features = false }
@@ -495,9 +484,7 @@ alloy-serde = { version = "1.7.3", default-features = false }
alloy-signer = { version = "1.7.3", default-features = false }
alloy-signer-local = { version = "1.7.3", default-features = false }
alloy-transport = { version = "1.7.3" }
alloy-transport-http = { version = "1.7.3", features = [
"reqwest-rustls-tls",
], default-features = false }
alloy-transport-http = { version = "1.7.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.7.3", default-features = false }
alloy-transport-ws = { version = "1.7.3", default-features = false }
@@ -511,10 +498,7 @@ either = { version = "1.15.0", default-features = false }
arrayvec = { version = "0.7.6", default-features = false }
aquamarine = "0.6"
auto_impl = "1"
backon = { version = "1.2", default-features = false, features = [
"std-blocking-sleep",
"tokio-sleep",
] }
backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] }
bincode = "1.3"
bitflags = "2.4"
boyer-moore-magiclen = "0.2.16"
@@ -538,13 +522,9 @@ linked_hash_set = "0.1"
libc = "0.2"
lz4 = "1.28.1"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = [
"macos_fsevent",
] }
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.8", default-features = false }
once_cell = { version = "1.19", default-features = false, features = [
"critical-section",
] }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
parking_lot = "0.12"
quanta = "0.12"
paste = "1.0"
@@ -558,16 +538,15 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
serde_with = { version = "3", default-features = false, features = ["macros"] }
sha2 = { version = "0.10", default-features = false }
shlex = "1.3"
slotmap = "1"
# https://github.com/orlp/slotmap/pull/148
slotmap = { git = "https://github.com/DaniPopes/slotmap.git", branch = "dani/shrink-methods" }
smallvec = "1"
strum = { version = "0.27", default-features = false }
strum_macros = "0.27"
syn = "2.0"
thiserror = { version = "2.0.0", default-features = false }
tar = "0.4.44"
tracing = { version = "0.1.0", default-features = false, features = [
"attributes",
] }
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
tracing-appender = "0.2"
url = { version = "2.3", default-features = false }
zstd = "0.13"
@@ -605,11 +584,7 @@ futures-util = { version = "0.3", default-features = false }
hyper = "1.3"
hyper-util = "0.1.5"
pin-project = "1.0.12"
reqwest = { version = "0.12", default-features = false, features = [
"rustls-tls",
"rustls-tls-native-roots",
"stream",
] }
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
tracing-futures = "0.2"
tower = "0.5"
tower-http = "0.6"
@@ -634,10 +609,7 @@ proptest-arbitrary-interop = "0.1.0"
# crypto
enr = { version = "0.13", default-features = false }
k256 = { version = "0.13", default-features = false, features = ["ecdsa"] }
secp256k1 = { version = "0.30", default-features = false, features = [
"global-context",
"recovery",
] }
secp256k1 = { version = "0.30", default-features = false, features = ["global-context", "recovery"] }
# rand 8 for secp256k1
rand_08 = { package = "rand", version = "0.8" }
@@ -780,15 +752,6 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "b0eb7e6" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "9bc2dba" }
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "24becc3" }
# revm from rakita/state-gas branch
revm = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
revm-database = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
revm-state = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }

View File

@@ -124,7 +124,6 @@ jemalloc = [
"reth-node-core/jemalloc",
"reth-node-metrics/jemalloc",
"reth-ethereum-cli/jemalloc",
"reth-provider/jemalloc",
]
jemalloc-prof = [
"reth-cli-util/jemalloc",

View File

@@ -36,6 +36,15 @@ pub use spec::{
DepositContract, ForkBaseFeeParams, DEV, HOLESKY, HOODI, MAINNET, SEPOLIA,
};
use reth_primitives_traits::sync::OnceLock;
/// Simple utility to create a thread-safe sync cell with a value set.
pub fn once_cell_set<T>(value: T) -> OnceLock<T> {
let once = OnceLock::new();
let _ = once.set(value);
once
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -187,6 +187,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
where
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
{
let prune_modes = config.prune.segments.clone();
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
db,
self.chain.clone(),
@@ -194,8 +195,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
rocksdb_provider,
runtime,
)?
.with_prune_modes(config.prune.segments.clone())
.with_minimum_pruning_distance(config.prune.minimum_pruning_distance);
.with_prune_modes(prune_modes.clone());
// Check for consistency between database and static files.
if !access.is_read_only_inconsistent() &&
@@ -229,13 +229,10 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
NoopBodiesDownloader::default(),
NoopEvmConfig::<N::Evm>::default(),
config.stages.clone(),
config.prune.segments.clone(),
prune_modes.clone(),
None,
))
.build(
factory.clone(),
StaticFileProducer::new(factory.clone(), config.prune.segments.clone()),
);
.build(factory.clone(), StaticFileProducer::new(factory.clone(), prune_modes));
// Move all applicable data from database to static files.
pipeline.move_to_static_files()?;

View File

@@ -199,7 +199,7 @@ pub(crate) fn config_for_selections(
}
return Config {
prune: PruneConfig { segments, ..Default::default() },
prune: PruneConfig { block_interval: PruneConfig::default().block_interval, segments },
static_files,
..Default::default()
};

View File

@@ -201,7 +201,6 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
let net =
NetworkConfigBuilder::<N::NetworkPrimitives>::new(p2p_secret_key, Runtime::test())
.peer_config(config.peers_config_with_basic_nodes_from_file(None))
.sessions_config(config.sessions)
.external_ip_resolver(self.network.nat.clone())
.network_id(self.network.network_id)
.boot_nodes(boot_nodes.clone())

View File

@@ -1,6 +1,6 @@
//! Configuration files.
use reth_network_types::{PeersConfig, SessionsConfig};
use reth_prune_types::{PruneModes, MINIMUM_UNWIND_SAFE_DISTANCE};
use reth_prune_types::PruneModes;
use reth_stages_types::ExecutionStageThresholds;
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
use std::{
@@ -540,24 +540,11 @@ pub struct PruneConfig {
/// Pruning configuration for every part of the data that can be pruned.
#[cfg_attr(feature = "serde", serde(alias = "parts"))]
pub segments: PruneModes,
/// Minimum distance from the tip required for pruning. Controls the safety margin for
/// reorgs and manual unwinds. Defaults to [`MINIMUM_UNWIND_SAFE_DISTANCE`].
#[cfg_attr(feature = "serde", serde(default = "default_minimum_pruning_distance"))]
pub minimum_pruning_distance: u64,
}
/// Returns the default minimum pruning distance.
const fn default_minimum_pruning_distance() -> u64 {
MINIMUM_UNWIND_SAFE_DISTANCE
}
impl Default for PruneConfig {
fn default() -> Self {
Self {
block_interval: DEFAULT_BLOCK_INTERVAL,
segments: PruneModes::default(),
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
}
Self { block_interval: DEFAULT_BLOCK_INTERVAL, segments: PruneModes::default() }
}
}
@@ -590,7 +577,6 @@ impl PruneConfig {
bodies_history,
receipts_log_filter,
},
minimum_pruning_distance,
} = other;
// Merge block_interval, only update if it's the default interval
@@ -598,11 +584,6 @@ impl PruneConfig {
self.block_interval = block_interval;
}
// Merge minimum_pruning_distance, only update if it's the default
if self.minimum_pruning_distance == MINIMUM_UNWIND_SAFE_DISTANCE {
self.minimum_pruning_distance = minimum_pruning_distance;
}
// Merge the various segment prune modes
self.segments.sender_recovery = self.segments.sender_recovery.or(sender_recovery);
self.segments.transaction_lookup = self.segments.transaction_lookup.or(transaction_lookup);
@@ -642,9 +623,7 @@ mod tests {
use crate::PruneConfig;
use alloy_primitives::Address;
use reth_network_peers::TrustedPeer;
use reth_prune_types::{
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
};
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig};
use std::{collections::BTreeMap, path::Path, str::FromStr, time::Duration};
fn with_tempdir(filename: &str, proc: fn(&std::path::Path)) {
@@ -1114,7 +1093,6 @@ receipts = { distance = 16384 }
fn test_prune_config_merge() {
let mut config1 = PruneConfig {
block_interval: 5,
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
segments: PruneModes {
sender_recovery: Some(PruneMode::Full),
transaction_lookup: None,
@@ -1131,7 +1109,6 @@ receipts = { distance = 16384 }
let config2 = PruneConfig {
block_interval: 10,
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
segments: PruneModes {
sender_recovery: Some(PruneMode::Distance(500)),
transaction_lookup: Some(PruneMode::Full),

View File

@@ -96,11 +96,6 @@ where
self
}
/// Sets the pruning arguments for the test nodes.
pub fn with_pruning(self, pruning: reth_node_core::args::PruningArgs) -> Self {
self.with_node_config_modifier(move |config| config.with_pruning(pruning.clone()))
}
/// Enables v2 storage defaults (`--storage.v2`), routing tx hashes, history
/// indices, etc. to `RocksDB` and changesets/senders to static files.
pub fn with_storage_v2(self) -> Self {

View File

@@ -38,8 +38,8 @@ op-alloy-rpc-types-engine = { workspace = true, optional = true }
workspace = true
[features]
# op = [
# "dep:op-alloy-rpc-types-engine",
# "reth-payload-primitives/op",
# "reth-primitives-traits/op",
# ]
op = [
"dep:op-alloy-rpc-types-engine",
"reth-payload-primitives/op",
"reth-primitives-traits/op",
]

View File

@@ -177,25 +177,18 @@ where
}
}
provider_rw.commit()?;
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
// Run the pruner in a separate provider so it reads committed RocksDB state
// that includes the history entries written by save_blocks above.
//
// The pruner reads the indices from rocksdb, filters it, and writes to indices, so it
// must be able to read anything written by save_blocks.
if self.pruner.is_pruning_needed(last.number) {
debug!(target: "engine::persistence", block_num=?last.number, "Running pruner");
let prune_start = Instant::now();
let provider_rw = self.provider.database_provider_rw()?;
let _ = self.pruner.run_with_provider(&provider_rw, last.number)?;
provider_rw.commit()?;
debug!(target: "engine::persistence", tip=?last.number, "Finished pruning after saving blocks");
self.metrics.prune_before_duration_seconds.record(prune_start.elapsed());
}
provider_rw.commit()?;
}
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
let elapsed = start_time.elapsed();
self.metrics.save_blocks_batch_size.record(block_count as f64);
self.metrics.save_blocks_duration_seconds.record(elapsed);
@@ -461,52 +454,4 @@ mod tests {
assert_eq!(last_hash, result.last_block.unwrap().hash);
}
}
/// Verifies that committing `save_blocks` history before running the pruner
/// prevents the pruner from overwriting new entries.
///
/// Previously, both `save_blocks` and the pruner pushed `RocksDB` batches before
/// a single commit. Both read committed state, so the pruner didn't see the
/// new entries and its batch overwrote them. The fix commits `save_blocks`
/// first, then runs the pruner against committed state in a separate provider.
#[test]
fn test_save_blocks_then_prune_preserves_new_history() {
use reth_db::{models::ShardedKey, tables, BlockNumberList};
use reth_provider::RocksDBProviderFactory;
reth_tracing::init_test_tracing();
let provider_factory = create_test_provider_factory();
let tracked_addr = alloy_primitives::Address::from([0xBE; 20]);
// Phase 1: Establish baseline history for blocks 0..20.
let rocksdb = provider_factory.rocksdb_provider();
{
let mut batch = rocksdb.batch();
let initial_blocks: Vec<u64> = (0..20).collect();
let shard = BlockNumberList::new_pre_sorted(initial_blocks.iter().copied());
batch
.put::<tables::AccountsHistory>(ShardedKey::new(tracked_addr, u64::MAX), &shard)
.unwrap();
batch.commit().unwrap();
}
// Phase 2: Simulate the fixed on_save_blocks flow.
// Step 1: save_blocks appends new entries 20..25 and commits immediately.
let mut batch1 = rocksdb.batch();
batch1.append_account_history_shard(tracked_addr, 20..25u64).unwrap();
batch1.commit().unwrap();
// Step 2: Pruner runs AFTER commit, so it reads state that includes 20..25.
// Prunes entries ≤ 14, leaving [15..25).
let mut batch2 = rocksdb.batch();
batch2.prune_account_history_to(tracked_addr, 14).unwrap();
batch2.commit().unwrap();
// Verify new entries survived pruning.
let shards = rocksdb.account_history_shards(tracked_addr).unwrap();
let entries: Vec<u64> = shards.iter().flat_map(|(_, list)| list.iter()).collect();
let expected: Vec<u64> = (15..25).collect();
assert_eq!(entries, expected, "new entries 20..25 must survive pruning");
}
}

View File

@@ -81,8 +81,8 @@ pub struct CacheEntry<S> {
}
impl<S> CacheEntry<S> {
const fn regular_gas_used(&self) -> u64 {
self.output.gas.limit() - self.output.gas.remaining()
const fn gas_used(&self) -> u64 {
self.output.gas_used
}
fn to_precompile_result(&self) -> PrecompileResult {
@@ -170,10 +170,10 @@ where
fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult {
if let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) &&
input.gas >= entry.regular_gas_used()
input.gas >= entry.gas_used()
{
self.increment_by_one_precompile_cache_hits();
return entry.to_precompile_result();
return entry.to_precompile_result()
}
let calldata = input.data;
@@ -228,14 +228,15 @@ mod tests {
use super::*;
use reth_evm::{EthEvmFactory, Evm, EvmEnv, EvmFactory};
use reth_revm::db::EmptyDB;
use revm::{context::TxEnv, interpreter::gas::GasTracker, precompile::PrecompileOutput};
use revm::{context::TxEnv, precompile::PrecompileOutput};
use revm_primitives::hardfork::SpecId;
#[test]
fn test_precompile_cache_basic() {
let dyn_precompile: DynPrecompile = (|_input: PrecompileInput<'_>| -> PrecompileResult {
Ok(PrecompileOutput {
gas: GasTracker::new(0, 0, 0),
gas_used: 0,
gas_refunded: 0,
bytes: Bytes::default(),
reverted: false,
})
@@ -246,7 +247,8 @@ mod tests {
CachedPrecompile::new(dyn_precompile, PrecompileCache::default(), SpecId::PRAGUE, None);
let output = PrecompileOutput {
gas: GasTracker::new(50, 0, 0),
gas_used: 50,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
reverted: false,
};
@@ -277,7 +279,8 @@ mod tests {
assert_eq!(input.data, input_data);
Ok(PrecompileOutput {
gas: GasTracker::new(5000, 0, 0),
gas_used: 5000,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
reverted: false,
})
@@ -291,7 +294,8 @@ mod tests {
assert_eq!(input.data, input_data);
Ok(PrecompileOutput {
gas: GasTracker::new(7000, 0, 0),
gas_used: 7000,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
reverted: false,
})

View File

@@ -152,7 +152,6 @@ jemalloc = [
"reth-cli-util?/jemalloc",
"reth-ethereum-cli?/jemalloc",
"reth-node-core?/jemalloc",
"reth-provider?/jemalloc",
]
jemalloc-prof = [
"jemalloc",

View File

@@ -62,4 +62,4 @@ test-utils = [
"reth-trie-common/test-utils",
"reth-ethereum-primitives/test-utils",
]
# op = ["alloy-evm/op", "reth-primitives-traits/op"]
op = ["alloy-evm/op", "reth-primitives-traits/op"]

View File

@@ -468,7 +468,7 @@ where
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
{
self.transactions.push(tx);
Ok(Some(gas_used.into()))
Ok(Some(gas_used))
} else {
Ok(None)
}

View File

@@ -122,10 +122,10 @@ test-utils = [
"reth-tasks/test-utils",
]
trie-debug = ["reth-engine-tree/trie-debug"]
# op = [
# "reth-db/op",
# "reth-db-api/op",
# "reth-engine-local/op",
# "reth-evm/op",
# "reth-primitives-traits/op",
# ]
op = [
"reth-db/op",
"reth-db-api/op",
"reth-engine-local/op",
"reth-evm/op",
"reth-primitives-traits/op",
]

View File

@@ -501,7 +501,6 @@ where
.build()?
};
let prune_config = self.prune_config();
let factory = ProviderFactory::new(
self.right().clone(),
self.chain_spec(),
@@ -509,8 +508,7 @@ where
rocksdb_provider,
self.task_executor().clone(),
)?
.with_prune_modes(prune_config.segments)
.with_minimum_pruning_distance(prune_config.minimum_pruning_distance)
.with_prune_modes(self.prune_modes())
.with_changeset_cache(changeset_cache);
// Check consistency between the database and static files, returning
@@ -1312,7 +1310,6 @@ mod tests {
bodies_distance: None,
receipts_log_filter: None,
bodies_before: None,
minimum_distance: None,
},
..NodeConfig::test()
};

View File

@@ -815,16 +815,14 @@ impl DiscoveryArgs {
SocketAddr::V6(addr) => Some(*addr.ip()),
});
let mut discv5_config_builder =
reth_discv5::discv5::ConfigBuilder::new(ListenConfig::from_two_sockets(
discv5_addr_ipv4.map(|addr| SocketAddrV4::new(addr, *discv5_port)),
discv5_addr_ipv6.map(|addr| SocketAddrV6::new(addr, *discv5_port_ipv6, 0, 0)),
));
if discv5_addr.is_some() || discv5_addr_ipv6.is_some() || self.disable_nat {
discv5_config_builder.disable_enr_update();
}
reth_discv5::Config::builder(rlpx_tcp_socket)
.discv5_config(discv5_config_builder.build())
.discv5_config(
reth_discv5::discv5::ConfigBuilder::new(ListenConfig::from_two_sockets(
discv5_addr_ipv4.map(|addr| SocketAddrV4::new(addr, *discv5_port)),
discv5_addr_ipv6.map(|addr| SocketAddrV6::new(addr, *discv5_port_ipv6, 0, 0)),
))
.build(),
)
.add_unsigned_boot_nodes(boot_nodes)
.lookup_interval(*discv5_lookup_interval)
.bootstrap_lookup_interval(*discv5_bootstrap_lookup_interval)

View File

@@ -196,11 +196,6 @@ pub struct PruningArgs {
/// pruned.
#[arg(long = "prune.bodies.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["bodies_distance", "bodies_pre_merge"])]
pub bodies_before: Option<BlockNumber>,
/// Minimum pruning distance from the tip. This controls the safety margin for reorgs and
/// manual unwinds.
#[arg(long = "prune.minimum-distance", value_name = "BLOCKS")]
pub minimum_distance: Option<u64>,
}
impl PruningArgs {
@@ -225,11 +220,7 @@ impl PruningArgs {
.block_number()
.map(PruneMode::Before);
}
config = PruneConfig {
block_interval: config.block_interval,
segments,
minimum_pruning_distance: config.minimum_pruning_distance,
}
config = PruneConfig { block_interval: config.block_interval, segments }
}
// If --minimal is set, use minimal storage mode with aggressive pruning.
@@ -237,7 +228,6 @@ impl PruningArgs {
config = PruneConfig {
block_interval: config.block_interval,
segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(),
minimum_pruning_distance: config.minimum_pruning_distance,
}
}
@@ -245,9 +235,6 @@ impl PruningArgs {
if let Some(block_interval) = self.block_interval {
config.block_interval = block_interval as usize;
}
if let Some(distance) = self.minimum_distance {
config.minimum_pruning_distance = distance;
}
if let Some(mode) = self.sender_recovery_prune_mode() {
config.segments.sender_recovery = Some(mode);
}

View File

@@ -21,7 +21,7 @@ alloy-primitives.workspace = true
alloy-consensus.workspace = true
tokio.workspace = true
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
tokio-tungstenite.workspace = true
futures-util.workspace = true
tokio-stream.workspace = true

View File

@@ -53,7 +53,7 @@ std = [
"either/std",
"alloy-consensus/std",
]
# op = [
# "dep:op-alloy-rpc-types-engine",
# "reth-primitives-traits/op",
# ]
op = [
"dep:op-alloy-rpc-types-engine",
"reth-primitives-traits/op",
]

View File

@@ -57,8 +57,8 @@ pub mod servers {
web3::Web3ApiServer,
};
pub use reth_rpc_eth_api::{
self as eth, EthApiServer, EthBundleApiServer, EthCallBundleApiServer, EthConfigApiServer,
EthFilterApiServer, EthPubSubApiServer, L2EthApiExtServer,
self as eth, EthApiServer, EthBundleApiServer, EthCallBundleApiServer, EthFilterApiServer,
EthPubSubApiServer, L2EthApiExtServer,
};
}
@@ -89,7 +89,7 @@ pub mod clients {
web3::Web3ApiClient,
};
pub use reth_rpc_eth_api::{
EthApiClient, EthBundleApiClient, EthCallBundleApiClient, EthConfigApiClient,
EthFilterApiClient, L2EthApiExtClient,
EthApiClient, EthBundleApiClient, EthCallBundleApiClient, EthFilterApiClient,
L2EthApiExtClient,
};
}

View File

@@ -43,10 +43,10 @@ serde_json.workspace = true
[features]
default = []
#op = [
# "dep:op-alloy-consensus",
# "dep:op-alloy-rpc-types",
# "reth-evm/op",
# "reth-primitives-traits/op",
# "alloy-evm/op",
#]
op = [
"dep:op-alloy-consensus",
"dep:op-alloy-rpc-types",
"reth-evm/op",
"reth-primitives-traits/op",
"alloy-evm/op",
]

View File

@@ -62,9 +62,9 @@ tracing.workspace = true
[features]
js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"]
client = ["jsonrpsee/client", "jsonrpsee/async-client"]
# op = [
# "reth-evm/op",
# "reth-primitives-traits/op",
# "reth-rpc-convert/op",
# "alloy-evm/op",
# ]
op = [
"reth-evm/op",
"reth-primitives-traits/op",
"reth-rpc-convert/op",
"alloy-evm/op",
]

View File

@@ -479,11 +479,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
.map_err(Self::Error::from_eth_err)?;
}
// Read fields from request before consuming it in create_txn_env
let request_has_gas_limit = request.as_ref().gas_limit().is_some();
let initial = request.as_ref().access_list().cloned().unwrap_or_default();
let mut tx_env = this.create_txn_env(&evm_env, request, &mut db)?;
let mut tx_env = this.create_txn_env(&evm_env, request.clone(), &mut db)?;
// we want to disable this in eth_createAccessList, since this is common practice used
// by other node impls and providers <https://github.com/foundry-rs/foundry/issues/4388>
@@ -506,18 +502,21 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
// per-tx cap (2^24 ≈ 16.7M post-Osaka).
evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX);
if !request_has_gas_limit && tx_env.gas_price() > 0 {
if request.as_ref().gas_limit().is_none() && tx_env.gas_price() > 0 {
let cap = this.caller_gas_allowance(&mut db, &evm_env, &tx_env)?;
// no gas limit was provided in the request, so we need to cap the request's gas
// limit
tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit()));
}
// can consume the list since we're not using the request anymore
let initial = request.as_ref().access_list().cloned().unwrap_or_default();
let mut inspector = AccessListInspector::new(initial);
let result = this.inspect(&mut db, evm_env.clone(), tx_env.clone(), &mut inspector)?;
let access_list = inspector.into_access_list();
let gas_used = result.result.tx_gas_used();
let gas_used = result.result.gas_used();
tx_env.set_access_list(access_list.clone());
if let Err(err) = Self::Error::ensure_success(result.result) {
return Ok(AccessListResult {
@@ -529,7 +528,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
// transact again to get the exact gas used
let result = this.transact(&mut db, evm_env, tx_env)?;
let gas_used = result.result.tx_gas_used();
let gas_used = result.result.gas_used();
let error = Self::Error::ensure_success(result.result).err().map(|e| e.to_string());
Ok(AccessListResult { access_list, gas_used: U256::from(gas_used), error })

View File

@@ -204,7 +204,7 @@ pub trait EstimateCall: Call {
// NOTE: this is the gas the transaction used, which is less than the
// transaction requires to succeed.
let mut gas_used = res.result.tx_gas_used();
let mut gas_used = res.result.gas_used();
// the lowest value is capped by the gas used by the unconstrained transaction
let mut lowest_gas_limit = gas_used.saturating_sub(1);
@@ -225,7 +225,7 @@ pub trait EstimateCall: Call {
res = evm.transact(optimistic_tx_env).map_err(Self::Error::from_evm_err)?;
// Update the gas used based on the new result.
gas_used = res.result.tx_gas_used();
gas_used = res.result.gas_used();
// Update the gas limit estimates (highest and lowest) based on the execution result.
update_estimated_gas_range(
res.result,

View File

@@ -318,8 +318,7 @@ pub trait LoadPendingBlock:
// There's only limited amount of blob space available per block, so we need to
// check if the EIP-4844 can still fit in the block
let tx_blob_gas = tx.blob_gas_used();
if let Some(tx_blob_gas) = tx_blob_gas &&
if let Some(tx_blob_gas) = tx.blob_gas_used() &&
sum_blob_gas_used + tx_blob_gas > blob_params.max_blob_gas_per_block()
{
// we can't fit this _blob_ transaction into the block, so we mark it as
@@ -336,7 +335,7 @@ pub trait LoadPendingBlock:
continue
}
let gas_used = match builder.execute_transaction(tx) {
let gas_used = match builder.execute_transaction(tx.clone()) {
Ok(gas_used) => gas_used,
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
error,
@@ -361,7 +360,7 @@ pub trait LoadPendingBlock:
};
// add to the total blob gas used if the transaction successfully executed
if let Some(tx_blob_gas) = tx_blob_gas {
if let Some(tx_blob_gas) = tx.blob_gas_used() {
sum_blob_gas_used += tx_blob_gas;
// if we've reached the max data gas per block, we can skip blob txs entirely

View File

@@ -25,7 +25,6 @@ pub use bundle::{EthBundleApiServer, EthCallBundleApiServer};
pub use core::{EthApiServer, FullEthApiServer};
pub use ext::L2EthApiExtServer;
pub use filter::{EngineEthFilter, EthFilterApiServer, QueryLimits};
pub use helpers::config::EthConfigApiServer;
pub use node::{RpcNodeCore, RpcNodeCoreExt};
pub use pubsub::EthPubSubApiServer;
pub use reth_rpc_convert::*;
@@ -42,7 +41,5 @@ pub use core::EthApiClient;
pub use ext::L2EthApiExtClient;
#[cfg(feature = "client")]
pub use filter::EthFilterApiClient;
#[cfg(feature = "client")]
pub use helpers::config::EthConfigApiClient;
use reth_trie_common as _;

View File

@@ -126,7 +126,7 @@ pub trait FromEvmError<Evm: ConfigureEvm>:
ExecutionResult::Success { output, .. } => Ok(output.into_data()),
ExecutionResult::Revert { output, .. } => Err(Self::from_revert(output)),
ExecutionResult::Halt { reason, gas, .. } => {
Err(Self::from_evm_halt(reason, gas.tx_gas_used()))
Err(Self::from_evm_halt(reason, gas.used()))
}
}
}

View File

@@ -553,7 +553,6 @@ where
EVMError::Header(err) => err.into(),
EVMError::Database(err) => err.into(),
EVMError::Custom(err) => Self::EvmCustom(err),
EVMError::CustomAny(err) => Self::EvmCustom(err.to_string()),
}
}
}

View File

@@ -175,8 +175,7 @@ where
};
if block_values.is_empty() {
// For empty blocks, use zero gas price to signal no demand
results.push(U256::ZERO);
results.push(U256::from(inner.last_price.price));
} else {
results.extend(block_values);
populated_blocks += 1;

View File

@@ -314,7 +314,7 @@ where
code: SIMULATE_VM_ERROR_CODE,
..SimulateError::invalid_params()
}),
gas_used: gas.tx_gas_used(),
gas_used: gas.used(),
logs: Vec::new(),
status: false,
..Default::default()
@@ -330,7 +330,7 @@ where
code: SIMULATE_REVERT_CODE,
..SimulateError::invalid_params()
}),
gas_used: gas.tx_gas_used(),
gas_used: gas.used(),
status: false,
logs: Vec::new(),
..Default::default()
@@ -342,7 +342,7 @@ where
SimCallResult {
return_data: output.into_data(),
error: None,
gas_used: gas.tx_gas_used(),
gas_used: gas.used(),
logs: logs
.into_iter()
.map(|log| {

View File

@@ -188,7 +188,7 @@ where
let gas_price = tx
.effective_tip_per_gas(basefee)
.expect("fee is always valid; execution succeeded");
let gas_used = result.tx_gas_used();
let gas_used = result.gas_used();
total_gas_used += gas_used;
let gas_fees = U256::from(gas_used) * U256::from(gas_price);

View File

@@ -478,24 +478,22 @@ where
return Err(ProviderError::HeaderNotFound(block_hash.into()).into())
};
// Read number and timestamp from cached block or provider header
let (block_number, block_timestamp) = if let Some(block) = &maybe_block {
(block.header().number(), block.header().timestamp())
// Get header - from cached block if available, otherwise from provider
let header = if let Some(block) = &maybe_block {
block.header().clone()
} else {
let header = self
.provider()
self.provider()
.header_by_hash_or_number(block_hash.into())?
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?;
(header.number(), header.timestamp())
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?
};
// Check if the block has been pruned (EIP-4444)
let earliest_block = self.provider().earliest_block_number()?;
if block_number < earliest_block {
if header.number() < earliest_block {
return Err(EthApiError::PrunedHistoryUnavailable.into());
}
let block_num_hash = BlockNumHash::new(block_number, block_hash);
let block_num_hash = BlockNumHash::new(header.number(), block_hash);
let mut all_logs = Vec::new();
append_matching_block_logs(
@@ -507,7 +505,7 @@ where
block_num_hash,
&receipts,
false,
block_timestamp,
header.timestamp(),
)?;
Ok(all_logs)

View File

@@ -289,7 +289,7 @@ where
.into());
}
let gas_used = result.tx_gas_used();
let gas_used = result.gas_used();
total_gas_used += gas_used;
// coinbase is always present in the result state

View File

@@ -88,8 +88,8 @@ arbitrary = [
"op-alloy-consensus?/arbitrary",
"reth-ethereum-primitives/arbitrary",
]
# op = [
# "dep:op-alloy-consensus",
# "reth-codecs/op",
# "reth-primitives-traits/op",
# ]
op = [
"dep:op-alloy-consensus",
"reth-codecs/op",
"reth-primitives-traits/op",
]

View File

@@ -93,8 +93,8 @@ arbitrary = [
"reth-primitives-traits/arbitrary",
"reth-prune-types/arbitrary",
]
# op = [
# "reth-db-api/op",
# "reth-primitives-traits/op",
# ]
op = [
"reth-db-api/op",
"reth-primitives-traits/op",
]
disable-lock = []

View File

@@ -64,7 +64,7 @@ tokio = { workspace = true, features = ["sync"], optional = true }
# parallel utils
rayon.workspace = true
rocksdb.workspace = true
rocksdb = { workspace = true, features = ["jemalloc"] }
[dev-dependencies]
reth-db = { workspace = true, features = ["test-utils"] }
@@ -86,7 +86,6 @@ rand.workspace = true
tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
[features]
jemalloc = ["rocksdb/jemalloc"]
test-utils = [
"reth-db/test-utils",
"reth-nippy-jar/test-utils",

View File

@@ -106,10 +106,8 @@ impl<N> ProviderFactoryBuilder<N> {
let db = open_db_read_only(db_dir, db_args)?;
let static_file_provider =
StaticFileProvider::read_only(static_files_dir, watch_static_files)?;
let rocksdb_provider = RocksDBProvider::builder(&rocksdb_dir)
.with_default_tables()
.with_read_only(true)
.build()?;
let rocksdb_provider =
RocksDBProvider::builder(&rocksdb_dir).with_default_tables().build()?;
ProviderFactory::new(db, chainspec, static_file_provider, rocksdb_provider, runtime)
.map_err(Into::into)
}

View File

@@ -90,7 +90,7 @@ pub(crate) struct DatabaseProviderMetrics {
/// Duration of `update_pipeline_stages` in `save_blocks`
save_blocks_update_pipeline_stages: Histogram,
/// Number of blocks per `save_blocks` call
save_blocks_batch_size: Histogram,
save_blocks_block_count: Histogram,
/// Duration of MDBX commit in `save_blocks`
save_blocks_commit_mdbx: Histogram,
/// Duration of static file commit in `save_blocks`
@@ -118,7 +118,7 @@ pub(crate) struct DatabaseProviderMetrics {
/// Last duration of `update_pipeline_stages` in `save_blocks`
save_blocks_update_pipeline_stages_last: Gauge,
/// Last number of blocks per `save_blocks` call
save_blocks_batch_size_last: Gauge,
save_blocks_block_count_last: Gauge,
/// Last duration of MDBX commit in `save_blocks`
save_blocks_commit_mdbx_last: Gauge,
/// Last duration of static file commit in `save_blocks`
@@ -140,7 +140,7 @@ pub(crate) struct SaveBlocksTimings {
pub write_trie_updates: Duration,
pub update_history_indices: Duration,
pub update_pipeline_stages: Duration,
pub batch_size: u64,
pub block_count: u64,
}
/// Timings collected during a `commit` call.
@@ -182,7 +182,7 @@ impl DatabaseProviderMetrics {
self.save_blocks_write_trie_updates.record(timings.write_trie_updates);
self.save_blocks_update_history_indices.record(timings.update_history_indices);
self.save_blocks_update_pipeline_stages.record(timings.update_pipeline_stages);
self.save_blocks_batch_size.record(timings.batch_size as f64);
self.save_blocks_block_count.record(timings.block_count as f64);
self.save_blocks_total_last.set(timings.total.as_secs_f64());
self.save_blocks_mdbx_last.set(timings.mdbx.as_secs_f64());
@@ -196,7 +196,7 @@ impl DatabaseProviderMetrics {
.set(timings.update_history_indices.as_secs_f64());
self.save_blocks_update_pipeline_stages_last
.set(timings.update_pipeline_stages.as_secs_f64());
self.save_blocks_batch_size_last.set(timings.batch_size as f64);
self.save_blocks_block_count_last.set(timings.block_count as f64);
}
/// Records all commit timings.

View File

@@ -24,12 +24,12 @@ use reth_node_types::{
BlockTy, HeaderTy, NodeTypesWithDB, NodeTypesWithDBAdapter, ReceiptTy, TxTy,
};
use reth_primitives_traits::{RecoveredBlock, SealedHeader};
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment, MINIMUM_UNWIND_SAFE_DISTANCE};
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment};
use reth_stages_types::{PipelineTarget, StageCheckpoint, StageId};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::{
BlockBodyIndicesProvider, ChainStateBlockReader, ChainStateBlockWriter, DBProvider,
NodePrimitivesProvider, StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider,
BlockBodyIndicesProvider, ChainStateBlockReader, ChainStateBlockWriter, NodePrimitivesProvider,
StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider,
};
use reth_storage_errors::provider::ProviderResult;
use reth_trie::HashedPostState;
@@ -80,8 +80,6 @@ pub struct ProviderFactory<N: NodeTypesWithDB> {
changeset_cache: ChangesetCache,
/// Task runtime for spawning parallel I/O work.
runtime: reth_tasks::Runtime,
/// Minimum distance from tip required before pruning can occur.
minimum_pruning_distance: u64,
}
impl<N: NodeTypesForProvider> ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>> {
@@ -136,7 +134,6 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
rocksdb_provider,
changeset_cache: ChangesetCache::new(),
runtime,
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
})
}
@@ -171,15 +168,6 @@ impl<N: NodeTypesWithDB> ProviderFactory<N> {
self
}
/// Sets the minimum pruning distance for an existing [`ProviderFactory`].
///
/// This controls the minimum distance from tip required before pruning can occur.
/// The default is [`MINIMUM_UNWIND_SAFE_DISTANCE`].
pub const fn with_minimum_pruning_distance(mut self, distance: u64) -> Self {
self.minimum_pruning_distance = distance;
self
}
/// Returns reference to the underlying database.
pub const fn db_ref(&self) -> &N::DB {
&self.db
@@ -258,8 +246,7 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
self.changeset_cache.clone(),
self.runtime.clone(),
self.db.path(),
)
.with_minimum_pruning_distance(self.minimum_pruning_distance))
))
}
/// Returns a provider with a created `DbTxMut` inside, which allows fetching and updating
@@ -268,21 +255,18 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
/// open.
#[track_caller]
pub fn provider_rw(&self) -> ProviderResult<DatabaseProviderRW<N::DB, N>> {
Ok(DatabaseProviderRW(
DatabaseProvider::new_rw(
self.db.tx_mut()?,
self.chain_spec.clone(),
self.static_file_provider.clone(),
self.prune_modes.clone(),
self.storage.clone(),
self.storage_settings.clone(),
self.rocksdb_provider.clone(),
self.changeset_cache.clone(),
self.runtime.clone(),
self.db.path(),
)
.with_minimum_pruning_distance(self.minimum_pruning_distance),
))
Ok(DatabaseProviderRW(DatabaseProvider::new_rw(
self.db.tx_mut()?,
self.chain_spec.clone(),
self.static_file_provider.clone(),
self.prune_modes.clone(),
self.storage.clone(),
self.storage_settings.clone(),
self.rocksdb_provider.clone(),
self.changeset_cache.clone(),
self.runtime.clone(),
self.db.path(),
)))
}
/// Returns a provider with a created `DbTxMut` inside, configured for unwind operations.
@@ -303,8 +287,7 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
self.changeset_cache.clone(),
self.runtime.clone(),
self.db.path(),
)
.with_minimum_pruning_distance(self.minimum_pruning_distance))
))
}
/// State provider for latest block
@@ -366,12 +349,7 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
/// consistency. I.e. this MAY result in writes to the static files.
#[instrument(err, skip(self))]
pub fn check_consistency(&self) -> ProviderResult<(Option<u64>, Option<u64>)> {
let provider_ro = self
.database_provider_ro()?
// Healing can run long-lived read transactions (e.g., iterating changesets
// over millions of blocks). Disable the default timeout so MDBX doesn't
// kill the transaction mid-heal, which causes a crash loop on startup.
.disable_long_read_transaction_safety();
let provider_ro = self.database_provider_ro()?;
// Step 1: heal file-level inconsistencies (no pruning)
self.static_file_provider().check_file_consistency(&provider_ro)?;
@@ -819,7 +797,6 @@ where
rocksdb_provider,
changeset_cache,
runtime,
minimum_pruning_distance,
} = self;
f.debug_struct("ProviderFactory")
.field("db", &db)
@@ -831,7 +808,6 @@ where
.field("rocksdb_provider", &rocksdb_provider)
.field("changeset_cache", &changeset_cache)
.field("runtime", &runtime)
.field("minimum_pruning_distance", &minimum_pruning_distance)
.finish()
}
}
@@ -848,7 +824,6 @@ impl<N: NodeTypesWithDB> Clone for ProviderFactory<N> {
rocksdb_provider: self.rocksdb_provider.clone(),
changeset_cache: self.changeset_cache.clone(),
runtime: self.runtime.clone(),
minimum_pruning_distance: self.minimum_pruning_distance,
}
}
}

View File

@@ -238,12 +238,6 @@ impl<TX, N: NodeTypes> DatabaseProvider<TX, N> {
pub const fn prune_modes_ref(&self) -> &PruneModes {
&self.prune_modes
}
/// Sets the minimum pruning distance.
pub const fn with_minimum_pruning_distance(mut self, distance: u64) -> Self {
self.minimum_pruning_distance = distance;
self
}
}
impl<TX: DbTx + 'static, N: NodeTypes> DatabaseProvider<TX, N> {
@@ -554,8 +548,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
nums
};
let mut timings =
metrics::SaveBlocksTimings { batch_size: block_count, ..Default::default() };
let mut timings = metrics::SaveBlocksTimings { block_count, ..Default::default() };
// avoid capturing &self.tx in scope below.
let sf_provider = &self.static_file_provider;

View File

@@ -9,9 +9,7 @@ use crate::StaticFileProviderFactory;
use alloy_eips::eip2718::Encodable2718;
use alloy_primitives::BlockNumber;
use rayon::prelude::*;
use reth_chainspec::{ChainSpecProvider, EthChainSpec};
use reth_db::models::{storage_sharded_key::StorageShardedKey, ShardedKey};
use reth_db_api::tables;
use reth_db_api::{table::Table, tables};
use reth_stages_types::StageId;
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::{
@@ -59,8 +57,7 @@ impl RocksDBProvider {
+ BlockBodyIndicesProvider
+ StorageChangeSetReader
+ ChangeSetReader
+ TransactionsProvider<Transaction: Encodable2718>
+ ChainSpecProvider,
+ TransactionsProvider<Transaction: Encodable2718>,
{
let mut unwind_target: Option<BlockNumber> = None;
@@ -117,14 +114,11 @@ impl RocksDBProvider {
// Fast path: clear any stale data and return.
if checkpoint == 0 {
if self.first::<tables::TransactionHashNumbers>()?.is_some() {
tracing::info!(
target: "reth::providers::rocksdb",
"TransactionHashNumbers: checkpoint is 0, clearing stale data"
);
self.clear::<tables::TransactionHashNumbers>()?;
}
tracing::info!(
target: "reth::providers::rocksdb",
"TransactionHashNumbers: checkpoint is 0, clearing stale data"
);
self.clear::<tables::TransactionHashNumbers>()?;
return Ok(None);
}
@@ -262,39 +256,21 @@ impl RocksDBProvider {
provider: &Provider,
) -> ProviderResult<Option<BlockNumber>>
where
Provider: DBProvider
+ StageCheckpointReader
+ StaticFileProviderFactory
+ StorageChangeSetReader
+ ChainSpecProvider,
Provider:
DBProvider + StageCheckpointReader + StaticFileProviderFactory + StorageChangeSetReader,
{
let checkpoint = provider
.get_stage_checkpoint(StageId::IndexStorageHistory)?
.map(|cp| cp.block_number)
.unwrap_or(0);
// Fast path: clear and re-insert genesis history.
// Fast path: clear any stale data and return.
if checkpoint == 0 {
tracing::info!(
target: "reth::providers::rocksdb",
"StoragesHistory: checkpoint is 0, clearing stale data"
);
self.clear::<tables::StoragesHistory>()?;
let chain_spec = provider.chain_spec();
let genesis = chain_spec.genesis();
let list = tables::BlockNumberList::new([0]).expect("single block always fits");
for (addr, account) in &genesis.alloc {
if let Some(storage) = &account.storage {
for key in storage.keys() {
self.put::<tables::StoragesHistory>(
StorageShardedKey::last(*addr, *key),
&list,
)?;
}
}
}
return Ok(None);
}
@@ -358,6 +334,7 @@ impl RocksDBProvider {
let batch = self.unwind_storage_history_indices(&indices)?;
self.commit_batch(batch)?;
self.flush(&[tables::StoragesHistory::NAME])?;
}
batch_start = batch_end + 1;
@@ -375,32 +352,20 @@ impl RocksDBProvider {
provider: &Provider,
) -> ProviderResult<Option<BlockNumber>>
where
Provider: DBProvider
+ StageCheckpointReader
+ StaticFileProviderFactory
+ ChangeSetReader
+ ChainSpecProvider,
Provider: DBProvider + StageCheckpointReader + StaticFileProviderFactory + ChangeSetReader,
{
let checkpoint = provider
.get_stage_checkpoint(StageId::IndexAccountHistory)?
.map(|cp| cp.block_number)
.unwrap_or(0);
// Fast path: clear and re-insert genesis history.
// Fast path: clear any stale data and return.
if checkpoint == 0 {
tracing::info!(
target: "reth::providers::rocksdb",
"AccountsHistory: checkpoint is 0, clearing stale data"
);
self.clear::<tables::AccountsHistory>()?;
let chain_spec = provider.chain_spec();
let genesis = chain_spec.genesis();
let list = tables::BlockNumberList::new([0]).expect("single block always fits");
for addr in genesis.alloc.keys() {
self.put::<tables::AccountsHistory>(ShardedKey::last(*addr), &list)?;
}
return Ok(None);
}
@@ -474,18 +439,13 @@ impl RocksDBProvider {
#[cfg(test)]
mod tests {
use std::sync::Arc;
use super::*;
use crate::{
init::insert_genesis_history,
providers::{rocksdb::RocksDBBuilder, static_file::StaticFileWriter},
test_utils::{create_test_provider_factory, create_test_provider_factory_with_chain_spec},
BlockWriter, DatabaseProviderFactory, RocksDBProviderFactory, StageCheckpointWriter,
TransactionsProvider,
test_utils::create_test_provider_factory,
BlockWriter, DatabaseProviderFactory, StageCheckpointWriter, TransactionsProvider,
};
use alloy_primitives::{Address, B256};
use reth_chainspec::MAINNET;
use reth_db::cursor::{DbCursorRO, DbCursorRW};
use reth_db_api::{
models::{storage_sharded_key::StorageShardedKey, StorageSettings},
@@ -569,14 +529,11 @@ mod tests {
let result = rocksdb.heal_storages_history(&provider).unwrap();
assert_eq!(result, None, "StoragesHistory should return early at checkpoint 0");
assert!(rocksdb.first::<tables::StoragesHistory>().unwrap().is_none());
let result = rocksdb.heal_accounts_history(&provider).unwrap();
assert_eq!(result, None, "AccountsHistory should return early at checkpoint 0");
// Genesis account history entries are re-inserted
assert_eq!(
rocksdb.iter::<tables::AccountsHistory>().unwrap().count(),
factory.chain_spec().genesis().alloc.len()
);
assert!(rocksdb.first::<tables::AccountsHistory>().unwrap().is_none());
}
#[test]
@@ -703,38 +660,35 @@ mod tests {
}
#[test]
fn test_check_consistency_storages_history_preserves_genesis_entries_at_checkpoint_zero(
) -> eyre::Result<()> {
// Modify mainnet chainspec to include a single genesis storage slot
let mut chain_spec = MAINNET.clone();
Arc::make_mut(&mut chain_spec).genesis.alloc.first_entry().unwrap().get_mut().storage =
Some(From::from([(B256::random(), B256::random())]));
fn test_check_consistency_storages_history_has_data_no_checkpoint_prunes_data() {
let temp_dir = TempDir::new().unwrap();
let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
// Insert data into RocksDB
let key = StorageShardedKey::new(Address::ZERO, B256::ZERO, 50);
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30, 50]);
rocksdb.put::<tables::StoragesHistory>(key, &block_list).unwrap();
// Verify data exists
assert!(rocksdb.last::<tables::StoragesHistory>().unwrap().is_some());
// Create a test provider factory for MDBX with NO checkpoint
let factory = create_test_provider_factory_with_chain_spec(chain_spec);
let rocksdb = factory.rocksdb_provider();
let factory = create_test_provider_factory();
factory.set_storage_settings_cache(StorageSettings::v2());
// Insert genesis history into RocksDB
let provider_rw = factory.database_provider_rw().unwrap();
insert_genesis_history(&provider_rw, factory.chain_spec().genesis.alloc.iter())?;
provider_rw.commit()?;
let provider = factory.database_provider_ro().unwrap();
// This should not prune anything because only genesis entries are present
let result = rocksdb.heal_storages_history(&provider).unwrap();
assert_eq!(result, None, "Should skip healing when only genesis entries present");
// RocksDB has data but checkpoint is 0
// This means RocksDB has stale data that should be pruned (healed)
let result = rocksdb.check_consistency(&provider).unwrap();
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
// Verify data was NOT deleted
// Verify data was pruned
assert!(
rocksdb.iter::<tables::StoragesHistory>().unwrap().count() > 0,
"Genesis entries should be preserved"
rocksdb.last::<tables::StoragesHistory>().unwrap().is_none(),
"RocksDB should be empty after pruning"
);
Ok(())
}
#[test]
fn test_check_consistency_mdbx_behind_checkpoint_needs_unwind() {
let temp_dir = TempDir::new().unwrap();
@@ -1114,31 +1068,36 @@ mod tests {
}
#[test]
fn test_check_consistency_accounts_history_preserves_genesis_entries_at_checkpoint_zero(
) -> eyre::Result<()> {
fn test_check_consistency_accounts_history_has_data_no_checkpoint_prunes_data() {
use reth_db_api::models::ShardedKey;
let temp_dir = TempDir::new().unwrap();
let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
// Insert data into RocksDB
let key = ShardedKey::new(Address::ZERO, 50);
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30, 50]);
rocksdb.put::<tables::AccountsHistory>(key, &block_list).unwrap();
// Verify data exists
assert!(rocksdb.last::<tables::AccountsHistory>().unwrap().is_some());
// Create a test provider factory for MDBX with NO checkpoint
let factory = create_test_provider_factory();
factory.set_storage_settings_cache(StorageSettings::v2());
let rocksdb = factory.rocksdb_provider();
// Insert genesis history into RocksDB
let provider_rw = factory.database_provider_rw().unwrap();
insert_genesis_history(&provider_rw, factory.chain_spec().genesis.alloc.iter())?;
provider_rw.commit()?;
let provider = factory.database_provider_ro().unwrap();
// This should not prune anything because only genesis entries are present
// RocksDB has data but checkpoint is 0
// This means RocksDB has stale data that should be pruned (healed)
let result = rocksdb.check_consistency(&provider).unwrap();
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
// Verify data was NOT deleted
// Verify data was pruned
assert!(
rocksdb.iter::<tables::AccountsHistory>().unwrap().count() > 0,
"Genesis entries should be preserved"
rocksdb.last::<tables::AccountsHistory>().unwrap().is_none(),
"RocksDB should be empty after pruning"
);
Ok(())
}
#[test]

View File

@@ -20,7 +20,7 @@ use reth_storage_errors::provider::ProviderResult;
use reth_trie::{
hashed_cursor::HashedPostStateCursorFactory,
proof::{Proof, StorageProof},
trie_cursor::InMemoryTrieCursorFactory,
trie_cursor::{masked::MaskedTrieCursorFactory, InMemoryTrieCursorFactory},
updates::TrieUpdates,
witness::TrieWitness,
AccountProof, HashedPostState, HashedPostStateSorted, HashedStorage, KeccakKeyHasher,
@@ -525,16 +525,18 @@ impl<
let nodes_sorted = input.nodes.into_sorted();
let state_sorted = input.state.into_sorted();
TrieWitness::new(
InMemoryTrieCursorFactory::new(
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
&nodes_sorted,
MaskedTrieCursorFactory::new(
InMemoryTrieCursorFactory::new(
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
&nodes_sorted,
),
input.prefix_sets.freeze(),
),
HashedPostStateCursorFactory::new(
reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()),
&state_sorted,
),
)
.with_prefix_sets_mut(input.prefix_sets)
.always_include_root_node()
.compute(target)
.map_err(ProviderError::from)

View File

@@ -11,7 +11,7 @@ use reth_storage_errors::provider::{ProviderError, ProviderResult};
use reth_trie::{
hashed_cursor::HashedPostStateCursorFactory,
proof::{Proof, StorageProof},
trie_cursor::InMemoryTrieCursorFactory,
trie_cursor::{masked::MaskedTrieCursorFactory, InMemoryTrieCursorFactory},
updates::TrieUpdates,
witness::TrieWitness,
AccountProof, HashedPostState, HashedStorage, KeccakKeyHasher, MultiProof, MultiProofTargets,
@@ -223,16 +223,18 @@ impl<Provider: DBProvider + StorageSettingsCache> StateProofProvider
let nodes_sorted = input.nodes.into_sorted();
let state_sorted = input.state.into_sorted();
Ok(TrieWitness::new(
InMemoryTrieCursorFactory::new(
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
&nodes_sorted,
MaskedTrieCursorFactory::new(
InMemoryTrieCursorFactory::new(
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
&nodes_sorted,
),
input.prefix_sets.freeze(),
),
HashedPostStateCursorFactory::new(
reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()),
&state_sorted,
),
)
.with_prefix_sets_mut(input.prefix_sets)
.always_include_root_node()
.compute(target)?
.into_values()

View File

@@ -1429,7 +1429,7 @@ pub fn ensure_intrinsic_gas<T: EthPoolTransaction>(
);
let gas_limit = transaction.gas_limit();
if gas_limit < gas.initial_total_gas || gas_limit < gas.floor_gas {
if gas_limit < gas.initial_gas || gas_limit < gas.floor_gas {
Err(InvalidPoolTransactionError::IntrinsicGasTooLow)
} else {
Ok(())

View File

@@ -9,7 +9,7 @@ use nodes::{
};
use crate::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie, SparseTrieUpdates};
use alloc::{borrow::Cow, boxed::Box, collections::VecDeque, vec::Vec};
use alloc::{borrow::Cow, boxed::Box, vec::Vec};
use alloy_primitives::{
keccak256,
map::{B256Map, HashMap, HashSet},
@@ -60,54 +60,6 @@ fn prefix_range(
begin..end
}
/// Compacts an arena by BFS-copying all reachable nodes into a fresh `SlotMap`, dropping
/// unreachable (pruned) slots. Parents are stored before children for cache-friendly top-down
/// traversal.
fn compact_arena(arena: &mut NodeArena, root: &mut Index) {
let mut new_arena = SlotMap::with_capacity(arena.len());
let mut queue = VecDeque::new();
let root_node = arena.remove(*root).expect("root exists");
let new_root = new_arena.insert(root_node);
queue.push_back(new_root);
while let Some(new_idx) = queue.pop_front() {
// Invariant: any node popped from `queue` has been moved into `new_arena` but
// its Branch.children have not been rewritten yet — every Revealed(idx) here is
// still an old-arena index, and the child is still present in `arena` because
// only this parent's iteration can remove it (each child has exactly one parent).
let old_children: SmallVec<[(usize, Index); 16]> = match &new_arena[new_idx] {
ArenaSparseNode::Branch(b) => b
.children
.iter()
.enumerate()
.filter_map(|(i, c)| match c {
ArenaSparseNodeBranchChild::Revealed(old_idx) => Some((i, *old_idx)),
_ => None,
})
.collect(),
_ => continue,
};
for (child_pos, old_child_idx) in old_children {
let child_node = arena.remove(old_child_idx).expect("child exists");
let new_child_idx = new_arena.insert(child_node);
let ArenaSparseNode::Branch(b) = &mut new_arena[new_idx] else { unreachable!() };
b.children[child_pos] = ArenaSparseNodeBranchChild::Revealed(new_child_idx);
queue.push_back(new_child_idx);
}
}
debug_assert!(
arena.is_empty(),
"compact_arena: {} orphaned nodes remaining after BFS drain",
arena.len(),
);
*arena = new_arena;
*root = new_root;
}
/// Reusable buffers shared by both [`ArenaSparseSubtrie`] and [`ArenaParallelSparseTrie`].
#[derive(Debug, Default, Clone)]
struct ArenaTrieBuffers {
@@ -174,7 +126,7 @@ impl ArenaSparseSubtrie {
}
fn clear(&mut self) {
self.arena = SlotMap::new();
self.arena.clear();
self.buffers.clear();
self.required_proofs.clear();
self.num_leaves = 0;
@@ -255,11 +207,6 @@ impl ArenaSparseSubtrie {
}
self.num_leaves -= pruned_leaves;
if pruned > 0 {
compact_arena(&mut self.arena, &mut self.root);
}
#[cfg(debug_assertions)]
self.debug_assert_counters();
pruned
@@ -1764,18 +1711,7 @@ impl ArenaParallelSparseTrie {
.filter(|(_, _, u)| matches!(u, LeafUpdate::Changed(v) if v.is_empty()))
.count() as u64;
// Touched is a no-op that doesn't alter trie structure, so it must be
// excluded when deciding whether "all updates are removals". This mirrors
// the `all_removals` / `might_empty_subtrie` filter in `update_leaves`.
// Without this, a batch of removals + Touched entries
// would fail the `num_removals != num_changed` check, skip the proof
// request for the blinded sibling, and later panic in
// `maybe_collapse_or_remove_branch` when the subtrie empties inline.
let num_changed =
subtrie_updates.iter().filter(|(_, _, u)| matches!(u, LeafUpdate::Changed(_))).count()
as u64;
if num_removals == 0 || num_removals != num_changed {
if num_removals == 0 || num_removals as usize != subtrie_updates.len() {
return None;
}
@@ -2594,7 +2530,7 @@ impl SparseTrie for ArenaParallelSparseTrie {
self.cleared_subtries.push(subtrie);
}
}
self.upper_arena = SlotMap::new();
self.upper_arena.clear();
self.root = self.upper_arena.insert(ArenaSparseNode::EmptyRoot);
if let Some(updates) = self.updates.as_mut() {
updates.clear()
@@ -2603,15 +2539,15 @@ impl SparseTrie for ArenaParallelSparseTrie {
}
fn shrink_nodes_to(&mut self, size: usize) {
// We do not shrink the upper trie for now.
//
// As soon as a trie rotates from a live subtrie into the
// cleared_subtrie it will be properly shrunk.
for s in &mut self.cleared_subtries {
if s.arena.capacity() > size {
s.arena = SlotMap::with_capacity(size);
self.upper_arena.shrink_to(size);
for (_, node) in &mut self.upper_arena {
if let ArenaSparseNode::Subtrie(s) = node {
s.arena.shrink_to(size);
}
}
for s in &mut self.cleared_subtries {
s.arena.shrink_to(size);
}
}
fn shrink_values_to(&mut self, _size: usize) {
@@ -2788,10 +2724,6 @@ impl SparseTrie for ArenaParallelSparseTrie {
}
}
if pruned > 0 {
compact_arena(&mut self.upper_arena, &mut self.root);
}
#[cfg(feature = "trie-debug")]
self.record_initial_state();

View File

@@ -271,7 +271,6 @@ sparse_trie_tests! {
test_branch_collapse_updates_leaf_key_len_across_subtries,
test_remove_leaf_does_not_reveal_blind_subtries,
test_branch_collapse_multi_empty_subtries_blinded_remaining,
test_subtrie_collapse_touched_with_blinded_sibling,
test_subtrie_emptied_by_deletes_with_touched,
// root

View File

@@ -1406,79 +1406,6 @@ pub(super) fn test_branch_collapse_multi_empty_subtries_blinded_remaining<T: Spa
);
}
/// Regression: subtrie emptied by deletes + `Touched` with a **blinded** sibling.
///
/// `check_subtrie_collapse_needs_proof` compared `num_removals` against the full
/// `subtrie_updates.len()`, which included `Touched` entries. When `Touched` was
/// present alongside removals, the lengths didn't match, so the function skipped
/// the proof request for the blinded sibling. The subtrie was then emptied inline
/// via `might_empty_subtrie`, and `maybe_collapse_or_remove_branch` hit the
/// blinded sibling and panicked.
pub(super) fn test_subtrie_collapse_touched_with_blinded_sibling<T: SparseTrie>(
new_trie: fn() -> T,
) {
// Trie shape: root branch has children at nibbles 0xa and 0xc.
// Under 0xa there is a branch with children at 0xab (subtrie, 2 leaves) and
// 0xac (blinded — we never reveal it).
//
// We delete both 0xab leaves and also send Touched for a third 0xab key.
// After the subtrie empties, the branch at 0xa has a single child (0xac) that
// is blinded. The code must request a proof for it instead of panicking.
let mut key_ab1 = B256::ZERO;
key_ab1[0] = 0xAB;
key_ab1[31] = 0x11;
let mut key_ab2 = B256::ZERO;
key_ab2[0] = 0xAB;
key_ab2[31] = 0x22;
let mut key_ab3 = B256::ZERO;
key_ab3[0] = 0xAB;
key_ab3[31] = 0x33;
let mut key_ac1 = B256::ZERO;
key_ac1[0] = 0xAC;
key_ac1[31] = 0x44;
let mut key_cd1 = B256::ZERO;
key_cd1[0] = 0xCD;
key_cd1[31] = 0x01;
let value = U256::from(1u64);
let base_storage: BTreeMap<B256, U256> =
[(key_ab1, value), (key_ab2, value), (key_ac1, value), (key_cd1, value)]
.into_iter()
.collect();
let harness = SuiteTestHarness::new(base_storage.clone());
// Reveal only the 0xAB keys and 0xCD — leave 0xAC blinded.
let revealed_keys = vec![key_ab1, key_ab2, key_cd1];
let mut trie: T = harness.init_trie_with_targets(&revealed_keys, false, new_trie);
// Verify initial root matches.
let root = trie.root();
assert_eq!(root, harness.original_root(), "initial root mismatch");
// Delete both 0xAB leaves + Touched on a third 0xAB key (not in the trie).
// The combination of Touched + removals with a blinded sibling (0xAC) is the
// trigger for the bug.
let mut leaf_updates: B256Map<LeafUpdate> = [
(key_ab1, LeafUpdate::Changed(Vec::new())),
(key_ab2, LeafUpdate::Changed(Vec::new())),
(key_ab3, LeafUpdate::Touched),
]
.into_iter()
.collect();
harness.reveal_and_update(&mut trie, &mut leaf_updates);
// Root should match reference trie with ab1 and ab2 removed.
let mut expected_storage = base_storage;
expected_storage.remove(&key_ab1);
expected_storage.remove(&key_ab2);
let expected_harness = SuiteTestHarness::new(expected_storage);
let actual_root = trie.root();
assert_eq!(actual_root, expected_harness.original_root(), "post-delete root mismatch");
}
/// Regression: subtrie emptied by deletes mixed with `LeafUpdate::Touched`.
///
/// When all `Changed` updates in a subtrie are removals and they would empty the subtrie,

View File

@@ -149,24 +149,15 @@ where
) -> Result<DecodedMultiProofV2, StateProofError> {
let MultiProofTargetsV2 { mut account_targets, storage_targets } = targets;
let storage_prefix_sets: B256Map<_> = self
.prefix_sets
.storage_prefix_sets
.into_iter()
.map(|(addr, ps)| (addr, ps.freeze()))
.collect();
// Compute account proofs using the V2 proof calculator with sync account encoding.
let account_trie_cursor = self.trie_cursor_factory.account_trie_cursor()?;
let hashed_account_cursor = self.hashed_cursor_factory.hashed_account_cursor()?;
let mut account_value_encoder = SyncAccountValueEncoder::new(
self.trie_cursor_factory.clone(),
self.hashed_cursor_factory.clone(),
)
.with_storage_prefix_sets(storage_prefix_sets.clone());
);
let mut account_calculator =
proof_v2::ProofCalculator::new(account_trie_cursor, hashed_account_cursor)
.with_prefix_set(self.prefix_sets.account_prefix_set.freeze());
proof_v2::ProofCalculator::new(account_trie_cursor, hashed_account_cursor);
let account_proofs =
account_calculator.proof(&mut account_value_encoder, &mut account_targets)?;
@@ -182,9 +173,6 @@ where
storage_trie_cursor,
hashed_storage_cursor,
);
if let Some(prefix_set) = storage_prefix_sets.get(&hashed_address) {
storage_calculator = storage_calculator.with_prefix_set(prefix_set.clone());
}
let proofs = storage_calculator.storage_proof(hashed_address, &mut targets)?;
storage_proofs.insert(hashed_address, proofs);
}

View File

@@ -0,0 +1,752 @@
use super::{TrieCursor, TrieCursorFactory, TrieStorageCursor};
use alloy_primitives::{map::B256Map, B256};
use reth_storage_errors::db::DatabaseError;
use reth_trie_common::{
prefix_set::{PrefixSet, TriePrefixSets},
BranchNodeCompact, Nibbles,
};
use std::sync::Arc;
/// A [`TrieCursorFactory`] wrapper that creates cursors which invalidate cached trie hash data
/// for children whose paths match the prefix sets in a [`TriePrefixSets`].
///
/// The `destroyed_accounts` field of the prefix sets is not used by the cursor — it is only
/// relevant during trie update finalization, not during cursor traversal.
#[derive(Debug, Clone)]
pub struct MaskedTrieCursorFactory<CF> {
/// Underlying trie cursor factory.
cursor_factory: CF,
/// Frozen prefix sets used for masking.
prefix_sets: TriePrefixSets,
}
impl<CF> MaskedTrieCursorFactory<CF> {
/// Create a new factory from an inner cursor factory and frozen prefix sets.
pub const fn new(cursor_factory: CF, prefix_sets: TriePrefixSets) -> Self {
Self { cursor_factory, prefix_sets }
}
}
impl<CF: TrieCursorFactory> TrieCursorFactory for MaskedTrieCursorFactory<CF> {
type AccountTrieCursor<'a>
= MaskedTrieCursor<CF::AccountTrieCursor<'a>>
where
Self: 'a;
type StorageTrieCursor<'a>
= MaskedTrieCursor<CF::StorageTrieCursor<'a>>
where
Self: 'a;
fn account_trie_cursor(&self) -> Result<Self::AccountTrieCursor<'_>, DatabaseError> {
let cursor = self.cursor_factory.account_trie_cursor()?;
Ok(MaskedTrieCursor::new(cursor, self.prefix_sets.account_prefix_set.clone()))
}
fn storage_trie_cursor(
&self,
hashed_address: B256,
) -> Result<Self::StorageTrieCursor<'_>, DatabaseError> {
let cursor = self.cursor_factory.storage_trie_cursor(hashed_address)?;
let prefix_set =
self.prefix_sets.storage_prefix_sets.get(&hashed_address).cloned().unwrap_or_default();
Ok(MaskedTrieCursor::new_storage(
cursor,
prefix_set,
self.prefix_sets.storage_prefix_sets.clone(),
))
}
}
/// A [`TrieCursor`] wrapper that invalidates cached trie hash data for children whose paths match
/// a [`PrefixSet`].
///
/// For each node returned by the inner cursor, hash bits are unset for children whose paths match
/// the prefix set, and the corresponding hashes are removed from the node. If a node's `hash_mask`
/// and `tree_mask` are both empty after masking, the node is skipped entirely.
#[derive(Debug)]
pub struct MaskedTrieCursor<C> {
/// The inner cursor.
cursor: C,
/// Prefix set used to determine which children's hashes to invalidate.
prefix_set: PrefixSet,
/// Storage prefix sets for swapping on `set_hashed_address`.
storage_prefix_sets: Option<B256Map<PrefixSet>>,
}
impl<C> MaskedTrieCursor<C> {
/// Create a new cursor wrapping `cursor`, masking hash bits for children whose paths match
/// `prefix_set`.
pub const fn new(cursor: C, prefix_set: PrefixSet) -> Self {
Self { cursor, prefix_set, storage_prefix_sets: None }
}
/// Create a new storage cursor that can swap its prefix set on `set_hashed_address`.
pub const fn new_storage(
cursor: C,
prefix_set: PrefixSet,
storage_prefix_sets: B256Map<PrefixSet>,
) -> Self {
Self { cursor, prefix_set, storage_prefix_sets: Some(storage_prefix_sets) }
}
}
impl<C: TrieCursor> MaskedTrieCursor<C> {
/// Mask hash bits on a node for children whose paths match the prefix set.
///
/// Returns `true` if the node should be kept, `false` if it should be skipped (both
/// `hash_mask` and `tree_mask` are empty after masking).
fn mask_node(&mut self, key: &Nibbles, node: &mut BranchNodeCompact) -> bool {
if !self.prefix_set.contains(key) {
return true;
}
// The subtree is modified — root hash is always invalid.
node.root_hash = None;
let original_hash_mask = node.hash_mask;
if original_hash_mask.is_empty() {
return true;
}
let mut new_hash_mask = original_hash_mask;
let mut child_path = *key;
let key_len = key.len();
for nibble in original_hash_mask.iter() {
child_path.truncate(key_len);
child_path.push(nibble);
if self.prefix_set.contains(&child_path) {
new_hash_mask.unset_bit(nibble);
}
}
if new_hash_mask != original_hash_mask {
// Remove hashes for unset bits in-place.
let hashes = Arc::make_mut(&mut node.hashes);
let mut write = 0;
for (read, nibble) in original_hash_mask.iter().enumerate() {
if new_hash_mask.is_bit_set(nibble) {
hashes[write] = hashes[read];
write += 1;
}
}
hashes.truncate(write);
node.hash_mask = new_hash_mask;
if node.hash_mask.is_empty() && node.tree_mask.is_empty() {
return false;
}
}
true
}
/// Apply masking to entries, advancing past fully-masked nodes.
fn mask_entries(
&mut self,
mut entry: Option<(Nibbles, BranchNodeCompact)>,
) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
while let Some((key, mut node)) = entry {
if self.mask_node(&key, &mut node) {
return Ok(Some((key, node)));
}
entry = self.cursor.next()?;
}
Ok(None)
}
}
impl<C: TrieCursor> TrieCursor for MaskedTrieCursor<C> {
fn seek_exact(
&mut self,
key: Nibbles,
) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
if let Some((key, mut node)) = self.cursor.seek_exact(key)? {
if self.mask_node(&key, &mut node) {
Ok(Some((key, node)))
} else {
Ok(None)
}
} else {
Ok(None)
}
}
fn seek(
&mut self,
key: Nibbles,
) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
let entry = self.cursor.seek(key)?;
self.mask_entries(entry)
}
fn next(&mut self) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
let entry = self.cursor.next()?;
self.mask_entries(entry)
}
fn current(&mut self) -> Result<Option<Nibbles>, DatabaseError> {
self.cursor.current()
}
fn reset(&mut self) {
self.cursor.reset();
}
}
impl<C: TrieStorageCursor> TrieStorageCursor for MaskedTrieCursor<C> {
fn set_hashed_address(&mut self, hashed_address: B256) {
self.cursor.set_hashed_address(hashed_address);
if let Some(storage_prefix_sets) = &self.storage_prefix_sets {
self.prefix_set = storage_prefix_sets.get(&hashed_address).cloned().unwrap_or_default();
}
}
}
#[cfg(test)]
mod tests {
use super::*;
use crate::trie_cursor::mock::MockTrieCursor;
use parking_lot::Mutex;
use reth_trie_common::prefix_set::PrefixSetMut;
use std::{collections::BTreeMap, sync::Arc};
fn make_cursor(nodes: Vec<(Nibbles, BranchNodeCompact)>) -> MockTrieCursor {
let map: BTreeMap<Nibbles, BranchNodeCompact> = nodes.into_iter().collect();
MockTrieCursor::new(Arc::new(map), Arc::new(Mutex::new(Vec::new())))
}
fn node(state_mask: u16) -> BranchNodeCompact {
BranchNodeCompact::new(state_mask, 0, 0, vec![], None)
}
fn node_with_hashes(state_mask: u16, hash_mask: u16, hashes: Vec<B256>) -> BranchNodeCompact {
BranchNodeCompact::new(state_mask, 0, hash_mask, hashes, None)
}
fn node_with_tree_mask(
state_mask: u16,
tree_mask: u16,
hash_mask: u16,
hashes: Vec<B256>,
) -> BranchNodeCompact {
BranchNodeCompact::new(state_mask, tree_mask, hash_mask, hashes, None)
}
fn hash(byte: u8) -> B256 {
B256::repeat_byte(byte)
}
#[test]
fn test_seek_masks_matching_child_hashes() {
// Node at [0x1] with children 2 and 5 hashed.
// Prefix set marks child 2 as changed.
let nodes = vec![(
Nibbles::from_nibbles([0x1]),
node_with_hashes(0b0000_0000_0010_0100, 0b0000_0000_0010_0100, vec![hash(2), hash(5)]),
)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x2]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek(Nibbles::default()).unwrap();
let (key, node) = result.unwrap();
assert_eq!(key, Nibbles::from_nibbles([0x1]));
// Hash bit 2 should be unset, only bit 5 remains.
assert!(!node.hash_mask.is_bit_set(2));
assert!(node.hash_mask.is_bit_set(5));
assert_eq!(&*node.hashes, &[hash(5)]);
}
#[test]
fn test_seek_skips_fully_masked_node() {
// Node at [0x1] with only child 3 hashed, tree_mask empty.
// Prefix set marks child 3 as changed → fully masked → skipped.
// Node at [0x2] is unaffected → returned.
let nodes = vec![
(
Nibbles::from_nibbles([0x1]),
node_with_hashes(0b0000_0000_0000_1000, 0b0000_0000_0000_1000, vec![hash(3)]),
),
(Nibbles::from_nibbles([0x2]), node(0b0000_0000_0000_0001)),
];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x3]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek(Nibbles::default()).unwrap();
assert_eq!(result, Some((Nibbles::from_nibbles([0x2]), node(0b0000_0000_0000_0001))));
}
#[test]
fn test_node_with_tree_mask_not_skipped() {
// Node at [0x1] with child 3 hashed, tree_mask has bit 3 set.
// Prefix set marks child 3 → hash cleared, but tree_mask keeps the node alive.
let nodes = vec![(
Nibbles::from_nibbles([0x1]),
node_with_tree_mask(
0b0000_0000_0000_1000,
0b0000_0000_0000_1000,
0b0000_0000_0000_1000,
vec![hash(3)],
),
)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x3]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek(Nibbles::default()).unwrap();
let (key, node) = result.unwrap();
assert_eq!(key, Nibbles::from_nibbles([0x1]));
assert!(node.hash_mask.is_empty());
assert!(node.tree_mask.is_bit_set(3));
assert!(node.hashes.is_empty());
}
#[test]
fn test_seek_exact_masks_hash_bits() {
let nodes = vec![(
Nibbles::from_nibbles([0x1]),
node_with_tree_mask(
0b0000_0000_0010_0100,
0b0000_0000_0010_0100,
0b0000_0000_0010_0100,
vec![hash(2), hash(5)],
),
)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x5]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek_exact(Nibbles::from_nibbles([0x1])).unwrap();
let (_, node) = result.unwrap();
assert!(node.hash_mask.is_bit_set(2));
assert!(!node.hash_mask.is_bit_set(5));
assert_eq!(&*node.hashes, &[hash(2)]);
}
#[test]
fn test_seek_exact_returns_none_for_fully_masked() {
let nodes = vec![(
Nibbles::from_nibbles([0x1]),
node_with_hashes(0b0000_0000_0000_0100, 0b0000_0000_0000_0100, vec![hash(2)]),
)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x2]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek_exact(Nibbles::from_nibbles([0x1])).unwrap();
assert_eq!(result, None);
}
#[test]
fn test_next_masks_and_skips() {
// Three nodes: [0x1] unaffected, [0x2] fully masked, [0x3] unaffected.
let nodes = vec![
(
Nibbles::from_nibbles([0x1]),
node_with_hashes(0b0000_0000_0000_0010, 0b0000_0000_0000_0010, vec![hash(1)]),
),
(
Nibbles::from_nibbles([0x2]),
node_with_hashes(0b0000_0000_0001_0000, 0b0000_0000_0001_0000, vec![hash(4)]),
),
(
Nibbles::from_nibbles([0x3]),
node_with_hashes(0b0000_0000_0100_0000, 0b0000_0000_0100_0000, vec![hash(6)]),
),
];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x2, 0x4]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
// seek to [0x1], no match → returned unchanged.
let result = cursor.seek(Nibbles::from_nibbles([0x1])).unwrap();
let (key, node) = result.unwrap();
assert_eq!(key, Nibbles::from_nibbles([0x1]));
assert_eq!(&*node.hashes, &[hash(1)]);
// next() should skip [0x2] (fully masked), returning [0x3].
let result = cursor.next().unwrap();
let (key, node) = result.unwrap();
assert_eq!(key, Nibbles::from_nibbles([0x3]));
assert_eq!(&*node.hashes, &[hash(6)]);
}
#[test]
fn test_no_match_returns_unchanged() {
let nodes = vec![(
Nibbles::from_nibbles([0x2]),
node_with_hashes(0b0000_0000_0000_0010, 0b0000_0000_0000_0010, vec![hash(1)]),
)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x3]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek(Nibbles::default()).unwrap();
let (key, node) = result.unwrap();
assert_eq!(key, Nibbles::from_nibbles([0x2]));
// Unchanged — prefix set doesn't match [0x2].
assert!(node.hash_mask.is_bit_set(1));
assert_eq!(&*node.hashes, &[hash(1)]);
}
#[test]
fn test_empty_prefix_set_returns_all_unchanged() {
let h1 = hash(1);
let h2 = hash(2);
let nodes = vec![
(
Nibbles::from_nibbles([0x1]),
node_with_hashes(0b0000_0000_0000_0010, 0b0000_0000_0000_0010, vec![h1]),
),
(
Nibbles::from_nibbles([0x2]),
node_with_hashes(0b0000_0000_0000_0100, 0b0000_0000_0000_0100, vec![h2]),
),
];
let ps = PrefixSetMut::default();
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let r1 = cursor.seek(Nibbles::default()).unwrap().unwrap();
assert_eq!(r1.0, Nibbles::from_nibbles([0x1]));
assert_eq!(&*r1.1.hashes, &[h1]);
let r2 = cursor.next().unwrap().unwrap();
assert_eq!(r2.0, Nibbles::from_nibbles([0x2]));
assert_eq!(&*r2.1.hashes, &[h2]);
assert_eq!(cursor.next().unwrap(), None);
}
#[test]
fn test_root_hash_cleared_on_mask() {
let mut n =
node_with_hashes(0b0000_0000_0010_0100, 0b0000_0000_0010_0100, vec![hash(2), hash(5)]);
n.root_hash = Some(hash(0xFF));
let nodes = vec![(Nibbles::from_nibbles([0x1]), n)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x2]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let (_, node) = cursor.seek(Nibbles::default()).unwrap().unwrap();
assert_eq!(node.root_hash, None);
}
#[test]
fn test_node_without_hashes_returned_unchanged() {
// Node with state_mask only (no hashes, no tree_mask) should pass through.
let nodes = vec![(Nibbles::from_nibbles([0x1]), node(0b0000_0000_0000_0011))];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x0]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let result = cursor.seek(Nibbles::default()).unwrap();
assert_eq!(result, Some((Nibbles::from_nibbles([0x1]), node(0b0000_0000_0000_0011))));
}
#[test]
fn test_empty_cursor_returns_none() {
let nodes = vec![];
let ps = PrefixSetMut::default();
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
assert_eq!(cursor.seek(Nibbles::default()).unwrap(), None);
}
#[test]
fn test_reset_delegates() {
let nodes =
vec![(Nibbles::from_nibbles([0x1]), node(1)), (Nibbles::from_nibbles([0x2]), node(2))];
let ps = PrefixSetMut::default();
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let _ = cursor.seek(Nibbles::from_nibbles([0x1])).unwrap();
assert_eq!(cursor.current().unwrap(), Some(Nibbles::from_nibbles([0x1])));
cursor.reset();
assert_eq!(cursor.current().unwrap(), None);
}
#[test]
fn test_partial_mask_preserves_remaining_hashes() {
// Node at [0x1] with children 0, 3, 7 hashed.
// Prefix set marks children 0 and 7 as changed.
// Only hash for child 3 should remain.
let nodes = vec![(
Nibbles::from_nibbles([0x1]),
node_with_tree_mask(
0b0000_0000_1000_1001,
0b0000_0000_1000_1001,
0b0000_0000_1000_1001,
vec![hash(0), hash(3), hash(7)],
),
)];
let mut ps = PrefixSetMut::default();
ps.insert(Nibbles::from_nibbles([0x1, 0x0]));
ps.insert(Nibbles::from_nibbles([0x1, 0x7]));
let inner = make_cursor(nodes);
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
let (key, node) = cursor.seek(Nibbles::default()).unwrap().unwrap();
assert_eq!(key, Nibbles::from_nibbles([0x1]));
assert!(!node.hash_mask.is_bit_set(0));
assert!(node.hash_mask.is_bit_set(3));
assert!(!node.hash_mask.is_bit_set(7));
assert_eq!(&*node.hashes, &[hash(3)]);
assert_eq!(node.root_hash, None);
}
mod proptest_tests {
use crate::{
hashed_cursor::{mock::MockHashedCursorFactory, HashedPostStateCursorFactory},
proof::Proof,
trie_cursor::{
masked::MaskedTrieCursorFactory, mock::MockTrieCursorFactory,
noop::NoopTrieCursorFactory,
},
StateRoot,
};
use alloy_primitives::{map::B256Set, B256, U256};
use proptest::prelude::*;
use reth_primitives_traits::Account;
use reth_trie_common::{HashedPostState, HashedStorage, MultiProofTargets};
fn account_strategy() -> impl Strategy<Value = Account> {
(any::<u64>(), any::<u64>(), any::<[u8; 32]>()).prop_map(
|(nonce, balance, code_hash)| Account {
nonce,
balance: U256::from(balance),
bytecode_hash: Some(B256::from(code_hash)),
},
)
}
fn storage_value_strategy() -> impl Strategy<Value = U256> {
any::<u64>().prop_filter("non-zero", |v| *v != 0).prop_map(U256::from)
}
/// Generates a base dataset of 1000 storage slots for account `B256::ZERO`,
/// a 200-entry changeset partially overlapping with the base, and random
/// proof targets partially overlapping with both.
#[allow(clippy::type_complexity)]
fn test_input_strategy(
) -> impl Strategy<Value = (Vec<(B256, U256)>, Account, Vec<(B256, Option<U256>)>, Vec<B256>)>
{
(
// 1000 base storage slots: unique keys with non-zero values
prop::collection::vec(
(any::<[u8; 32]>().prop_map(B256::from), storage_value_strategy()),
1000,
),
account_strategy(),
// 200 changeset entries: (key, Option<value>) where None = removal
prop::collection::vec(
(
any::<[u8; 32]>().prop_map(B256::from),
prop::option::of(storage_value_strategy()),
),
200,
),
// Extra random keys for proof targets
prop::collection::vec(any::<[u8; 32]>().prop_map(B256::from), 50),
)
.prop_flat_map(
|(base_slots, account, changeset_raw, extra_targets)| {
// Dedup base slots by key
let mut base_map = alloy_primitives::map::B256Map::default();
for (k, v) in &base_slots {
base_map.insert(*k, *v);
}
let base_deduped: Vec<(B256, U256)> =
base_map.iter().map(|(&k, &v)| (k, v)).collect();
let base_keys: Vec<B256> = base_deduped.iter().map(|(k, _)| *k).collect();
// Build changeset: 50% overlap with base keys, 50% new keys
let changeset_len = changeset_raw.len();
let half = changeset_len / 2;
let base_keys_for_overlap = base_keys.clone();
// Use indices to select from base keys for overlap portion
let overlap_indices =
prop::collection::vec(0..base_keys_for_overlap.len().max(1), half);
overlap_indices.prop_map(move |indices| {
let mut changeset: Vec<(B256, Option<U256>)> = Vec::new();
// First half: overlapping with base keys
for (i, (_, value)) in
indices.iter().zip(changeset_raw.iter()).take(half)
{
let key = if base_keys_for_overlap.is_empty() {
changeset_raw[*i].0
} else {
base_keys_for_overlap[*i % base_keys_for_overlap.len()]
};
changeset.push((key, *value));
}
// Second half: new keys from changeset_raw
for (key, value) in changeset_raw.iter().skip(half) {
changeset.push((*key, *value));
}
// Build proof targets: mix of base keys, changeset keys, and randoms
let changeset_keys: Vec<B256> =
changeset.iter().map(|(k, _)| *k).collect();
let mut proof_slot_targets: Vec<B256> = Vec::new();
// ~40% from base
for k in base_keys.iter().take(40) {
proof_slot_targets.push(*k);
}
// ~30% from changeset
for k in changeset_keys.iter().take(30) {
proof_slot_targets.push(*k);
}
// ~30% random
for k in extra_targets.iter().take(30) {
proof_slot_targets.push(*k);
}
(base_deduped.clone(), account, changeset, proof_slot_targets)
})
},
)
}
proptest! {
#![proptest_config(ProptestConfig::with_cases(50))]
#[test]
fn proptest_masked_cursor_multiproof_equivalence(
(base_slots, account, changeset, proof_slot_targets) in test_input_strategy()
) {
reth_tracing::init_test_tracing();
let hashed_address = B256::ZERO;
// Step 1: Create the base hashed post state with a single account
// and 1000 storage slots.
let base_state = HashedPostState {
accounts: std::iter::once((hashed_address, Some(account))).collect(),
storages: std::iter::once((
hashed_address,
HashedStorage::from_iter(false, base_slots),
))
.collect(),
};
// Step 2: Compute trie updates from state root over the full base state.
let base_hashed_cursor_factory =
MockHashedCursorFactory::from_hashed_post_state(base_state);
let (_, trie_updates) = StateRoot::new(
NoopTrieCursorFactory,
base_hashed_cursor_factory.clone(),
)
.root_with_updates()
.expect("state root computation should succeed");
// Step 3: Create a MockTrieCursorFactory from those trie updates.
let mock_trie_cursor_factory =
MockTrieCursorFactory::from_trie_updates(trie_updates);
// Step 4: Build the changeset post state. Removals use U256::ZERO.
let changeset_storage: Vec<(B256, U256)> = changeset
.iter()
.map(|(k, v)| (*k, v.unwrap_or(U256::ZERO)))
.collect();
let changeset_state = HashedPostState {
accounts: std::iter::once((hashed_address, Some(account))).collect(),
storages: std::iter::once((
hashed_address,
HashedStorage::from_iter(false, changeset_storage),
))
.collect(),
};
// Step 5: Generate prefix sets from the changeset.
let prefix_sets_mut = changeset_state.construct_prefix_sets();
// Step 6: Build proof targets.
let slot_targets: B256Set = proof_slot_targets.into_iter().collect();
let targets =
MultiProofTargets::from_iter([(hashed_address, slot_targets)]);
// Step 7: Create the HashedPostStateCursorFactory overlaying changeset
// on the base.
let changeset_sorted = changeset_state.into_sorted();
let overlay_cursor_factory = HashedPostStateCursorFactory::new(
base_hashed_cursor_factory,
&changeset_sorted,
);
// Step 8a: Approach A — prefix sets passed to Proof directly.
let proof_a = Proof::new(
mock_trie_cursor_factory.clone(),
overlay_cursor_factory.clone(),
)
.with_prefix_sets_mut(prefix_sets_mut.clone());
let multiproof_a = proof_a
.multiproof(targets.clone())
.expect("multiproof A should succeed");
// Step 8b: Approach B — MaskedTrieCursorFactory, no prefix sets on Proof.
let masked_trie_cursor_factory = MaskedTrieCursorFactory::new(
mock_trie_cursor_factory,
prefix_sets_mut.freeze(),
);
let proof_b = Proof::new(
masked_trie_cursor_factory,
overlay_cursor_factory,
);
let multiproof_b = proof_b
.multiproof(targets)
.expect("multiproof B should succeed");
// Step 9: Compare results.
assert_eq!(
multiproof_a, multiproof_b,
"multiproof with prefix sets should equal multiproof with masked cursor"
);
}
}
}
}

View File

@@ -11,6 +11,9 @@ pub mod subnode;
/// Noop trie cursor implementations.
pub mod noop;
/// Masked trie cursor wrapper that skips nodes matching a prefix set.
pub mod masked;
/// Depth-first trie iterator.
pub mod depth_first;

View File

@@ -1,23 +1,29 @@
use crate::{
hashed_cursor::{HashedCursor, HashedCursorFactory},
prefix_set::TriePrefixSetsMut,
proof::Proof,
proof_v2,
proof::{Proof, ProofTrieNodeProviderFactory},
trie_cursor::TrieCursorFactory,
TRIE_ACCOUNT_RLP_MAX_SIZE,
};
use alloy_rlp::EMPTY_STRING_CODE;
use alloy_trie::EMPTY_ROOT_HASH;
use reth_trie_common::HashedPostState;
use reth_trie_sparse::SparseTrie;
use alloy_primitives::{
keccak256,
map::{B256Map, HashMap},
Bytes, B256, U256,
map::{B256Map, B256Set, Entry, HashMap},
Bytes, B256,
};
use alloy_rlp::{Encodable, EMPTY_STRING_CODE};
use alloy_trie::{nodes::BranchNodeRef, EMPTY_ROOT_HASH};
use reth_execution_errors::{SparseStateTrieErrorKind, StateProofError, TrieWitnessError};
use reth_trie_common::{
DecodedMultiProofV2, HashedPostState, MultiProofTargetsV2, ProofV2Target, TrieNodeV2,
use itertools::Itertools;
use reth_execution_errors::{
SparseStateTrieErrorKind, SparseTrieError, SparseTrieErrorKind, StateProofError,
TrieWitnessError,
};
use reth_trie_sparse::{LeafUpdate, SparseStateTrie, SparseTrie as _};
use reth_trie_common::{MultiProofTargets, Nibbles};
use reth_trie_sparse::{
provider::{RevealedNode, TrieNodeProvider, TrieNodeProviderFactory},
SparseStateTrie,
};
use std::sync::mpsc;
/// State transition witness for the trie.
#[derive(Debug)]
@@ -26,8 +32,6 @@ pub struct TrieWitness<T, H> {
trie_cursor_factory: T,
/// The factory for hashed cursors.
hashed_cursor_factory: H,
/// A set of prefix sets that have changes.
prefix_sets: TriePrefixSetsMut,
/// Flag indicating whether the root node should always be included (even if the target state
/// is empty). This setting is useful if the caller wants to verify the witness against the
/// parent state root.
@@ -43,7 +47,6 @@ impl<T, H> TrieWitness<T, H> {
Self {
trie_cursor_factory,
hashed_cursor_factory,
prefix_sets: TriePrefixSetsMut::default(),
always_include_root_node: false,
witness: HashMap::default(),
}
@@ -54,7 +57,6 @@ impl<T, H> TrieWitness<T, H> {
TrieWitness {
trie_cursor_factory,
hashed_cursor_factory: self.hashed_cursor_factory,
prefix_sets: self.prefix_sets,
always_include_root_node: self.always_include_root_node,
witness: self.witness,
}
@@ -65,18 +67,11 @@ impl<T, H> TrieWitness<T, H> {
TrieWitness {
trie_cursor_factory: self.trie_cursor_factory,
hashed_cursor_factory,
prefix_sets: self.prefix_sets,
always_include_root_node: self.always_include_root_node,
witness: self.witness,
}
}
/// Set the prefix sets. They have to be mutable in order to allow extension with proof target.
pub fn with_prefix_sets_mut(mut self, prefix_sets: TriePrefixSetsMut) -> Self {
self.prefix_sets = prefix_sets;
self
}
/// Set `always_include_root_node` to true. Root node will be included even in empty state.
/// This setting is useful if the caller wants to verify the witness against the
/// parent state root.
@@ -97,131 +92,84 @@ where
/// # Arguments
///
/// `state` - state transition containing both modified and touched accounts and storage slots.
pub fn compute(
mut self,
mut state: HashedPostState,
) -> Result<B256Map<Bytes>, TrieWitnessError> {
pub fn compute(mut self, state: HashedPostState) -> Result<B256Map<Bytes>, TrieWitnessError> {
let is_state_empty = state.is_empty();
if is_state_empty && !self.always_include_root_node {
return Ok(Default::default())
}
// Expand wiped storages into explicit zero-value entries for every existing slot,
// so that downstream code can treat all storages uniformly.
self.expand_wiped_storages(&mut state)?;
let proof_targets = if is_state_empty {
MultiProofTargetsV2 {
account_targets: vec![ProofV2Target::new(B256::ZERO)],
..Default::default()
}
MultiProofTargets::account(B256::ZERO)
} else {
Self::get_proof_targets(&state)
self.get_proof_targets(&state)?
};
let multiproof =
Proof::new(self.trie_cursor_factory.clone(), self.hashed_cursor_factory.clone())
.with_prefix_sets_mut(self.prefix_sets.clone())
.multiproof_v2(proof_targets)?;
.multiproof(proof_targets.clone())?;
// No need to reconstruct the rest of the trie, we just need to include
// the root node and return.
if is_state_empty {
let (root_hash, root_node) = if let Some(root_node) =
multiproof.account_proofs.into_iter().find(|n| n.path.is_empty())
multiproof.account_subtree.into_inner().remove(&Nibbles::default())
{
let mut encoded = Vec::new();
root_node.node.encode(&mut encoded);
let bytes = Bytes::from(encoded);
(keccak256(&bytes), bytes)
(keccak256(&root_node), root_node)
} else {
(EMPTY_ROOT_HASH, Bytes::from([EMPTY_STRING_CODE]))
};
return Ok(B256Map::from_iter([(root_hash, root_node)]))
}
// Record all nodes from multiproof in the witness.
self.record_multiproof_nodes(&multiproof);
// Record all nodes from multiproof in the witness
for account_node in multiproof.account_subtree.values() {
if let Entry::Vacant(entry) = self.witness.entry(keccak256(account_node.as_ref())) {
entry.insert(account_node.clone());
}
}
for storage_node in multiproof.storages.values().flat_map(|s| s.subtree.values()) {
if let Entry::Vacant(entry) = self.witness.entry(keccak256(storage_node.as_ref())) {
entry.insert(storage_node.clone());
}
}
let (tx, rx) = mpsc::channel();
let blinded_provider_factory = WitnessTrieNodeProviderFactory::new(
ProofTrieNodeProviderFactory::new(self.trie_cursor_factory, self.hashed_cursor_factory),
tx,
);
let mut sparse_trie = SparseStateTrie::new();
sparse_trie.reveal_decoded_multiproof_v2(multiproof)?;
sparse_trie.reveal_multiproof(multiproof)?;
// Build storage leaf updates for all accounts with storage changes, split into
// removals and upserts. Removals must be applied first so that branch collapse
// detection fires correctly: if a removal and an insertion target siblings under
// the same branch, processing the removal first may reduce the branch to a single
// blinded child, triggering a proof fetch for the sibling. Processing the insertion
// first would add a new child that keeps the count above one, masking the need.
let mut storage_removals: B256Map<B256Map<LeafUpdate>> = B256Map::default();
let mut storage_upserts: B256Map<B256Map<LeafUpdate>> = B256Map::default();
for (hashed_address, storage) in &state.storages {
for (&hashed_slot, value) in &storage.storage {
if value.is_zero() {
storage_removals
.entry(*hashed_address)
.or_default()
.insert(hashed_slot, LeafUpdate::Changed(vec![]));
// Attempt to update state trie to gather additional information for the witness.
for (hashed_address, hashed_slots) in
proof_targets.into_iter().sorted_unstable_by_key(|(ha, _)| *ha)
{
// Update storage trie first.
let provider = blinded_provider_factory.storage_node_provider(hashed_address);
let storage = state.storages.get(&hashed_address);
let storage_trie = sparse_trie.storage_trie_mut(&hashed_address).ok_or(
SparseStateTrieErrorKind::SparseStorageTrie(
hashed_address,
SparseTrieErrorKind::Blind,
),
)?;
for hashed_slot in hashed_slots.into_iter().sorted_unstable() {
let storage_nibbles = Nibbles::unpack(hashed_slot);
let maybe_leaf_value = storage
.and_then(|s| s.storage.get(&hashed_slot))
.filter(|v| !v.is_zero())
.map(|v| alloy_rlp::encode_fixed_size(v).to_vec());
if let Some(value) = maybe_leaf_value {
storage_trie.update_leaf(storage_nibbles, value, &provider).map_err(|err| {
SparseStateTrieErrorKind::SparseStorageTrie(hashed_address, err.into_kind())
})?;
} else {
storage_upserts.entry(*hashed_address).or_default().insert(
hashed_slot,
LeafUpdate::Changed(alloy_rlp::encode_fixed_size(value).to_vec()),
);
storage_trie.remove_leaf(&storage_nibbles, &provider).map_err(|err| {
SparseStateTrieErrorKind::SparseStorageTrie(hashed_address, err.into_kind())
})?;
}
}
}
// Apply storage removals first, then upserts, fetching additional proofs as needed.
for storage_updates in [&mut storage_removals, &mut storage_upserts] {
loop {
let mut targets = MultiProofTargetsV2::default();
for (&hashed_address, slot_updates) in storage_updates.iter_mut() {
if slot_updates.is_empty() {
continue;
}
let storage_trie = sparse_trie
.storage_trie_mut(&hashed_address)
.expect("storage trie was revealed from multiproof");
storage_trie
.update_leaves(slot_updates, |key, min_len| {
targets
.storage_targets
.entry(hashed_address)
.or_default()
.push(ProofV2Target::new(key).with_min_len(min_len));
})
.map_err(|err| {
SparseStateTrieErrorKind::SparseStorageTrie(
hashed_address,
err.into_kind(),
)
})?;
}
if targets.is_empty() {
break;
}
let multiproof = Proof::new(
self.trie_cursor_factory.clone(),
self.hashed_cursor_factory.clone(),
)
.with_prefix_sets_mut(self.prefix_sets.clone())
.multiproof_v2(targets)?;
self.record_multiproof_nodes(&multiproof);
sparse_trie.reveal_decoded_multiproof_v2(multiproof)?;
}
}
// Build account leaf updates, split into removals and upserts (same reasoning
// as for storage updates above).
let mut account_removals: B256Map<LeafUpdate> = B256Map::default();
let mut account_upserts: B256Map<LeafUpdate> = B256Map::default();
for &hashed_address in state.accounts.keys().chain(state.storages.keys()) {
if account_removals.contains_key(&hashed_address) ||
account_upserts.contains_key(&hashed_address)
{
continue;
}
let account = state
.accounts
@@ -229,149 +177,105 @@ where
.ok_or(TrieWitnessError::MissingAccount(hashed_address))?
.unwrap_or_default();
let storage_root =
if let Some(storage_trie) = sparse_trie.storage_trie_mut(&hashed_address) {
storage_trie.root()
} else {
self.account_storage_root(hashed_address)?
};
if account.is_empty() && storage_root == EMPTY_ROOT_HASH {
account_removals.insert(hashed_address, LeafUpdate::Changed(vec![]));
} else {
let mut rlp = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE);
account.into_trie_account(storage_root).encode(&mut rlp);
account_upserts.insert(hashed_address, LeafUpdate::Changed(rlp));
if !sparse_trie.update_account(hashed_address, account, &blinded_provider_factory)? {
let nibbles = Nibbles::unpack(hashed_address);
sparse_trie.remove_account_leaf(&nibbles, &blinded_provider_factory)?;
}
}
// Apply account removals first, then upserts, fetching additional proofs as needed.
for account_updates in [&mut account_removals, &mut account_upserts] {
loop {
let mut targets = MultiProofTargetsV2::default();
sparse_trie
.trie_mut()
.update_leaves(account_updates, |key, min_len| {
targets.account_targets.push(ProofV2Target::new(key).with_min_len(min_len));
})
.map_err(SparseStateTrieErrorKind::from)?;
if targets.is_empty() {
break;
}
let multiproof = Proof::new(
self.trie_cursor_factory.clone(),
self.hashed_cursor_factory.clone(),
)
.with_prefix_sets_mut(self.prefix_sets.clone())
.multiproof_v2(targets)?;
self.record_multiproof_nodes(&multiproof);
sparse_trie.reveal_decoded_multiproof_v2(multiproof)?;
while let Ok(node) = rx.try_recv() {
self.witness.insert(keccak256(&node), node);
}
}
Ok(self.witness)
}
/// Record all nodes from a V2 decoded multiproof in the witness.
fn record_multiproof_nodes(&mut self, multiproof: &DecodedMultiProofV2) {
let mut encoded = Vec::new();
for proof_node in &multiproof.account_proofs {
self.record_witness_node(&proof_node.node, &mut encoded);
}
for proof_nodes in multiproof.storage_proofs.values() {
for proof_node in proof_nodes {
self.record_witness_node(&proof_node.node, &mut encoded);
}
}
}
/// Record a single [`TrieNodeV2`] in the witness.
fn record_witness_node(&mut self, node: &TrieNodeV2, encoded: &mut Vec<u8>) {
encoded.clear();
node.encode(encoded);
let bytes = Bytes::from(encoded.clone());
self.witness.entry(keccak256(&bytes)).or_insert(bytes);
if let TrieNodeV2::Branch(branch) = node &&
!branch.key.is_empty()
{
encoded.clear();
BranchNodeRef::new(&branch.stack, branch.state_mask).encode(encoded);
let bytes = Bytes::from(encoded.clone());
self.witness.entry(keccak256(&bytes)).or_insert(bytes);
}
}
/// Compute the storage root for an account by walking the storage trie using the cursor
/// factories and trie input prefix sets. Records the root node in the witness.
fn account_storage_root(&mut self, hashed_address: B256) -> Result<B256, TrieWitnessError> {
let storage_trie_cursor = self
.trie_cursor_factory
.storage_trie_cursor(hashed_address)
.map_err(StateProofError::from)?;
let hashed_storage_cursor = self
.hashed_cursor_factory
.hashed_storage_cursor(hashed_address)
.map_err(StateProofError::from)?;
let mut calculator = proof_v2::StorageProofCalculator::new_storage(
storage_trie_cursor,
hashed_storage_cursor,
);
if let Some(prefix_set) = self.prefix_sets.storage_prefix_sets.get(&hashed_address) {
calculator = calculator.with_prefix_set(prefix_set.clone().freeze());
}
let root_node = calculator.storage_root_node(hashed_address)?;
let root_hash = calculator
.compute_root_hash(core::slice::from_ref(&root_node))?
.unwrap_or(EMPTY_ROOT_HASH);
drop(calculator);
let mut encoded = Vec::new();
self.record_witness_node(&root_node.node, &mut encoded);
Ok(root_hash)
}
/// Expand wiped storages into explicit zero-value entries for every existing slot in the
/// database. After this, all storages can be treated uniformly without special wiped handling.
fn expand_wiped_storages(&self, state: &mut HashedPostState) -> Result<(), StateProofError> {
for (hashed_address, storage) in &mut state.storages {
if !storage.wiped {
continue;
}
let mut storage_cursor =
self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?;
let mut current_entry = storage_cursor.seek(B256::ZERO)?;
while let Some((hashed_slot, _)) = current_entry {
storage.storage.entry(hashed_slot).or_insert(U256::ZERO);
current_entry = storage_cursor.next()?;
}
storage.wiped = false;
}
Ok(())
}
/// Retrieve proof targets for incoming hashed state.
/// Aggregates all accounts and slots present in the state. Wiped storages must have been
/// expanded via [`Self::expand_wiped_storages`] before calling this.
fn get_proof_targets(state: &HashedPostState) -> MultiProofTargetsV2 {
let mut targets = MultiProofTargetsV2::default();
for &hashed_address in state.accounts.keys() {
targets.account_targets.push(ProofV2Target::new(hashed_address));
/// This method will aggregate all accounts and slots present in the hash state as well as
/// select all existing slots from the database for the accounts that have been destroyed.
fn get_proof_targets(
&self,
state: &HashedPostState,
) -> Result<MultiProofTargets, StateProofError> {
let mut proof_targets = MultiProofTargets::default();
for hashed_address in state.accounts.keys() {
proof_targets.insert(*hashed_address, B256Set::default());
}
for (&hashed_address, storage) in &state.storages {
if !state.accounts.contains_key(&hashed_address) {
targets.account_targets.push(ProofV2Target::new(hashed_address));
for (hashed_address, storage) in &state.storages {
let mut storage_keys = storage.storage.keys().copied().collect::<B256Set>();
if storage.wiped {
// storage for this account was destroyed, gather all slots from the current state
let mut storage_cursor =
self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?;
// position cursor at the start
let mut current_entry = storage_cursor.seek(B256::ZERO)?;
while let Some((hashed_slot, _)) = current_entry {
storage_keys.insert(hashed_slot);
current_entry = storage_cursor.next()?;
}
}
// Skip accounts with no storage slot changes — an empty target set would produce
// an empty proof vec which cannot be revealed (no root node).
if storage.storage.is_empty() {
continue;
}
let storage_keys = storage.storage.keys().map(|k| ProofV2Target::new(*k)).collect();
targets.storage_targets.insert(hashed_address, storage_keys);
proof_targets.insert(*hashed_address, storage_keys);
}
targets
Ok(proof_targets)
}
}
#[derive(Debug, Clone)]
struct WitnessTrieNodeProviderFactory<F> {
/// Trie node provider factory.
provider_factory: F,
/// Sender for forwarding fetched trie node.
tx: mpsc::Sender<Bytes>,
}
impl<F> WitnessTrieNodeProviderFactory<F> {
const fn new(provider_factory: F, tx: mpsc::Sender<Bytes>) -> Self {
Self { provider_factory, tx }
}
}
impl<F> TrieNodeProviderFactory for WitnessTrieNodeProviderFactory<F>
where
F: TrieNodeProviderFactory,
F::AccountNodeProvider: TrieNodeProvider,
F::StorageNodeProvider: TrieNodeProvider,
{
type AccountNodeProvider = WitnessTrieNodeProvider<F::AccountNodeProvider>;
type StorageNodeProvider = WitnessTrieNodeProvider<F::StorageNodeProvider>;
fn account_node_provider(&self) -> Self::AccountNodeProvider {
let provider = self.provider_factory.account_node_provider();
WitnessTrieNodeProvider::new(provider, self.tx.clone())
}
fn storage_node_provider(&self, account: B256) -> Self::StorageNodeProvider {
let provider = self.provider_factory.storage_node_provider(account);
WitnessTrieNodeProvider::new(provider, self.tx.clone())
}
}
#[derive(Debug)]
struct WitnessTrieNodeProvider<P> {
/// Proof-based blinded.
provider: P,
/// Sender for forwarding fetched blinded node.
tx: mpsc::Sender<Bytes>,
}
impl<P> WitnessTrieNodeProvider<P> {
const fn new(provider: P, tx: mpsc::Sender<Bytes>) -> Self {
Self { provider, tx }
}
}
impl<P: TrieNodeProvider> TrieNodeProvider for WitnessTrieNodeProvider<P> {
fn trie_node(&self, path: &Nibbles) -> Result<Option<RevealedNode>, SparseTrieError> {
let maybe_node = self.provider.trie_node(path)?;
if let Some(node) = &maybe_node {
self.tx
.send(node.node.clone())
.map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?;
}
Ok(maybe_node)
}
}

View File

@@ -914,9 +914,6 @@ Pruning:
--prune.bodies.before <BLOCK_NUMBER>
Prune storage history before the specified block number. The specified block number is not pruned
--prune.minimum-distance <BLOCKS>
Minimum pruning distance from the tip. This controls the safety margin for reorgs and manual unwinds
Engine:
--engine.persistence-threshold <PERSISTENCE_THRESHOLD>
Configure persistence threshold for the engine. This determines how many canonical blocks must be in-memory, ahead of the last persisted block, before flushing canonical blocks to disk again.

View File

@@ -669,7 +669,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "avg(reth_storage_providers_database_save_blocks_batch_size{$instance_label=\"$instance\", quantile=\"$quantile\"})",
"expr": "avg(reth_storage_providers_database_save_blocks_block_count{$instance_label=\"$instance\", quantile=\"$quantile\"})",
"legendFormat": "Blocks per save_blocks call",
"range": true,
"refId": "A"
@@ -1309,7 +1309,7 @@
"uid": "${datasource}"
},
"editorMode": "code",
"expr": "avg(reth_storage_providers_database_save_blocks_batch_size_last{$instance_label=\"$instance\"})",
"expr": "avg(reth_storage_providers_database_save_blocks_block_count_last{$instance_label=\"$instance\"})",
"legendFormat": "Blocks per save_blocks call",
"range": true,
"refId": "A"

View File

@@ -5,7 +5,7 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_evm::{
block::{BlockExecutorFactory, BlockExecutorFor, ExecutableTx, GasOutput},
block::{BlockExecutorFactory, BlockExecutorFor, ExecutableTx},
eth::{EthBlockExecutionCtx, EthBlockExecutor, EthTxResult},
precompiles::PrecompilesMap,
revm::context::Block as _,
@@ -211,10 +211,7 @@ where
self.inner.execute_transaction_without_commit(tx)
}
fn commit_transaction(
&mut self,
output: Self::Result,
) -> Result<GasOutput, BlockExecutionError> {
fn commit_transaction(&mut self, output: Self::Result) -> Result<u64, BlockExecutionError> {
self.inner.commit_transaction(output)
}

View File

@@ -110,7 +110,7 @@ pub fn prague_custom() -> &'static Precompiles {
let precompile = Precompile::new(
PrecompileId::custom("custom"),
address!("0x0000000000000000000000000000000000000999"),
|_, _| PrecompileResult::Ok(PrecompileOutput::new(0, 0, Bytes::new())),
|_, _| PrecompileResult::Ok(PrecompileOutput::new(0, Bytes::new())),
);
precompiles.extend([precompile]);
precompiles

View File

@@ -1,149 +0,0 @@
#!/usr/bin/env bash
set -euo pipefail
usage() {
echo "Usage: $0 [-n NUM_BLOCKS] [-b BLOCK] <remote_rpc_url>"
echo ""
echo "Fetches debug_executionWitness from both localhost:8545 and the"
echo "given remote RPC, then compares them."
echo ""
echo "Options:"
echo " -n NUM Compare the last NUM blocks (default: 20)"
echo " -b BLOCK Compare only the given block number"
exit 1
}
NUM_BLOCKS=20
SINGLE_BLOCK=""
while getopts "n:b:h" opt; do
case "$opt" in
n) NUM_BLOCKS="$OPTARG" ;;
b) SINGLE_BLOCK="$OPTARG" ;;
*) usage ;;
esac
done
shift $((OPTIND - 1))
if [[ $# -lt 1 ]]; then
echo "Error: remote_rpc_url is required" >&2
usage
fi
REMOTE_RPC="$1"
LOCAL_RPC="http://127.0.0.1:8545"
log() { echo "[$(date '+%H:%M:%S')] $*" >&2; }
if [[ -n "$SINGLE_BLOCK" ]]; then
start=$SINGLE_BLOCK
latest=$SINGLE_BLOCK
log "Comparing block $SINGLE_BLOCK"
else
# Get latest block number from local node
if ! latest=$(cast bn --rpc-url "$LOCAL_RPC" --rpc-timeout 600 2>&1); then
log "FATAL: failed to get block number from local RPC: $latest"
exit 1
fi
start=$((latest - NUM_BLOCKS + 1))
if (( start < 0 )); then
start=0
fi
log "Comparing blocks $start..$latest ($NUM_BLOCKS blocks)"
fi
errors=0
for (( block = start; block <= latest; block++ )); do
block_hex=$(printf '"0x%x"' "$block")
log "Checking block $block ($block_hex)"
# Fetch witness from both RPCs
local_witness=$(cast rpc debug_executionWitness "$block_hex" --rpc-url "$LOCAL_RPC" --rpc-timeout 600 2>&1) || {
log "WARN: failed to get witness from local RPC for block $block: $local_witness"
((errors++)) || true
continue
}
remote_witness=$(cast rpc debug_executionWitness "$block_hex" --rpc-url "$REMOTE_RPC" --rpc-timeout 600 2>&1) || {
log "WARN: failed to get witness from remote RPC for block $block: $remote_witness"
((errors++)) || true
continue
}
# Normalize: sort all arrays of objects by a stable key so ordering doesn't cause false diffs
normalize='walk(if type == "array" then sort_by(if type == "object" then (keys | join(",")) + ":" + (to_entries | map(.value | tostring) | join(",")) else tostring end) else . end) | . as $root | $root'
local_file=$(mktemp)
remote_file=$(mktemp)
trap "rm -f '$local_file' '$remote_file'" EXIT
echo "$local_witness" | jq -S "$normalize" > "$local_file"
echo "$remote_witness" | jq -S "$normalize" > "$remote_file"
# Compare: for "state", local may contain extra nodes (superset OK).
# For "codes", "keys", "headers", require exact set equality.
has_error=false
# Check exact-match fields (as sorted sets)
for field in codes keys headers; do
local_set=$(jq -r --arg f "$field" '.[$f] // [] | sort | .[]' "$local_file")
remote_set=$(jq -r --arg f "$field" '.[$f] // [] | sort | .[]' "$remote_file")
if [[ "$local_set" != "$remote_set" ]]; then
log "ERROR: block $block field '$field' differs"
diff <(echo "$remote_set") <(echo "$local_set") | head -30 || true
has_error=true
fi
done
# Check state: every remote node must be present in local (extras OK)
missing=$(jq -r -n \
--slurpfile l "$local_file" \
--slurpfile r "$remote_file" \
'($l[0].state // [] | map({(.):true}) | add // {}) as $local_set |
[$r[0].state // [] | .[] | select($local_set[.] | not)] |
if length == 0 then empty else .[] end')
if [[ -n "$missing" ]]; then
n_missing=$(echo "$missing" | wc -l)
log "ERROR: block $block state has $n_missing missing node(s) (present in remote, absent in local):"
echo "$missing" | head -20
has_error=true
fi
extra=$(jq -r -n \
--slurpfile l "$local_file" \
--slurpfile r "$remote_file" \
'($r[0].state // [] | map({(.):true}) | add // {}) as $remote_set |
[$l[0].state // [] | .[] | select($remote_set[.] | not)] |
if length == 0 then empty else .[] end')
n_extra=0
if [[ -n "$extra" ]]; then
n_extra=$(echo "$extra" | wc -l)
fi
if ! $has_error; then
if [[ $n_extra -gt 0 ]]; then
log "OK: block $block witnesses match ($n_extra extra state node(s) in local)"
else
log "OK: block $block witnesses match exactly"
fi
else
cp "$local_file" "witness-local-${block}.json"
cp "$remote_file" "witness-remote-${block}.json"
log "Wrote witness-local-${block}.json and witness-remote-${block}.json"
if [[ $n_extra -gt 0 ]]; then
log " (local also has $n_extra extra state node(s))"
fi
((errors++)) || true
log "---"
fi
rm -f "$local_file" "$remote_file"
done
total=$((latest - start + 1))
if (( errors > 0 )); then
log "DONE: $errors block(s) had errors out of $total"
exit 1
else
log "DONE: all $total block(s) matched"
fi