mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
1 Commits
eip8037
...
klkvr/flus
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
3a18325e0e |
@@ -1,5 +0,0 @@
|
||||
---
|
||||
reth-trie-sparse: patch
|
||||
---
|
||||
|
||||
Fixed another branch collapse edge case where `check_subtrie_collapse_needs_proof` incorrectly compared removal count against total update count (including `Touched` entries), causing it to skip proof requests for blinded siblings and panic when the subtrie emptied. Added a regression test covering the removals + `Touched` + blinded sibling scenario.
|
||||
132
.github/scripts/bench-scheduled-refs.sh
vendored
132
.github/scripts/bench-scheduled-refs.sh
vendored
@@ -1,132 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
#
|
||||
# Resolves baseline and feature refs for nightly regression benchmark runs.
|
||||
#
|
||||
# Queries the latest successful scheduled docker.yml run via GitHub API
|
||||
# to find the commit that built the nightly Docker image. Compares with
|
||||
# the last successful feature ref (from GH Actions cache) to determine
|
||||
# baseline, detect staleness, and decide whether to skip.
|
||||
#
|
||||
# Usage: bench-nightly-refs.sh [--force]
|
||||
#
|
||||
# Outputs (via GITHUB_OUTPUT):
|
||||
# baseline-ref — commit SHA for baseline
|
||||
# feature-ref — commit SHA for feature (current nightly)
|
||||
# should-skip — "true" if no new nightly since last run
|
||||
# is-stale — "true" if latest nightly build is >24h old
|
||||
# stale-age-hours — age of the nightly build in hours (only if stale)
|
||||
# nightly-created — ISO timestamp of the nightly build
|
||||
#
|
||||
# Reads:
|
||||
# .nightly-state/last-feature-ref (from GH Actions cache, may not exist)
|
||||
#
|
||||
# Requires: gh (GitHub CLI), jq, date
|
||||
set -euo pipefail
|
||||
|
||||
FORCE="${1:-false}"
|
||||
REPO="${GITHUB_REPOSITORY:-paradigmxyz/reth}"
|
||||
|
||||
# --- Step 1: Query latest successful scheduled docker.yml run ---
|
||||
echo "::group::Querying latest nightly docker build"
|
||||
|
||||
RUNS_JSON=$(gh run list \
|
||||
-R "$REPO" \
|
||||
--workflow=docker.yml \
|
||||
--event=schedule \
|
||||
--status=completed \
|
||||
--limit 5 \
|
||||
--json headSha,createdAt,conclusion)
|
||||
|
||||
# Find the most recent successful run
|
||||
LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
|
||||
|
||||
if [ -z "$LATEST" ]; then
|
||||
echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
|
||||
echo "Runs found: $RUNS_JSON"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
|
||||
CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt')
|
||||
echo "Latest nightly commit: $FEATURE_REF"
|
||||
echo "Built at: $CREATED_AT"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 2: Staleness check ---
|
||||
echo "::group::Checking staleness"
|
||||
NOW_EPOCH=$(date +%s)
|
||||
# Handle both GNU date (-d) and BSD date (-j -f) for cross-platform compat
|
||||
CREATED_EPOCH=$(date -d "$CREATED_AT" +%s 2>/dev/null || \
|
||||
date -j -f "%Y-%m-%dT%H:%M:%SZ" "$CREATED_AT" +%s 2>/dev/null || \
|
||||
date -j -f "%Y-%m-%dT%T%z" "$CREATED_AT" +%s 2>/dev/null || \
|
||||
{ echo "::error::Cannot parse date: $CREATED_AT"; exit 1; })
|
||||
|
||||
AGE_SECONDS=$(( NOW_EPOCH - CREATED_EPOCH ))
|
||||
AGE_HOURS=$(( AGE_SECONDS / 3600 ))
|
||||
IS_STALE="false"
|
||||
|
||||
if [ "$AGE_HOURS" -gt 24 ]; then
|
||||
IS_STALE="true"
|
||||
echo "::warning::STALE NIGHTLY: Build is ${AGE_HOURS}h old (>24h threshold)"
|
||||
echo "This indicates the nightly docker build failed — no new image was produced"
|
||||
else
|
||||
echo "Nightly build age: ${AGE_HOURS}h (within 24h threshold)"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 3: Read last successful feature ref from cache ---
|
||||
echo "::group::Reading cached state"
|
||||
LAST_FEATURE_REF=""
|
||||
STATE_FILE=".nightly-state/last-feature-ref"
|
||||
if [ -f "$STATE_FILE" ]; then
|
||||
LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
|
||||
echo "Previous feature ref: $LAST_FEATURE_REF"
|
||||
else
|
||||
echo "No cached state found (first run)"
|
||||
fi
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 4: Determine baseline and skip logic ---
|
||||
echo "::group::Resolving refs"
|
||||
SHOULD_SKIP="false"
|
||||
BASELINE_REF="$FEATURE_REF" # default for first run
|
||||
|
||||
if [ "$IS_STALE" = "true" ]; then
|
||||
# Stale = error path, don't skip (will alert and fail downstream)
|
||||
SHOULD_SKIP="false"
|
||||
BASELINE_REF="${LAST_FEATURE_REF:-$FEATURE_REF}"
|
||||
echo "Stale nightly detected — will alert and fail"
|
||||
elif [ -z "$LAST_FEATURE_REF" ]; then
|
||||
# First run: baseline = feature (self-comparison to establish baseline)
|
||||
BASELINE_REF="$FEATURE_REF"
|
||||
echo "First run — will benchmark nightly against itself to establish baseline"
|
||||
elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
|
||||
# No new nightly since last successful run
|
||||
if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
|
||||
echo "No new nightly, but force=true — running anyway"
|
||||
BASELINE_REF="$LAST_FEATURE_REF"
|
||||
else
|
||||
SHOULD_SKIP="true"
|
||||
echo "No new nightly since last run — will skip"
|
||||
fi
|
||||
else
|
||||
# Normal case: new nightly available
|
||||
BASELINE_REF="$LAST_FEATURE_REF"
|
||||
echo "New nightly detected"
|
||||
fi
|
||||
|
||||
echo "Baseline: $BASELINE_REF"
|
||||
echo "Feature: $FEATURE_REF"
|
||||
echo "Skip: $SHOULD_SKIP"
|
||||
echo "Stale: $IS_STALE"
|
||||
echo "::endgroup::"
|
||||
|
||||
# --- Step 5: Write outputs ---
|
||||
{
|
||||
echo "baseline-ref=$BASELINE_REF"
|
||||
echo "feature-ref=$FEATURE_REF"
|
||||
echo "should-skip=$SHOULD_SKIP"
|
||||
echo "is-stale=$IS_STALE"
|
||||
echo "stale-age-hours=$AGE_HOURS"
|
||||
echo "nightly-created=$CREATED_AT"
|
||||
} >> "$GITHUB_OUTPUT"
|
||||
865
.github/workflows/bench-scheduled.yml
vendored
865
.github/workflows/bench-scheduled.yml
vendored
@@ -1,865 +0,0 @@
|
||||
# Nightly regression benchmark.
|
||||
#
|
||||
# Compares the previous nightly build against the current nightly build to
|
||||
# detect performance regressions. Runs daily after docker.yml produces a new
|
||||
# nightly image at 01:00 UTC.
|
||||
#
|
||||
# State is persisted between runs via GitHub Actions cache: each successful
|
||||
# run saves the feature commit SHA so the next run knows what to compare against.
|
||||
|
||||
on:
|
||||
schedule:
|
||||
- cron: "30 5 * * *" # 06:30 UTC daily
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
force:
|
||||
description: "Force run even if no new nightly (bypass skip logic)"
|
||||
required: false
|
||||
default: false
|
||||
type: boolean
|
||||
no_slack:
|
||||
description: "Suppress Slack notifications"
|
||||
required: false
|
||||
default: true
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
RUSTC_WRAPPER: "sccache"
|
||||
|
||||
name: bench-scheduled
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
actions: read
|
||||
|
||||
jobs:
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job 1: Resolve nightly refs, check staleness, manage state
|
||||
# ---------------------------------------------------------------------------
|
||||
resolve-refs:
|
||||
name: resolve-refs
|
||||
runs-on: ubuntu-latest
|
||||
outputs:
|
||||
baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
|
||||
feature-ref: ${{ steps.refs.outputs.feature-ref }}
|
||||
should-skip: ${{ steps.refs.outputs.should-skip }}
|
||||
is-stale: ${{ steps.refs.outputs.is-stale }}
|
||||
stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
|
||||
nightly-created: ${{ steps.refs.outputs.nightly-created }}
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
sparse-checkout: .github/scripts
|
||||
sparse-checkout-cone-mode: true
|
||||
|
||||
- name: Restore nightly state
|
||||
id: state-cache
|
||||
uses: actions/cache/restore@v4
|
||||
with:
|
||||
path: .nightly-state
|
||||
key: bench-scheduled-state-dummy
|
||||
restore-keys: |
|
||||
bench-scheduled-state-
|
||||
|
||||
- name: Resolve nightly refs
|
||||
id: refs
|
||||
env:
|
||||
GH_TOKEN: ${{ github.token }}
|
||||
GITHUB_REPOSITORY: ${{ github.repository }}
|
||||
run: |
|
||||
FORCE="${{ inputs.force || 'false' }}"
|
||||
.github/scripts/bench-scheduled-refs.sh "$FORCE"
|
||||
|
||||
- name: Alert on stale nightly
|
||||
if: steps.refs.outputs.is-stale == 'true'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
if (!token || !channel) {
|
||||
core.warning('Slack credentials not set, skipping stale nightly alert');
|
||||
return;
|
||||
}
|
||||
|
||||
const ageHours = '${{ steps.refs.outputs.stale-age-hours }}';
|
||||
const created = '${{ steps.refs.outputs.nightly-created }}';
|
||||
const featureRef = '${{ steps.refs.outputs.feature-ref }}';
|
||||
const shortSha = featureRef.slice(0, 8);
|
||||
const repo = '${{ github.repository }}';
|
||||
const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: ':rotating_light: Nightly Regression: nightly build is stale', emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: {
|
||||
type: 'mrkdwn',
|
||||
text: [
|
||||
'*Nightly regression did not run* — nightly build is stale',
|
||||
'',
|
||||
`The latest nightly image was built from a commit that is *${ageHours}h old* (threshold: 24h).`,
|
||||
`This means today's nightly docker build likely failed and no new image was produced.`,
|
||||
'',
|
||||
`Stale commit: \`${shortSha}\` (built at ${created})`,
|
||||
'',
|
||||
'*Action required:* Check the <https://github.com/' + repo + '/actions/workflows/docker.yml|docker.yml> workflow for failures.',
|
||||
].join('\n'),
|
||||
},
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: [
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
|
||||
url: runUrl,
|
||||
action_id: 'ci_button',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const resp = await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel,
|
||||
blocks,
|
||||
text: 'Nightly regression: nightly build is stale',
|
||||
unfurl_links: false,
|
||||
}),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.ok) {
|
||||
core.warning(`Slack API error: ${JSON.stringify(data)}`);
|
||||
}
|
||||
|
||||
- name: Fail on stale nightly
|
||||
if: steps.refs.outputs.is-stale == 'true'
|
||||
run: |
|
||||
echo "::error::Nightly build is stale (>24h old). Aborting."
|
||||
exit 1
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job 2: Run the benchmark
|
||||
# ---------------------------------------------------------------------------
|
||||
bench-scheduled:
|
||||
needs: resolve-refs
|
||||
if: |
|
||||
needs.resolve-refs.outputs.should-skip != 'true' &&
|
||||
needs.resolve-refs.outputs.is-stale != 'true'
|
||||
name: bench-scheduled
|
||||
runs-on: [self-hosted, Linux, X64, available]
|
||||
timeout-minutes: 120
|
||||
env:
|
||||
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
||||
SCHELK_MOUNT: /reth-bench
|
||||
BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
|
||||
BENCH_PR: ""
|
||||
BENCH_ACTOR: "nightly-regression"
|
||||
BENCH_BLOCKS: "2000"
|
||||
BENCH_WARMUP_BLOCKS: "500"
|
||||
BENCH_SAMPLY: "false"
|
||||
BENCH_CORES: "0"
|
||||
BENCH_BIG_BLOCKS: "false"
|
||||
BENCH_RETH_NEW_PAYLOAD: "true"
|
||||
BENCH_WAIT_TIME: ""
|
||||
BENCH_BASELINE_ARGS: ""
|
||||
BENCH_FEATURE_ARGS: ""
|
||||
BENCH_ABBA: "true"
|
||||
BENCH_COMMENT_ID: ""
|
||||
BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
|
||||
BENCH_METRICS_ADDR: "127.0.0.1:9100"
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
|
||||
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
|
||||
BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
|
||||
FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
steps:
|
||||
- name: Clean up previous bench-work
|
||||
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
|
||||
- uses: actions/checkout@v6
|
||||
with:
|
||||
submodules: true
|
||||
fetch-depth: 0
|
||||
ref: ${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
|
||||
- name: Resolve job URL
|
||||
id: job-url
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
run_id: context.runId,
|
||||
});
|
||||
const job = jobs.jobs.find(j => j.name === 'bench-scheduled');
|
||||
const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
core.exportVariable('BENCH_JOB_URL', jobUrl);
|
||||
|
||||
- uses: dtolnay/rust-toolchain@stable
|
||||
- uses: mozilla-actions/sccache-action@v0.0.9
|
||||
continue-on-error: true
|
||||
|
||||
- name: Install dependencies
|
||||
env:
|
||||
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
|
||||
run: |
|
||||
mkdir -p "$HOME/.local/bin"
|
||||
|
||||
# apt packages
|
||||
sudo apt-get update -qq
|
||||
sudo apt-get install -y --no-install-recommends \
|
||||
python3 make jq zstd curl dmsetup \
|
||||
linux-tools-"$(uname -r)" || \
|
||||
sudo apt-get install -y --no-install-recommends linux-tools-generic
|
||||
|
||||
# mc (MinIO client)
|
||||
if ! command -v mc &>/dev/null; then
|
||||
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
|
||||
chmod +x "$HOME/.local/bin/mc"
|
||||
fi
|
||||
|
||||
# uv (Python package manager)
|
||||
if ! command -v uv &>/dev/null; then
|
||||
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
|
||||
fi
|
||||
|
||||
# Configure git auth for private repos
|
||||
git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
|
||||
|
||||
# thin-provisioning-tools (era_invalidate, required by schelk)
|
||||
if ! command -v era_invalidate &>/dev/null; then
|
||||
git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
|
||||
sudo make -C /tmp/tpt install
|
||||
rm -rf /tmp/tpt
|
||||
fi
|
||||
|
||||
# schelk (snapshot rollback tool, invoked via sudo)
|
||||
if ! sudo sh -c 'command -v schelk' &>/dev/null; then
|
||||
cargo install --git https://github.com/tempoxyz/schelk --locked
|
||||
sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
|
||||
fi
|
||||
|
||||
- name: Check dependencies
|
||||
run: |
|
||||
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
||||
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
||||
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
||||
missing=()
|
||||
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
|
||||
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
||||
done
|
||||
if [ ${#missing[@]} -gt 0 ]; then
|
||||
echo "::error::Missing required tools: ${missing[*]}"
|
||||
exit 1
|
||||
fi
|
||||
echo "All dependencies found"
|
||||
|
||||
- name: Resolve display names
|
||||
id: refs
|
||||
run: |
|
||||
BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
|
||||
FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
|
||||
echo "baseline-name=nightly-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-name=nightly-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
|
||||
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
|
||||
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
||||
|
||||
- name: Check if snapshot needs update
|
||||
id: snapshot-check
|
||||
run: |
|
||||
if .github/scripts/bench-reth-snapshot.sh --check; then
|
||||
echo "needed=false" >> "$GITHUB_OUTPUT"
|
||||
else
|
||||
echo "needed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Prepare source dirs
|
||||
run: |
|
||||
if [ -d ../reth-baseline ]; then
|
||||
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
||||
else
|
||||
git clone . ../reth-baseline
|
||||
fi
|
||||
git -C ../reth-baseline checkout "$BASELINE_REF"
|
||||
|
||||
if [ -d ../reth-feature ]; then
|
||||
git -C ../reth-feature fetch origin "$FEATURE_REF"
|
||||
else
|
||||
git clone . ../reth-feature
|
||||
fi
|
||||
git -C ../reth-feature checkout "$FEATURE_REF"
|
||||
|
||||
- name: Build binaries and download snapshot in parallel
|
||||
id: build
|
||||
env:
|
||||
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
BENCH_REPO: ${{ github.repository }}
|
||||
SNAPSHOT_NEEDED: ${{ steps.snapshot-check.outputs.needed }}
|
||||
run: |
|
||||
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
|
||||
FEATURE_DIR="$(cd ../reth-feature && pwd)"
|
||||
|
||||
.github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "$BASELINE_REF" &
|
||||
PID_BASELINE=$!
|
||||
.github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "$FEATURE_REF" &
|
||||
PID_FEATURE=$!
|
||||
|
||||
PID_SNAPSHOT=
|
||||
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
|
||||
.github/scripts/bench-reth-snapshot.sh &
|
||||
PID_SNAPSHOT=$!
|
||||
fi
|
||||
|
||||
FAIL=0
|
||||
wait $PID_BASELINE || FAIL=1
|
||||
wait $PID_FEATURE || FAIL=1
|
||||
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
|
||||
if [ $FAIL -ne 0 ]; then
|
||||
echo "::error::One or more parallel tasks failed (builds / snapshot download)"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# System tuning for reproducible benchmarks
|
||||
- name: System setup
|
||||
run: |
|
||||
sudo cpupower frequency-set -g performance || true
|
||||
# Disable turbo boost (Intel and AMD paths)
|
||||
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
|
||||
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
|
||||
sudo swapoff -a || true
|
||||
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
|
||||
# Disable SMT (hyperthreading)
|
||||
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
|
||||
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
|
||||
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
|
||||
if [ "$current" != "$first" ]; then
|
||||
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
|
||||
fi
|
||||
done
|
||||
echo "Online CPUs: $(nproc)"
|
||||
# Disable transparent huge pages
|
||||
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
||||
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
||||
done || true
|
||||
# Prevent deep C-states
|
||||
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
||||
# Move all IRQs to core 0
|
||||
for irq in /proc/irq/*/smp_affinity_list; do
|
||||
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
||||
done
|
||||
# Stop noisy background services
|
||||
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
|
||||
echo "=== Benchmark environment ==="
|
||||
uname -r
|
||||
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
|
||||
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
|
||||
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
|
||||
free -h
|
||||
|
||||
- name: Pre-flight cleanup
|
||||
run: |
|
||||
sudo pkill -9 reth || true
|
||||
sleep 1
|
||||
if mountpoint -q "$SCHELK_MOUNT"; then
|
||||
sudo umount -l "$SCHELK_MOUNT" || true
|
||||
sudo schelk recover -y || true
|
||||
fi
|
||||
rm -rf "$BENCH_WORK_DIR"
|
||||
mkdir -p "$BENCH_WORK_DIR"
|
||||
|
||||
- name: Start metrics proxy
|
||||
run: |
|
||||
BENCH_ID="nightly-${{ github.run_id }}"
|
||||
BENCH_REFERENCE_EPOCH=$(date +%s)
|
||||
echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
|
||||
echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
|
||||
|
||||
LABELS_FILE="/tmp/bench-metrics-labels.json"
|
||||
echo '{}' > "$LABELS_FILE"
|
||||
echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
|
||||
|
||||
python3 .github/scripts/bench-metrics-proxy.py \
|
||||
--labels "$LABELS_FILE" \
|
||||
--upstream "http://${BENCH_METRICS_ADDR}/" \
|
||||
--subnet 10.10.0.0/24 \
|
||||
--port 9090 &
|
||||
PROXY_PID=$!
|
||||
echo "BENCH_METRICS_PROXY_PID=${PROXY_PID}" >> "$GITHUB_ENV"
|
||||
echo "Metrics proxy started (PID $PROXY_PID)"
|
||||
|
||||
# Interleaved run order (B-F-F-B) to reduce systematic bias
|
||||
- name: "Run benchmark: baseline (1/2)"
|
||||
id: run-baseline-1
|
||||
run: |
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
|
||||
|
||||
- name: "Run benchmark: feature (1/2)"
|
||||
id: run-feature-1
|
||||
run: |
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
|
||||
|
||||
- name: "Run benchmark: feature (2/2)"
|
||||
id: run-feature-2
|
||||
run: |
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
|
||||
|
||||
- name: "Run benchmark: baseline (2/2)"
|
||||
id: run-baseline-2
|
||||
run: |
|
||||
LAST_RUN_START=$(date +%s)
|
||||
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
|
||||
cat > "$BENCH_LABELS_FILE" <<LABELS
|
||||
{"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
|
||||
LABELS
|
||||
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
|
||||
|
||||
- name: Stop metrics proxy & generate Grafana URL
|
||||
id: metrics
|
||||
if: "!cancelled()"
|
||||
run: |
|
||||
kill "$BENCH_METRICS_PROXY_PID" 2>/dev/null || true
|
||||
|
||||
LAST_RUN_DURATION=$(( $(date +%s) - BENCH_LAST_RUN_START ))
|
||||
FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
|
||||
TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
|
||||
GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=ef57fux92e9z4e&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
|
||||
echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
|
||||
echo "Grafana URL: ${GRAFANA_URL}"
|
||||
|
||||
- name: Scan logs for errors
|
||||
if: "!cancelled()"
|
||||
run: |
|
||||
ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
|
||||
found=false
|
||||
for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
|
||||
LOG="$BENCH_WORK_DIR/$run_dir/node.log"
|
||||
if [ ! -f "$LOG" ]; then continue; fi
|
||||
|
||||
panics=$(grep -c -E 'panicked at' "$LOG" || true)
|
||||
errors=$(grep -c ' ERROR ' "$LOG" || true)
|
||||
|
||||
if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
|
||||
if [ "$found" = false ]; then
|
||||
printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
|
||||
found=true
|
||||
fi
|
||||
printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
|
||||
if [ "$panics" -gt 0 ]; then
|
||||
printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
|
||||
grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
|
||||
printf '```\n' >> "$ERRORS_FILE"
|
||||
fi
|
||||
if [ "$errors" -gt 0 ]; then
|
||||
printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
|
||||
grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
|
||||
printf '```\n' >> "$ERRORS_FILE"
|
||||
fi
|
||||
printf '\n</details>\n\n' >> "$ERRORS_FILE"
|
||||
fi
|
||||
done
|
||||
|
||||
- name: Parse results
|
||||
id: results
|
||||
if: success()
|
||||
env:
|
||||
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
||||
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
||||
run: |
|
||||
SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
|
||||
|
||||
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
|
||||
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
|
||||
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
|
||||
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
|
||||
|
||||
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
|
||||
if [ -n "$GRAFANA_URL" ]; then
|
||||
SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
|
||||
fi
|
||||
# shellcheck disable=SC2086
|
||||
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
|
||||
|
||||
- name: Generate charts
|
||||
if: success()
|
||||
env:
|
||||
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
||||
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
||||
run: |
|
||||
CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
|
||||
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
|
||||
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
|
||||
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
|
||||
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
|
||||
CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
|
||||
CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
|
||||
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
|
||||
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
|
||||
# shellcheck disable=SC2086
|
||||
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
|
||||
|
||||
- name: Upload results
|
||||
if: "!cancelled()"
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: bench-scheduled-results
|
||||
path: ${{ env.BENCH_WORK_DIR }}
|
||||
|
||||
- name: Push charts
|
||||
id: push-charts
|
||||
if: success()
|
||||
run: |
|
||||
RUN_ID=${{ github.run_id }}
|
||||
CHART_DIR="nightly/${RUN_ID}"
|
||||
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
|
||||
|
||||
TMP_DIR=$(mktemp -d)
|
||||
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
|
||||
true
|
||||
else
|
||||
git init "${TMP_DIR}"
|
||||
git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
|
||||
fi
|
||||
|
||||
mkdir -p "${TMP_DIR}/${CHART_DIR}"
|
||||
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
|
||||
git -C "${TMP_DIR}" add "${CHART_DIR}"
|
||||
git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
||||
commit -m "nightly bench charts for run ${RUN_ID}"
|
||||
git -C "${TMP_DIR}" push origin HEAD:main
|
||||
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
||||
rm -rf "${TMP_DIR}"
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const { verdict, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
|
||||
|
||||
let summary;
|
||||
try {
|
||||
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
|
||||
} catch (e) {
|
||||
await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
|
||||
return;
|
||||
}
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
const { emoji, label } = verdict(summary.changes);
|
||||
const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
|
||||
const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
|
||||
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
|
||||
|
||||
let md = `# ${emoji} Nightly Regression: ${label}\n\n`;
|
||||
md += `**Baseline:** ${baselineLink}\n`;
|
||||
md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
|
||||
md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
|
||||
|
||||
const rows = metricRows(summary);
|
||||
md += `| Metric | Baseline | Feature | Change |\n`;
|
||||
md += `|--------|----------|---------|--------|\n`;
|
||||
for (const r of rows) {
|
||||
md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
|
||||
}
|
||||
md += '\n';
|
||||
|
||||
const wtRows = waitTimeRows(summary);
|
||||
if (wtRows.length > 0) {
|
||||
md += `### Wait Time Breakdown\n\n`;
|
||||
md += `| Metric | Baseline | Feature |\n`;
|
||||
md += `|--------|----------|--------|\n`;
|
||||
for (const r of wtRows) {
|
||||
md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
|
||||
}
|
||||
md += '\n';
|
||||
}
|
||||
|
||||
// Charts
|
||||
const chartSha = '${{ steps.push-charts.outputs.sha }}';
|
||||
if (chartSha) {
|
||||
const runId = '${{ github.run_id }}';
|
||||
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`;
|
||||
const charts = [
|
||||
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
|
||||
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
|
||||
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
|
||||
];
|
||||
md += `### Charts\n\n`;
|
||||
for (const chart of charts) {
|
||||
md += `<details><summary>${chart.label}</summary>\n\n`;
|
||||
md += `\n\n`;
|
||||
md += `</details>\n\n`;
|
||||
}
|
||||
}
|
||||
|
||||
const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}';
|
||||
if (grafanaUrl) {
|
||||
md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
|
||||
}
|
||||
|
||||
try {
|
||||
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
|
||||
if (errors.trim()) md += '\n' + errors + '\n';
|
||||
} catch {}
|
||||
|
||||
await core.summary.addRaw(md).write();
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && env.BENCH_NO_SLACK != 'true'
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
const { verdict, fmtChange, fmtMs, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
|
||||
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
if (!token || !channel) {
|
||||
core.info('Slack credentials not set, skipping notification');
|
||||
return;
|
||||
}
|
||||
|
||||
let summary;
|
||||
try {
|
||||
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
|
||||
} catch (e) {
|
||||
core.warning('Could not read summary.json for Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
// Only notify on significant changes (regression OR improvement)
|
||||
const changes = summary.changes || {};
|
||||
const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
|
||||
if (!hasSignificant) {
|
||||
core.info('No significant changes detected, skipping nightly Slack notification');
|
||||
return;
|
||||
}
|
||||
|
||||
const SLACK_VERDICT = {
|
||||
'⚠️': ':warning:',
|
||||
'❌': ':x:',
|
||||
'✅': ':white_check_mark:',
|
||||
'⚪': ':white_circle:',
|
||||
};
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const { emoji, label } = verdict(changes);
|
||||
const headerEmoji = SLACK_VERDICT[emoji] || emoji;
|
||||
const commitUrl = `https://github.com/${repo}/commit`;
|
||||
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
|
||||
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
|
||||
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
|
||||
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
|
||||
|
||||
const sectionText = [
|
||||
'*Nightly Regression*',
|
||||
'',
|
||||
`*Baseline:* ${baselineLink}`,
|
||||
`*Feature:* ${featureLink}`,
|
||||
blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '),
|
||||
].join('\n');
|
||||
|
||||
const rows = metricRows(summary);
|
||||
const tableRows = [
|
||||
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
|
||||
...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
|
||||
];
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: `${headerEmoji} Nightly: ${label}`, emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: sectionText },
|
||||
},
|
||||
{
|
||||
type: 'table',
|
||||
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }, { align: 'right' }],
|
||||
rows: tableRows,
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: [
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
|
||||
url: jobUrl,
|
||||
action_id: 'ci_button',
|
||||
},
|
||||
{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
|
||||
url: diffUrl,
|
||||
action_id: 'diff_button',
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
const text = `Nightly Regression: ${summary.baseline.name} vs ${summary.feature.name}`;
|
||||
const resp = await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({ channel, blocks, text, unfurl_links: false }),
|
||||
});
|
||||
const data = await resp.json();
|
||||
if (!data.ok) {
|
||||
core.warning(`Slack API error: ${JSON.stringify(data)}`);
|
||||
return;
|
||||
}
|
||||
|
||||
// Post wait time breakdown as threaded reply
|
||||
const wtRows = waitTimeRows(summary);
|
||||
if (data.ts && wtRows.length > 0) {
|
||||
const waitTableRows = [
|
||||
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
|
||||
...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
|
||||
];
|
||||
await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel,
|
||||
thread_ts: data.ts,
|
||||
blocks: [{
|
||||
type: 'table',
|
||||
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }],
|
||||
rows: waitTableRows,
|
||||
}],
|
||||
text: 'Wait time breakdown',
|
||||
unfurl_links: false,
|
||||
}),
|
||||
});
|
||||
}
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure()
|
||||
uses: actions/github-script@v8
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
with:
|
||||
script: |
|
||||
const token = process.env.SLACK_BENCH_BOT_TOKEN;
|
||||
const channel = process.env.SLACK_BENCH_CHANNEL;
|
||||
if (!token || !channel) return;
|
||||
|
||||
const steps_status = [
|
||||
['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
|
||||
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
|
||||
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
|
||||
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
|
||||
['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
|
||||
];
|
||||
const failed = steps_status.find(([, o]) => o === 'failure');
|
||||
const failedStep = failed ? failed[0] : 'unknown step';
|
||||
|
||||
const repo = `${context.repo.owner}/${context.repo.repo}`;
|
||||
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const blocks = [
|
||||
{
|
||||
type: 'header',
|
||||
text: { type: 'plain_text', text: ':rotating_light: Nightly Bench Failed', emoji: true },
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*Nightly regression* failed while *${failedStep}*` },
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
elements: [{
|
||||
type: 'button',
|
||||
text: { type: 'plain_text', text: 'View Logs :github:', emoji: true },
|
||||
url: jobUrl,
|
||||
action_id: 'ci_button',
|
||||
}],
|
||||
},
|
||||
];
|
||||
|
||||
await fetch('https://slack.com/api/chat.postMessage', {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Authorization': `Bearer ${token}`,
|
||||
'Content-Type': 'application/json',
|
||||
},
|
||||
body: JSON.stringify({
|
||||
channel,
|
||||
blocks,
|
||||
text: `Nightly bench failed while ${failedStep}`,
|
||||
unfurl_links: false,
|
||||
}),
|
||||
});
|
||||
|
||||
- name: Restore system settings
|
||||
if: always()
|
||||
run: |
|
||||
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Job 3: Save state on success
|
||||
# ---------------------------------------------------------------------------
|
||||
save-state:
|
||||
needs: [resolve-refs, bench-scheduled]
|
||||
if: success()
|
||||
name: save-state
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- name: Write state file
|
||||
run: |
|
||||
mkdir -p .nightly-state
|
||||
echo "${{ needs.resolve-refs.outputs.feature-ref }}" > .nightly-state/last-feature-ref
|
||||
|
||||
- name: Save nightly state
|
||||
uses: actions/cache/save@v4
|
||||
with:
|
||||
path: .nightly-state
|
||||
key: bench-scheduled-state-${{ needs.resolve-refs.outputs.feature-ref }}
|
||||
32
.github/workflows/bench.yml
vendored
32
.github/workflows/bench.yml
vendored
@@ -71,11 +71,6 @@ on:
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
otlp:
|
||||
description: "Export OTLP traces and logs"
|
||||
required: false
|
||||
default: "true"
|
||||
type: boolean
|
||||
|
||||
env:
|
||||
CARGO_TERM_COLOR: always
|
||||
@@ -115,7 +110,6 @@ jobs:
|
||||
baseline-args: ${{ steps.args.outputs.baseline-args }}
|
||||
feature-args: ${{ steps.args.outputs.feature-args }}
|
||||
abba: ${{ steps.args.outputs.abba }}
|
||||
otlp: ${{ steps.args.outputs.otlp }}
|
||||
comment-id: ${{ steps.ack.outputs.comment-id }}
|
||||
steps:
|
||||
- name: Check org membership
|
||||
@@ -157,7 +151,6 @@ jobs:
|
||||
bigBlocks = blocks === 'big' ? 'true' : 'false';
|
||||
var rethNewPayload = '${{ github.event.inputs.reth_newPayload }}' !== 'false' ? 'true' : 'false';
|
||||
var abba = '${{ github.event.inputs.abba }}' !== 'false' ? 'true' : 'false';
|
||||
var otlp = '${{ github.event.inputs.otlp }}' !== 'false' ? 'true' : 'false';
|
||||
var waitTime = '${{ github.event.inputs.wait_time }}' || '';
|
||||
var baselineNodeArgs = '${{ github.event.inputs.baseline_args }}' || '';
|
||||
var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || '';
|
||||
@@ -184,10 +177,10 @@ jobs:
|
||||
const intOrKeywordArgs = new Map([['blocks', new Set(['big'])]]);
|
||||
const refArgs = new Set(['baseline', 'feature']);
|
||||
const boolArgs = new Set(['samply', 'no-slack']);
|
||||
const boolDefaultTrue = new Set(['reth_newPayload', 'abba', 'otlp']);
|
||||
const boolDefaultTrue = new Set(['reth_newPayload', 'abba']);
|
||||
const durationArgs = new Set(['wait-time']);
|
||||
const stringArgs = new Set(['baseline-args', 'feature-args']);
|
||||
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const unknown = [];
|
||||
const invalid = [];
|
||||
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
|
||||
@@ -257,7 +250,7 @@ jobs:
|
||||
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
|
||||
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
|
||||
if (errors.length) {
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [otlp=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
|
||||
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
|
||||
await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
repo: context.repo.repo,
|
||||
@@ -277,7 +270,6 @@ jobs:
|
||||
bigBlocks = blocks === 'big' ? 'true' : 'false';
|
||||
var rethNewPayload = defaults.reth_newPayload;
|
||||
var abba = defaults.abba;
|
||||
var otlp = defaults.otlp;
|
||||
var waitTime = defaults['wait-time'];
|
||||
var baselineNodeArgs = defaults['baseline-args'];
|
||||
var featureNodeArgs = defaults['feature-args'];
|
||||
@@ -316,7 +308,6 @@ jobs:
|
||||
core.setOutput('baseline-args', baselineNodeArgs);
|
||||
core.setOutput('feature-args', featureNodeArgs);
|
||||
core.setOutput('abba', abba);
|
||||
core.setOutput('otlp', otlp);
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
@@ -384,8 +375,6 @@ jobs:
|
||||
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
|
||||
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
|
||||
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
|
||||
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
|
||||
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
|
||||
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
|
||||
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
|
||||
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
|
||||
@@ -393,7 +382,7 @@ jobs:
|
||||
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
|
||||
const { data: comment } = await github.rest.issues.createComment({
|
||||
owner: context.repo.owner,
|
||||
@@ -428,8 +417,6 @@ jobs:
|
||||
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
|
||||
const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
|
||||
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
|
||||
const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
|
||||
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
|
||||
const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
|
||||
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
|
||||
const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
|
||||
@@ -437,7 +424,7 @@ jobs:
|
||||
const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
|
||||
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
||||
|
||||
const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
|
||||
@@ -507,12 +494,11 @@ jobs:
|
||||
BENCH_BASELINE_ARGS: ${{ needs.reth-bench-ack.outputs.baseline-args }}
|
||||
BENCH_FEATURE_ARGS: ${{ needs.reth-bench-ack.outputs.feature-args }}
|
||||
BENCH_ABBA: ${{ needs.reth-bench-ack.outputs.abba }}
|
||||
BENCH_OTLP: ${{ needs.reth-bench-ack.outputs.otlp }}
|
||||
BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
|
||||
BENCH_NO_SLACK: ${{ needs.reth-bench-ack.outputs.no-slack }}
|
||||
BENCH_METRICS_ADDR: "127.0.0.1:9100"
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_TRACES_ENDPOINT || '' }}
|
||||
BENCH_OTLP_LOGS_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_LOGS_ENDPOINT || '' }}
|
||||
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
|
||||
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
|
||||
steps:
|
||||
- name: Clean up previous bench-work
|
||||
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
|
||||
@@ -573,8 +559,6 @@ jobs:
|
||||
const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
|
||||
const abbaEnabled = (process.env.BENCH_ABBA || 'true') !== 'false';
|
||||
const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
|
||||
const otlpEnabled = (process.env.BENCH_OTLP || 'true') !== 'false';
|
||||
const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
|
||||
const waitTimeVal = process.env.BENCH_WAIT_TIME || '';
|
||||
const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
|
||||
const baselineArgsVal = process.env.BENCH_BASELINE_ARGS || '';
|
||||
@@ -582,7 +566,7 @@ jobs:
|
||||
const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
|
||||
const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
|
||||
const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
|
||||
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
|
||||
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
|
||||
|
||||
const { buildBody } = require('./.github/scripts/bench-update-status.js');
|
||||
await github.rest.issues.updateComment({
|
||||
|
||||
150
Cargo.lock
generated
150
Cargo.lock
generated
@@ -290,17 +290,21 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "alloy-evm"
|
||||
version = "0.29.2"
|
||||
source = "git+https://github.com/alloy-rs/evm?rev=b0eb7e6#b0eb7e617f964f7090c504f21a5977cc440117f7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3cb6ba2dafd6327f78f2b59ae539bd5c39c57a01dc76763e92942619d934a7bb"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-eips",
|
||||
"alloy-hardforks 0.4.7",
|
||||
"alloy-op-hardforks",
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-engine",
|
||||
"alloy-rpc-types-eth",
|
||||
"alloy-sol-types",
|
||||
"auto_impl",
|
||||
"derive_more",
|
||||
"op-alloy",
|
||||
"op-revm",
|
||||
"revm",
|
||||
"thiserror 2.0.18",
|
||||
"tracing",
|
||||
@@ -436,6 +440,18 @@ dependencies = [
|
||||
"url",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloy-op-hardforks"
|
||||
version = "0.4.7"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "6472c610150c4c4c15be9e1b964c9b78068f933bda25fb9cdf09b9ac2bb66f36"
|
||||
dependencies = [
|
||||
"alloy-chains",
|
||||
"alloy-hardforks 0.4.7",
|
||||
"alloy-primitives",
|
||||
"auto_impl",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "alloy-primitives"
|
||||
version = "1.5.7"
|
||||
@@ -1021,7 +1037,7 @@ version = "1.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc"
|
||||
dependencies = [
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -1032,7 +1048,7 @@ checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d"
|
||||
dependencies = [
|
||||
"anstyle",
|
||||
"once_cell_polyfill",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -3456,7 +3472,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4270,8 +4286,8 @@ dependencies = [
|
||||
"libc",
|
||||
"log",
|
||||
"rustversion",
|
||||
"windows-link 0.1.3",
|
||||
"windows-result 0.3.4",
|
||||
"windows-link 0.2.1",
|
||||
"windows-result 0.4.1",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -4827,7 +4843,7 @@ dependencies = [
|
||||
"js-sys",
|
||||
"log",
|
||||
"wasm-bindgen",
|
||||
"windows-core 0.61.2",
|
||||
"windows-core 0.62.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -5179,7 +5195,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46"
|
||||
dependencies = [
|
||||
"hermit-abi",
|
||||
"libc",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6134,7 +6150,7 @@ version = "0.50.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "7957b9740744892f114936ab4a57b3f487491bbeafaf8083688b16841a4240e5"
|
||||
dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -6339,6 +6355,19 @@ version = "11.1.5"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e"
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "a95dd0974d5e60ffe9342a70cc0033d299244fab01cb16a958eb7352ddba1fa7"
|
||||
dependencies = [
|
||||
"op-alloy-consensus",
|
||||
"op-alloy-network",
|
||||
"op-alloy-provider",
|
||||
"op-alloy-rpc-types",
|
||||
"op-alloy-rpc-types-engine",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-consensus"
|
||||
version = "0.24.0"
|
||||
@@ -6359,6 +6388,37 @@ dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-network"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c8ea44162d493219cc678aaca1253d46c3aa73aa361326dfa9d406f086dfa135"
|
||||
dependencies = [
|
||||
"alloy-consensus",
|
||||
"alloy-network",
|
||||
"alloy-primitives",
|
||||
"alloy-provider",
|
||||
"alloy-rpc-types-eth",
|
||||
"alloy-signer",
|
||||
"op-alloy-consensus",
|
||||
"op-alloy-rpc-types",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-provider"
|
||||
version = "0.24.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "83aa8dc34bdf077c8e6d48ff75beff4ac14b428d982c9722483ccd7473c0e114"
|
||||
dependencies = [
|
||||
"alloy-network",
|
||||
"alloy-primitives",
|
||||
"alloy-provider",
|
||||
"alloy-rpc-types-engine",
|
||||
"alloy-transport",
|
||||
"async-trait",
|
||||
"op-alloy-rpc-types-engine",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "op-alloy-rpc-types"
|
||||
version = "0.24.0"
|
||||
@@ -6400,6 +6460,17 @@ dependencies = [
|
||||
"thiserror 2.0.18",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "op-revm"
|
||||
version = "17.0.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "57a98f3a512a7e02a1dcf1242b57302d83657b265a665d50ad98d0b158efaf2c"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"revm",
|
||||
"serde",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "opaque-debug"
|
||||
version = "0.3.1"
|
||||
@@ -10592,7 +10663,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm"
|
||||
version = "36.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "b0abc15d09cd211e9e73410ada10134069c794d4bcdb787dfc16a1bf0939849c"
|
||||
dependencies = [
|
||||
"revm-bytecode",
|
||||
"revm-context",
|
||||
@@ -10610,7 +10682,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-bytecode"
|
||||
version = "9.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e86e468df3cf5cf59fa7ef71a3e9ccabb76bb336401ea2c0674f563104cf3c5e"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"phf",
|
||||
@@ -10621,7 +10694,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-context"
|
||||
version = "15.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9eb1f0a76b14d684a444fc52f7bf6b7564bf882599d91ee62e76d602e7a187c7"
|
||||
dependencies = [
|
||||
"bitvec",
|
||||
"cfg-if",
|
||||
@@ -10637,7 +10711,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-context-interface"
|
||||
version = "16.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fc256b27743e2912ca16899568e6652a372eb5d1d573e6edb16c7836b16cf487"
|
||||
dependencies = [
|
||||
"alloy-eip2930",
|
||||
"alloy-eip7702",
|
||||
@@ -10652,7 +10727,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-database"
|
||||
version = "12.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "2c0a7d6da41061f2c50f99a2632571026b23684b5449ff319914151f4449b6c8"
|
||||
dependencies = [
|
||||
"alloy-eips",
|
||||
"revm-bytecode",
|
||||
@@ -10665,7 +10741,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-database-interface"
|
||||
version = "10.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bd497a38a79057b94a049552cb1f925ad15078bc1a479c132aeeebd1d2ccc768"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"either",
|
||||
@@ -10678,7 +10755,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-handler"
|
||||
version = "17.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "9f1eed729ca9b228ae98688f352235871e9b8be3d568d488e4070f64c56e9d3d"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"derive-where",
|
||||
@@ -10696,7 +10774,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-inspector"
|
||||
version = "17.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cbf5102391706513689f91cb3cb3d97b5f13a02e8647e6e9cb7620877ef84847"
|
||||
dependencies = [
|
||||
"auto_impl",
|
||||
"either",
|
||||
@@ -10712,8 +10791,9 @@ dependencies = [
|
||||
|
||||
[[package]]
|
||||
name = "revm-inspectors"
|
||||
version = "0.36.1"
|
||||
source = "git+https://github.com/paradigmxyz/revm-inspectors?rev=24becc3#24becc35973c6c1d4e1c1475fa51a83d36d50d48"
|
||||
version = "0.36.0"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cfb0f462c8a3d9989d3dbc62d7cca4dfecd7072cfa5d563ab90ced60590ed1da"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"alloy-rpc-types-eth",
|
||||
@@ -10732,7 +10812,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-interpreter"
|
||||
version = "34.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "cf22f80612bb8f58fd1f578750281f2afadb6c93835b14ae6a4d6b75ca26f445"
|
||||
dependencies = [
|
||||
"revm-bytecode",
|
||||
"revm-context-interface",
|
||||
@@ -10744,7 +10825,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-precompile"
|
||||
version = "32.1.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "e2ec11f45deec71e4945e1809736bb20d454285f9167ab53c5159dae1deb603f"
|
||||
dependencies = [
|
||||
"ark-bls12-381",
|
||||
"ark-bn254",
|
||||
@@ -10758,7 +10840,6 @@ dependencies = [
|
||||
"cfg-if",
|
||||
"k256",
|
||||
"p256",
|
||||
"revm-context-interface",
|
||||
"revm-primitives",
|
||||
"ripemd",
|
||||
"secp256k1 0.31.1",
|
||||
@@ -10768,7 +10849,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-primitives"
|
||||
version = "22.1.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "4bcfb5ce6cf18b118932bcdb7da05cd9c250f2cb9f64131396b55f3fe3537c35"
|
||||
dependencies = [
|
||||
"alloy-primitives",
|
||||
"num_enum",
|
||||
@@ -10779,7 +10861,8 @@ dependencies = [
|
||||
[[package]]
|
||||
name = "revm-state"
|
||||
version = "10.0.0"
|
||||
source = "git+https://github.com/bluealloy/revm?rev=712dac7a#712dac7a3461f5f80682627ee8fc9032c337c51b"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "d29404707763da607e5d6e4771cb203998c28159279c2f64cc32de08d2814651"
|
||||
dependencies = [
|
||||
"alloy-eip7928",
|
||||
"bitflags 2.11.0",
|
||||
@@ -11022,7 +11105,7 @@ dependencies = [
|
||||
"errno",
|
||||
"libc",
|
||||
"linux-raw-sys",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11613,8 +11696,7 @@ checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5"
|
||||
[[package]]
|
||||
name = "slotmap"
|
||||
version = "1.1.1"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "bdd58c3c93c3d278ca835519292445cb4b0d4dc59ccfdf7ceadaab3f8aeb4038"
|
||||
source = "git+https://github.com/DaniPopes/slotmap.git?branch=dani%2Fshrink-methods#09fbd360968f9ecef19fc9559037cf869d33858b"
|
||||
dependencies = [
|
||||
"version_check",
|
||||
]
|
||||
@@ -11680,7 +11762,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3a766e1110788c36f4fa1c2b71b387a7815aa65f88ce0229841826633d93723e"
|
||||
dependencies = [
|
||||
"libc",
|
||||
"windows-sys 0.60.2",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -11880,7 +11962,7 @@ dependencies = [
|
||||
"getrandom 0.4.2",
|
||||
"once_cell",
|
||||
"rustix",
|
||||
"windows-sys 0.59.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12221,11 +12303,7 @@ checksum = "d25a406cddcc431a75d3d9afc6a7c0f7428d4891dd973e4d54c56b46127bf857"
|
||||
dependencies = [
|
||||
"futures-util",
|
||||
"log",
|
||||
"rustls",
|
||||
"rustls-native-certs",
|
||||
"rustls-pki-types",
|
||||
"tokio",
|
||||
"tokio-rustls",
|
||||
"tungstenite 0.28.0",
|
||||
]
|
||||
|
||||
@@ -12605,7 +12683,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c5f7c95348f20c1c913d72157b3c6dee6ea3e30b3d19502c5a7f6d3f160dacbf"
|
||||
dependencies = [
|
||||
"cc",
|
||||
"windows-targets 0.48.5",
|
||||
"windows-targets 0.52.6",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
@@ -12680,8 +12758,6 @@ dependencies = [
|
||||
"httparse",
|
||||
"log",
|
||||
"rand 0.9.2",
|
||||
"rustls",
|
||||
"rustls-pki-types",
|
||||
"sha1",
|
||||
"thiserror 2.0.18",
|
||||
"utf-8",
|
||||
@@ -13184,7 +13260,7 @@ version = "0.1.11"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22"
|
||||
dependencies = [
|
||||
"windows-sys 0.48.0",
|
||||
"windows-sys 0.61.2",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
|
||||
71
Cargo.toml
71
Cargo.toml
@@ -399,9 +399,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
|
||||
reth-payload-primitives = { path = "crates/payload/primitives" }
|
||||
reth-payload-validator = { path = "crates/payload/validator" }
|
||||
reth-payload-util = { path = "crates/payload/util" }
|
||||
reth-primitives = { path = "crates/primitives", default-features = false, features = [
|
||||
"__internal",
|
||||
] }
|
||||
reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
|
||||
reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
|
||||
reth-provider = { path = "crates/storage/provider" }
|
||||
reth-prune = { path = "crates/prune/prune" }
|
||||
@@ -446,22 +444,18 @@ revm-state = { version = "10.0.0", default-features = false }
|
||||
revm-primitives = { version = "22.1.0", default-features = false }
|
||||
revm-interpreter = { version = "34.0.0", default-features = false }
|
||||
revm-database-interface = { version = "10.0.0", default-features = false }
|
||||
revm-inspectors = "0.36.1"
|
||||
revm-inspectors = "0.36.0"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.6"
|
||||
alloy-primitives = { version = "1.5.6", default-features = false, features = [
|
||||
"map-foldhash",
|
||||
] }
|
||||
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
|
||||
alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
|
||||
alloy-chains = { version = "0.2.5", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.29.2", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = [
|
||||
"core-net",
|
||||
] }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
alloy-hardforks = "0.4.5"
|
||||
@@ -473,15 +467,10 @@ alloy-genesis = { version = "1.7.3", default-features = false }
|
||||
alloy-json-rpc = { version = "1.7.3", default-features = false }
|
||||
alloy-network = { version = "1.7.3", default-features = false }
|
||||
alloy-network-primitives = { version = "1.7.3", default-features = false }
|
||||
alloy-provider = { version = "1.7.3", features = [
|
||||
"reqwest",
|
||||
"debug-api",
|
||||
], default-features = false }
|
||||
alloy-provider = { version = "1.7.3", features = ["reqwest", "debug-api"], default-features = false }
|
||||
alloy-pubsub = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-client = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types = { version = "1.7.3", features = [
|
||||
"eth",
|
||||
], default-features = false }
|
||||
alloy-rpc-types = { version = "1.7.3", features = ["eth"], default-features = false }
|
||||
alloy-rpc-types-admin = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-anvil = { version = "1.7.3", default-features = false }
|
||||
alloy-rpc-types-beacon = { version = "1.7.3", default-features = false }
|
||||
@@ -495,9 +484,7 @@ alloy-serde = { version = "1.7.3", default-features = false }
|
||||
alloy-signer = { version = "1.7.3", default-features = false }
|
||||
alloy-signer-local = { version = "1.7.3", default-features = false }
|
||||
alloy-transport = { version = "1.7.3" }
|
||||
alloy-transport-http = { version = "1.7.3", features = [
|
||||
"reqwest-rustls-tls",
|
||||
], default-features = false }
|
||||
alloy-transport-http = { version = "1.7.3", features = ["reqwest-rustls-tls"], default-features = false }
|
||||
alloy-transport-ipc = { version = "1.7.3", default-features = false }
|
||||
alloy-transport-ws = { version = "1.7.3", default-features = false }
|
||||
|
||||
@@ -511,10 +498,7 @@ either = { version = "1.15.0", default-features = false }
|
||||
arrayvec = { version = "0.7.6", default-features = false }
|
||||
aquamarine = "0.6"
|
||||
auto_impl = "1"
|
||||
backon = { version = "1.2", default-features = false, features = [
|
||||
"std-blocking-sleep",
|
||||
"tokio-sleep",
|
||||
] }
|
||||
backon = { version = "1.2", default-features = false, features = ["std-blocking-sleep", "tokio-sleep"] }
|
||||
bincode = "1.3"
|
||||
bitflags = "2.4"
|
||||
boyer-moore-magiclen = "0.2.16"
|
||||
@@ -538,13 +522,9 @@ linked_hash_set = "0.1"
|
||||
libc = "0.2"
|
||||
lz4 = "1.28.1"
|
||||
modular-bitfield = "0.13.1"
|
||||
notify = { version = "8.0.0", default-features = false, features = [
|
||||
"macos_fsevent",
|
||||
] }
|
||||
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
|
||||
nybbles = { version = "0.4.8", default-features = false }
|
||||
once_cell = { version = "1.19", default-features = false, features = [
|
||||
"critical-section",
|
||||
] }
|
||||
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
|
||||
parking_lot = "0.12"
|
||||
quanta = "0.12"
|
||||
paste = "1.0"
|
||||
@@ -558,16 +538,15 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
|
||||
serde_with = { version = "3", default-features = false, features = ["macros"] }
|
||||
sha2 = { version = "0.10", default-features = false }
|
||||
shlex = "1.3"
|
||||
slotmap = "1"
|
||||
# https://github.com/orlp/slotmap/pull/148
|
||||
slotmap = { git = "https://github.com/DaniPopes/slotmap.git", branch = "dani/shrink-methods" }
|
||||
smallvec = "1"
|
||||
strum = { version = "0.27", default-features = false }
|
||||
strum_macros = "0.27"
|
||||
syn = "2.0"
|
||||
thiserror = { version = "2.0.0", default-features = false }
|
||||
tar = "0.4.44"
|
||||
tracing = { version = "0.1.0", default-features = false, features = [
|
||||
"attributes",
|
||||
] }
|
||||
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
|
||||
tracing-appender = "0.2"
|
||||
url = { version = "2.3", default-features = false }
|
||||
zstd = "0.13"
|
||||
@@ -605,11 +584,7 @@ futures-util = { version = "0.3", default-features = false }
|
||||
hyper = "1.3"
|
||||
hyper-util = "0.1.5"
|
||||
pin-project = "1.0.12"
|
||||
reqwest = { version = "0.12", default-features = false, features = [
|
||||
"rustls-tls",
|
||||
"rustls-tls-native-roots",
|
||||
"stream",
|
||||
] }
|
||||
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
|
||||
tracing-futures = "0.2"
|
||||
tower = "0.5"
|
||||
tower-http = "0.6"
|
||||
@@ -634,10 +609,7 @@ proptest-arbitrary-interop = "0.1.0"
|
||||
# crypto
|
||||
enr = { version = "0.13", default-features = false }
|
||||
k256 = { version = "0.13", default-features = false, features = ["ecdsa"] }
|
||||
secp256k1 = { version = "0.30", default-features = false, features = [
|
||||
"global-context",
|
||||
"recovery",
|
||||
] }
|
||||
secp256k1 = { version = "0.30", default-features = false, features = ["global-context", "recovery"] }
|
||||
# rand 8 for secp256k1
|
||||
rand_08 = { package = "rand", version = "0.8" }
|
||||
|
||||
@@ -780,15 +752,6 @@ ipnet = "2.11"
|
||||
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
|
||||
|
||||
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "b0eb7e6" }
|
||||
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "9bc2dba" }
|
||||
|
||||
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "24becc3" }
|
||||
|
||||
# revm from rakita/state-gas branch
|
||||
revm = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
revm-database = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
revm-state = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "712dac7a" }
|
||||
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
|
||||
|
||||
@@ -124,7 +124,6 @@ jemalloc = [
|
||||
"reth-node-core/jemalloc",
|
||||
"reth-node-metrics/jemalloc",
|
||||
"reth-ethereum-cli/jemalloc",
|
||||
"reth-provider/jemalloc",
|
||||
]
|
||||
jemalloc-prof = [
|
||||
"reth-cli-util/jemalloc",
|
||||
|
||||
@@ -36,6 +36,15 @@ pub use spec::{
|
||||
DepositContract, ForkBaseFeeParams, DEV, HOLESKY, HOODI, MAINNET, SEPOLIA,
|
||||
};
|
||||
|
||||
use reth_primitives_traits::sync::OnceLock;
|
||||
|
||||
/// Simple utility to create a thread-safe sync cell with a value set.
|
||||
pub fn once_cell_set<T>(value: T) -> OnceLock<T> {
|
||||
let once = OnceLock::new();
|
||||
let _ = once.set(value);
|
||||
once
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -187,6 +187,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
where
|
||||
C: ChainSpecParser<ChainSpec = N::ChainSpec>,
|
||||
{
|
||||
let prune_modes = config.prune.segments.clone();
|
||||
let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
|
||||
db,
|
||||
self.chain.clone(),
|
||||
@@ -194,8 +195,7 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
rocksdb_provider,
|
||||
runtime,
|
||||
)?
|
||||
.with_prune_modes(config.prune.segments.clone())
|
||||
.with_minimum_pruning_distance(config.prune.minimum_pruning_distance);
|
||||
.with_prune_modes(prune_modes.clone());
|
||||
|
||||
// Check for consistency between database and static files.
|
||||
if !access.is_read_only_inconsistent() &&
|
||||
@@ -229,13 +229,10 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
|
||||
NoopBodiesDownloader::default(),
|
||||
NoopEvmConfig::<N::Evm>::default(),
|
||||
config.stages.clone(),
|
||||
config.prune.segments.clone(),
|
||||
prune_modes.clone(),
|
||||
None,
|
||||
))
|
||||
.build(
|
||||
factory.clone(),
|
||||
StaticFileProducer::new(factory.clone(), config.prune.segments.clone()),
|
||||
);
|
||||
.build(factory.clone(), StaticFileProducer::new(factory.clone(), prune_modes));
|
||||
|
||||
// Move all applicable data from database to static files.
|
||||
pipeline.move_to_static_files()?;
|
||||
|
||||
@@ -199,7 +199,7 @@ pub(crate) fn config_for_selections(
|
||||
}
|
||||
|
||||
return Config {
|
||||
prune: PruneConfig { segments, ..Default::default() },
|
||||
prune: PruneConfig { block_interval: PruneConfig::default().block_interval, segments },
|
||||
static_files,
|
||||
..Default::default()
|
||||
};
|
||||
|
||||
@@ -201,7 +201,6 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
|
||||
let net =
|
||||
NetworkConfigBuilder::<N::NetworkPrimitives>::new(p2p_secret_key, Runtime::test())
|
||||
.peer_config(config.peers_config_with_basic_nodes_from_file(None))
|
||||
.sessions_config(config.sessions)
|
||||
.external_ip_resolver(self.network.nat.clone())
|
||||
.network_id(self.network.network_id)
|
||||
.boot_nodes(boot_nodes.clone())
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
//! Configuration files.
|
||||
use reth_network_types::{PeersConfig, SessionsConfig};
|
||||
use reth_prune_types::{PruneModes, MINIMUM_UNWIND_SAFE_DISTANCE};
|
||||
use reth_prune_types::PruneModes;
|
||||
use reth_stages_types::ExecutionStageThresholds;
|
||||
use reth_static_file_types::{StaticFileMap, StaticFileSegment};
|
||||
use std::{
|
||||
@@ -540,24 +540,11 @@ pub struct PruneConfig {
|
||||
/// Pruning configuration for every part of the data that can be pruned.
|
||||
#[cfg_attr(feature = "serde", serde(alias = "parts"))]
|
||||
pub segments: PruneModes,
|
||||
/// Minimum distance from the tip required for pruning. Controls the safety margin for
|
||||
/// reorgs and manual unwinds. Defaults to [`MINIMUM_UNWIND_SAFE_DISTANCE`].
|
||||
#[cfg_attr(feature = "serde", serde(default = "default_minimum_pruning_distance"))]
|
||||
pub minimum_pruning_distance: u64,
|
||||
}
|
||||
|
||||
/// Returns the default minimum pruning distance.
|
||||
const fn default_minimum_pruning_distance() -> u64 {
|
||||
MINIMUM_UNWIND_SAFE_DISTANCE
|
||||
}
|
||||
|
||||
impl Default for PruneConfig {
|
||||
fn default() -> Self {
|
||||
Self {
|
||||
block_interval: DEFAULT_BLOCK_INTERVAL,
|
||||
segments: PruneModes::default(),
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
}
|
||||
Self { block_interval: DEFAULT_BLOCK_INTERVAL, segments: PruneModes::default() }
|
||||
}
|
||||
}
|
||||
|
||||
@@ -590,7 +577,6 @@ impl PruneConfig {
|
||||
bodies_history,
|
||||
receipts_log_filter,
|
||||
},
|
||||
minimum_pruning_distance,
|
||||
} = other;
|
||||
|
||||
// Merge block_interval, only update if it's the default interval
|
||||
@@ -598,11 +584,6 @@ impl PruneConfig {
|
||||
self.block_interval = block_interval;
|
||||
}
|
||||
|
||||
// Merge minimum_pruning_distance, only update if it's the default
|
||||
if self.minimum_pruning_distance == MINIMUM_UNWIND_SAFE_DISTANCE {
|
||||
self.minimum_pruning_distance = minimum_pruning_distance;
|
||||
}
|
||||
|
||||
// Merge the various segment prune modes
|
||||
self.segments.sender_recovery = self.segments.sender_recovery.or(sender_recovery);
|
||||
self.segments.transaction_lookup = self.segments.transaction_lookup.or(transaction_lookup);
|
||||
@@ -642,9 +623,7 @@ mod tests {
|
||||
use crate::PruneConfig;
|
||||
use alloy_primitives::Address;
|
||||
use reth_network_peers::TrustedPeer;
|
||||
use reth_prune_types::{
|
||||
PruneMode, PruneModes, ReceiptsLogPruneConfig, MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
};
|
||||
use reth_prune_types::{PruneMode, PruneModes, ReceiptsLogPruneConfig};
|
||||
use std::{collections::BTreeMap, path::Path, str::FromStr, time::Duration};
|
||||
|
||||
fn with_tempdir(filename: &str, proc: fn(&std::path::Path)) {
|
||||
@@ -1114,7 +1093,6 @@ receipts = { distance = 16384 }
|
||||
fn test_prune_config_merge() {
|
||||
let mut config1 = PruneConfig {
|
||||
block_interval: 5,
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Full),
|
||||
transaction_lookup: None,
|
||||
@@ -1131,7 +1109,6 @@ receipts = { distance = 16384 }
|
||||
|
||||
let config2 = PruneConfig {
|
||||
block_interval: 10,
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
segments: PruneModes {
|
||||
sender_recovery: Some(PruneMode::Distance(500)),
|
||||
transaction_lookup: Some(PruneMode::Full),
|
||||
|
||||
@@ -96,11 +96,6 @@ where
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the pruning arguments for the test nodes.
|
||||
pub fn with_pruning(self, pruning: reth_node_core::args::PruningArgs) -> Self {
|
||||
self.with_node_config_modifier(move |config| config.with_pruning(pruning.clone()))
|
||||
}
|
||||
|
||||
/// Enables v2 storage defaults (`--storage.v2`), routing tx hashes, history
|
||||
/// indices, etc. to `RocksDB` and changesets/senders to static files.
|
||||
pub fn with_storage_v2(self) -> Self {
|
||||
|
||||
@@ -38,8 +38,8 @@ op-alloy-rpc-types-engine = { workspace = true, optional = true }
|
||||
workspace = true
|
||||
|
||||
[features]
|
||||
# op = [
|
||||
# "dep:op-alloy-rpc-types-engine",
|
||||
# "reth-payload-primitives/op",
|
||||
# "reth-primitives-traits/op",
|
||||
# ]
|
||||
op = [
|
||||
"dep:op-alloy-rpc-types-engine",
|
||||
"reth-payload-primitives/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
|
||||
@@ -177,25 +177,18 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
// Run the pruner in a separate provider so it reads committed RocksDB state
|
||||
// that includes the history entries written by save_blocks above.
|
||||
//
|
||||
// The pruner reads the indices from rocksdb, filters it, and writes to indices, so it
|
||||
// must be able to read anything written by save_blocks.
|
||||
if self.pruner.is_pruning_needed(last.number) {
|
||||
debug!(target: "engine::persistence", block_num=?last.number, "Running pruner");
|
||||
let prune_start = Instant::now();
|
||||
let provider_rw = self.provider.database_provider_rw()?;
|
||||
let _ = self.pruner.run_with_provider(&provider_rw, last.number)?;
|
||||
provider_rw.commit()?;
|
||||
debug!(target: "engine::persistence", tip=?last.number, "Finished pruning after saving blocks");
|
||||
self.metrics.prune_before_duration_seconds.record(prune_start.elapsed());
|
||||
}
|
||||
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
|
||||
debug!(target: "engine::persistence", first=?first_block, last=?last_block, "Saved range of blocks");
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
self.metrics.save_blocks_batch_size.record(block_count as f64);
|
||||
self.metrics.save_blocks_duration_seconds.record(elapsed);
|
||||
@@ -461,52 +454,4 @@ mod tests {
|
||||
assert_eq!(last_hash, result.last_block.unwrap().hash);
|
||||
}
|
||||
}
|
||||
|
||||
/// Verifies that committing `save_blocks` history before running the pruner
|
||||
/// prevents the pruner from overwriting new entries.
|
||||
///
|
||||
/// Previously, both `save_blocks` and the pruner pushed `RocksDB` batches before
|
||||
/// a single commit. Both read committed state, so the pruner didn't see the
|
||||
/// new entries and its batch overwrote them. The fix commits `save_blocks`
|
||||
/// first, then runs the pruner against committed state in a separate provider.
|
||||
#[test]
|
||||
fn test_save_blocks_then_prune_preserves_new_history() {
|
||||
use reth_db::{models::ShardedKey, tables, BlockNumberList};
|
||||
use reth_provider::RocksDBProviderFactory;
|
||||
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let provider_factory = create_test_provider_factory();
|
||||
let tracked_addr = alloy_primitives::Address::from([0xBE; 20]);
|
||||
|
||||
// Phase 1: Establish baseline history for blocks 0..20.
|
||||
let rocksdb = provider_factory.rocksdb_provider();
|
||||
{
|
||||
let mut batch = rocksdb.batch();
|
||||
let initial_blocks: Vec<u64> = (0..20).collect();
|
||||
let shard = BlockNumberList::new_pre_sorted(initial_blocks.iter().copied());
|
||||
batch
|
||||
.put::<tables::AccountsHistory>(ShardedKey::new(tracked_addr, u64::MAX), &shard)
|
||||
.unwrap();
|
||||
batch.commit().unwrap();
|
||||
}
|
||||
|
||||
// Phase 2: Simulate the fixed on_save_blocks flow.
|
||||
// Step 1: save_blocks appends new entries 20..25 and commits immediately.
|
||||
let mut batch1 = rocksdb.batch();
|
||||
batch1.append_account_history_shard(tracked_addr, 20..25u64).unwrap();
|
||||
batch1.commit().unwrap();
|
||||
|
||||
// Step 2: Pruner runs AFTER commit, so it reads state that includes 20..25.
|
||||
// Prunes entries ≤ 14, leaving [15..25).
|
||||
let mut batch2 = rocksdb.batch();
|
||||
batch2.prune_account_history_to(tracked_addr, 14).unwrap();
|
||||
batch2.commit().unwrap();
|
||||
|
||||
// Verify new entries survived pruning.
|
||||
let shards = rocksdb.account_history_shards(tracked_addr).unwrap();
|
||||
let entries: Vec<u64> = shards.iter().flat_map(|(_, list)| list.iter()).collect();
|
||||
let expected: Vec<u64> = (15..25).collect();
|
||||
assert_eq!(entries, expected, "new entries 20..25 must survive pruning");
|
||||
}
|
||||
}
|
||||
|
||||
@@ -81,8 +81,8 @@ pub struct CacheEntry<S> {
|
||||
}
|
||||
|
||||
impl<S> CacheEntry<S> {
|
||||
const fn regular_gas_used(&self) -> u64 {
|
||||
self.output.gas.limit() - self.output.gas.remaining()
|
||||
const fn gas_used(&self) -> u64 {
|
||||
self.output.gas_used
|
||||
}
|
||||
|
||||
fn to_precompile_result(&self) -> PrecompileResult {
|
||||
@@ -170,10 +170,10 @@ where
|
||||
|
||||
fn call(&self, input: PrecompileInput<'_>) -> PrecompileResult {
|
||||
if let Some(entry) = &self.cache.get(input.data, self.spec_id.clone()) &&
|
||||
input.gas >= entry.regular_gas_used()
|
||||
input.gas >= entry.gas_used()
|
||||
{
|
||||
self.increment_by_one_precompile_cache_hits();
|
||||
return entry.to_precompile_result();
|
||||
return entry.to_precompile_result()
|
||||
}
|
||||
|
||||
let calldata = input.data;
|
||||
@@ -228,14 +228,15 @@ mod tests {
|
||||
use super::*;
|
||||
use reth_evm::{EthEvmFactory, Evm, EvmEnv, EvmFactory};
|
||||
use reth_revm::db::EmptyDB;
|
||||
use revm::{context::TxEnv, interpreter::gas::GasTracker, precompile::PrecompileOutput};
|
||||
use revm::{context::TxEnv, precompile::PrecompileOutput};
|
||||
use revm_primitives::hardfork::SpecId;
|
||||
|
||||
#[test]
|
||||
fn test_precompile_cache_basic() {
|
||||
let dyn_precompile: DynPrecompile = (|_input: PrecompileInput<'_>| -> PrecompileResult {
|
||||
Ok(PrecompileOutput {
|
||||
gas: GasTracker::new(0, 0, 0),
|
||||
gas_used: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: Bytes::default(),
|
||||
reverted: false,
|
||||
})
|
||||
@@ -246,7 +247,8 @@ mod tests {
|
||||
CachedPrecompile::new(dyn_precompile, PrecompileCache::default(), SpecId::PRAGUE, None);
|
||||
|
||||
let output = PrecompileOutput {
|
||||
gas: GasTracker::new(50, 0, 0),
|
||||
gas_used: 50,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
|
||||
reverted: false,
|
||||
};
|
||||
@@ -277,7 +279,8 @@ mod tests {
|
||||
assert_eq!(input.data, input_data);
|
||||
|
||||
Ok(PrecompileOutput {
|
||||
gas: GasTracker::new(5000, 0, 0),
|
||||
gas_used: 5000,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
|
||||
reverted: false,
|
||||
})
|
||||
@@ -291,7 +294,8 @@ mod tests {
|
||||
assert_eq!(input.data, input_data);
|
||||
|
||||
Ok(PrecompileOutput {
|
||||
gas: GasTracker::new(7000, 0, 0),
|
||||
gas_used: 7000,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
|
||||
reverted: false,
|
||||
})
|
||||
|
||||
@@ -152,7 +152,6 @@ jemalloc = [
|
||||
"reth-cli-util?/jemalloc",
|
||||
"reth-ethereum-cli?/jemalloc",
|
||||
"reth-node-core?/jemalloc",
|
||||
"reth-provider?/jemalloc",
|
||||
]
|
||||
jemalloc-prof = [
|
||||
"jemalloc",
|
||||
|
||||
@@ -62,4 +62,4 @@ test-utils = [
|
||||
"reth-trie-common/test-utils",
|
||||
"reth-ethereum-primitives/test-utils",
|
||||
]
|
||||
# op = ["alloy-evm/op", "reth-primitives-traits/op"]
|
||||
op = ["alloy-evm/op", "reth-primitives-traits/op"]
|
||||
|
||||
@@ -468,7 +468,7 @@ where
|
||||
self.executor.execute_transaction_with_commit_condition((tx_env, &tx), f)?
|
||||
{
|
||||
self.transactions.push(tx);
|
||||
Ok(Some(gas_used.into()))
|
||||
Ok(Some(gas_used))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
|
||||
@@ -122,10 +122,10 @@ test-utils = [
|
||||
"reth-tasks/test-utils",
|
||||
]
|
||||
trie-debug = ["reth-engine-tree/trie-debug"]
|
||||
# op = [
|
||||
# "reth-db/op",
|
||||
# "reth-db-api/op",
|
||||
# "reth-engine-local/op",
|
||||
# "reth-evm/op",
|
||||
# "reth-primitives-traits/op",
|
||||
# ]
|
||||
op = [
|
||||
"reth-db/op",
|
||||
"reth-db-api/op",
|
||||
"reth-engine-local/op",
|
||||
"reth-evm/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
|
||||
@@ -501,7 +501,6 @@ where
|
||||
.build()?
|
||||
};
|
||||
|
||||
let prune_config = self.prune_config();
|
||||
let factory = ProviderFactory::new(
|
||||
self.right().clone(),
|
||||
self.chain_spec(),
|
||||
@@ -509,8 +508,7 @@ where
|
||||
rocksdb_provider,
|
||||
self.task_executor().clone(),
|
||||
)?
|
||||
.with_prune_modes(prune_config.segments)
|
||||
.with_minimum_pruning_distance(prune_config.minimum_pruning_distance)
|
||||
.with_prune_modes(self.prune_modes())
|
||||
.with_changeset_cache(changeset_cache);
|
||||
|
||||
// Check consistency between the database and static files, returning
|
||||
@@ -1312,7 +1310,6 @@ mod tests {
|
||||
bodies_distance: None,
|
||||
receipts_log_filter: None,
|
||||
bodies_before: None,
|
||||
minimum_distance: None,
|
||||
},
|
||||
..NodeConfig::test()
|
||||
};
|
||||
|
||||
@@ -815,16 +815,14 @@ impl DiscoveryArgs {
|
||||
SocketAddr::V6(addr) => Some(*addr.ip()),
|
||||
});
|
||||
|
||||
let mut discv5_config_builder =
|
||||
reth_discv5::discv5::ConfigBuilder::new(ListenConfig::from_two_sockets(
|
||||
discv5_addr_ipv4.map(|addr| SocketAddrV4::new(addr, *discv5_port)),
|
||||
discv5_addr_ipv6.map(|addr| SocketAddrV6::new(addr, *discv5_port_ipv6, 0, 0)),
|
||||
));
|
||||
if discv5_addr.is_some() || discv5_addr_ipv6.is_some() || self.disable_nat {
|
||||
discv5_config_builder.disable_enr_update();
|
||||
}
|
||||
reth_discv5::Config::builder(rlpx_tcp_socket)
|
||||
.discv5_config(discv5_config_builder.build())
|
||||
.discv5_config(
|
||||
reth_discv5::discv5::ConfigBuilder::new(ListenConfig::from_two_sockets(
|
||||
discv5_addr_ipv4.map(|addr| SocketAddrV4::new(addr, *discv5_port)),
|
||||
discv5_addr_ipv6.map(|addr| SocketAddrV6::new(addr, *discv5_port_ipv6, 0, 0)),
|
||||
))
|
||||
.build(),
|
||||
)
|
||||
.add_unsigned_boot_nodes(boot_nodes)
|
||||
.lookup_interval(*discv5_lookup_interval)
|
||||
.bootstrap_lookup_interval(*discv5_bootstrap_lookup_interval)
|
||||
|
||||
@@ -196,11 +196,6 @@ pub struct PruningArgs {
|
||||
/// pruned.
|
||||
#[arg(long = "prune.bodies.before", value_name = "BLOCK_NUMBER", conflicts_with_all = &["bodies_distance", "bodies_pre_merge"])]
|
||||
pub bodies_before: Option<BlockNumber>,
|
||||
|
||||
/// Minimum pruning distance from the tip. This controls the safety margin for reorgs and
|
||||
/// manual unwinds.
|
||||
#[arg(long = "prune.minimum-distance", value_name = "BLOCKS")]
|
||||
pub minimum_distance: Option<u64>,
|
||||
}
|
||||
|
||||
impl PruningArgs {
|
||||
@@ -225,11 +220,7 @@ impl PruningArgs {
|
||||
.block_number()
|
||||
.map(PruneMode::Before);
|
||||
}
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments,
|
||||
minimum_pruning_distance: config.minimum_pruning_distance,
|
||||
}
|
||||
config = PruneConfig { block_interval: config.block_interval, segments }
|
||||
}
|
||||
|
||||
// If --minimal is set, use minimal storage mode with aggressive pruning.
|
||||
@@ -237,7 +228,6 @@ impl PruningArgs {
|
||||
config = PruneConfig {
|
||||
block_interval: config.block_interval,
|
||||
segments: DefaultPruningValues::get_global().minimal_prune_modes.clone(),
|
||||
minimum_pruning_distance: config.minimum_pruning_distance,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -245,9 +235,6 @@ impl PruningArgs {
|
||||
if let Some(block_interval) = self.block_interval {
|
||||
config.block_interval = block_interval as usize;
|
||||
}
|
||||
if let Some(distance) = self.minimum_distance {
|
||||
config.minimum_pruning_distance = distance;
|
||||
}
|
||||
if let Some(mode) = self.sender_recovery_prune_mode() {
|
||||
config.segments.sender_recovery = Some(mode);
|
||||
}
|
||||
|
||||
@@ -21,7 +21,7 @@ alloy-primitives.workspace = true
|
||||
alloy-consensus.workspace = true
|
||||
|
||||
tokio.workspace = true
|
||||
tokio-tungstenite = { workspace = true, features = ["rustls-tls-native-roots"] }
|
||||
tokio-tungstenite.workspace = true
|
||||
futures-util.workspace = true
|
||||
tokio-stream.workspace = true
|
||||
|
||||
|
||||
@@ -53,7 +53,7 @@ std = [
|
||||
"either/std",
|
||||
"alloy-consensus/std",
|
||||
]
|
||||
# op = [
|
||||
# "dep:op-alloy-rpc-types-engine",
|
||||
# "reth-primitives-traits/op",
|
||||
# ]
|
||||
op = [
|
||||
"dep:op-alloy-rpc-types-engine",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
|
||||
@@ -57,8 +57,8 @@ pub mod servers {
|
||||
web3::Web3ApiServer,
|
||||
};
|
||||
pub use reth_rpc_eth_api::{
|
||||
self as eth, EthApiServer, EthBundleApiServer, EthCallBundleApiServer, EthConfigApiServer,
|
||||
EthFilterApiServer, EthPubSubApiServer, L2EthApiExtServer,
|
||||
self as eth, EthApiServer, EthBundleApiServer, EthCallBundleApiServer, EthFilterApiServer,
|
||||
EthPubSubApiServer, L2EthApiExtServer,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -89,7 +89,7 @@ pub mod clients {
|
||||
web3::Web3ApiClient,
|
||||
};
|
||||
pub use reth_rpc_eth_api::{
|
||||
EthApiClient, EthBundleApiClient, EthCallBundleApiClient, EthConfigApiClient,
|
||||
EthFilterApiClient, L2EthApiExtClient,
|
||||
EthApiClient, EthBundleApiClient, EthCallBundleApiClient, EthFilterApiClient,
|
||||
L2EthApiExtClient,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -43,10 +43,10 @@ serde_json.workspace = true
|
||||
|
||||
[features]
|
||||
default = []
|
||||
#op = [
|
||||
# "dep:op-alloy-consensus",
|
||||
# "dep:op-alloy-rpc-types",
|
||||
# "reth-evm/op",
|
||||
# "reth-primitives-traits/op",
|
||||
# "alloy-evm/op",
|
||||
#]
|
||||
op = [
|
||||
"dep:op-alloy-consensus",
|
||||
"dep:op-alloy-rpc-types",
|
||||
"reth-evm/op",
|
||||
"reth-primitives-traits/op",
|
||||
"alloy-evm/op",
|
||||
]
|
||||
|
||||
@@ -62,9 +62,9 @@ tracing.workspace = true
|
||||
[features]
|
||||
js-tracer = ["revm-inspectors/js-tracer", "reth-rpc-eth-types/js-tracer"]
|
||||
client = ["jsonrpsee/client", "jsonrpsee/async-client"]
|
||||
# op = [
|
||||
# "reth-evm/op",
|
||||
# "reth-primitives-traits/op",
|
||||
# "reth-rpc-convert/op",
|
||||
# "alloy-evm/op",
|
||||
# ]
|
||||
op = [
|
||||
"reth-evm/op",
|
||||
"reth-primitives-traits/op",
|
||||
"reth-rpc-convert/op",
|
||||
"alloy-evm/op",
|
||||
]
|
||||
|
||||
@@ -479,11 +479,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
|
||||
.map_err(Self::Error::from_eth_err)?;
|
||||
}
|
||||
|
||||
// Read fields from request before consuming it in create_txn_env
|
||||
let request_has_gas_limit = request.as_ref().gas_limit().is_some();
|
||||
let initial = request.as_ref().access_list().cloned().unwrap_or_default();
|
||||
|
||||
let mut tx_env = this.create_txn_env(&evm_env, request, &mut db)?;
|
||||
let mut tx_env = this.create_txn_env(&evm_env, request.clone(), &mut db)?;
|
||||
|
||||
// we want to disable this in eth_createAccessList, since this is common practice used
|
||||
// by other node impls and providers <https://github.com/foundry-rs/foundry/issues/4388>
|
||||
@@ -506,18 +502,21 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
|
||||
// per-tx cap (2^24 ≈ 16.7M post-Osaka).
|
||||
evm_env.cfg_env.tx_gas_limit_cap = Some(u64::MAX);
|
||||
|
||||
if !request_has_gas_limit && tx_env.gas_price() > 0 {
|
||||
if request.as_ref().gas_limit().is_none() && tx_env.gas_price() > 0 {
|
||||
let cap = this.caller_gas_allowance(&mut db, &evm_env, &tx_env)?;
|
||||
// no gas limit was provided in the request, so we need to cap the request's gas
|
||||
// limit
|
||||
tx_env.set_gas_limit(cap.min(evm_env.block_env.gas_limit()));
|
||||
}
|
||||
|
||||
// can consume the list since we're not using the request anymore
|
||||
let initial = request.as_ref().access_list().cloned().unwrap_or_default();
|
||||
|
||||
let mut inspector = AccessListInspector::new(initial);
|
||||
|
||||
let result = this.inspect(&mut db, evm_env.clone(), tx_env.clone(), &mut inspector)?;
|
||||
let access_list = inspector.into_access_list();
|
||||
let gas_used = result.result.tx_gas_used();
|
||||
let gas_used = result.result.gas_used();
|
||||
tx_env.set_access_list(access_list.clone());
|
||||
if let Err(err) = Self::Error::ensure_success(result.result) {
|
||||
return Ok(AccessListResult {
|
||||
@@ -529,7 +528,7 @@ pub trait EthCall: EstimateCall + Call + LoadPendingBlock + LoadBlock + FullEthA
|
||||
|
||||
// transact again to get the exact gas used
|
||||
let result = this.transact(&mut db, evm_env, tx_env)?;
|
||||
let gas_used = result.result.tx_gas_used();
|
||||
let gas_used = result.result.gas_used();
|
||||
let error = Self::Error::ensure_success(result.result).err().map(|e| e.to_string());
|
||||
|
||||
Ok(AccessListResult { access_list, gas_used: U256::from(gas_used), error })
|
||||
|
||||
@@ -204,7 +204,7 @@ pub trait EstimateCall: Call {
|
||||
|
||||
// NOTE: this is the gas the transaction used, which is less than the
|
||||
// transaction requires to succeed.
|
||||
let mut gas_used = res.result.tx_gas_used();
|
||||
let mut gas_used = res.result.gas_used();
|
||||
// the lowest value is capped by the gas used by the unconstrained transaction
|
||||
let mut lowest_gas_limit = gas_used.saturating_sub(1);
|
||||
|
||||
@@ -225,7 +225,7 @@ pub trait EstimateCall: Call {
|
||||
res = evm.transact(optimistic_tx_env).map_err(Self::Error::from_evm_err)?;
|
||||
|
||||
// Update the gas used based on the new result.
|
||||
gas_used = res.result.tx_gas_used();
|
||||
gas_used = res.result.gas_used();
|
||||
// Update the gas limit estimates (highest and lowest) based on the execution result.
|
||||
update_estimated_gas_range(
|
||||
res.result,
|
||||
|
||||
@@ -318,8 +318,7 @@ pub trait LoadPendingBlock:
|
||||
|
||||
// There's only limited amount of blob space available per block, so we need to
|
||||
// check if the EIP-4844 can still fit in the block
|
||||
let tx_blob_gas = tx.blob_gas_used();
|
||||
if let Some(tx_blob_gas) = tx_blob_gas &&
|
||||
if let Some(tx_blob_gas) = tx.blob_gas_used() &&
|
||||
sum_blob_gas_used + tx_blob_gas > blob_params.max_blob_gas_per_block()
|
||||
{
|
||||
// we can't fit this _blob_ transaction into the block, so we mark it as
|
||||
@@ -336,7 +335,7 @@ pub trait LoadPendingBlock:
|
||||
continue
|
||||
}
|
||||
|
||||
let gas_used = match builder.execute_transaction(tx) {
|
||||
let gas_used = match builder.execute_transaction(tx.clone()) {
|
||||
Ok(gas_used) => gas_used,
|
||||
Err(BlockExecutionError::Validation(BlockValidationError::InvalidTx {
|
||||
error,
|
||||
@@ -361,7 +360,7 @@ pub trait LoadPendingBlock:
|
||||
};
|
||||
|
||||
// add to the total blob gas used if the transaction successfully executed
|
||||
if let Some(tx_blob_gas) = tx_blob_gas {
|
||||
if let Some(tx_blob_gas) = tx.blob_gas_used() {
|
||||
sum_blob_gas_used += tx_blob_gas;
|
||||
|
||||
// if we've reached the max data gas per block, we can skip blob txs entirely
|
||||
|
||||
@@ -25,7 +25,6 @@ pub use bundle::{EthBundleApiServer, EthCallBundleApiServer};
|
||||
pub use core::{EthApiServer, FullEthApiServer};
|
||||
pub use ext::L2EthApiExtServer;
|
||||
pub use filter::{EngineEthFilter, EthFilterApiServer, QueryLimits};
|
||||
pub use helpers::config::EthConfigApiServer;
|
||||
pub use node::{RpcNodeCore, RpcNodeCoreExt};
|
||||
pub use pubsub::EthPubSubApiServer;
|
||||
pub use reth_rpc_convert::*;
|
||||
@@ -42,7 +41,5 @@ pub use core::EthApiClient;
|
||||
pub use ext::L2EthApiExtClient;
|
||||
#[cfg(feature = "client")]
|
||||
pub use filter::EthFilterApiClient;
|
||||
#[cfg(feature = "client")]
|
||||
pub use helpers::config::EthConfigApiClient;
|
||||
|
||||
use reth_trie_common as _;
|
||||
|
||||
@@ -126,7 +126,7 @@ pub trait FromEvmError<Evm: ConfigureEvm>:
|
||||
ExecutionResult::Success { output, .. } => Ok(output.into_data()),
|
||||
ExecutionResult::Revert { output, .. } => Err(Self::from_revert(output)),
|
||||
ExecutionResult::Halt { reason, gas, .. } => {
|
||||
Err(Self::from_evm_halt(reason, gas.tx_gas_used()))
|
||||
Err(Self::from_evm_halt(reason, gas.used()))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -553,7 +553,6 @@ where
|
||||
EVMError::Header(err) => err.into(),
|
||||
EVMError::Database(err) => err.into(),
|
||||
EVMError::Custom(err) => Self::EvmCustom(err),
|
||||
EVMError::CustomAny(err) => Self::EvmCustom(err.to_string()),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,8 +175,7 @@ where
|
||||
};
|
||||
|
||||
if block_values.is_empty() {
|
||||
// For empty blocks, use zero gas price to signal no demand
|
||||
results.push(U256::ZERO);
|
||||
results.push(U256::from(inner.last_price.price));
|
||||
} else {
|
||||
results.extend(block_values);
|
||||
populated_blocks += 1;
|
||||
|
||||
@@ -314,7 +314,7 @@ where
|
||||
code: SIMULATE_VM_ERROR_CODE,
|
||||
..SimulateError::invalid_params()
|
||||
}),
|
||||
gas_used: gas.tx_gas_used(),
|
||||
gas_used: gas.used(),
|
||||
logs: Vec::new(),
|
||||
status: false,
|
||||
..Default::default()
|
||||
@@ -330,7 +330,7 @@ where
|
||||
code: SIMULATE_REVERT_CODE,
|
||||
..SimulateError::invalid_params()
|
||||
}),
|
||||
gas_used: gas.tx_gas_used(),
|
||||
gas_used: gas.used(),
|
||||
status: false,
|
||||
logs: Vec::new(),
|
||||
..Default::default()
|
||||
@@ -342,7 +342,7 @@ where
|
||||
SimCallResult {
|
||||
return_data: output.into_data(),
|
||||
error: None,
|
||||
gas_used: gas.tx_gas_used(),
|
||||
gas_used: gas.used(),
|
||||
logs: logs
|
||||
.into_iter()
|
||||
.map(|log| {
|
||||
|
||||
@@ -188,7 +188,7 @@ where
|
||||
let gas_price = tx
|
||||
.effective_tip_per_gas(basefee)
|
||||
.expect("fee is always valid; execution succeeded");
|
||||
let gas_used = result.tx_gas_used();
|
||||
let gas_used = result.gas_used();
|
||||
total_gas_used += gas_used;
|
||||
|
||||
let gas_fees = U256::from(gas_used) * U256::from(gas_price);
|
||||
|
||||
@@ -478,24 +478,22 @@ where
|
||||
return Err(ProviderError::HeaderNotFound(block_hash.into()).into())
|
||||
};
|
||||
|
||||
// Read number and timestamp from cached block or provider header
|
||||
let (block_number, block_timestamp) = if let Some(block) = &maybe_block {
|
||||
(block.header().number(), block.header().timestamp())
|
||||
// Get header - from cached block if available, otherwise from provider
|
||||
let header = if let Some(block) = &maybe_block {
|
||||
block.header().clone()
|
||||
} else {
|
||||
let header = self
|
||||
.provider()
|
||||
self.provider()
|
||||
.header_by_hash_or_number(block_hash.into())?
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?;
|
||||
(header.number(), header.timestamp())
|
||||
.ok_or_else(|| ProviderError::HeaderNotFound(block_hash.into()))?
|
||||
};
|
||||
|
||||
// Check if the block has been pruned (EIP-4444)
|
||||
let earliest_block = self.provider().earliest_block_number()?;
|
||||
if block_number < earliest_block {
|
||||
if header.number() < earliest_block {
|
||||
return Err(EthApiError::PrunedHistoryUnavailable.into());
|
||||
}
|
||||
|
||||
let block_num_hash = BlockNumHash::new(block_number, block_hash);
|
||||
let block_num_hash = BlockNumHash::new(header.number(), block_hash);
|
||||
|
||||
let mut all_logs = Vec::new();
|
||||
append_matching_block_logs(
|
||||
@@ -507,7 +505,7 @@ where
|
||||
block_num_hash,
|
||||
&receipts,
|
||||
false,
|
||||
block_timestamp,
|
||||
header.timestamp(),
|
||||
)?;
|
||||
|
||||
Ok(all_logs)
|
||||
|
||||
@@ -289,7 +289,7 @@ where
|
||||
.into());
|
||||
}
|
||||
|
||||
let gas_used = result.tx_gas_used();
|
||||
let gas_used = result.gas_used();
|
||||
total_gas_used += gas_used;
|
||||
|
||||
// coinbase is always present in the result state
|
||||
|
||||
@@ -88,8 +88,8 @@ arbitrary = [
|
||||
"op-alloy-consensus?/arbitrary",
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
]
|
||||
# op = [
|
||||
# "dep:op-alloy-consensus",
|
||||
# "reth-codecs/op",
|
||||
# "reth-primitives-traits/op",
|
||||
# ]
|
||||
op = [
|
||||
"dep:op-alloy-consensus",
|
||||
"reth-codecs/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
|
||||
@@ -93,8 +93,8 @@ arbitrary = [
|
||||
"reth-primitives-traits/arbitrary",
|
||||
"reth-prune-types/arbitrary",
|
||||
]
|
||||
# op = [
|
||||
# "reth-db-api/op",
|
||||
# "reth-primitives-traits/op",
|
||||
# ]
|
||||
op = [
|
||||
"reth-db-api/op",
|
||||
"reth-primitives-traits/op",
|
||||
]
|
||||
disable-lock = []
|
||||
|
||||
@@ -64,7 +64,7 @@ tokio = { workspace = true, features = ["sync"], optional = true }
|
||||
# parallel utils
|
||||
rayon.workspace = true
|
||||
|
||||
rocksdb.workspace = true
|
||||
rocksdb = { workspace = true, features = ["jemalloc"] }
|
||||
|
||||
[dev-dependencies]
|
||||
reth-db = { workspace = true, features = ["test-utils"] }
|
||||
@@ -86,7 +86,6 @@ rand.workspace = true
|
||||
tokio = { workspace = true, features = ["sync", "macros", "rt-multi-thread"] }
|
||||
|
||||
[features]
|
||||
jemalloc = ["rocksdb/jemalloc"]
|
||||
test-utils = [
|
||||
"reth-db/test-utils",
|
||||
"reth-nippy-jar/test-utils",
|
||||
|
||||
@@ -106,10 +106,8 @@ impl<N> ProviderFactoryBuilder<N> {
|
||||
let db = open_db_read_only(db_dir, db_args)?;
|
||||
let static_file_provider =
|
||||
StaticFileProvider::read_only(static_files_dir, watch_static_files)?;
|
||||
let rocksdb_provider = RocksDBProvider::builder(&rocksdb_dir)
|
||||
.with_default_tables()
|
||||
.with_read_only(true)
|
||||
.build()?;
|
||||
let rocksdb_provider =
|
||||
RocksDBProvider::builder(&rocksdb_dir).with_default_tables().build()?;
|
||||
ProviderFactory::new(db, chainspec, static_file_provider, rocksdb_provider, runtime)
|
||||
.map_err(Into::into)
|
||||
}
|
||||
|
||||
@@ -90,7 +90,7 @@ pub(crate) struct DatabaseProviderMetrics {
|
||||
/// Duration of `update_pipeline_stages` in `save_blocks`
|
||||
save_blocks_update_pipeline_stages: Histogram,
|
||||
/// Number of blocks per `save_blocks` call
|
||||
save_blocks_batch_size: Histogram,
|
||||
save_blocks_block_count: Histogram,
|
||||
/// Duration of MDBX commit in `save_blocks`
|
||||
save_blocks_commit_mdbx: Histogram,
|
||||
/// Duration of static file commit in `save_blocks`
|
||||
@@ -118,7 +118,7 @@ pub(crate) struct DatabaseProviderMetrics {
|
||||
/// Last duration of `update_pipeline_stages` in `save_blocks`
|
||||
save_blocks_update_pipeline_stages_last: Gauge,
|
||||
/// Last number of blocks per `save_blocks` call
|
||||
save_blocks_batch_size_last: Gauge,
|
||||
save_blocks_block_count_last: Gauge,
|
||||
/// Last duration of MDBX commit in `save_blocks`
|
||||
save_blocks_commit_mdbx_last: Gauge,
|
||||
/// Last duration of static file commit in `save_blocks`
|
||||
@@ -140,7 +140,7 @@ pub(crate) struct SaveBlocksTimings {
|
||||
pub write_trie_updates: Duration,
|
||||
pub update_history_indices: Duration,
|
||||
pub update_pipeline_stages: Duration,
|
||||
pub batch_size: u64,
|
||||
pub block_count: u64,
|
||||
}
|
||||
|
||||
/// Timings collected during a `commit` call.
|
||||
@@ -182,7 +182,7 @@ impl DatabaseProviderMetrics {
|
||||
self.save_blocks_write_trie_updates.record(timings.write_trie_updates);
|
||||
self.save_blocks_update_history_indices.record(timings.update_history_indices);
|
||||
self.save_blocks_update_pipeline_stages.record(timings.update_pipeline_stages);
|
||||
self.save_blocks_batch_size.record(timings.batch_size as f64);
|
||||
self.save_blocks_block_count.record(timings.block_count as f64);
|
||||
|
||||
self.save_blocks_total_last.set(timings.total.as_secs_f64());
|
||||
self.save_blocks_mdbx_last.set(timings.mdbx.as_secs_f64());
|
||||
@@ -196,7 +196,7 @@ impl DatabaseProviderMetrics {
|
||||
.set(timings.update_history_indices.as_secs_f64());
|
||||
self.save_blocks_update_pipeline_stages_last
|
||||
.set(timings.update_pipeline_stages.as_secs_f64());
|
||||
self.save_blocks_batch_size_last.set(timings.batch_size as f64);
|
||||
self.save_blocks_block_count_last.set(timings.block_count as f64);
|
||||
}
|
||||
|
||||
/// Records all commit timings.
|
||||
|
||||
@@ -24,12 +24,12 @@ use reth_node_types::{
|
||||
BlockTy, HeaderTy, NodeTypesWithDB, NodeTypesWithDBAdapter, ReceiptTy, TxTy,
|
||||
};
|
||||
use reth_primitives_traits::{RecoveredBlock, SealedHeader};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment, MINIMUM_UNWIND_SAFE_DISTANCE};
|
||||
use reth_prune_types::{PruneCheckpoint, PruneModes, PruneSegment};
|
||||
use reth_stages_types::{PipelineTarget, StageCheckpoint, StageId};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::{
|
||||
BlockBodyIndicesProvider, ChainStateBlockReader, ChainStateBlockWriter, DBProvider,
|
||||
NodePrimitivesProvider, StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider,
|
||||
BlockBodyIndicesProvider, ChainStateBlockReader, ChainStateBlockWriter, NodePrimitivesProvider,
|
||||
StorageSettings, StorageSettingsCache, TryIntoHistoricalStateProvider,
|
||||
};
|
||||
use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_trie::HashedPostState;
|
||||
@@ -80,8 +80,6 @@ pub struct ProviderFactory<N: NodeTypesWithDB> {
|
||||
changeset_cache: ChangesetCache,
|
||||
/// Task runtime for spawning parallel I/O work.
|
||||
runtime: reth_tasks::Runtime,
|
||||
/// Minimum distance from tip required before pruning can occur.
|
||||
minimum_pruning_distance: u64,
|
||||
}
|
||||
|
||||
impl<N: NodeTypesForProvider> ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>> {
|
||||
@@ -136,7 +134,6 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
rocksdb_provider,
|
||||
changeset_cache: ChangesetCache::new(),
|
||||
runtime,
|
||||
minimum_pruning_distance: MINIMUM_UNWIND_SAFE_DISTANCE,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -171,15 +168,6 @@ impl<N: NodeTypesWithDB> ProviderFactory<N> {
|
||||
self
|
||||
}
|
||||
|
||||
/// Sets the minimum pruning distance for an existing [`ProviderFactory`].
|
||||
///
|
||||
/// This controls the minimum distance from tip required before pruning can occur.
|
||||
/// The default is [`MINIMUM_UNWIND_SAFE_DISTANCE`].
|
||||
pub const fn with_minimum_pruning_distance(mut self, distance: u64) -> Self {
|
||||
self.minimum_pruning_distance = distance;
|
||||
self
|
||||
}
|
||||
|
||||
/// Returns reference to the underlying database.
|
||||
pub const fn db_ref(&self) -> &N::DB {
|
||||
&self.db
|
||||
@@ -258,8 +246,7 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
self.changeset_cache.clone(),
|
||||
self.runtime.clone(),
|
||||
self.db.path(),
|
||||
)
|
||||
.with_minimum_pruning_distance(self.minimum_pruning_distance))
|
||||
))
|
||||
}
|
||||
|
||||
/// Returns a provider with a created `DbTxMut` inside, which allows fetching and updating
|
||||
@@ -268,21 +255,18 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
/// open.
|
||||
#[track_caller]
|
||||
pub fn provider_rw(&self) -> ProviderResult<DatabaseProviderRW<N::DB, N>> {
|
||||
Ok(DatabaseProviderRW(
|
||||
DatabaseProvider::new_rw(
|
||||
self.db.tx_mut()?,
|
||||
self.chain_spec.clone(),
|
||||
self.static_file_provider.clone(),
|
||||
self.prune_modes.clone(),
|
||||
self.storage.clone(),
|
||||
self.storage_settings.clone(),
|
||||
self.rocksdb_provider.clone(),
|
||||
self.changeset_cache.clone(),
|
||||
self.runtime.clone(),
|
||||
self.db.path(),
|
||||
)
|
||||
.with_minimum_pruning_distance(self.minimum_pruning_distance),
|
||||
))
|
||||
Ok(DatabaseProviderRW(DatabaseProvider::new_rw(
|
||||
self.db.tx_mut()?,
|
||||
self.chain_spec.clone(),
|
||||
self.static_file_provider.clone(),
|
||||
self.prune_modes.clone(),
|
||||
self.storage.clone(),
|
||||
self.storage_settings.clone(),
|
||||
self.rocksdb_provider.clone(),
|
||||
self.changeset_cache.clone(),
|
||||
self.runtime.clone(),
|
||||
self.db.path(),
|
||||
)))
|
||||
}
|
||||
|
||||
/// Returns a provider with a created `DbTxMut` inside, configured for unwind operations.
|
||||
@@ -303,8 +287,7 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
self.changeset_cache.clone(),
|
||||
self.runtime.clone(),
|
||||
self.db.path(),
|
||||
)
|
||||
.with_minimum_pruning_distance(self.minimum_pruning_distance))
|
||||
))
|
||||
}
|
||||
|
||||
/// State provider for latest block
|
||||
@@ -366,12 +349,7 @@ impl<N: ProviderNodeTypes> ProviderFactory<N> {
|
||||
/// consistency. I.e. this MAY result in writes to the static files.
|
||||
#[instrument(err, skip(self))]
|
||||
pub fn check_consistency(&self) -> ProviderResult<(Option<u64>, Option<u64>)> {
|
||||
let provider_ro = self
|
||||
.database_provider_ro()?
|
||||
// Healing can run long-lived read transactions (e.g., iterating changesets
|
||||
// over millions of blocks). Disable the default timeout so MDBX doesn't
|
||||
// kill the transaction mid-heal, which causes a crash loop on startup.
|
||||
.disable_long_read_transaction_safety();
|
||||
let provider_ro = self.database_provider_ro()?;
|
||||
|
||||
// Step 1: heal file-level inconsistencies (no pruning)
|
||||
self.static_file_provider().check_file_consistency(&provider_ro)?;
|
||||
@@ -819,7 +797,6 @@ where
|
||||
rocksdb_provider,
|
||||
changeset_cache,
|
||||
runtime,
|
||||
minimum_pruning_distance,
|
||||
} = self;
|
||||
f.debug_struct("ProviderFactory")
|
||||
.field("db", &db)
|
||||
@@ -831,7 +808,6 @@ where
|
||||
.field("rocksdb_provider", &rocksdb_provider)
|
||||
.field("changeset_cache", &changeset_cache)
|
||||
.field("runtime", &runtime)
|
||||
.field("minimum_pruning_distance", &minimum_pruning_distance)
|
||||
.finish()
|
||||
}
|
||||
}
|
||||
@@ -848,7 +824,6 @@ impl<N: NodeTypesWithDB> Clone for ProviderFactory<N> {
|
||||
rocksdb_provider: self.rocksdb_provider.clone(),
|
||||
changeset_cache: self.changeset_cache.clone(),
|
||||
runtime: self.runtime.clone(),
|
||||
minimum_pruning_distance: self.minimum_pruning_distance,
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -238,12 +238,6 @@ impl<TX, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
pub const fn prune_modes_ref(&self) -> &PruneModes {
|
||||
&self.prune_modes
|
||||
}
|
||||
|
||||
/// Sets the minimum pruning distance.
|
||||
pub const fn with_minimum_pruning_distance(mut self, distance: u64) -> Self {
|
||||
self.minimum_pruning_distance = distance;
|
||||
self
|
||||
}
|
||||
}
|
||||
|
||||
impl<TX: DbTx + 'static, N: NodeTypes> DatabaseProvider<TX, N> {
|
||||
@@ -554,8 +548,7 @@ impl<TX: DbTx + DbTxMut + 'static, N: NodeTypesForProvider> DatabaseProvider<TX,
|
||||
nums
|
||||
};
|
||||
|
||||
let mut timings =
|
||||
metrics::SaveBlocksTimings { batch_size: block_count, ..Default::default() };
|
||||
let mut timings = metrics::SaveBlocksTimings { block_count, ..Default::default() };
|
||||
|
||||
// avoid capturing &self.tx in scope below.
|
||||
let sf_provider = &self.static_file_provider;
|
||||
|
||||
@@ -9,9 +9,7 @@ use crate::StaticFileProviderFactory;
|
||||
use alloy_eips::eip2718::Encodable2718;
|
||||
use alloy_primitives::BlockNumber;
|
||||
use rayon::prelude::*;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec};
|
||||
use reth_db::models::{storage_sharded_key::StorageShardedKey, ShardedKey};
|
||||
use reth_db_api::tables;
|
||||
use reth_db_api::{table::Table, tables};
|
||||
use reth_stages_types::StageId;
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::{
|
||||
@@ -59,8 +57,7 @@ impl RocksDBProvider {
|
||||
+ BlockBodyIndicesProvider
|
||||
+ StorageChangeSetReader
|
||||
+ ChangeSetReader
|
||||
+ TransactionsProvider<Transaction: Encodable2718>
|
||||
+ ChainSpecProvider,
|
||||
+ TransactionsProvider<Transaction: Encodable2718>,
|
||||
{
|
||||
let mut unwind_target: Option<BlockNumber> = None;
|
||||
|
||||
@@ -117,14 +114,11 @@ impl RocksDBProvider {
|
||||
|
||||
// Fast path: clear any stale data and return.
|
||||
if checkpoint == 0 {
|
||||
if self.first::<tables::TransactionHashNumbers>()?.is_some() {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
"TransactionHashNumbers: checkpoint is 0, clearing stale data"
|
||||
);
|
||||
self.clear::<tables::TransactionHashNumbers>()?;
|
||||
}
|
||||
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
"TransactionHashNumbers: checkpoint is 0, clearing stale data"
|
||||
);
|
||||
self.clear::<tables::TransactionHashNumbers>()?;
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
@@ -262,39 +256,21 @@ impl RocksDBProvider {
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ StaticFileProviderFactory
|
||||
+ StorageChangeSetReader
|
||||
+ ChainSpecProvider,
|
||||
Provider:
|
||||
DBProvider + StageCheckpointReader + StaticFileProviderFactory + StorageChangeSetReader,
|
||||
{
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::IndexStorageHistory)?
|
||||
.map(|cp| cp.block_number)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Fast path: clear and re-insert genesis history.
|
||||
// Fast path: clear any stale data and return.
|
||||
if checkpoint == 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
"StoragesHistory: checkpoint is 0, clearing stale data"
|
||||
);
|
||||
self.clear::<tables::StoragesHistory>()?;
|
||||
|
||||
let chain_spec = provider.chain_spec();
|
||||
let genesis = chain_spec.genesis();
|
||||
let list = tables::BlockNumberList::new([0]).expect("single block always fits");
|
||||
for (addr, account) in &genesis.alloc {
|
||||
if let Some(storage) = &account.storage {
|
||||
for key in storage.keys() {
|
||||
self.put::<tables::StoragesHistory>(
|
||||
StorageShardedKey::last(*addr, *key),
|
||||
&list,
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
@@ -358,6 +334,7 @@ impl RocksDBProvider {
|
||||
|
||||
let batch = self.unwind_storage_history_indices(&indices)?;
|
||||
self.commit_batch(batch)?;
|
||||
self.flush(&[tables::StoragesHistory::NAME])?;
|
||||
}
|
||||
|
||||
batch_start = batch_end + 1;
|
||||
@@ -375,32 +352,20 @@ impl RocksDBProvider {
|
||||
provider: &Provider,
|
||||
) -> ProviderResult<Option<BlockNumber>>
|
||||
where
|
||||
Provider: DBProvider
|
||||
+ StageCheckpointReader
|
||||
+ StaticFileProviderFactory
|
||||
+ ChangeSetReader
|
||||
+ ChainSpecProvider,
|
||||
Provider: DBProvider + StageCheckpointReader + StaticFileProviderFactory + ChangeSetReader,
|
||||
{
|
||||
let checkpoint = provider
|
||||
.get_stage_checkpoint(StageId::IndexAccountHistory)?
|
||||
.map(|cp| cp.block_number)
|
||||
.unwrap_or(0);
|
||||
|
||||
// Fast path: clear and re-insert genesis history.
|
||||
// Fast path: clear any stale data and return.
|
||||
if checkpoint == 0 {
|
||||
tracing::info!(
|
||||
target: "reth::providers::rocksdb",
|
||||
"AccountsHistory: checkpoint is 0, clearing stale data"
|
||||
);
|
||||
self.clear::<tables::AccountsHistory>()?;
|
||||
|
||||
let chain_spec = provider.chain_spec();
|
||||
let genesis = chain_spec.genesis();
|
||||
let list = tables::BlockNumberList::new([0]).expect("single block always fits");
|
||||
for addr in genesis.alloc.keys() {
|
||||
self.put::<tables::AccountsHistory>(ShardedKey::last(*addr), &list)?;
|
||||
}
|
||||
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
@@ -474,18 +439,13 @@ impl RocksDBProvider {
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use std::sync::Arc;
|
||||
|
||||
use super::*;
|
||||
use crate::{
|
||||
init::insert_genesis_history,
|
||||
providers::{rocksdb::RocksDBBuilder, static_file::StaticFileWriter},
|
||||
test_utils::{create_test_provider_factory, create_test_provider_factory_with_chain_spec},
|
||||
BlockWriter, DatabaseProviderFactory, RocksDBProviderFactory, StageCheckpointWriter,
|
||||
TransactionsProvider,
|
||||
test_utils::create_test_provider_factory,
|
||||
BlockWriter, DatabaseProviderFactory, StageCheckpointWriter, TransactionsProvider,
|
||||
};
|
||||
use alloy_primitives::{Address, B256};
|
||||
use reth_chainspec::MAINNET;
|
||||
use reth_db::cursor::{DbCursorRO, DbCursorRW};
|
||||
use reth_db_api::{
|
||||
models::{storage_sharded_key::StorageShardedKey, StorageSettings},
|
||||
@@ -569,14 +529,11 @@ mod tests {
|
||||
|
||||
let result = rocksdb.heal_storages_history(&provider).unwrap();
|
||||
assert_eq!(result, None, "StoragesHistory should return early at checkpoint 0");
|
||||
assert!(rocksdb.first::<tables::StoragesHistory>().unwrap().is_none());
|
||||
|
||||
let result = rocksdb.heal_accounts_history(&provider).unwrap();
|
||||
assert_eq!(result, None, "AccountsHistory should return early at checkpoint 0");
|
||||
// Genesis account history entries are re-inserted
|
||||
assert_eq!(
|
||||
rocksdb.iter::<tables::AccountsHistory>().unwrap().count(),
|
||||
factory.chain_spec().genesis().alloc.len()
|
||||
);
|
||||
assert!(rocksdb.first::<tables::AccountsHistory>().unwrap().is_none());
|
||||
}
|
||||
|
||||
#[test]
|
||||
@@ -703,38 +660,35 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_storages_history_preserves_genesis_entries_at_checkpoint_zero(
|
||||
) -> eyre::Result<()> {
|
||||
// Modify mainnet chainspec to include a single genesis storage slot
|
||||
let mut chain_spec = MAINNET.clone();
|
||||
Arc::make_mut(&mut chain_spec).genesis.alloc.first_entry().unwrap().get_mut().storage =
|
||||
Some(From::from([(B256::random(), B256::random())]));
|
||||
fn test_check_consistency_storages_history_has_data_no_checkpoint_prunes_data() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
// Insert data into RocksDB
|
||||
let key = StorageShardedKey::new(Address::ZERO, B256::ZERO, 50);
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30, 50]);
|
||||
rocksdb.put::<tables::StoragesHistory>(key, &block_list).unwrap();
|
||||
|
||||
// Verify data exists
|
||||
assert!(rocksdb.last::<tables::StoragesHistory>().unwrap().is_some());
|
||||
|
||||
// Create a test provider factory for MDBX with NO checkpoint
|
||||
let factory = create_test_provider_factory_with_chain_spec(chain_spec);
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
|
||||
// Insert genesis history into RocksDB
|
||||
let provider_rw = factory.database_provider_rw().unwrap();
|
||||
insert_genesis_history(&provider_rw, factory.chain_spec().genesis.alloc.iter())?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// This should not prune anything because only genesis entries are present
|
||||
let result = rocksdb.heal_storages_history(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should skip healing when only genesis entries present");
|
||||
// RocksDB has data but checkpoint is 0
|
||||
// This means RocksDB has stale data that should be pruned (healed)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify data was NOT deleted
|
||||
// Verify data was pruned
|
||||
assert!(
|
||||
rocksdb.iter::<tables::StoragesHistory>().unwrap().count() > 0,
|
||||
"Genesis entries should be preserved"
|
||||
rocksdb.last::<tables::StoragesHistory>().unwrap().is_none(),
|
||||
"RocksDB should be empty after pruning"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_mdbx_behind_checkpoint_needs_unwind() {
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
@@ -1114,31 +1068,36 @@ mod tests {
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_check_consistency_accounts_history_preserves_genesis_entries_at_checkpoint_zero(
|
||||
) -> eyre::Result<()> {
|
||||
fn test_check_consistency_accounts_history_has_data_no_checkpoint_prunes_data() {
|
||||
use reth_db_api::models::ShardedKey;
|
||||
|
||||
let temp_dir = TempDir::new().unwrap();
|
||||
let rocksdb = RocksDBBuilder::new(temp_dir.path()).with_default_tables().build().unwrap();
|
||||
|
||||
// Insert data into RocksDB
|
||||
let key = ShardedKey::new(Address::ZERO, 50);
|
||||
let block_list = BlockNumberList::new_pre_sorted([10, 20, 30, 50]);
|
||||
rocksdb.put::<tables::AccountsHistory>(key, &block_list).unwrap();
|
||||
|
||||
// Verify data exists
|
||||
assert!(rocksdb.last::<tables::AccountsHistory>().unwrap().is_some());
|
||||
|
||||
// Create a test provider factory for MDBX with NO checkpoint
|
||||
let factory = create_test_provider_factory();
|
||||
factory.set_storage_settings_cache(StorageSettings::v2());
|
||||
let rocksdb = factory.rocksdb_provider();
|
||||
|
||||
// Insert genesis history into RocksDB
|
||||
let provider_rw = factory.database_provider_rw().unwrap();
|
||||
insert_genesis_history(&provider_rw, factory.chain_spec().genesis.alloc.iter())?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
let provider = factory.database_provider_ro().unwrap();
|
||||
|
||||
// This should not prune anything because only genesis entries are present
|
||||
// RocksDB has data but checkpoint is 0
|
||||
// This means RocksDB has stale data that should be pruned (healed)
|
||||
let result = rocksdb.check_consistency(&provider).unwrap();
|
||||
assert_eq!(result, None, "Should heal by pruning, no unwind needed");
|
||||
|
||||
// Verify data was NOT deleted
|
||||
// Verify data was pruned
|
||||
assert!(
|
||||
rocksdb.iter::<tables::AccountsHistory>().unwrap().count() > 0,
|
||||
"Genesis entries should be preserved"
|
||||
rocksdb.last::<tables::AccountsHistory>().unwrap().is_none(),
|
||||
"RocksDB should be empty after pruning"
|
||||
);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
@@ -20,7 +20,7 @@ use reth_storage_errors::provider::ProviderResult;
|
||||
use reth_trie::{
|
||||
hashed_cursor::HashedPostStateCursorFactory,
|
||||
proof::{Proof, StorageProof},
|
||||
trie_cursor::InMemoryTrieCursorFactory,
|
||||
trie_cursor::{masked::MaskedTrieCursorFactory, InMemoryTrieCursorFactory},
|
||||
updates::TrieUpdates,
|
||||
witness::TrieWitness,
|
||||
AccountProof, HashedPostState, HashedPostStateSorted, HashedStorage, KeccakKeyHasher,
|
||||
@@ -525,16 +525,18 @@ impl<
|
||||
let nodes_sorted = input.nodes.into_sorted();
|
||||
let state_sorted = input.state.into_sorted();
|
||||
TrieWitness::new(
|
||||
InMemoryTrieCursorFactory::new(
|
||||
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
|
||||
&nodes_sorted,
|
||||
MaskedTrieCursorFactory::new(
|
||||
InMemoryTrieCursorFactory::new(
|
||||
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
|
||||
&nodes_sorted,
|
||||
),
|
||||
input.prefix_sets.freeze(),
|
||||
),
|
||||
HashedPostStateCursorFactory::new(
|
||||
reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()),
|
||||
&state_sorted,
|
||||
),
|
||||
)
|
||||
.with_prefix_sets_mut(input.prefix_sets)
|
||||
.always_include_root_node()
|
||||
.compute(target)
|
||||
.map_err(ProviderError::from)
|
||||
|
||||
@@ -11,7 +11,7 @@ use reth_storage_errors::provider::{ProviderError, ProviderResult};
|
||||
use reth_trie::{
|
||||
hashed_cursor::HashedPostStateCursorFactory,
|
||||
proof::{Proof, StorageProof},
|
||||
trie_cursor::InMemoryTrieCursorFactory,
|
||||
trie_cursor::{masked::MaskedTrieCursorFactory, InMemoryTrieCursorFactory},
|
||||
updates::TrieUpdates,
|
||||
witness::TrieWitness,
|
||||
AccountProof, HashedPostState, HashedStorage, KeccakKeyHasher, MultiProof, MultiProofTargets,
|
||||
@@ -223,16 +223,18 @@ impl<Provider: DBProvider + StorageSettingsCache> StateProofProvider
|
||||
let nodes_sorted = input.nodes.into_sorted();
|
||||
let state_sorted = input.state.into_sorted();
|
||||
Ok(TrieWitness::new(
|
||||
InMemoryTrieCursorFactory::new(
|
||||
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
|
||||
&nodes_sorted,
|
||||
MaskedTrieCursorFactory::new(
|
||||
InMemoryTrieCursorFactory::new(
|
||||
reth_trie_db::DatabaseTrieCursorFactory::<_, A>::new(self.tx()),
|
||||
&nodes_sorted,
|
||||
),
|
||||
input.prefix_sets.freeze(),
|
||||
),
|
||||
HashedPostStateCursorFactory::new(
|
||||
reth_trie_db::DatabaseHashedCursorFactory::new(self.tx()),
|
||||
&state_sorted,
|
||||
),
|
||||
)
|
||||
.with_prefix_sets_mut(input.prefix_sets)
|
||||
.always_include_root_node()
|
||||
.compute(target)?
|
||||
.into_values()
|
||||
|
||||
@@ -1429,7 +1429,7 @@ pub fn ensure_intrinsic_gas<T: EthPoolTransaction>(
|
||||
);
|
||||
|
||||
let gas_limit = transaction.gas_limit();
|
||||
if gas_limit < gas.initial_total_gas || gas_limit < gas.floor_gas {
|
||||
if gas_limit < gas.initial_gas || gas_limit < gas.floor_gas {
|
||||
Err(InvalidPoolTransactionError::IntrinsicGasTooLow)
|
||||
} else {
|
||||
Ok(())
|
||||
|
||||
@@ -9,7 +9,7 @@ use nodes::{
|
||||
};
|
||||
|
||||
use crate::{LeafLookup, LeafLookupError, LeafUpdate, SparseTrie, SparseTrieUpdates};
|
||||
use alloc::{borrow::Cow, boxed::Box, collections::VecDeque, vec::Vec};
|
||||
use alloc::{borrow::Cow, boxed::Box, vec::Vec};
|
||||
use alloy_primitives::{
|
||||
keccak256,
|
||||
map::{B256Map, HashMap, HashSet},
|
||||
@@ -60,54 +60,6 @@ fn prefix_range(
|
||||
begin..end
|
||||
}
|
||||
|
||||
/// Compacts an arena by BFS-copying all reachable nodes into a fresh `SlotMap`, dropping
|
||||
/// unreachable (pruned) slots. Parents are stored before children for cache-friendly top-down
|
||||
/// traversal.
|
||||
fn compact_arena(arena: &mut NodeArena, root: &mut Index) {
|
||||
let mut new_arena = SlotMap::with_capacity(arena.len());
|
||||
let mut queue = VecDeque::new();
|
||||
|
||||
let root_node = arena.remove(*root).expect("root exists");
|
||||
let new_root = new_arena.insert(root_node);
|
||||
queue.push_back(new_root);
|
||||
|
||||
while let Some(new_idx) = queue.pop_front() {
|
||||
// Invariant: any node popped from `queue` has been moved into `new_arena` but
|
||||
// its Branch.children have not been rewritten yet — every Revealed(idx) here is
|
||||
// still an old-arena index, and the child is still present in `arena` because
|
||||
// only this parent's iteration can remove it (each child has exactly one parent).
|
||||
let old_children: SmallVec<[(usize, Index); 16]> = match &new_arena[new_idx] {
|
||||
ArenaSparseNode::Branch(b) => b
|
||||
.children
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter_map(|(i, c)| match c {
|
||||
ArenaSparseNodeBranchChild::Revealed(old_idx) => Some((i, *old_idx)),
|
||||
_ => None,
|
||||
})
|
||||
.collect(),
|
||||
_ => continue,
|
||||
};
|
||||
|
||||
for (child_pos, old_child_idx) in old_children {
|
||||
let child_node = arena.remove(old_child_idx).expect("child exists");
|
||||
let new_child_idx = new_arena.insert(child_node);
|
||||
let ArenaSparseNode::Branch(b) = &mut new_arena[new_idx] else { unreachable!() };
|
||||
b.children[child_pos] = ArenaSparseNodeBranchChild::Revealed(new_child_idx);
|
||||
queue.push_back(new_child_idx);
|
||||
}
|
||||
}
|
||||
|
||||
debug_assert!(
|
||||
arena.is_empty(),
|
||||
"compact_arena: {} orphaned nodes remaining after BFS drain",
|
||||
arena.len(),
|
||||
);
|
||||
|
||||
*arena = new_arena;
|
||||
*root = new_root;
|
||||
}
|
||||
|
||||
/// Reusable buffers shared by both [`ArenaSparseSubtrie`] and [`ArenaParallelSparseTrie`].
|
||||
#[derive(Debug, Default, Clone)]
|
||||
struct ArenaTrieBuffers {
|
||||
@@ -174,7 +126,7 @@ impl ArenaSparseSubtrie {
|
||||
}
|
||||
|
||||
fn clear(&mut self) {
|
||||
self.arena = SlotMap::new();
|
||||
self.arena.clear();
|
||||
self.buffers.clear();
|
||||
self.required_proofs.clear();
|
||||
self.num_leaves = 0;
|
||||
@@ -255,11 +207,6 @@ impl ArenaSparseSubtrie {
|
||||
}
|
||||
|
||||
self.num_leaves -= pruned_leaves;
|
||||
|
||||
if pruned > 0 {
|
||||
compact_arena(&mut self.arena, &mut self.root);
|
||||
}
|
||||
|
||||
#[cfg(debug_assertions)]
|
||||
self.debug_assert_counters();
|
||||
pruned
|
||||
@@ -1764,18 +1711,7 @@ impl ArenaParallelSparseTrie {
|
||||
.filter(|(_, _, u)| matches!(u, LeafUpdate::Changed(v) if v.is_empty()))
|
||||
.count() as u64;
|
||||
|
||||
// Touched is a no-op that doesn't alter trie structure, so it must be
|
||||
// excluded when deciding whether "all updates are removals". This mirrors
|
||||
// the `all_removals` / `might_empty_subtrie` filter in `update_leaves`.
|
||||
// Without this, a batch of removals + Touched entries
|
||||
// would fail the `num_removals != num_changed` check, skip the proof
|
||||
// request for the blinded sibling, and later panic in
|
||||
// `maybe_collapse_or_remove_branch` when the subtrie empties inline.
|
||||
let num_changed =
|
||||
subtrie_updates.iter().filter(|(_, _, u)| matches!(u, LeafUpdate::Changed(_))).count()
|
||||
as u64;
|
||||
|
||||
if num_removals == 0 || num_removals != num_changed {
|
||||
if num_removals == 0 || num_removals as usize != subtrie_updates.len() {
|
||||
return None;
|
||||
}
|
||||
|
||||
@@ -2594,7 +2530,7 @@ impl SparseTrie for ArenaParallelSparseTrie {
|
||||
self.cleared_subtries.push(subtrie);
|
||||
}
|
||||
}
|
||||
self.upper_arena = SlotMap::new();
|
||||
self.upper_arena.clear();
|
||||
self.root = self.upper_arena.insert(ArenaSparseNode::EmptyRoot);
|
||||
if let Some(updates) = self.updates.as_mut() {
|
||||
updates.clear()
|
||||
@@ -2603,15 +2539,15 @@ impl SparseTrie for ArenaParallelSparseTrie {
|
||||
}
|
||||
|
||||
fn shrink_nodes_to(&mut self, size: usize) {
|
||||
// We do not shrink the upper trie for now.
|
||||
//
|
||||
// As soon as a trie rotates from a live subtrie into the
|
||||
// cleared_subtrie it will be properly shrunk.
|
||||
for s in &mut self.cleared_subtries {
|
||||
if s.arena.capacity() > size {
|
||||
s.arena = SlotMap::with_capacity(size);
|
||||
self.upper_arena.shrink_to(size);
|
||||
for (_, node) in &mut self.upper_arena {
|
||||
if let ArenaSparseNode::Subtrie(s) = node {
|
||||
s.arena.shrink_to(size);
|
||||
}
|
||||
}
|
||||
for s in &mut self.cleared_subtries {
|
||||
s.arena.shrink_to(size);
|
||||
}
|
||||
}
|
||||
|
||||
fn shrink_values_to(&mut self, _size: usize) {
|
||||
@@ -2788,10 +2724,6 @@ impl SparseTrie for ArenaParallelSparseTrie {
|
||||
}
|
||||
}
|
||||
|
||||
if pruned > 0 {
|
||||
compact_arena(&mut self.upper_arena, &mut self.root);
|
||||
}
|
||||
|
||||
#[cfg(feature = "trie-debug")]
|
||||
self.record_initial_state();
|
||||
|
||||
|
||||
@@ -271,7 +271,6 @@ sparse_trie_tests! {
|
||||
test_branch_collapse_updates_leaf_key_len_across_subtries,
|
||||
test_remove_leaf_does_not_reveal_blind_subtries,
|
||||
test_branch_collapse_multi_empty_subtries_blinded_remaining,
|
||||
test_subtrie_collapse_touched_with_blinded_sibling,
|
||||
test_subtrie_emptied_by_deletes_with_touched,
|
||||
|
||||
// root
|
||||
|
||||
@@ -1406,79 +1406,6 @@ pub(super) fn test_branch_collapse_multi_empty_subtries_blinded_remaining<T: Spa
|
||||
);
|
||||
}
|
||||
|
||||
/// Regression: subtrie emptied by deletes + `Touched` with a **blinded** sibling.
|
||||
///
|
||||
/// `check_subtrie_collapse_needs_proof` compared `num_removals` against the full
|
||||
/// `subtrie_updates.len()`, which included `Touched` entries. When `Touched` was
|
||||
/// present alongside removals, the lengths didn't match, so the function skipped
|
||||
/// the proof request for the blinded sibling. The subtrie was then emptied inline
|
||||
/// via `might_empty_subtrie`, and `maybe_collapse_or_remove_branch` hit the
|
||||
/// blinded sibling and panicked.
|
||||
pub(super) fn test_subtrie_collapse_touched_with_blinded_sibling<T: SparseTrie>(
|
||||
new_trie: fn() -> T,
|
||||
) {
|
||||
// Trie shape: root branch has children at nibbles 0xa and 0xc.
|
||||
// Under 0xa there is a branch with children at 0xab (subtrie, 2 leaves) and
|
||||
// 0xac (blinded — we never reveal it).
|
||||
//
|
||||
// We delete both 0xab leaves and also send Touched for a third 0xab key.
|
||||
// After the subtrie empties, the branch at 0xa has a single child (0xac) that
|
||||
// is blinded. The code must request a proof for it instead of panicking.
|
||||
let mut key_ab1 = B256::ZERO;
|
||||
key_ab1[0] = 0xAB;
|
||||
key_ab1[31] = 0x11;
|
||||
let mut key_ab2 = B256::ZERO;
|
||||
key_ab2[0] = 0xAB;
|
||||
key_ab2[31] = 0x22;
|
||||
let mut key_ab3 = B256::ZERO;
|
||||
key_ab3[0] = 0xAB;
|
||||
key_ab3[31] = 0x33;
|
||||
let mut key_ac1 = B256::ZERO;
|
||||
key_ac1[0] = 0xAC;
|
||||
key_ac1[31] = 0x44;
|
||||
let mut key_cd1 = B256::ZERO;
|
||||
key_cd1[0] = 0xCD;
|
||||
key_cd1[31] = 0x01;
|
||||
|
||||
let value = U256::from(1u64);
|
||||
|
||||
let base_storage: BTreeMap<B256, U256> =
|
||||
[(key_ab1, value), (key_ab2, value), (key_ac1, value), (key_cd1, value)]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
let harness = SuiteTestHarness::new(base_storage.clone());
|
||||
// Reveal only the 0xAB keys and 0xCD — leave 0xAC blinded.
|
||||
let revealed_keys = vec![key_ab1, key_ab2, key_cd1];
|
||||
let mut trie: T = harness.init_trie_with_targets(&revealed_keys, false, new_trie);
|
||||
|
||||
// Verify initial root matches.
|
||||
let root = trie.root();
|
||||
assert_eq!(root, harness.original_root(), "initial root mismatch");
|
||||
|
||||
// Delete both 0xAB leaves + Touched on a third 0xAB key (not in the trie).
|
||||
// The combination of Touched + removals with a blinded sibling (0xAC) is the
|
||||
// trigger for the bug.
|
||||
let mut leaf_updates: B256Map<LeafUpdate> = [
|
||||
(key_ab1, LeafUpdate::Changed(Vec::new())),
|
||||
(key_ab2, LeafUpdate::Changed(Vec::new())),
|
||||
(key_ab3, LeafUpdate::Touched),
|
||||
]
|
||||
.into_iter()
|
||||
.collect();
|
||||
|
||||
harness.reveal_and_update(&mut trie, &mut leaf_updates);
|
||||
|
||||
// Root should match reference trie with ab1 and ab2 removed.
|
||||
let mut expected_storage = base_storage;
|
||||
expected_storage.remove(&key_ab1);
|
||||
expected_storage.remove(&key_ab2);
|
||||
let expected_harness = SuiteTestHarness::new(expected_storage);
|
||||
|
||||
let actual_root = trie.root();
|
||||
assert_eq!(actual_root, expected_harness.original_root(), "post-delete root mismatch");
|
||||
}
|
||||
|
||||
/// Regression: subtrie emptied by deletes mixed with `LeafUpdate::Touched`.
|
||||
///
|
||||
/// When all `Changed` updates in a subtrie are removals and they would empty the subtrie,
|
||||
|
||||
@@ -149,24 +149,15 @@ where
|
||||
) -> Result<DecodedMultiProofV2, StateProofError> {
|
||||
let MultiProofTargetsV2 { mut account_targets, storage_targets } = targets;
|
||||
|
||||
let storage_prefix_sets: B256Map<_> = self
|
||||
.prefix_sets
|
||||
.storage_prefix_sets
|
||||
.into_iter()
|
||||
.map(|(addr, ps)| (addr, ps.freeze()))
|
||||
.collect();
|
||||
|
||||
// Compute account proofs using the V2 proof calculator with sync account encoding.
|
||||
let account_trie_cursor = self.trie_cursor_factory.account_trie_cursor()?;
|
||||
let hashed_account_cursor = self.hashed_cursor_factory.hashed_account_cursor()?;
|
||||
let mut account_value_encoder = SyncAccountValueEncoder::new(
|
||||
self.trie_cursor_factory.clone(),
|
||||
self.hashed_cursor_factory.clone(),
|
||||
)
|
||||
.with_storage_prefix_sets(storage_prefix_sets.clone());
|
||||
);
|
||||
let mut account_calculator =
|
||||
proof_v2::ProofCalculator::new(account_trie_cursor, hashed_account_cursor)
|
||||
.with_prefix_set(self.prefix_sets.account_prefix_set.freeze());
|
||||
proof_v2::ProofCalculator::new(account_trie_cursor, hashed_account_cursor);
|
||||
let account_proofs =
|
||||
account_calculator.proof(&mut account_value_encoder, &mut account_targets)?;
|
||||
|
||||
@@ -182,9 +173,6 @@ where
|
||||
storage_trie_cursor,
|
||||
hashed_storage_cursor,
|
||||
);
|
||||
if let Some(prefix_set) = storage_prefix_sets.get(&hashed_address) {
|
||||
storage_calculator = storage_calculator.with_prefix_set(prefix_set.clone());
|
||||
}
|
||||
let proofs = storage_calculator.storage_proof(hashed_address, &mut targets)?;
|
||||
storage_proofs.insert(hashed_address, proofs);
|
||||
}
|
||||
|
||||
752
crates/trie/trie/src/trie_cursor/masked.rs
Normal file
752
crates/trie/trie/src/trie_cursor/masked.rs
Normal file
@@ -0,0 +1,752 @@
|
||||
use super::{TrieCursor, TrieCursorFactory, TrieStorageCursor};
|
||||
use alloy_primitives::{map::B256Map, B256};
|
||||
use reth_storage_errors::db::DatabaseError;
|
||||
use reth_trie_common::{
|
||||
prefix_set::{PrefixSet, TriePrefixSets},
|
||||
BranchNodeCompact, Nibbles,
|
||||
};
|
||||
use std::sync::Arc;
|
||||
|
||||
/// A [`TrieCursorFactory`] wrapper that creates cursors which invalidate cached trie hash data
|
||||
/// for children whose paths match the prefix sets in a [`TriePrefixSets`].
|
||||
///
|
||||
/// The `destroyed_accounts` field of the prefix sets is not used by the cursor — it is only
|
||||
/// relevant during trie update finalization, not during cursor traversal.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct MaskedTrieCursorFactory<CF> {
|
||||
/// Underlying trie cursor factory.
|
||||
cursor_factory: CF,
|
||||
/// Frozen prefix sets used for masking.
|
||||
prefix_sets: TriePrefixSets,
|
||||
}
|
||||
|
||||
impl<CF> MaskedTrieCursorFactory<CF> {
|
||||
/// Create a new factory from an inner cursor factory and frozen prefix sets.
|
||||
pub const fn new(cursor_factory: CF, prefix_sets: TriePrefixSets) -> Self {
|
||||
Self { cursor_factory, prefix_sets }
|
||||
}
|
||||
}
|
||||
|
||||
impl<CF: TrieCursorFactory> TrieCursorFactory for MaskedTrieCursorFactory<CF> {
|
||||
type AccountTrieCursor<'a>
|
||||
= MaskedTrieCursor<CF::AccountTrieCursor<'a>>
|
||||
where
|
||||
Self: 'a;
|
||||
|
||||
type StorageTrieCursor<'a>
|
||||
= MaskedTrieCursor<CF::StorageTrieCursor<'a>>
|
||||
where
|
||||
Self: 'a;
|
||||
|
||||
fn account_trie_cursor(&self) -> Result<Self::AccountTrieCursor<'_>, DatabaseError> {
|
||||
let cursor = self.cursor_factory.account_trie_cursor()?;
|
||||
Ok(MaskedTrieCursor::new(cursor, self.prefix_sets.account_prefix_set.clone()))
|
||||
}
|
||||
|
||||
fn storage_trie_cursor(
|
||||
&self,
|
||||
hashed_address: B256,
|
||||
) -> Result<Self::StorageTrieCursor<'_>, DatabaseError> {
|
||||
let cursor = self.cursor_factory.storage_trie_cursor(hashed_address)?;
|
||||
let prefix_set =
|
||||
self.prefix_sets.storage_prefix_sets.get(&hashed_address).cloned().unwrap_or_default();
|
||||
Ok(MaskedTrieCursor::new_storage(
|
||||
cursor,
|
||||
prefix_set,
|
||||
self.prefix_sets.storage_prefix_sets.clone(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
/// A [`TrieCursor`] wrapper that invalidates cached trie hash data for children whose paths match
|
||||
/// a [`PrefixSet`].
|
||||
///
|
||||
/// For each node returned by the inner cursor, hash bits are unset for children whose paths match
|
||||
/// the prefix set, and the corresponding hashes are removed from the node. If a node's `hash_mask`
|
||||
/// and `tree_mask` are both empty after masking, the node is skipped entirely.
|
||||
#[derive(Debug)]
|
||||
pub struct MaskedTrieCursor<C> {
|
||||
/// The inner cursor.
|
||||
cursor: C,
|
||||
/// Prefix set used to determine which children's hashes to invalidate.
|
||||
prefix_set: PrefixSet,
|
||||
/// Storage prefix sets for swapping on `set_hashed_address`.
|
||||
storage_prefix_sets: Option<B256Map<PrefixSet>>,
|
||||
}
|
||||
|
||||
impl<C> MaskedTrieCursor<C> {
|
||||
/// Create a new cursor wrapping `cursor`, masking hash bits for children whose paths match
|
||||
/// `prefix_set`.
|
||||
pub const fn new(cursor: C, prefix_set: PrefixSet) -> Self {
|
||||
Self { cursor, prefix_set, storage_prefix_sets: None }
|
||||
}
|
||||
|
||||
/// Create a new storage cursor that can swap its prefix set on `set_hashed_address`.
|
||||
pub const fn new_storage(
|
||||
cursor: C,
|
||||
prefix_set: PrefixSet,
|
||||
storage_prefix_sets: B256Map<PrefixSet>,
|
||||
) -> Self {
|
||||
Self { cursor, prefix_set, storage_prefix_sets: Some(storage_prefix_sets) }
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: TrieCursor> MaskedTrieCursor<C> {
|
||||
/// Mask hash bits on a node for children whose paths match the prefix set.
|
||||
///
|
||||
/// Returns `true` if the node should be kept, `false` if it should be skipped (both
|
||||
/// `hash_mask` and `tree_mask` are empty after masking).
|
||||
fn mask_node(&mut self, key: &Nibbles, node: &mut BranchNodeCompact) -> bool {
|
||||
if !self.prefix_set.contains(key) {
|
||||
return true;
|
||||
}
|
||||
|
||||
// The subtree is modified — root hash is always invalid.
|
||||
node.root_hash = None;
|
||||
|
||||
let original_hash_mask = node.hash_mask;
|
||||
if original_hash_mask.is_empty() {
|
||||
return true;
|
||||
}
|
||||
|
||||
let mut new_hash_mask = original_hash_mask;
|
||||
let mut child_path = *key;
|
||||
let key_len = key.len();
|
||||
|
||||
for nibble in original_hash_mask.iter() {
|
||||
child_path.truncate(key_len);
|
||||
child_path.push(nibble);
|
||||
|
||||
if self.prefix_set.contains(&child_path) {
|
||||
new_hash_mask.unset_bit(nibble);
|
||||
}
|
||||
}
|
||||
|
||||
if new_hash_mask != original_hash_mask {
|
||||
// Remove hashes for unset bits in-place.
|
||||
let hashes = Arc::make_mut(&mut node.hashes);
|
||||
let mut write = 0;
|
||||
for (read, nibble) in original_hash_mask.iter().enumerate() {
|
||||
if new_hash_mask.is_bit_set(nibble) {
|
||||
hashes[write] = hashes[read];
|
||||
write += 1;
|
||||
}
|
||||
}
|
||||
hashes.truncate(write);
|
||||
|
||||
node.hash_mask = new_hash_mask;
|
||||
|
||||
if node.hash_mask.is_empty() && node.tree_mask.is_empty() {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
true
|
||||
}
|
||||
|
||||
/// Apply masking to entries, advancing past fully-masked nodes.
|
||||
fn mask_entries(
|
||||
&mut self,
|
||||
mut entry: Option<(Nibbles, BranchNodeCompact)>,
|
||||
) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
|
||||
while let Some((key, mut node)) = entry {
|
||||
if self.mask_node(&key, &mut node) {
|
||||
return Ok(Some((key, node)));
|
||||
}
|
||||
entry = self.cursor.next()?;
|
||||
}
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: TrieCursor> TrieCursor for MaskedTrieCursor<C> {
|
||||
fn seek_exact(
|
||||
&mut self,
|
||||
key: Nibbles,
|
||||
) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
|
||||
if let Some((key, mut node)) = self.cursor.seek_exact(key)? {
|
||||
if self.mask_node(&key, &mut node) {
|
||||
Ok(Some((key, node)))
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
} else {
|
||||
Ok(None)
|
||||
}
|
||||
}
|
||||
|
||||
fn seek(
|
||||
&mut self,
|
||||
key: Nibbles,
|
||||
) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
|
||||
let entry = self.cursor.seek(key)?;
|
||||
self.mask_entries(entry)
|
||||
}
|
||||
|
||||
fn next(&mut self) -> Result<Option<(Nibbles, BranchNodeCompact)>, DatabaseError> {
|
||||
let entry = self.cursor.next()?;
|
||||
self.mask_entries(entry)
|
||||
}
|
||||
|
||||
fn current(&mut self) -> Result<Option<Nibbles>, DatabaseError> {
|
||||
self.cursor.current()
|
||||
}
|
||||
|
||||
fn reset(&mut self) {
|
||||
self.cursor.reset();
|
||||
}
|
||||
}
|
||||
|
||||
impl<C: TrieStorageCursor> TrieStorageCursor for MaskedTrieCursor<C> {
|
||||
fn set_hashed_address(&mut self, hashed_address: B256) {
|
||||
self.cursor.set_hashed_address(hashed_address);
|
||||
if let Some(storage_prefix_sets) = &self.storage_prefix_sets {
|
||||
self.prefix_set = storage_prefix_sets.get(&hashed_address).cloned().unwrap_or_default();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use crate::trie_cursor::mock::MockTrieCursor;
|
||||
use parking_lot::Mutex;
|
||||
use reth_trie_common::prefix_set::PrefixSetMut;
|
||||
use std::{collections::BTreeMap, sync::Arc};
|
||||
|
||||
fn make_cursor(nodes: Vec<(Nibbles, BranchNodeCompact)>) -> MockTrieCursor {
|
||||
let map: BTreeMap<Nibbles, BranchNodeCompact> = nodes.into_iter().collect();
|
||||
MockTrieCursor::new(Arc::new(map), Arc::new(Mutex::new(Vec::new())))
|
||||
}
|
||||
|
||||
fn node(state_mask: u16) -> BranchNodeCompact {
|
||||
BranchNodeCompact::new(state_mask, 0, 0, vec![], None)
|
||||
}
|
||||
|
||||
fn node_with_hashes(state_mask: u16, hash_mask: u16, hashes: Vec<B256>) -> BranchNodeCompact {
|
||||
BranchNodeCompact::new(state_mask, 0, hash_mask, hashes, None)
|
||||
}
|
||||
|
||||
fn node_with_tree_mask(
|
||||
state_mask: u16,
|
||||
tree_mask: u16,
|
||||
hash_mask: u16,
|
||||
hashes: Vec<B256>,
|
||||
) -> BranchNodeCompact {
|
||||
BranchNodeCompact::new(state_mask, tree_mask, hash_mask, hashes, None)
|
||||
}
|
||||
|
||||
fn hash(byte: u8) -> B256 {
|
||||
B256::repeat_byte(byte)
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_seek_masks_matching_child_hashes() {
|
||||
// Node at [0x1] with children 2 and 5 hashed.
|
||||
// Prefix set marks child 2 as changed.
|
||||
let nodes = vec![(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_hashes(0b0000_0000_0010_0100, 0b0000_0000_0010_0100, vec![hash(2), hash(5)]),
|
||||
)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x2]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek(Nibbles::default()).unwrap();
|
||||
let (key, node) = result.unwrap();
|
||||
assert_eq!(key, Nibbles::from_nibbles([0x1]));
|
||||
// Hash bit 2 should be unset, only bit 5 remains.
|
||||
assert!(!node.hash_mask.is_bit_set(2));
|
||||
assert!(node.hash_mask.is_bit_set(5));
|
||||
assert_eq!(&*node.hashes, &[hash(5)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_seek_skips_fully_masked_node() {
|
||||
// Node at [0x1] with only child 3 hashed, tree_mask empty.
|
||||
// Prefix set marks child 3 as changed → fully masked → skipped.
|
||||
// Node at [0x2] is unaffected → returned.
|
||||
let nodes = vec![
|
||||
(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_hashes(0b0000_0000_0000_1000, 0b0000_0000_0000_1000, vec![hash(3)]),
|
||||
),
|
||||
(Nibbles::from_nibbles([0x2]), node(0b0000_0000_0000_0001)),
|
||||
];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x3]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek(Nibbles::default()).unwrap();
|
||||
assert_eq!(result, Some((Nibbles::from_nibbles([0x2]), node(0b0000_0000_0000_0001))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_node_with_tree_mask_not_skipped() {
|
||||
// Node at [0x1] with child 3 hashed, tree_mask has bit 3 set.
|
||||
// Prefix set marks child 3 → hash cleared, but tree_mask keeps the node alive.
|
||||
let nodes = vec![(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_tree_mask(
|
||||
0b0000_0000_0000_1000,
|
||||
0b0000_0000_0000_1000,
|
||||
0b0000_0000_0000_1000,
|
||||
vec![hash(3)],
|
||||
),
|
||||
)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x3]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek(Nibbles::default()).unwrap();
|
||||
let (key, node) = result.unwrap();
|
||||
assert_eq!(key, Nibbles::from_nibbles([0x1]));
|
||||
assert!(node.hash_mask.is_empty());
|
||||
assert!(node.tree_mask.is_bit_set(3));
|
||||
assert!(node.hashes.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_seek_exact_masks_hash_bits() {
|
||||
let nodes = vec![(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_tree_mask(
|
||||
0b0000_0000_0010_0100,
|
||||
0b0000_0000_0010_0100,
|
||||
0b0000_0000_0010_0100,
|
||||
vec![hash(2), hash(5)],
|
||||
),
|
||||
)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x5]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek_exact(Nibbles::from_nibbles([0x1])).unwrap();
|
||||
let (_, node) = result.unwrap();
|
||||
assert!(node.hash_mask.is_bit_set(2));
|
||||
assert!(!node.hash_mask.is_bit_set(5));
|
||||
assert_eq!(&*node.hashes, &[hash(2)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_seek_exact_returns_none_for_fully_masked() {
|
||||
let nodes = vec![(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_hashes(0b0000_0000_0000_0100, 0b0000_0000_0000_0100, vec![hash(2)]),
|
||||
)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x2]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek_exact(Nibbles::from_nibbles([0x1])).unwrap();
|
||||
assert_eq!(result, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_next_masks_and_skips() {
|
||||
// Three nodes: [0x1] unaffected, [0x2] fully masked, [0x3] unaffected.
|
||||
let nodes = vec![
|
||||
(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_hashes(0b0000_0000_0000_0010, 0b0000_0000_0000_0010, vec![hash(1)]),
|
||||
),
|
||||
(
|
||||
Nibbles::from_nibbles([0x2]),
|
||||
node_with_hashes(0b0000_0000_0001_0000, 0b0000_0000_0001_0000, vec![hash(4)]),
|
||||
),
|
||||
(
|
||||
Nibbles::from_nibbles([0x3]),
|
||||
node_with_hashes(0b0000_0000_0100_0000, 0b0000_0000_0100_0000, vec![hash(6)]),
|
||||
),
|
||||
];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x2, 0x4]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
// seek to [0x1], no match → returned unchanged.
|
||||
let result = cursor.seek(Nibbles::from_nibbles([0x1])).unwrap();
|
||||
let (key, node) = result.unwrap();
|
||||
assert_eq!(key, Nibbles::from_nibbles([0x1]));
|
||||
assert_eq!(&*node.hashes, &[hash(1)]);
|
||||
|
||||
// next() should skip [0x2] (fully masked), returning [0x3].
|
||||
let result = cursor.next().unwrap();
|
||||
let (key, node) = result.unwrap();
|
||||
assert_eq!(key, Nibbles::from_nibbles([0x3]));
|
||||
assert_eq!(&*node.hashes, &[hash(6)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_no_match_returns_unchanged() {
|
||||
let nodes = vec![(
|
||||
Nibbles::from_nibbles([0x2]),
|
||||
node_with_hashes(0b0000_0000_0000_0010, 0b0000_0000_0000_0010, vec![hash(1)]),
|
||||
)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x3]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek(Nibbles::default()).unwrap();
|
||||
let (key, node) = result.unwrap();
|
||||
assert_eq!(key, Nibbles::from_nibbles([0x2]));
|
||||
// Unchanged — prefix set doesn't match [0x2].
|
||||
assert!(node.hash_mask.is_bit_set(1));
|
||||
assert_eq!(&*node.hashes, &[hash(1)]);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_prefix_set_returns_all_unchanged() {
|
||||
let h1 = hash(1);
|
||||
let h2 = hash(2);
|
||||
let nodes = vec![
|
||||
(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_hashes(0b0000_0000_0000_0010, 0b0000_0000_0000_0010, vec![h1]),
|
||||
),
|
||||
(
|
||||
Nibbles::from_nibbles([0x2]),
|
||||
node_with_hashes(0b0000_0000_0000_0100, 0b0000_0000_0000_0100, vec![h2]),
|
||||
),
|
||||
];
|
||||
|
||||
let ps = PrefixSetMut::default();
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let r1 = cursor.seek(Nibbles::default()).unwrap().unwrap();
|
||||
assert_eq!(r1.0, Nibbles::from_nibbles([0x1]));
|
||||
assert_eq!(&*r1.1.hashes, &[h1]);
|
||||
|
||||
let r2 = cursor.next().unwrap().unwrap();
|
||||
assert_eq!(r2.0, Nibbles::from_nibbles([0x2]));
|
||||
assert_eq!(&*r2.1.hashes, &[h2]);
|
||||
|
||||
assert_eq!(cursor.next().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_root_hash_cleared_on_mask() {
|
||||
let mut n =
|
||||
node_with_hashes(0b0000_0000_0010_0100, 0b0000_0000_0010_0100, vec![hash(2), hash(5)]);
|
||||
n.root_hash = Some(hash(0xFF));
|
||||
|
||||
let nodes = vec![(Nibbles::from_nibbles([0x1]), n)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x2]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let (_, node) = cursor.seek(Nibbles::default()).unwrap().unwrap();
|
||||
assert_eq!(node.root_hash, None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_node_without_hashes_returned_unchanged() {
|
||||
// Node with state_mask only (no hashes, no tree_mask) should pass through.
|
||||
let nodes = vec![(Nibbles::from_nibbles([0x1]), node(0b0000_0000_0000_0011))];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x0]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let result = cursor.seek(Nibbles::default()).unwrap();
|
||||
assert_eq!(result, Some((Nibbles::from_nibbles([0x1]), node(0b0000_0000_0000_0011))));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_empty_cursor_returns_none() {
|
||||
let nodes = vec![];
|
||||
let ps = PrefixSetMut::default();
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
assert_eq!(cursor.seek(Nibbles::default()).unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_reset_delegates() {
|
||||
let nodes =
|
||||
vec![(Nibbles::from_nibbles([0x1]), node(1)), (Nibbles::from_nibbles([0x2]), node(2))];
|
||||
|
||||
let ps = PrefixSetMut::default();
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let _ = cursor.seek(Nibbles::from_nibbles([0x1])).unwrap();
|
||||
assert_eq!(cursor.current().unwrap(), Some(Nibbles::from_nibbles([0x1])));
|
||||
|
||||
cursor.reset();
|
||||
assert_eq!(cursor.current().unwrap(), None);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn test_partial_mask_preserves_remaining_hashes() {
|
||||
// Node at [0x1] with children 0, 3, 7 hashed.
|
||||
// Prefix set marks children 0 and 7 as changed.
|
||||
// Only hash for child 3 should remain.
|
||||
let nodes = vec![(
|
||||
Nibbles::from_nibbles([0x1]),
|
||||
node_with_tree_mask(
|
||||
0b0000_0000_1000_1001,
|
||||
0b0000_0000_1000_1001,
|
||||
0b0000_0000_1000_1001,
|
||||
vec![hash(0), hash(3), hash(7)],
|
||||
),
|
||||
)];
|
||||
|
||||
let mut ps = PrefixSetMut::default();
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x0]));
|
||||
ps.insert(Nibbles::from_nibbles([0x1, 0x7]));
|
||||
|
||||
let inner = make_cursor(nodes);
|
||||
let mut cursor = MaskedTrieCursor::new(inner, ps.freeze());
|
||||
|
||||
let (key, node) = cursor.seek(Nibbles::default()).unwrap().unwrap();
|
||||
assert_eq!(key, Nibbles::from_nibbles([0x1]));
|
||||
assert!(!node.hash_mask.is_bit_set(0));
|
||||
assert!(node.hash_mask.is_bit_set(3));
|
||||
assert!(!node.hash_mask.is_bit_set(7));
|
||||
assert_eq!(&*node.hashes, &[hash(3)]);
|
||||
assert_eq!(node.root_hash, None);
|
||||
}
|
||||
|
||||
mod proptest_tests {
|
||||
use crate::{
|
||||
hashed_cursor::{mock::MockHashedCursorFactory, HashedPostStateCursorFactory},
|
||||
proof::Proof,
|
||||
trie_cursor::{
|
||||
masked::MaskedTrieCursorFactory, mock::MockTrieCursorFactory,
|
||||
noop::NoopTrieCursorFactory,
|
||||
},
|
||||
StateRoot,
|
||||
};
|
||||
use alloy_primitives::{map::B256Set, B256, U256};
|
||||
use proptest::prelude::*;
|
||||
use reth_primitives_traits::Account;
|
||||
use reth_trie_common::{HashedPostState, HashedStorage, MultiProofTargets};
|
||||
|
||||
fn account_strategy() -> impl Strategy<Value = Account> {
|
||||
(any::<u64>(), any::<u64>(), any::<[u8; 32]>()).prop_map(
|
||||
|(nonce, balance, code_hash)| Account {
|
||||
nonce,
|
||||
balance: U256::from(balance),
|
||||
bytecode_hash: Some(B256::from(code_hash)),
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
fn storage_value_strategy() -> impl Strategy<Value = U256> {
|
||||
any::<u64>().prop_filter("non-zero", |v| *v != 0).prop_map(U256::from)
|
||||
}
|
||||
|
||||
/// Generates a base dataset of 1000 storage slots for account `B256::ZERO`,
|
||||
/// a 200-entry changeset partially overlapping with the base, and random
|
||||
/// proof targets partially overlapping with both.
|
||||
#[allow(clippy::type_complexity)]
|
||||
fn test_input_strategy(
|
||||
) -> impl Strategy<Value = (Vec<(B256, U256)>, Account, Vec<(B256, Option<U256>)>, Vec<B256>)>
|
||||
{
|
||||
(
|
||||
// 1000 base storage slots: unique keys with non-zero values
|
||||
prop::collection::vec(
|
||||
(any::<[u8; 32]>().prop_map(B256::from), storage_value_strategy()),
|
||||
1000,
|
||||
),
|
||||
account_strategy(),
|
||||
// 200 changeset entries: (key, Option<value>) where None = removal
|
||||
prop::collection::vec(
|
||||
(
|
||||
any::<[u8; 32]>().prop_map(B256::from),
|
||||
prop::option::of(storage_value_strategy()),
|
||||
),
|
||||
200,
|
||||
),
|
||||
// Extra random keys for proof targets
|
||||
prop::collection::vec(any::<[u8; 32]>().prop_map(B256::from), 50),
|
||||
)
|
||||
.prop_flat_map(
|
||||
|(base_slots, account, changeset_raw, extra_targets)| {
|
||||
// Dedup base slots by key
|
||||
let mut base_map = alloy_primitives::map::B256Map::default();
|
||||
for (k, v) in &base_slots {
|
||||
base_map.insert(*k, *v);
|
||||
}
|
||||
let base_deduped: Vec<(B256, U256)> =
|
||||
base_map.iter().map(|(&k, &v)| (k, v)).collect();
|
||||
let base_keys: Vec<B256> = base_deduped.iter().map(|(k, _)| *k).collect();
|
||||
|
||||
// Build changeset: 50% overlap with base keys, 50% new keys
|
||||
let changeset_len = changeset_raw.len();
|
||||
let half = changeset_len / 2;
|
||||
let base_keys_for_overlap = base_keys.clone();
|
||||
|
||||
// Use indices to select from base keys for overlap portion
|
||||
let overlap_indices =
|
||||
prop::collection::vec(0..base_keys_for_overlap.len().max(1), half);
|
||||
|
||||
overlap_indices.prop_map(move |indices| {
|
||||
let mut changeset: Vec<(B256, Option<U256>)> = Vec::new();
|
||||
|
||||
// First half: overlapping with base keys
|
||||
for (i, (_, value)) in
|
||||
indices.iter().zip(changeset_raw.iter()).take(half)
|
||||
{
|
||||
let key = if base_keys_for_overlap.is_empty() {
|
||||
changeset_raw[*i].0
|
||||
} else {
|
||||
base_keys_for_overlap[*i % base_keys_for_overlap.len()]
|
||||
};
|
||||
changeset.push((key, *value));
|
||||
}
|
||||
|
||||
// Second half: new keys from changeset_raw
|
||||
for (key, value) in changeset_raw.iter().skip(half) {
|
||||
changeset.push((*key, *value));
|
||||
}
|
||||
|
||||
// Build proof targets: mix of base keys, changeset keys, and randoms
|
||||
let changeset_keys: Vec<B256> =
|
||||
changeset.iter().map(|(k, _)| *k).collect();
|
||||
let mut proof_slot_targets: Vec<B256> = Vec::new();
|
||||
|
||||
// ~40% from base
|
||||
for k in base_keys.iter().take(40) {
|
||||
proof_slot_targets.push(*k);
|
||||
}
|
||||
// ~30% from changeset
|
||||
for k in changeset_keys.iter().take(30) {
|
||||
proof_slot_targets.push(*k);
|
||||
}
|
||||
// ~30% random
|
||||
for k in extra_targets.iter().take(30) {
|
||||
proof_slot_targets.push(*k);
|
||||
}
|
||||
|
||||
(base_deduped.clone(), account, changeset, proof_slot_targets)
|
||||
})
|
||||
},
|
||||
)
|
||||
}
|
||||
|
||||
proptest! {
|
||||
#![proptest_config(ProptestConfig::with_cases(50))]
|
||||
#[test]
|
||||
fn proptest_masked_cursor_multiproof_equivalence(
|
||||
(base_slots, account, changeset, proof_slot_targets) in test_input_strategy()
|
||||
) {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let hashed_address = B256::ZERO;
|
||||
|
||||
// Step 1: Create the base hashed post state with a single account
|
||||
// and 1000 storage slots.
|
||||
let base_state = HashedPostState {
|
||||
accounts: std::iter::once((hashed_address, Some(account))).collect(),
|
||||
storages: std::iter::once((
|
||||
hashed_address,
|
||||
HashedStorage::from_iter(false, base_slots),
|
||||
))
|
||||
.collect(),
|
||||
};
|
||||
|
||||
// Step 2: Compute trie updates from state root over the full base state.
|
||||
let base_hashed_cursor_factory =
|
||||
MockHashedCursorFactory::from_hashed_post_state(base_state);
|
||||
let (_, trie_updates) = StateRoot::new(
|
||||
NoopTrieCursorFactory,
|
||||
base_hashed_cursor_factory.clone(),
|
||||
)
|
||||
.root_with_updates()
|
||||
.expect("state root computation should succeed");
|
||||
|
||||
// Step 3: Create a MockTrieCursorFactory from those trie updates.
|
||||
let mock_trie_cursor_factory =
|
||||
MockTrieCursorFactory::from_trie_updates(trie_updates);
|
||||
|
||||
// Step 4: Build the changeset post state. Removals use U256::ZERO.
|
||||
let changeset_storage: Vec<(B256, U256)> = changeset
|
||||
.iter()
|
||||
.map(|(k, v)| (*k, v.unwrap_or(U256::ZERO)))
|
||||
.collect();
|
||||
let changeset_state = HashedPostState {
|
||||
accounts: std::iter::once((hashed_address, Some(account))).collect(),
|
||||
storages: std::iter::once((
|
||||
hashed_address,
|
||||
HashedStorage::from_iter(false, changeset_storage),
|
||||
))
|
||||
.collect(),
|
||||
};
|
||||
|
||||
// Step 5: Generate prefix sets from the changeset.
|
||||
let prefix_sets_mut = changeset_state.construct_prefix_sets();
|
||||
|
||||
// Step 6: Build proof targets.
|
||||
let slot_targets: B256Set = proof_slot_targets.into_iter().collect();
|
||||
let targets =
|
||||
MultiProofTargets::from_iter([(hashed_address, slot_targets)]);
|
||||
|
||||
// Step 7: Create the HashedPostStateCursorFactory overlaying changeset
|
||||
// on the base.
|
||||
let changeset_sorted = changeset_state.into_sorted();
|
||||
let overlay_cursor_factory = HashedPostStateCursorFactory::new(
|
||||
base_hashed_cursor_factory,
|
||||
&changeset_sorted,
|
||||
);
|
||||
|
||||
// Step 8a: Approach A — prefix sets passed to Proof directly.
|
||||
let proof_a = Proof::new(
|
||||
mock_trie_cursor_factory.clone(),
|
||||
overlay_cursor_factory.clone(),
|
||||
)
|
||||
.with_prefix_sets_mut(prefix_sets_mut.clone());
|
||||
let multiproof_a = proof_a
|
||||
.multiproof(targets.clone())
|
||||
.expect("multiproof A should succeed");
|
||||
|
||||
// Step 8b: Approach B — MaskedTrieCursorFactory, no prefix sets on Proof.
|
||||
let masked_trie_cursor_factory = MaskedTrieCursorFactory::new(
|
||||
mock_trie_cursor_factory,
|
||||
prefix_sets_mut.freeze(),
|
||||
);
|
||||
let proof_b = Proof::new(
|
||||
masked_trie_cursor_factory,
|
||||
overlay_cursor_factory,
|
||||
);
|
||||
let multiproof_b = proof_b
|
||||
.multiproof(targets)
|
||||
.expect("multiproof B should succeed");
|
||||
|
||||
// Step 9: Compare results.
|
||||
assert_eq!(
|
||||
multiproof_a, multiproof_b,
|
||||
"multiproof with prefix sets should equal multiproof with masked cursor"
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -11,6 +11,9 @@ pub mod subnode;
|
||||
/// Noop trie cursor implementations.
|
||||
pub mod noop;
|
||||
|
||||
/// Masked trie cursor wrapper that skips nodes matching a prefix set.
|
||||
pub mod masked;
|
||||
|
||||
/// Depth-first trie iterator.
|
||||
pub mod depth_first;
|
||||
|
||||
|
||||
@@ -1,23 +1,29 @@
|
||||
use crate::{
|
||||
hashed_cursor::{HashedCursor, HashedCursorFactory},
|
||||
prefix_set::TriePrefixSetsMut,
|
||||
proof::Proof,
|
||||
proof_v2,
|
||||
proof::{Proof, ProofTrieNodeProviderFactory},
|
||||
trie_cursor::TrieCursorFactory,
|
||||
TRIE_ACCOUNT_RLP_MAX_SIZE,
|
||||
};
|
||||
use alloy_rlp::EMPTY_STRING_CODE;
|
||||
use alloy_trie::EMPTY_ROOT_HASH;
|
||||
use reth_trie_common::HashedPostState;
|
||||
use reth_trie_sparse::SparseTrie;
|
||||
|
||||
use alloy_primitives::{
|
||||
keccak256,
|
||||
map::{B256Map, HashMap},
|
||||
Bytes, B256, U256,
|
||||
map::{B256Map, B256Set, Entry, HashMap},
|
||||
Bytes, B256,
|
||||
};
|
||||
use alloy_rlp::{Encodable, EMPTY_STRING_CODE};
|
||||
use alloy_trie::{nodes::BranchNodeRef, EMPTY_ROOT_HASH};
|
||||
use reth_execution_errors::{SparseStateTrieErrorKind, StateProofError, TrieWitnessError};
|
||||
use reth_trie_common::{
|
||||
DecodedMultiProofV2, HashedPostState, MultiProofTargetsV2, ProofV2Target, TrieNodeV2,
|
||||
use itertools::Itertools;
|
||||
use reth_execution_errors::{
|
||||
SparseStateTrieErrorKind, SparseTrieError, SparseTrieErrorKind, StateProofError,
|
||||
TrieWitnessError,
|
||||
};
|
||||
use reth_trie_sparse::{LeafUpdate, SparseStateTrie, SparseTrie as _};
|
||||
use reth_trie_common::{MultiProofTargets, Nibbles};
|
||||
use reth_trie_sparse::{
|
||||
provider::{RevealedNode, TrieNodeProvider, TrieNodeProviderFactory},
|
||||
SparseStateTrie,
|
||||
};
|
||||
use std::sync::mpsc;
|
||||
|
||||
/// State transition witness for the trie.
|
||||
#[derive(Debug)]
|
||||
@@ -26,8 +32,6 @@ pub struct TrieWitness<T, H> {
|
||||
trie_cursor_factory: T,
|
||||
/// The factory for hashed cursors.
|
||||
hashed_cursor_factory: H,
|
||||
/// A set of prefix sets that have changes.
|
||||
prefix_sets: TriePrefixSetsMut,
|
||||
/// Flag indicating whether the root node should always be included (even if the target state
|
||||
/// is empty). This setting is useful if the caller wants to verify the witness against the
|
||||
/// parent state root.
|
||||
@@ -43,7 +47,6 @@ impl<T, H> TrieWitness<T, H> {
|
||||
Self {
|
||||
trie_cursor_factory,
|
||||
hashed_cursor_factory,
|
||||
prefix_sets: TriePrefixSetsMut::default(),
|
||||
always_include_root_node: false,
|
||||
witness: HashMap::default(),
|
||||
}
|
||||
@@ -54,7 +57,6 @@ impl<T, H> TrieWitness<T, H> {
|
||||
TrieWitness {
|
||||
trie_cursor_factory,
|
||||
hashed_cursor_factory: self.hashed_cursor_factory,
|
||||
prefix_sets: self.prefix_sets,
|
||||
always_include_root_node: self.always_include_root_node,
|
||||
witness: self.witness,
|
||||
}
|
||||
@@ -65,18 +67,11 @@ impl<T, H> TrieWitness<T, H> {
|
||||
TrieWitness {
|
||||
trie_cursor_factory: self.trie_cursor_factory,
|
||||
hashed_cursor_factory,
|
||||
prefix_sets: self.prefix_sets,
|
||||
always_include_root_node: self.always_include_root_node,
|
||||
witness: self.witness,
|
||||
}
|
||||
}
|
||||
|
||||
/// Set the prefix sets. They have to be mutable in order to allow extension with proof target.
|
||||
pub fn with_prefix_sets_mut(mut self, prefix_sets: TriePrefixSetsMut) -> Self {
|
||||
self.prefix_sets = prefix_sets;
|
||||
self
|
||||
}
|
||||
|
||||
/// Set `always_include_root_node` to true. Root node will be included even in empty state.
|
||||
/// This setting is useful if the caller wants to verify the witness against the
|
||||
/// parent state root.
|
||||
@@ -97,131 +92,84 @@ where
|
||||
/// # Arguments
|
||||
///
|
||||
/// `state` - state transition containing both modified and touched accounts and storage slots.
|
||||
pub fn compute(
|
||||
mut self,
|
||||
mut state: HashedPostState,
|
||||
) -> Result<B256Map<Bytes>, TrieWitnessError> {
|
||||
pub fn compute(mut self, state: HashedPostState) -> Result<B256Map<Bytes>, TrieWitnessError> {
|
||||
let is_state_empty = state.is_empty();
|
||||
if is_state_empty && !self.always_include_root_node {
|
||||
return Ok(Default::default())
|
||||
}
|
||||
|
||||
// Expand wiped storages into explicit zero-value entries for every existing slot,
|
||||
// so that downstream code can treat all storages uniformly.
|
||||
self.expand_wiped_storages(&mut state)?;
|
||||
|
||||
let proof_targets = if is_state_empty {
|
||||
MultiProofTargetsV2 {
|
||||
account_targets: vec![ProofV2Target::new(B256::ZERO)],
|
||||
..Default::default()
|
||||
}
|
||||
MultiProofTargets::account(B256::ZERO)
|
||||
} else {
|
||||
Self::get_proof_targets(&state)
|
||||
self.get_proof_targets(&state)?
|
||||
};
|
||||
let multiproof =
|
||||
Proof::new(self.trie_cursor_factory.clone(), self.hashed_cursor_factory.clone())
|
||||
.with_prefix_sets_mut(self.prefix_sets.clone())
|
||||
.multiproof_v2(proof_targets)?;
|
||||
.multiproof(proof_targets.clone())?;
|
||||
|
||||
// No need to reconstruct the rest of the trie, we just need to include
|
||||
// the root node and return.
|
||||
if is_state_empty {
|
||||
let (root_hash, root_node) = if let Some(root_node) =
|
||||
multiproof.account_proofs.into_iter().find(|n| n.path.is_empty())
|
||||
multiproof.account_subtree.into_inner().remove(&Nibbles::default())
|
||||
{
|
||||
let mut encoded = Vec::new();
|
||||
root_node.node.encode(&mut encoded);
|
||||
let bytes = Bytes::from(encoded);
|
||||
(keccak256(&bytes), bytes)
|
||||
(keccak256(&root_node), root_node)
|
||||
} else {
|
||||
(EMPTY_ROOT_HASH, Bytes::from([EMPTY_STRING_CODE]))
|
||||
};
|
||||
return Ok(B256Map::from_iter([(root_hash, root_node)]))
|
||||
}
|
||||
|
||||
// Record all nodes from multiproof in the witness.
|
||||
self.record_multiproof_nodes(&multiproof);
|
||||
// Record all nodes from multiproof in the witness
|
||||
for account_node in multiproof.account_subtree.values() {
|
||||
if let Entry::Vacant(entry) = self.witness.entry(keccak256(account_node.as_ref())) {
|
||||
entry.insert(account_node.clone());
|
||||
}
|
||||
}
|
||||
for storage_node in multiproof.storages.values().flat_map(|s| s.subtree.values()) {
|
||||
if let Entry::Vacant(entry) = self.witness.entry(keccak256(storage_node.as_ref())) {
|
||||
entry.insert(storage_node.clone());
|
||||
}
|
||||
}
|
||||
|
||||
let (tx, rx) = mpsc::channel();
|
||||
let blinded_provider_factory = WitnessTrieNodeProviderFactory::new(
|
||||
ProofTrieNodeProviderFactory::new(self.trie_cursor_factory, self.hashed_cursor_factory),
|
||||
tx,
|
||||
);
|
||||
let mut sparse_trie = SparseStateTrie::new();
|
||||
sparse_trie.reveal_decoded_multiproof_v2(multiproof)?;
|
||||
sparse_trie.reveal_multiproof(multiproof)?;
|
||||
|
||||
// Build storage leaf updates for all accounts with storage changes, split into
|
||||
// removals and upserts. Removals must be applied first so that branch collapse
|
||||
// detection fires correctly: if a removal and an insertion target siblings under
|
||||
// the same branch, processing the removal first may reduce the branch to a single
|
||||
// blinded child, triggering a proof fetch for the sibling. Processing the insertion
|
||||
// first would add a new child that keeps the count above one, masking the need.
|
||||
let mut storage_removals: B256Map<B256Map<LeafUpdate>> = B256Map::default();
|
||||
let mut storage_upserts: B256Map<B256Map<LeafUpdate>> = B256Map::default();
|
||||
for (hashed_address, storage) in &state.storages {
|
||||
for (&hashed_slot, value) in &storage.storage {
|
||||
if value.is_zero() {
|
||||
storage_removals
|
||||
.entry(*hashed_address)
|
||||
.or_default()
|
||||
.insert(hashed_slot, LeafUpdate::Changed(vec![]));
|
||||
// Attempt to update state trie to gather additional information for the witness.
|
||||
for (hashed_address, hashed_slots) in
|
||||
proof_targets.into_iter().sorted_unstable_by_key(|(ha, _)| *ha)
|
||||
{
|
||||
// Update storage trie first.
|
||||
let provider = blinded_provider_factory.storage_node_provider(hashed_address);
|
||||
let storage = state.storages.get(&hashed_address);
|
||||
let storage_trie = sparse_trie.storage_trie_mut(&hashed_address).ok_or(
|
||||
SparseStateTrieErrorKind::SparseStorageTrie(
|
||||
hashed_address,
|
||||
SparseTrieErrorKind::Blind,
|
||||
),
|
||||
)?;
|
||||
for hashed_slot in hashed_slots.into_iter().sorted_unstable() {
|
||||
let storage_nibbles = Nibbles::unpack(hashed_slot);
|
||||
let maybe_leaf_value = storage
|
||||
.and_then(|s| s.storage.get(&hashed_slot))
|
||||
.filter(|v| !v.is_zero())
|
||||
.map(|v| alloy_rlp::encode_fixed_size(v).to_vec());
|
||||
|
||||
if let Some(value) = maybe_leaf_value {
|
||||
storage_trie.update_leaf(storage_nibbles, value, &provider).map_err(|err| {
|
||||
SparseStateTrieErrorKind::SparseStorageTrie(hashed_address, err.into_kind())
|
||||
})?;
|
||||
} else {
|
||||
storage_upserts.entry(*hashed_address).or_default().insert(
|
||||
hashed_slot,
|
||||
LeafUpdate::Changed(alloy_rlp::encode_fixed_size(value).to_vec()),
|
||||
);
|
||||
storage_trie.remove_leaf(&storage_nibbles, &provider).map_err(|err| {
|
||||
SparseStateTrieErrorKind::SparseStorageTrie(hashed_address, err.into_kind())
|
||||
})?;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Apply storage removals first, then upserts, fetching additional proofs as needed.
|
||||
for storage_updates in [&mut storage_removals, &mut storage_upserts] {
|
||||
loop {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
|
||||
for (&hashed_address, slot_updates) in storage_updates.iter_mut() {
|
||||
if slot_updates.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let storage_trie = sparse_trie
|
||||
.storage_trie_mut(&hashed_address)
|
||||
.expect("storage trie was revealed from multiproof");
|
||||
storage_trie
|
||||
.update_leaves(slot_updates, |key, min_len| {
|
||||
targets
|
||||
.storage_targets
|
||||
.entry(hashed_address)
|
||||
.or_default()
|
||||
.push(ProofV2Target::new(key).with_min_len(min_len));
|
||||
})
|
||||
.map_err(|err| {
|
||||
SparseStateTrieErrorKind::SparseStorageTrie(
|
||||
hashed_address,
|
||||
err.into_kind(),
|
||||
)
|
||||
})?;
|
||||
}
|
||||
|
||||
if targets.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let multiproof = Proof::new(
|
||||
self.trie_cursor_factory.clone(),
|
||||
self.hashed_cursor_factory.clone(),
|
||||
)
|
||||
.with_prefix_sets_mut(self.prefix_sets.clone())
|
||||
.multiproof_v2(targets)?;
|
||||
self.record_multiproof_nodes(&multiproof);
|
||||
sparse_trie.reveal_decoded_multiproof_v2(multiproof)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Build account leaf updates, split into removals and upserts (same reasoning
|
||||
// as for storage updates above).
|
||||
let mut account_removals: B256Map<LeafUpdate> = B256Map::default();
|
||||
let mut account_upserts: B256Map<LeafUpdate> = B256Map::default();
|
||||
for &hashed_address in state.accounts.keys().chain(state.storages.keys()) {
|
||||
if account_removals.contains_key(&hashed_address) ||
|
||||
account_upserts.contains_key(&hashed_address)
|
||||
{
|
||||
continue;
|
||||
}
|
||||
|
||||
let account = state
|
||||
.accounts
|
||||
@@ -229,149 +177,105 @@ where
|
||||
.ok_or(TrieWitnessError::MissingAccount(hashed_address))?
|
||||
.unwrap_or_default();
|
||||
|
||||
let storage_root =
|
||||
if let Some(storage_trie) = sparse_trie.storage_trie_mut(&hashed_address) {
|
||||
storage_trie.root()
|
||||
} else {
|
||||
self.account_storage_root(hashed_address)?
|
||||
};
|
||||
|
||||
if account.is_empty() && storage_root == EMPTY_ROOT_HASH {
|
||||
account_removals.insert(hashed_address, LeafUpdate::Changed(vec![]));
|
||||
} else {
|
||||
let mut rlp = Vec::with_capacity(TRIE_ACCOUNT_RLP_MAX_SIZE);
|
||||
account.into_trie_account(storage_root).encode(&mut rlp);
|
||||
account_upserts.insert(hashed_address, LeafUpdate::Changed(rlp));
|
||||
if !sparse_trie.update_account(hashed_address, account, &blinded_provider_factory)? {
|
||||
let nibbles = Nibbles::unpack(hashed_address);
|
||||
sparse_trie.remove_account_leaf(&nibbles, &blinded_provider_factory)?;
|
||||
}
|
||||
}
|
||||
|
||||
// Apply account removals first, then upserts, fetching additional proofs as needed.
|
||||
for account_updates in [&mut account_removals, &mut account_upserts] {
|
||||
loop {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
|
||||
sparse_trie
|
||||
.trie_mut()
|
||||
.update_leaves(account_updates, |key, min_len| {
|
||||
targets.account_targets.push(ProofV2Target::new(key).with_min_len(min_len));
|
||||
})
|
||||
.map_err(SparseStateTrieErrorKind::from)?;
|
||||
|
||||
if targets.is_empty() {
|
||||
break;
|
||||
}
|
||||
|
||||
let multiproof = Proof::new(
|
||||
self.trie_cursor_factory.clone(),
|
||||
self.hashed_cursor_factory.clone(),
|
||||
)
|
||||
.with_prefix_sets_mut(self.prefix_sets.clone())
|
||||
.multiproof_v2(targets)?;
|
||||
self.record_multiproof_nodes(&multiproof);
|
||||
sparse_trie.reveal_decoded_multiproof_v2(multiproof)?;
|
||||
while let Ok(node) = rx.try_recv() {
|
||||
self.witness.insert(keccak256(&node), node);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(self.witness)
|
||||
}
|
||||
|
||||
/// Record all nodes from a V2 decoded multiproof in the witness.
|
||||
fn record_multiproof_nodes(&mut self, multiproof: &DecodedMultiProofV2) {
|
||||
let mut encoded = Vec::new();
|
||||
for proof_node in &multiproof.account_proofs {
|
||||
self.record_witness_node(&proof_node.node, &mut encoded);
|
||||
}
|
||||
for proof_nodes in multiproof.storage_proofs.values() {
|
||||
for proof_node in proof_nodes {
|
||||
self.record_witness_node(&proof_node.node, &mut encoded);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Record a single [`TrieNodeV2`] in the witness.
|
||||
fn record_witness_node(&mut self, node: &TrieNodeV2, encoded: &mut Vec<u8>) {
|
||||
encoded.clear();
|
||||
node.encode(encoded);
|
||||
let bytes = Bytes::from(encoded.clone());
|
||||
self.witness.entry(keccak256(&bytes)).or_insert(bytes);
|
||||
|
||||
if let TrieNodeV2::Branch(branch) = node &&
|
||||
!branch.key.is_empty()
|
||||
{
|
||||
encoded.clear();
|
||||
BranchNodeRef::new(&branch.stack, branch.state_mask).encode(encoded);
|
||||
let bytes = Bytes::from(encoded.clone());
|
||||
self.witness.entry(keccak256(&bytes)).or_insert(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Compute the storage root for an account by walking the storage trie using the cursor
|
||||
/// factories and trie input prefix sets. Records the root node in the witness.
|
||||
fn account_storage_root(&mut self, hashed_address: B256) -> Result<B256, TrieWitnessError> {
|
||||
let storage_trie_cursor = self
|
||||
.trie_cursor_factory
|
||||
.storage_trie_cursor(hashed_address)
|
||||
.map_err(StateProofError::from)?;
|
||||
let hashed_storage_cursor = self
|
||||
.hashed_cursor_factory
|
||||
.hashed_storage_cursor(hashed_address)
|
||||
.map_err(StateProofError::from)?;
|
||||
let mut calculator = proof_v2::StorageProofCalculator::new_storage(
|
||||
storage_trie_cursor,
|
||||
hashed_storage_cursor,
|
||||
);
|
||||
if let Some(prefix_set) = self.prefix_sets.storage_prefix_sets.get(&hashed_address) {
|
||||
calculator = calculator.with_prefix_set(prefix_set.clone().freeze());
|
||||
}
|
||||
let root_node = calculator.storage_root_node(hashed_address)?;
|
||||
let root_hash = calculator
|
||||
.compute_root_hash(core::slice::from_ref(&root_node))?
|
||||
.unwrap_or(EMPTY_ROOT_HASH);
|
||||
drop(calculator);
|
||||
let mut encoded = Vec::new();
|
||||
self.record_witness_node(&root_node.node, &mut encoded);
|
||||
Ok(root_hash)
|
||||
}
|
||||
|
||||
/// Expand wiped storages into explicit zero-value entries for every existing slot in the
|
||||
/// database. After this, all storages can be treated uniformly without special wiped handling.
|
||||
fn expand_wiped_storages(&self, state: &mut HashedPostState) -> Result<(), StateProofError> {
|
||||
for (hashed_address, storage) in &mut state.storages {
|
||||
if !storage.wiped {
|
||||
continue;
|
||||
}
|
||||
let mut storage_cursor =
|
||||
self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?;
|
||||
let mut current_entry = storage_cursor.seek(B256::ZERO)?;
|
||||
while let Some((hashed_slot, _)) = current_entry {
|
||||
storage.storage.entry(hashed_slot).or_insert(U256::ZERO);
|
||||
current_entry = storage_cursor.next()?;
|
||||
}
|
||||
storage.wiped = false;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Retrieve proof targets for incoming hashed state.
|
||||
/// Aggregates all accounts and slots present in the state. Wiped storages must have been
|
||||
/// expanded via [`Self::expand_wiped_storages`] before calling this.
|
||||
fn get_proof_targets(state: &HashedPostState) -> MultiProofTargetsV2 {
|
||||
let mut targets = MultiProofTargetsV2::default();
|
||||
for &hashed_address in state.accounts.keys() {
|
||||
targets.account_targets.push(ProofV2Target::new(hashed_address));
|
||||
/// This method will aggregate all accounts and slots present in the hash state as well as
|
||||
/// select all existing slots from the database for the accounts that have been destroyed.
|
||||
fn get_proof_targets(
|
||||
&self,
|
||||
state: &HashedPostState,
|
||||
) -> Result<MultiProofTargets, StateProofError> {
|
||||
let mut proof_targets = MultiProofTargets::default();
|
||||
for hashed_address in state.accounts.keys() {
|
||||
proof_targets.insert(*hashed_address, B256Set::default());
|
||||
}
|
||||
for (&hashed_address, storage) in &state.storages {
|
||||
if !state.accounts.contains_key(&hashed_address) {
|
||||
targets.account_targets.push(ProofV2Target::new(hashed_address));
|
||||
for (hashed_address, storage) in &state.storages {
|
||||
let mut storage_keys = storage.storage.keys().copied().collect::<B256Set>();
|
||||
if storage.wiped {
|
||||
// storage for this account was destroyed, gather all slots from the current state
|
||||
let mut storage_cursor =
|
||||
self.hashed_cursor_factory.hashed_storage_cursor(*hashed_address)?;
|
||||
// position cursor at the start
|
||||
let mut current_entry = storage_cursor.seek(B256::ZERO)?;
|
||||
while let Some((hashed_slot, _)) = current_entry {
|
||||
storage_keys.insert(hashed_slot);
|
||||
current_entry = storage_cursor.next()?;
|
||||
}
|
||||
}
|
||||
// Skip accounts with no storage slot changes — an empty target set would produce
|
||||
// an empty proof vec which cannot be revealed (no root node).
|
||||
if storage.storage.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let storage_keys = storage.storage.keys().map(|k| ProofV2Target::new(*k)).collect();
|
||||
targets.storage_targets.insert(hashed_address, storage_keys);
|
||||
proof_targets.insert(*hashed_address, storage_keys);
|
||||
}
|
||||
targets
|
||||
Ok(proof_targets)
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone)]
|
||||
struct WitnessTrieNodeProviderFactory<F> {
|
||||
/// Trie node provider factory.
|
||||
provider_factory: F,
|
||||
/// Sender for forwarding fetched trie node.
|
||||
tx: mpsc::Sender<Bytes>,
|
||||
}
|
||||
|
||||
impl<F> WitnessTrieNodeProviderFactory<F> {
|
||||
const fn new(provider_factory: F, tx: mpsc::Sender<Bytes>) -> Self {
|
||||
Self { provider_factory, tx }
|
||||
}
|
||||
}
|
||||
|
||||
impl<F> TrieNodeProviderFactory for WitnessTrieNodeProviderFactory<F>
|
||||
where
|
||||
F: TrieNodeProviderFactory,
|
||||
F::AccountNodeProvider: TrieNodeProvider,
|
||||
F::StorageNodeProvider: TrieNodeProvider,
|
||||
{
|
||||
type AccountNodeProvider = WitnessTrieNodeProvider<F::AccountNodeProvider>;
|
||||
type StorageNodeProvider = WitnessTrieNodeProvider<F::StorageNodeProvider>;
|
||||
|
||||
fn account_node_provider(&self) -> Self::AccountNodeProvider {
|
||||
let provider = self.provider_factory.account_node_provider();
|
||||
WitnessTrieNodeProvider::new(provider, self.tx.clone())
|
||||
}
|
||||
|
||||
fn storage_node_provider(&self, account: B256) -> Self::StorageNodeProvider {
|
||||
let provider = self.provider_factory.storage_node_provider(account);
|
||||
WitnessTrieNodeProvider::new(provider, self.tx.clone())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug)]
|
||||
struct WitnessTrieNodeProvider<P> {
|
||||
/// Proof-based blinded.
|
||||
provider: P,
|
||||
/// Sender for forwarding fetched blinded node.
|
||||
tx: mpsc::Sender<Bytes>,
|
||||
}
|
||||
|
||||
impl<P> WitnessTrieNodeProvider<P> {
|
||||
const fn new(provider: P, tx: mpsc::Sender<Bytes>) -> Self {
|
||||
Self { provider, tx }
|
||||
}
|
||||
}
|
||||
|
||||
impl<P: TrieNodeProvider> TrieNodeProvider for WitnessTrieNodeProvider<P> {
|
||||
fn trie_node(&self, path: &Nibbles) -> Result<Option<RevealedNode>, SparseTrieError> {
|
||||
let maybe_node = self.provider.trie_node(path)?;
|
||||
if let Some(node) = &maybe_node {
|
||||
self.tx
|
||||
.send(node.node.clone())
|
||||
.map_err(|error| SparseTrieErrorKind::Other(Box::new(error)))?;
|
||||
}
|
||||
Ok(maybe_node)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -914,9 +914,6 @@ Pruning:
|
||||
--prune.bodies.before <BLOCK_NUMBER>
|
||||
Prune storage history before the specified block number. The specified block number is not pruned
|
||||
|
||||
--prune.minimum-distance <BLOCKS>
|
||||
Minimum pruning distance from the tip. This controls the safety margin for reorgs and manual unwinds
|
||||
|
||||
Engine:
|
||||
--engine.persistence-threshold <PERSISTENCE_THRESHOLD>
|
||||
Configure persistence threshold for the engine. This determines how many canonical blocks must be in-memory, ahead of the last persisted block, before flushing canonical blocks to disk again.
|
||||
|
||||
@@ -669,7 +669,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "avg(reth_storage_providers_database_save_blocks_batch_size{$instance_label=\"$instance\", quantile=\"$quantile\"})",
|
||||
"expr": "avg(reth_storage_providers_database_save_blocks_block_count{$instance_label=\"$instance\", quantile=\"$quantile\"})",
|
||||
"legendFormat": "Blocks per save_blocks call",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
@@ -1309,7 +1309,7 @@
|
||||
"uid": "${datasource}"
|
||||
},
|
||||
"editorMode": "code",
|
||||
"expr": "avg(reth_storage_providers_database_save_blocks_batch_size_last{$instance_label=\"$instance\"})",
|
||||
"expr": "avg(reth_storage_providers_database_save_blocks_block_count_last{$instance_label=\"$instance\"})",
|
||||
"legendFormat": "Blocks per save_blocks call",
|
||||
"range": true,
|
||||
"refId": "A"
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_evm::{
|
||||
block::{BlockExecutorFactory, BlockExecutorFor, ExecutableTx, GasOutput},
|
||||
block::{BlockExecutorFactory, BlockExecutorFor, ExecutableTx},
|
||||
eth::{EthBlockExecutionCtx, EthBlockExecutor, EthTxResult},
|
||||
precompiles::PrecompilesMap,
|
||||
revm::context::Block as _,
|
||||
@@ -211,10 +211,7 @@ where
|
||||
self.inner.execute_transaction_without_commit(tx)
|
||||
}
|
||||
|
||||
fn commit_transaction(
|
||||
&mut self,
|
||||
output: Self::Result,
|
||||
) -> Result<GasOutput, BlockExecutionError> {
|
||||
fn commit_transaction(&mut self, output: Self::Result) -> Result<u64, BlockExecutionError> {
|
||||
self.inner.commit_transaction(output)
|
||||
}
|
||||
|
||||
|
||||
@@ -110,7 +110,7 @@ pub fn prague_custom() -> &'static Precompiles {
|
||||
let precompile = Precompile::new(
|
||||
PrecompileId::custom("custom"),
|
||||
address!("0x0000000000000000000000000000000000000999"),
|
||||
|_, _| PrecompileResult::Ok(PrecompileOutput::new(0, 0, Bytes::new())),
|
||||
|_, _| PrecompileResult::Ok(PrecompileOutput::new(0, Bytes::new())),
|
||||
);
|
||||
precompiles.extend([precompile]);
|
||||
precompiles
|
||||
|
||||
@@ -1,149 +0,0 @@
|
||||
#!/usr/bin/env bash
|
||||
set -euo pipefail
|
||||
|
||||
usage() {
|
||||
echo "Usage: $0 [-n NUM_BLOCKS] [-b BLOCK] <remote_rpc_url>"
|
||||
echo ""
|
||||
echo "Fetches debug_executionWitness from both localhost:8545 and the"
|
||||
echo "given remote RPC, then compares them."
|
||||
echo ""
|
||||
echo "Options:"
|
||||
echo " -n NUM Compare the last NUM blocks (default: 20)"
|
||||
echo " -b BLOCK Compare only the given block number"
|
||||
exit 1
|
||||
}
|
||||
|
||||
NUM_BLOCKS=20
|
||||
SINGLE_BLOCK=""
|
||||
|
||||
while getopts "n:b:h" opt; do
|
||||
case "$opt" in
|
||||
n) NUM_BLOCKS="$OPTARG" ;;
|
||||
b) SINGLE_BLOCK="$OPTARG" ;;
|
||||
*) usage ;;
|
||||
esac
|
||||
done
|
||||
shift $((OPTIND - 1))
|
||||
|
||||
if [[ $# -lt 1 ]]; then
|
||||
echo "Error: remote_rpc_url is required" >&2
|
||||
usage
|
||||
fi
|
||||
REMOTE_RPC="$1"
|
||||
LOCAL_RPC="http://127.0.0.1:8545"
|
||||
|
||||
log() { echo "[$(date '+%H:%M:%S')] $*" >&2; }
|
||||
|
||||
if [[ -n "$SINGLE_BLOCK" ]]; then
|
||||
start=$SINGLE_BLOCK
|
||||
latest=$SINGLE_BLOCK
|
||||
log "Comparing block $SINGLE_BLOCK"
|
||||
else
|
||||
# Get latest block number from local node
|
||||
if ! latest=$(cast bn --rpc-url "$LOCAL_RPC" --rpc-timeout 600 2>&1); then
|
||||
log "FATAL: failed to get block number from local RPC: $latest"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
start=$((latest - NUM_BLOCKS + 1))
|
||||
if (( start < 0 )); then
|
||||
start=0
|
||||
fi
|
||||
|
||||
log "Comparing blocks $start..$latest ($NUM_BLOCKS blocks)"
|
||||
fi
|
||||
|
||||
errors=0
|
||||
|
||||
for (( block = start; block <= latest; block++ )); do
|
||||
block_hex=$(printf '"0x%x"' "$block")
|
||||
log "Checking block $block ($block_hex)"
|
||||
|
||||
# Fetch witness from both RPCs
|
||||
local_witness=$(cast rpc debug_executionWitness "$block_hex" --rpc-url "$LOCAL_RPC" --rpc-timeout 600 2>&1) || {
|
||||
log "WARN: failed to get witness from local RPC for block $block: $local_witness"
|
||||
((errors++)) || true
|
||||
continue
|
||||
}
|
||||
|
||||
remote_witness=$(cast rpc debug_executionWitness "$block_hex" --rpc-url "$REMOTE_RPC" --rpc-timeout 600 2>&1) || {
|
||||
log "WARN: failed to get witness from remote RPC for block $block: $remote_witness"
|
||||
((errors++)) || true
|
||||
continue
|
||||
}
|
||||
|
||||
# Normalize: sort all arrays of objects by a stable key so ordering doesn't cause false diffs
|
||||
normalize='walk(if type == "array" then sort_by(if type == "object" then (keys | join(",")) + ":" + (to_entries | map(.value | tostring) | join(",")) else tostring end) else . end) | . as $root | $root'
|
||||
local_file=$(mktemp)
|
||||
remote_file=$(mktemp)
|
||||
trap "rm -f '$local_file' '$remote_file'" EXIT
|
||||
echo "$local_witness" | jq -S "$normalize" > "$local_file"
|
||||
echo "$remote_witness" | jq -S "$normalize" > "$remote_file"
|
||||
|
||||
# Compare: for "state", local may contain extra nodes (superset OK).
|
||||
# For "codes", "keys", "headers", require exact set equality.
|
||||
has_error=false
|
||||
|
||||
# Check exact-match fields (as sorted sets)
|
||||
for field in codes keys headers; do
|
||||
local_set=$(jq -r --arg f "$field" '.[$f] // [] | sort | .[]' "$local_file")
|
||||
remote_set=$(jq -r --arg f "$field" '.[$f] // [] | sort | .[]' "$remote_file")
|
||||
if [[ "$local_set" != "$remote_set" ]]; then
|
||||
log "ERROR: block $block field '$field' differs"
|
||||
diff <(echo "$remote_set") <(echo "$local_set") | head -30 || true
|
||||
has_error=true
|
||||
fi
|
||||
done
|
||||
|
||||
# Check state: every remote node must be present in local (extras OK)
|
||||
missing=$(jq -r -n \
|
||||
--slurpfile l "$local_file" \
|
||||
--slurpfile r "$remote_file" \
|
||||
'($l[0].state // [] | map({(.):true}) | add // {}) as $local_set |
|
||||
[$r[0].state // [] | .[] | select($local_set[.] | not)] |
|
||||
if length == 0 then empty else .[] end')
|
||||
if [[ -n "$missing" ]]; then
|
||||
n_missing=$(echo "$missing" | wc -l)
|
||||
log "ERROR: block $block state has $n_missing missing node(s) (present in remote, absent in local):"
|
||||
echo "$missing" | head -20
|
||||
has_error=true
|
||||
fi
|
||||
|
||||
extra=$(jq -r -n \
|
||||
--slurpfile l "$local_file" \
|
||||
--slurpfile r "$remote_file" \
|
||||
'($r[0].state // [] | map({(.):true}) | add // {}) as $remote_set |
|
||||
[$l[0].state // [] | .[] | select($remote_set[.] | not)] |
|
||||
if length == 0 then empty else .[] end')
|
||||
n_extra=0
|
||||
if [[ -n "$extra" ]]; then
|
||||
n_extra=$(echo "$extra" | wc -l)
|
||||
fi
|
||||
|
||||
if ! $has_error; then
|
||||
if [[ $n_extra -gt 0 ]]; then
|
||||
log "OK: block $block witnesses match ($n_extra extra state node(s) in local)"
|
||||
else
|
||||
log "OK: block $block witnesses match exactly"
|
||||
fi
|
||||
else
|
||||
cp "$local_file" "witness-local-${block}.json"
|
||||
cp "$remote_file" "witness-remote-${block}.json"
|
||||
log "Wrote witness-local-${block}.json and witness-remote-${block}.json"
|
||||
if [[ $n_extra -gt 0 ]]; then
|
||||
log " (local also has $n_extra extra state node(s))"
|
||||
fi
|
||||
((errors++)) || true
|
||||
log "---"
|
||||
fi
|
||||
|
||||
rm -f "$local_file" "$remote_file"
|
||||
done
|
||||
|
||||
total=$((latest - start + 1))
|
||||
if (( errors > 0 )); then
|
||||
log "DONE: $errors block(s) had errors out of $total"
|
||||
exit 1
|
||||
else
|
||||
log "DONE: all $total block(s) matched"
|
||||
fi
|
||||
Reference in New Issue
Block a user