mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-02-19 03:04:27 -05:00
b49cadb346
Co-authored-by: Georgios Konstantopoulos <me@gakonst.com> Co-authored-by: Amp <amp@ampcode.com>
472 lines
19 KiB
YAML
472 lines
19 KiB
YAML
# Runs benchmarks.
|
|
#
|
|
# The reth-bench job replays real blocks via the Engine API against a reth node
|
|
# backed by a local snapshot managed with schelk.
|
|
#
|
|
# It runs the baseline binary and the feature (candidate) binary on the
|
|
# same block range (snapshot recovered between runs) to compare performance.
|
|
|
|
on:
|
|
# TODO: Disabled temporarily for https://github.com/CodSpeedHQ/runner/issues/55
|
|
# merge_group:
|
|
push:
|
|
branches: [main]
|
|
issue_comment:
|
|
types: [created, edited]
|
|
workflow_dispatch:
|
|
inputs:
|
|
blocks:
|
|
description: "Number of blocks to benchmark"
|
|
required: false
|
|
default: "50"
|
|
type: string
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
BASELINE: base
|
|
SEED: reth
|
|
RUSTC_WRAPPER: "sccache"
|
|
|
|
name: bench
|
|
|
|
permissions:
|
|
contents: write
|
|
pull-requests: write
|
|
|
|
concurrency:
|
|
group: bench-${{ github.workflow }}-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
|
|
jobs:
|
|
codspeed:
|
|
if: github.event_name != 'issue_comment'
|
|
runs-on: depot-ubuntu-latest
|
|
strategy:
|
|
matrix:
|
|
partition: [1, 2]
|
|
total_partitions: [2]
|
|
include:
|
|
- partition: 1
|
|
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
|
|
- partition: 2
|
|
crates: "-p reth-trie"
|
|
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: true
|
|
ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/merge', github.event.issue.number) || '' }}
|
|
- uses: rui314/setup-mold@v1
|
|
- uses: dtolnay/rust-toolchain@stable
|
|
- uses: mozilla-actions/sccache-action@v0.0.9
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
cache-on-failure: true
|
|
- name: Install cargo-codspeed
|
|
uses: taiki-e/install-action@v2
|
|
with:
|
|
tool: cargo-codspeed
|
|
- name: Build the benchmark target(s)
|
|
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
|
|
- name: Run the benchmarks
|
|
uses: CodSpeedHQ/action@v4
|
|
with:
|
|
run: cargo codspeed run ${{ matrix.crates }}
|
|
mode: instrumentation
|
|
token: ${{ secrets.CODSPEED_TOKEN }}
|
|
|
|
reth-bench:
|
|
if: github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, 'derek bench')
|
|
name: reth-bench
|
|
runs-on: [self-hosted, Linux, X64]
|
|
timeout-minutes: 120
|
|
env:
|
|
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
|
SCHELK_MOUNT: /reth-bench
|
|
steps:
|
|
- name: Check org membership
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
script: |
|
|
const user = context.payload.comment.user.login;
|
|
try {
|
|
const { status } = await github.rest.orgs.checkMembershipForUser({
|
|
org: 'paradigmxyz',
|
|
username: user,
|
|
});
|
|
if (status !== 204 && status !== 302) {
|
|
core.setFailed(`@${user} is not a member of paradigmxyz`);
|
|
}
|
|
} catch (e) {
|
|
core.setFailed(`@${user} is not a member of paradigmxyz`);
|
|
}
|
|
|
|
- name: Parse arguments
|
|
id: args
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const body = context.payload.comment.body.trim();
|
|
const intArgs = new Set(['blocks', 'warmup']);
|
|
const refArgs = new Set(['baseline', 'feature']);
|
|
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '' };
|
|
const unknown = [];
|
|
const invalid = [];
|
|
const args = body.replace(/^derek bench\s*/, '');
|
|
for (const part of args.split(/\s+/).filter(Boolean)) {
|
|
const eq = part.indexOf('=');
|
|
if (eq === -1) {
|
|
unknown.push(part);
|
|
continue;
|
|
}
|
|
const key = part.slice(0, eq);
|
|
const value = part.slice(eq + 1);
|
|
if (intArgs.has(key)) {
|
|
if (!/^\d+$/.test(value)) {
|
|
invalid.push(`\`${key}=${value}\` (must be a positive integer)`);
|
|
} else {
|
|
defaults[key] = value;
|
|
}
|
|
} else if (refArgs.has(key)) {
|
|
if (!value) {
|
|
invalid.push(`\`${key}=\` (must be a git ref)`);
|
|
} else {
|
|
defaults[key] = value;
|
|
}
|
|
} else {
|
|
unknown.push(key);
|
|
}
|
|
}
|
|
const errors = [];
|
|
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
|
|
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
|
|
if (errors.length) {
|
|
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`derek bench [blocks=N] [warmup=N] [baseline=REF] [feature=REF]\``;
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body: msg,
|
|
});
|
|
core.setFailed(msg);
|
|
return;
|
|
}
|
|
core.setOutput('blocks', defaults.blocks);
|
|
core.setOutput('warmup', defaults.warmup);
|
|
core.setOutput('baseline', defaults.baseline);
|
|
core.setOutput('feature', defaults.feature);
|
|
core.exportVariable('BENCH_BLOCKS', defaults.blocks);
|
|
core.exportVariable('BENCH_WARMUP_BLOCKS', defaults.warmup);
|
|
|
|
- name: Acknowledge request
|
|
id: ack
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
await github.rest.reactions.createForIssueComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: context.payload.comment.id,
|
|
content: 'eyes',
|
|
});
|
|
|
|
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
const blocks = '${{ steps.args.outputs.blocks }}';
|
|
const warmup = '${{ steps.args.outputs.warmup }}';
|
|
const baseline = '${{ steps.args.outputs.baseline }}' || 'merge-base';
|
|
const feature = '${{ steps.args.outputs.feature }}' || 'PR head';
|
|
const { data: comment } = await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body: `🚀 Benchmark started! [View run](${runUrl})\n\n⏳ **Status:** Building binaries...\n\n**Config:** ${blocks} blocks, ${warmup} warmup blocks, baseline: \`${baseline}\`, feature: \`${feature}\``,
|
|
});
|
|
core.setOutput('comment-id', comment.id);
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 0
|
|
ref: ${{ format('refs/pull/{0}/merge', github.event.issue.number) }}
|
|
|
|
- uses: dtolnay/rust-toolchain@stable
|
|
- uses: mozilla-actions/sccache-action@v0.0.9
|
|
continue-on-error: true
|
|
|
|
# Verify all required tools are available
|
|
- name: Check dependencies
|
|
run: |
|
|
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
|
missing=()
|
|
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv; do
|
|
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
|
done
|
|
if [ ${#missing[@]} -gt 0 ]; then
|
|
echo "::error::Missing required tools: ${missing[*]}"
|
|
exit 1
|
|
fi
|
|
echo "All dependencies found"
|
|
|
|
# Build binaries
|
|
- name: Resolve PR head branch
|
|
id: pr-info
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const { data: pr } = await github.rest.pulls.get({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
pull_number: context.issue.number,
|
|
});
|
|
core.setOutput('head-ref', pr.head.ref);
|
|
core.setOutput('head-sha', pr.head.sha);
|
|
|
|
- name: Resolve refs
|
|
id: refs
|
|
run: |
|
|
BASELINE_ARG="${{ steps.args.outputs.baseline }}"
|
|
FEATURE_ARG="${{ steps.args.outputs.feature }}"
|
|
|
|
if [ -n "$BASELINE_ARG" ]; then
|
|
git fetch origin "$BASELINE_ARG" --quiet 2>/dev/null || true
|
|
BASELINE_REF=$(git rev-parse "$BASELINE_ARG" 2>/dev/null || git rev-parse "origin/$BASELINE_ARG" 2>/dev/null)
|
|
BASELINE_NAME="$BASELINE_ARG"
|
|
else
|
|
BASELINE_REF=$(git merge-base HEAD origin/main 2>/dev/null || echo "${{ github.sha }}")
|
|
BASELINE_NAME="main"
|
|
fi
|
|
|
|
if [ -n "$FEATURE_ARG" ]; then
|
|
git fetch origin "$FEATURE_ARG" --quiet 2>/dev/null || true
|
|
FEATURE_REF=$(git rev-parse "$FEATURE_ARG" 2>/dev/null || git rev-parse "origin/$FEATURE_ARG" 2>/dev/null)
|
|
FEATURE_NAME="$FEATURE_ARG"
|
|
else
|
|
FEATURE_REF="${{ steps.pr-info.outputs.head-sha }}"
|
|
FEATURE_NAME="${{ steps.pr-info.outputs.head-ref }}"
|
|
fi
|
|
|
|
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
|
|
echo "baseline-name=$BASELINE_NAME" >> "$GITHUB_OUTPUT"
|
|
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
|
echo "feature-name=$FEATURE_NAME" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Fetch or build baseline binaries
|
|
run: .github/scripts/bench-reth-build.sh baseline "${{ steps.refs.outputs.baseline-ref }}"
|
|
- name: Fetch or build feature binaries
|
|
run: .github/scripts/bench-reth-build.sh feature "${{ steps.refs.outputs.feature-ref }}"
|
|
|
|
# System tuning for reproducible benchmarks
|
|
- name: System setup
|
|
run: |
|
|
sudo cpupower frequency-set -g performance || true
|
|
# Disable turbo boost (Intel and AMD paths)
|
|
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
|
|
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
|
|
sudo swapoff -a || true
|
|
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
|
|
# Disable SMT (hyperthreading)
|
|
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
|
|
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
|
|
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
|
|
if [ "$current" != "$first" ]; then
|
|
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
|
|
fi
|
|
done
|
|
echo "Online CPUs: $(nproc)"
|
|
# Disable transparent huge pages (compaction causes latency spikes)
|
|
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
|
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
|
done || true
|
|
# Prevent deep C-states (avoids wake-up latency jitter)
|
|
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
|
# Move all IRQs to core 0 (housekeeping core)
|
|
for irq in /proc/irq/*/smp_affinity_list; do
|
|
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
|
done
|
|
# Stop noisy background services
|
|
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
|
|
# Log environment for reproducibility
|
|
echo "=== Benchmark environment ==="
|
|
uname -r
|
|
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
|
|
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
|
|
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
|
|
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
|
|
free -h
|
|
|
|
# Clean up any leftover state
|
|
- name: Pre-flight cleanup
|
|
run: |
|
|
pkill -9 reth || true
|
|
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
|
|
|
|
- name: Update status (running benchmarks)
|
|
if: steps.ack.outputs.comment-id
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: ${{ steps.ack.outputs.comment-id || 0 }},
|
|
body: `🚀 Benchmark started! [View run](${runUrl})\n\n⏳ **Status:** Running benchmarks (2 runs)...`,
|
|
});
|
|
|
|
- name: "Run benchmark: baseline"
|
|
run: taskset -c 0 .github/scripts/bench-reth-run.sh baseline target/profiling-baseline/reth /tmp/bench-results-baseline
|
|
|
|
- name: "Run benchmark: feature"
|
|
run: taskset -c 0 .github/scripts/bench-reth-run.sh feature target/profiling/reth /tmp/bench-results-feature
|
|
|
|
# Results & charts
|
|
- name: Parse results
|
|
id: results
|
|
if: success()
|
|
env:
|
|
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
|
|
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
|
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
|
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
|
|
run: |
|
|
git fetch origin "${BASELINE_NAME}" --quiet 2>/dev/null || true
|
|
BASELINE_HEAD=$(git rev-parse "origin/${BASELINE_NAME}" 2>/dev/null || echo "")
|
|
BEHIND_BASELINE=0
|
|
if [ -n "$BASELINE_HEAD" ] && [ "$BASELINE_REF" != "$BASELINE_HEAD" ]; then
|
|
BEHIND_BASELINE=$(git rev-list --count "${BASELINE_REF}..${BASELINE_HEAD}" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
SUMMARY_ARGS="--output-summary /tmp/bench-summary.json"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown /tmp/bench-comment.md"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv /tmp/bench-results-baseline/combined_latency.csv"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv /tmp/bench-results-feature/combined_latency.csv"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv /tmp/bench-results-feature/total_gas.csv"
|
|
if [ "$BEHIND_BASELINE" -gt 0 ]; then
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --behind-baseline $BEHIND_BASELINE"
|
|
fi
|
|
# shellcheck disable=SC2086
|
|
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
|
|
|
|
- name: Generate charts
|
|
if: success()
|
|
env:
|
|
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
|
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
|
run: |
|
|
CHART_ARGS="/tmp/bench-results-feature/combined_latency.csv --output-dir /tmp/bench-charts"
|
|
CHART_ARGS="$CHART_ARGS --baseline /tmp/bench-results-baseline/combined_latency.csv"
|
|
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
|
|
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
|
|
# shellcheck disable=SC2086
|
|
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
|
|
|
|
- name: Upload results
|
|
if: success()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: bench-reth-results
|
|
path: |
|
|
/tmp/bench-results-baseline/
|
|
/tmp/bench-results-feature/
|
|
/tmp/bench-summary.json
|
|
/tmp/bench-charts/
|
|
|
|
- name: Push charts
|
|
id: push-charts
|
|
if: success()
|
|
run: |
|
|
PR_NUMBER=${{ github.event.issue.number }}
|
|
RUN_ID=${{ github.run_id }}
|
|
CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}"
|
|
|
|
if git fetch origin bench-charts 2>/dev/null; then
|
|
git checkout bench-charts
|
|
else
|
|
git checkout --orphan bench-charts
|
|
git rm -rf . 2>/dev/null || true
|
|
fi
|
|
|
|
mkdir -p "${CHART_DIR}"
|
|
cp /tmp/bench-charts/*.png "${CHART_DIR}/"
|
|
git add "${CHART_DIR}"
|
|
git -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
|
commit -m "bench charts for PR #${PR_NUMBER} run ${RUN_ID}"
|
|
git push origin bench-charts
|
|
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Compare & comment
|
|
if: success()
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
|
|
let comment = '';
|
|
try {
|
|
comment = fs.readFileSync('/tmp/bench-comment.md', 'utf8');
|
|
} catch (e) {
|
|
comment = '⚠️ Engine benchmark completed but failed to generate comparison.';
|
|
}
|
|
|
|
const sha = '${{ steps.push-charts.outputs.sha }}';
|
|
const prNumber = context.issue.number;
|
|
const runId = '${{ github.run_id }}';
|
|
const baseUrl = `https://raw.githubusercontent.com/${context.repo.owner}/${context.repo.repo}/${sha}/pr/${prNumber}/${runId}`;
|
|
|
|
const charts = [
|
|
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
|
|
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
|
|
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
|
|
];
|
|
|
|
let chartMarkdown = '\n\n### Charts\n\n';
|
|
for (const chart of charts) {
|
|
chartMarkdown += `<details><summary>${chart.label}</summary>\n\n`;
|
|
chartMarkdown += `\n\n`;
|
|
chartMarkdown += `</details>\n\n`;
|
|
}
|
|
|
|
comment += chartMarkdown;
|
|
|
|
const requestedBy = '${{ github.event.comment.user.login }}';
|
|
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
const body = `cc @${requestedBy}\n\n✅ Benchmark complete! [View run](${runUrl})\n\n${comment}`;
|
|
const ackCommentId = '${{ steps.ack.outputs.comment-id }}';
|
|
|
|
if (ackCommentId) {
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: parseInt(ackCommentId),
|
|
body,
|
|
});
|
|
} else {
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body,
|
|
});
|
|
}
|
|
|
|
- name: Upload node log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: reth-node-log
|
|
path: |
|
|
/tmp/reth-bench-node-baseline.log
|
|
/tmp/reth-bench-node-feature.log
|
|
|
|
- name: Restore system settings
|
|
if: always()
|
|
run: |
|
|
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|