mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-02-19 03:04:27 -05:00
739 lines
31 KiB
YAML
739 lines
31 KiB
YAML
# Runs benchmarks.
|
|
#
|
|
# The reth-bench job replays real blocks via the Engine API against a reth node
|
|
# backed by a local snapshot managed with schelk.
|
|
#
|
|
# It runs the baseline binary and the feature (candidate) binary on the
|
|
# same block range (snapshot recovered between runs) to compare performance.
|
|
|
|
on:
|
|
# TODO: Disabled temporarily for https://github.com/CodSpeedHQ/runner/issues/55
|
|
# merge_group:
|
|
push:
|
|
branches: [main]
|
|
issue_comment:
|
|
types: [created, edited]
|
|
workflow_dispatch:
|
|
inputs:
|
|
blocks:
|
|
description: "Number of blocks to benchmark"
|
|
required: false
|
|
default: "500"
|
|
type: string
|
|
warmup:
|
|
description: "Number of warmup blocks"
|
|
required: false
|
|
default: "100"
|
|
type: string
|
|
baseline:
|
|
description: "Baseline git ref (default: merge-base)"
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
feature:
|
|
description: "Feature git ref (default: branch head)"
|
|
required: false
|
|
default: ""
|
|
type: string
|
|
|
|
env:
|
|
CARGO_TERM_COLOR: always
|
|
BASELINE: base
|
|
SEED: reth
|
|
RUSTC_WRAPPER: "sccache"
|
|
|
|
name: bench
|
|
|
|
permissions:
|
|
contents: write
|
|
pull-requests: write
|
|
|
|
jobs:
|
|
codspeed:
|
|
if: github.event_name == 'push'
|
|
runs-on: depot-ubuntu-latest
|
|
concurrency:
|
|
group: bench-codspeed-${{ github.head_ref || github.run_id }}
|
|
cancel-in-progress: true
|
|
strategy:
|
|
matrix:
|
|
partition: [1, 2]
|
|
total_partitions: [2]
|
|
include:
|
|
- partition: 1
|
|
crates: "-p reth-primitives -p reth-trie-common -p reth-trie-sparse"
|
|
- partition: 2
|
|
crates: "-p reth-trie"
|
|
name: codspeed (${{ matrix.partition }}/${{ matrix.total_partitions }})
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: true
|
|
ref: ${{ github.event_name == 'issue_comment' && format('refs/pull/{0}/merge', github.event.issue.number) || '' }}
|
|
- uses: rui314/setup-mold@v1
|
|
- uses: dtolnay/rust-toolchain@stable
|
|
- uses: mozilla-actions/sccache-action@v0.0.9
|
|
- uses: Swatinem/rust-cache@v2
|
|
with:
|
|
cache-on-failure: true
|
|
- name: Install cargo-codspeed
|
|
uses: taiki-e/install-action@v2
|
|
with:
|
|
tool: cargo-codspeed
|
|
- name: Build the benchmark target(s)
|
|
run: cargo codspeed build --profile profiling --features test-utils ${{ matrix.crates }}
|
|
- name: Run the benchmarks
|
|
uses: CodSpeedHQ/action@v4
|
|
with:
|
|
run: cargo codspeed run ${{ matrix.crates }}
|
|
mode: instrumentation
|
|
token: ${{ secrets.CODSPEED_TOKEN }}
|
|
|
|
reth-bench-ack:
|
|
if: |
|
|
(github.event_name == 'issue_comment' && github.event.issue.pull_request && startsWith(github.event.comment.body, 'derek bench')) ||
|
|
github.event_name == 'workflow_dispatch'
|
|
name: reth-bench-ack
|
|
runs-on: ubuntu-latest
|
|
outputs:
|
|
pr: ${{ steps.args.outputs.pr }}
|
|
actor: ${{ steps.args.outputs.actor }}
|
|
blocks: ${{ steps.args.outputs.blocks }}
|
|
warmup: ${{ steps.args.outputs.warmup }}
|
|
baseline: ${{ steps.args.outputs.baseline }}
|
|
feature: ${{ steps.args.outputs.feature }}
|
|
baseline-name: ${{ steps.args.outputs.baseline-name }}
|
|
feature-name: ${{ steps.args.outputs.feature-name }}
|
|
comment-id: ${{ steps.ack.outputs.comment-id }}
|
|
steps:
|
|
- name: Check org membership
|
|
if: github.event_name == 'issue_comment'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
github-token: ${{ secrets.GITHUB_TOKEN }}
|
|
script: |
|
|
const user = context.payload.comment.user.login;
|
|
try {
|
|
const { status } = await github.rest.orgs.checkMembershipForUser({
|
|
org: 'paradigmxyz',
|
|
username: user,
|
|
});
|
|
if (status !== 204 && status !== 302) {
|
|
core.setFailed(`@${user} is not a member of paradigmxyz`);
|
|
}
|
|
} catch (e) {
|
|
core.setFailed(`@${user} is not a member of paradigmxyz`);
|
|
}
|
|
|
|
- name: Parse arguments
|
|
id: args
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
let pr, actor, blocks, warmup, baseline, feature;
|
|
|
|
if (context.eventName === 'workflow_dispatch') {
|
|
actor = '${{ github.actor }}';
|
|
blocks = '${{ github.event.inputs.blocks }}' || '500';
|
|
warmup = '${{ github.event.inputs.warmup }}' || '100';
|
|
baseline = '${{ github.event.inputs.baseline }}';
|
|
feature = '${{ github.event.inputs.feature }}';
|
|
|
|
// Find PR for the selected branch
|
|
const branch = '${{ github.ref_name }}';
|
|
const { data: prs } = await github.rest.pulls.list({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
head: `${context.repo.owner}:${branch}`,
|
|
state: 'open',
|
|
per_page: 1,
|
|
});
|
|
pr = prs.length ? String(prs[0].number) : '';
|
|
if (!pr) {
|
|
core.info(`No open PR found for branch '${branch}', results will be in job summary`);
|
|
}
|
|
} else {
|
|
pr = String(context.issue.number);
|
|
actor = context.payload.comment.user.login;
|
|
|
|
const body = context.payload.comment.body.trim();
|
|
const intArgs = new Set(['blocks', 'warmup']);
|
|
const refArgs = new Set(['baseline', 'feature']);
|
|
const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '' };
|
|
const unknown = [];
|
|
const invalid = [];
|
|
const args = body.replace(/^derek bench\s*/, '');
|
|
for (const part of args.split(/\s+/).filter(Boolean)) {
|
|
const eq = part.indexOf('=');
|
|
if (eq === -1) {
|
|
unknown.push(part);
|
|
continue;
|
|
}
|
|
const key = part.slice(0, eq);
|
|
const value = part.slice(eq + 1);
|
|
if (intArgs.has(key)) {
|
|
if (!/^\d+$/.test(value)) {
|
|
invalid.push(`\`${key}=${value}\` (must be a positive integer)`);
|
|
} else {
|
|
defaults[key] = value;
|
|
}
|
|
} else if (refArgs.has(key)) {
|
|
if (!value) {
|
|
invalid.push(`\`${key}=\` (must be a git ref)`);
|
|
} else {
|
|
defaults[key] = value;
|
|
}
|
|
} else {
|
|
unknown.push(key);
|
|
}
|
|
}
|
|
const errors = [];
|
|
if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
|
|
if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
|
|
if (errors.length) {
|
|
const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`derek bench [blocks=N] [warmup=N] [baseline=REF] [feature=REF]\``;
|
|
await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: context.issue.number,
|
|
body: msg,
|
|
});
|
|
core.setFailed(msg);
|
|
return;
|
|
}
|
|
blocks = defaults.blocks;
|
|
warmup = defaults.warmup;
|
|
baseline = defaults.baseline;
|
|
feature = defaults.feature;
|
|
}
|
|
|
|
// Resolve display names for baseline/feature
|
|
let baselineName = baseline || 'main';
|
|
let featureName = feature;
|
|
if (!featureName) {
|
|
if (pr) {
|
|
const { data: prData } = await github.rest.pulls.get({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
pull_number: parseInt(pr),
|
|
});
|
|
featureName = prData.head.ref;
|
|
} else {
|
|
featureName = '${{ github.ref_name }}';
|
|
}
|
|
}
|
|
|
|
core.setOutput('pr', pr || '');
|
|
core.setOutput('actor', actor);
|
|
core.setOutput('blocks', blocks);
|
|
core.setOutput('warmup', warmup);
|
|
core.setOutput('baseline', baseline);
|
|
core.setOutput('feature', feature);
|
|
core.setOutput('baseline-name', baselineName);
|
|
core.setOutput('feature-name', featureName);
|
|
|
|
- name: Acknowledge request
|
|
id: ack
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
if (context.eventName === 'issue_comment') {
|
|
await github.rest.reactions.createForIssueComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: context.payload.comment.id,
|
|
content: 'eyes',
|
|
});
|
|
}
|
|
|
|
const pr = '${{ steps.args.outputs.pr }}';
|
|
if (!pr) return;
|
|
|
|
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
|
|
// Count queued/waiting bench runs ahead of this one
|
|
let queueMsg = '';
|
|
let ahead = 0;
|
|
try {
|
|
const statuses = ['queued', 'in_progress', 'waiting', 'requested', 'pending'];
|
|
const allRuns = [];
|
|
for (const status of statuses) {
|
|
const { data: { workflow_runs: r } } = await github.rest.actions.listWorkflowRuns({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
workflow_id: 'bench.yml',
|
|
status,
|
|
per_page: 100,
|
|
});
|
|
allRuns.push(...r);
|
|
}
|
|
// Only count runs that trigger reth-bench (not push-triggered codspeed runs)
|
|
const benchRuns = allRuns.filter(r => r.event === 'issue_comment' || r.event === 'workflow_dispatch');
|
|
const thisRun = benchRuns.find(r => r.id === context.runId);
|
|
const thisCreatedAt = thisRun ? new Date(thisRun.created_at) : new Date();
|
|
ahead = benchRuns.filter(r => r.id !== context.runId && new Date(r.created_at) <= thisCreatedAt).length;
|
|
if (ahead > 0) {
|
|
queueMsg = `\n🔢 **Queue position:** \`#${ahead + 1}\` (${ahead} job(s) ahead)`;
|
|
}
|
|
} catch (e) {
|
|
// Non-fatal — queue info is best-effort
|
|
core.info(`Could not fetch queue info: ${e.message}`);
|
|
}
|
|
|
|
const actor = '${{ steps.args.outputs.actor }}';
|
|
const blocks = '${{ steps.args.outputs.blocks }}';
|
|
const warmup = '${{ steps.args.outputs.warmup }}';
|
|
const baseline = '${{ steps.args.outputs.baseline-name }}';
|
|
const feature = '${{ steps.args.outputs.feature-name }}';
|
|
const config = `**Config:** ${blocks} blocks, ${warmup} warmup blocks, baseline: \`${baseline}\`, feature: \`${feature}\``;
|
|
|
|
const { data: comment } = await github.rest.issues.createComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
issue_number: parseInt(pr),
|
|
body: `cc @${actor}\n\n🚀 Benchmark queued! [View run](${runUrl})\n\n⏳ **Status:** Waiting for runner...${queueMsg}\n\n${config}`,
|
|
});
|
|
core.setOutput('comment-id', String(comment.id));
|
|
core.setOutput('queue-position', String(ahead || 0));
|
|
|
|
- name: Poll queue position
|
|
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const pr = '${{ steps.args.outputs.pr }}';
|
|
const commentId = parseInt('${{ steps.ack.outputs.comment-id }}');
|
|
const actor = '${{ steps.args.outputs.actor }}';
|
|
const blocks = '${{ steps.args.outputs.blocks }}';
|
|
const warmup = '${{ steps.args.outputs.warmup }}';
|
|
const baseline = '${{ steps.args.outputs.baseline-name }}';
|
|
const feature = '${{ steps.args.outputs.feature-name }}';
|
|
const config = `**Config:** ${blocks} blocks, ${warmup} warmup blocks, baseline: \`${baseline}\`, feature: \`${feature}\``;
|
|
const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
|
|
async function getQueuePosition() {
|
|
const statuses = ['queued', 'in_progress', 'waiting', 'requested', 'pending'];
|
|
const allRuns = [];
|
|
for (const status of statuses) {
|
|
const { data: { workflow_runs: r } } = await github.rest.actions.listWorkflowRuns({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
workflow_id: 'bench.yml',
|
|
status,
|
|
per_page: 100,
|
|
});
|
|
allRuns.push(...r);
|
|
}
|
|
const benchRuns = allRuns.filter(r => r.event === 'issue_comment' || r.event === 'workflow_dispatch');
|
|
const thisRun = benchRuns.find(r => r.id === context.runId);
|
|
const thisCreatedAt = thisRun ? new Date(thisRun.created_at) : new Date();
|
|
return benchRuns.filter(r => r.id !== context.runId && new Date(r.created_at) <= thisCreatedAt).length;
|
|
}
|
|
|
|
let lastPosition = parseInt('${{ steps.ack.outputs.queue-position }}');
|
|
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
|
|
|
while (true) {
|
|
await sleep(10_000);
|
|
try {
|
|
const ahead = await getQueuePosition();
|
|
if (ahead !== lastPosition) {
|
|
lastPosition = ahead;
|
|
const queueMsg = ahead > 0
|
|
? `\n🔢 **Queue position:** \`#${ahead + 1}\` (${ahead} job(s) ahead)`
|
|
: '';
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: commentId,
|
|
body: `cc @${actor}\n\n🚀 Benchmark queued! [View run](${runUrl})\n\n⏳ **Status:** Waiting for runner...${queueMsg}\n\n${config}`,
|
|
});
|
|
}
|
|
if (ahead === 0) break;
|
|
} catch (e) {
|
|
core.info(`Queue poll error: ${e.message}`);
|
|
}
|
|
}
|
|
|
|
reth-bench:
|
|
needs: reth-bench-ack
|
|
name: reth-bench
|
|
runs-on: [self-hosted, Linux, X64]
|
|
timeout-minutes: 120
|
|
concurrency:
|
|
group: reth-bench-queue
|
|
cancel-in-progress: false
|
|
env:
|
|
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
|
|
SCHELK_MOUNT: /reth-bench
|
|
BENCH_PR: ${{ needs.reth-bench-ack.outputs.pr }}
|
|
BENCH_ACTOR: ${{ needs.reth-bench-ack.outputs.actor }}
|
|
BENCH_BLOCKS: ${{ needs.reth-bench-ack.outputs.blocks }}
|
|
BENCH_WARMUP_BLOCKS: ${{ needs.reth-bench-ack.outputs.warmup }}
|
|
BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
|
|
steps:
|
|
- uses: actions/checkout@v6
|
|
with:
|
|
submodules: true
|
|
fetch-depth: 0
|
|
ref: ${{ env.BENCH_PR && format('refs/pull/{0}/merge', env.BENCH_PR) || github.ref }}
|
|
|
|
- name: Resolve job URL and update status
|
|
if: env.BENCH_COMMENT_ID
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
run_id: context.runId,
|
|
});
|
|
const job = jobs.jobs.find(j => j.name === 'reth-bench');
|
|
const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
core.exportVariable('BENCH_JOB_URL', jobUrl);
|
|
|
|
const blocks = process.env.BENCH_BLOCKS;
|
|
const warmup = process.env.BENCH_WARMUP_BLOCKS;
|
|
const baseline = '${{ needs.reth-bench-ack.outputs.baseline-name }}';
|
|
const feature = '${{ needs.reth-bench-ack.outputs.feature-name }}';
|
|
core.exportVariable('BENCH_CONFIG', `**Config:** ${blocks} blocks, ${warmup} warmup blocks, baseline: \`${baseline}\`, feature: \`${feature}\``);
|
|
|
|
const { buildBody } = require('./.github/scripts/bench-update-status.js');
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
|
|
body: buildBody('Building binaries...'),
|
|
});
|
|
|
|
- uses: dtolnay/rust-toolchain@stable
|
|
- uses: mozilla-actions/sccache-action@v0.0.9
|
|
continue-on-error: true
|
|
|
|
# Verify all required tools are available
|
|
- name: Check dependencies
|
|
run: |
|
|
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
|
|
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
|
|
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
|
|
missing=()
|
|
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv; do
|
|
command -v "$cmd" &>/dev/null || missing+=("$cmd")
|
|
done
|
|
if [ ${#missing[@]} -gt 0 ]; then
|
|
echo "::error::Missing required tools: ${missing[*]}"
|
|
exit 1
|
|
fi
|
|
echo "All dependencies found"
|
|
|
|
# Build binaries
|
|
- name: Resolve PR head branch
|
|
id: pr-info
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
if (process.env.BENCH_PR) {
|
|
const { data: pr } = await github.rest.pulls.get({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
pull_number: parseInt(process.env.BENCH_PR),
|
|
});
|
|
core.setOutput('head-ref', pr.head.ref);
|
|
core.setOutput('head-sha', pr.head.sha);
|
|
} else {
|
|
core.setOutput('head-ref', '${{ github.ref_name }}');
|
|
core.setOutput('head-sha', '${{ github.sha }}');
|
|
}
|
|
|
|
- name: Resolve refs
|
|
id: refs
|
|
run: |
|
|
BASELINE_ARG="${{ needs.reth-bench-ack.outputs.baseline }}"
|
|
FEATURE_ARG="${{ needs.reth-bench-ack.outputs.feature }}"
|
|
|
|
if [ -n "$BASELINE_ARG" ]; then
|
|
git fetch origin "$BASELINE_ARG" --quiet 2>/dev/null || true
|
|
BASELINE_REF=$(git rev-parse "$BASELINE_ARG" 2>/dev/null || git rev-parse "origin/$BASELINE_ARG" 2>/dev/null)
|
|
BASELINE_NAME="$BASELINE_ARG"
|
|
else
|
|
BASELINE_REF=$(git merge-base HEAD origin/main 2>/dev/null || echo "${{ github.sha }}")
|
|
BASELINE_NAME="main"
|
|
fi
|
|
|
|
if [ -n "$FEATURE_ARG" ]; then
|
|
git fetch origin "$FEATURE_ARG" --quiet 2>/dev/null || true
|
|
FEATURE_REF=$(git rev-parse "$FEATURE_ARG" 2>/dev/null || git rev-parse "origin/$FEATURE_ARG" 2>/dev/null)
|
|
FEATURE_NAME="$FEATURE_ARG"
|
|
else
|
|
FEATURE_REF="${{ steps.pr-info.outputs.head-sha }}"
|
|
FEATURE_NAME="${{ steps.pr-info.outputs.head-ref }}"
|
|
fi
|
|
|
|
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
|
|
echo "baseline-name=$BASELINE_NAME" >> "$GITHUB_OUTPUT"
|
|
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
|
|
echo "feature-name=$FEATURE_NAME" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Prepare source dirs
|
|
run: |
|
|
BASELINE_REF="${{ steps.refs.outputs.baseline-ref }}"
|
|
if [ -d ../reth-baseline ]; then
|
|
git -C ../reth-baseline fetch origin "$BASELINE_REF"
|
|
else
|
|
git clone . ../reth-baseline
|
|
fi
|
|
git -C ../reth-baseline checkout "$BASELINE_REF"
|
|
ln -sfn "$(pwd)" ../reth-feature
|
|
|
|
- name: Build baseline and feature binaries in parallel
|
|
id: build
|
|
run: |
|
|
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
|
|
FEATURE_DIR="$(cd ../reth-feature && pwd)"
|
|
|
|
.github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "${{ steps.refs.outputs.baseline-ref }}" &
|
|
PID_BASELINE=$!
|
|
.github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "${{ steps.refs.outputs.feature-ref }}" &
|
|
PID_FEATURE=$!
|
|
|
|
FAIL=0
|
|
wait $PID_BASELINE || FAIL=1
|
|
wait $PID_FEATURE || FAIL=1
|
|
if [ $FAIL -ne 0 ]; then
|
|
echo "::error::One or both builds failed"
|
|
exit 1
|
|
fi
|
|
|
|
# System tuning for reproducible benchmarks
|
|
- name: System setup
|
|
run: |
|
|
sudo cpupower frequency-set -g performance || true
|
|
# Disable turbo boost (Intel and AMD paths)
|
|
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
|
|
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
|
|
sudo swapoff -a || true
|
|
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
|
|
# Disable SMT (hyperthreading)
|
|
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
|
|
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
|
|
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
|
|
if [ "$current" != "$first" ]; then
|
|
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
|
|
fi
|
|
done
|
|
echo "Online CPUs: $(nproc)"
|
|
# Disable transparent huge pages (compaction causes latency spikes)
|
|
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
|
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
|
done || true
|
|
# Prevent deep C-states (avoids wake-up latency jitter)
|
|
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
|
# Move all IRQs to core 0 (housekeeping core)
|
|
for irq in /proc/irq/*/smp_affinity_list; do
|
|
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
|
done
|
|
# Stop noisy background services
|
|
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
|
|
# Log environment for reproducibility
|
|
echo "=== Benchmark environment ==="
|
|
uname -r
|
|
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
|
|
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
|
|
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
|
|
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
|
|
free -h
|
|
|
|
# Clean up any leftover state
|
|
- name: Pre-flight cleanup
|
|
run: |
|
|
pkill -9 reth || true
|
|
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
|
|
|
|
- name: Update status (running baseline benchmark)
|
|
if: success() && env.BENCH_COMMENT_ID
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const s = require('./.github/scripts/bench-update-status.js');
|
|
await s({github, context, status: 'Running baseline benchmark...'});
|
|
|
|
- name: "Run benchmark: baseline"
|
|
id: run-baseline
|
|
run: taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth /tmp/bench-results-baseline
|
|
|
|
- name: Update status (running feature benchmark)
|
|
if: success() && env.BENCH_COMMENT_ID
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const s = require('./.github/scripts/bench-update-status.js');
|
|
await s({github, context, status: 'Running feature benchmark...'});
|
|
|
|
- name: "Run benchmark: feature"
|
|
id: run-feature
|
|
run: taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth /tmp/bench-results-feature
|
|
|
|
# Results & charts
|
|
- name: Parse results
|
|
id: results
|
|
if: success()
|
|
env:
|
|
BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
|
|
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
|
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
|
FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
|
|
run: |
|
|
git fetch origin "${BASELINE_NAME}" --quiet 2>/dev/null || true
|
|
BASELINE_HEAD=$(git rev-parse "origin/${BASELINE_NAME}" 2>/dev/null || echo "")
|
|
BEHIND_BASELINE=0
|
|
if [ -n "$BASELINE_HEAD" ] && [ "$BASELINE_REF" != "$BASELINE_HEAD" ]; then
|
|
BEHIND_BASELINE=$(git rev-list --count "${BASELINE_REF}..${BASELINE_HEAD}" 2>/dev/null || echo "0")
|
|
fi
|
|
|
|
SUMMARY_ARGS="--output-summary /tmp/bench-summary.json"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown /tmp/bench-comment.md"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv /tmp/bench-results-baseline/combined_latency.csv"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv /tmp/bench-results-feature/combined_latency.csv"
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv /tmp/bench-results-feature/total_gas.csv"
|
|
if [ "$BEHIND_BASELINE" -gt 0 ]; then
|
|
SUMMARY_ARGS="$SUMMARY_ARGS --behind-baseline $BEHIND_BASELINE"
|
|
fi
|
|
# shellcheck disable=SC2086
|
|
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
|
|
|
|
- name: Generate charts
|
|
if: success()
|
|
env:
|
|
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
|
|
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
|
|
run: |
|
|
CHART_ARGS="--output-dir /tmp/bench-charts"
|
|
CHART_ARGS="$CHART_ARGS --feature /tmp/bench-results-feature-1/combined_latency.csv /tmp/bench-results-feature-2/combined_latency.csv"
|
|
CHART_ARGS="$CHART_ARGS --baseline /tmp/bench-results-baseline-1/combined_latency.csv /tmp/bench-results-baseline-2/combined_latency.csv"
|
|
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
|
|
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
|
|
# shellcheck disable=SC2086
|
|
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
|
|
|
|
- name: Upload results
|
|
if: success()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: bench-reth-results
|
|
path: |
|
|
/tmp/bench-results-baseline/
|
|
/tmp/bench-results-feature/
|
|
/tmp/bench-summary.json
|
|
/tmp/bench-charts/
|
|
|
|
- name: Push charts
|
|
id: push-charts
|
|
if: success() && env.BENCH_PR
|
|
run: |
|
|
PR_NUMBER=${{ env.BENCH_PR }}
|
|
RUN_ID=${{ github.run_id }}
|
|
CHART_DIR="pr/${PR_NUMBER}/${RUN_ID}"
|
|
|
|
if git fetch origin bench-charts 2>/dev/null; then
|
|
git checkout bench-charts
|
|
else
|
|
git checkout --orphan bench-charts
|
|
git rm -rf . 2>/dev/null || true
|
|
fi
|
|
|
|
mkdir -p "${CHART_DIR}"
|
|
cp /tmp/bench-charts/*.png "${CHART_DIR}/"
|
|
git add "${CHART_DIR}"
|
|
git -c user.name="github-actions" -c user.email="github-actions@github.com" \
|
|
commit -m "bench charts for PR #${PR_NUMBER} run ${RUN_ID}"
|
|
git push origin bench-charts
|
|
echo "sha=$(git rev-parse HEAD)" >> "$GITHUB_OUTPUT"
|
|
|
|
- name: Compare & comment
|
|
if: success()
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const fs = require('fs');
|
|
|
|
let comment = '';
|
|
try {
|
|
comment = fs.readFileSync('/tmp/bench-comment.md', 'utf8');
|
|
} catch (e) {
|
|
comment = '⚠️ Engine benchmark completed but failed to generate comparison.';
|
|
}
|
|
|
|
const sha = '${{ steps.push-charts.outputs.sha }}';
|
|
const prNumber = process.env.BENCH_PR;
|
|
const runId = '${{ github.run_id }}';
|
|
|
|
if (sha && prNumber) {
|
|
const baseUrl = `https://raw.githubusercontent.com/${context.repo.owner}/${context.repo.repo}/${sha}/pr/${prNumber}/${runId}`;
|
|
const charts = [
|
|
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
|
|
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
|
|
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
|
|
];
|
|
let chartMarkdown = '\n\n### Charts\n\n';
|
|
for (const chart of charts) {
|
|
chartMarkdown += `<details><summary>${chart.label}</summary>\n\n`;
|
|
chartMarkdown += `\n\n`;
|
|
chartMarkdown += `</details>\n\n`;
|
|
}
|
|
comment += chartMarkdown;
|
|
}
|
|
|
|
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
|
|
const body = `cc @${process.env.BENCH_ACTOR}\n\n✅ Benchmark complete! [View job](${jobUrl})\n\n${comment}`;
|
|
const ackCommentId = process.env.BENCH_COMMENT_ID;
|
|
|
|
if (ackCommentId) {
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner,
|
|
repo: context.repo.repo,
|
|
comment_id: parseInt(ackCommentId),
|
|
body,
|
|
});
|
|
} else {
|
|
// No PR — write results to job summary
|
|
await core.summary.addRaw(body).write();
|
|
}
|
|
|
|
- name: Update status (failed)
|
|
if: failure() && env.BENCH_COMMENT_ID
|
|
uses: actions/github-script@v7
|
|
with:
|
|
script: |
|
|
const steps_status = [
|
|
['building binaries', '${{ steps.build.outcome }}'],
|
|
['running baseline benchmark', '${{ steps.run-baseline.outcome }}'],
|
|
['running feature benchmark', '${{ steps.run-feature.outcome }}'],
|
|
];
|
|
const failed = steps_status.find(([, o]) => o === 'failure');
|
|
const failedStep = failed ? failed[0] : 'unknown step';
|
|
|
|
await github.rest.issues.updateComment({
|
|
owner: context.repo.owner, repo: context.repo.repo,
|
|
comment_id: parseInt(process.env.BENCH_COMMENT_ID),
|
|
body: `cc @${process.env.BENCH_ACTOR}\n\n❌ Benchmark failed while ${failedStep}. [View logs](${process.env.BENCH_JOB_URL})`,
|
|
});
|
|
|
|
- name: Upload node log
|
|
if: failure()
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: reth-node-log
|
|
path: |
|
|
/tmp/reth-bench-node-baseline.log
|
|
/tmp/reth-bench-node-feature.log
|
|
|
|
- name: Restore system settings
|
|
if: always()
|
|
run: |
|
|
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|