Compare commits

..

1 Commits

Author SHA1 Message Date
yongkangc
91c4c80da7 perf(engine): release cache lock before waiting for state root validation
Move valid_block_rx.recv() out from under the execution cache write lock
in save_cache(). The original code held the lock across the blocking recv,
which stalled concurrent readers (get_cache_for) for ~100ms+ during state
root validation.

The usage guard (Arc<()> strong_count) still prevents concurrent access
to the shared ExecutionCache while the lock is released: is_available()
returns false as long as the SavedCache is alive, so get_cache_for()
returns None. split() is called inside update_with_guard to release the
guard and publish the new cache atomically under the write lock.
2026-02-20 07:21:52 +00:00
671 changed files with 20959 additions and 39336 deletions

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: patch
---
Fixed a bug in `merge_subtrie_updates` where source insertions did not cancel destination removals (and vice versa), causing inconsistent trie updates accumulated across multiple `root()` calls without intermediate `take_updates()`. Added a test covering the cross-cancellation behavior.

View File

@@ -1,5 +0,0 @@
---
reth-transaction-pool: minor
---
Added support for optional custom stateless and stateful validation hooks in `EthTransactionValidator` via `set_additional_stateless_validation` and `set_additional_stateful_validation` methods. Also implemented a manual `Debug` impl to handle the non-`Debug` function pointer fields.

View File

@@ -1,5 +0,0 @@
---
reth-tasks: patch
---
Added panic handler to all rayon thread pools that logs panics via `tracing::error` instead of aborting the process.

View File

@@ -1,5 +0,0 @@
---
reth-network-types: patch
---
Increased default maximum concurrent outbound dials from 15 to 30.

View File

@@ -1,6 +0,0 @@
---
reth-trie-common: minor
reth-trie: minor
---
Added `contains_range` method to `PrefixSet` for checking if any key falls within a half-open range. Added prefix set support to `ProofCalculator` via `with_prefix_set`, enabling stale cached hash invalidation and branch collapse detection when keys are inserted or removed; propagated storage prefix sets through `SyncAccountValueEncoder`.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: patch
---
Refactored arena trie internals by adding a `BranchChildIdx::sibling()` helper, deduplicating `Index`/`NodeArena` type aliases, and replacing `is_empty()` with a `drop_root()` method. Fixed a bug where `cursor.pop()` was called before checking if the leaf was the root node, which could cause incorrect dirty-state propagation.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: minor
---
Fixed a bug in `ArenaParallelSparseTrie` where subtrie updates that would completely empty a subtrie were incorrectly dispatched to parallel workers instead of being processed inline, preventing correct branch collapse detection when blinded siblings are present. Refactored the `SparseTrie` test suite to accept a `fn() -> T` factory instead of requiring `T: Default`, enabling a new `arena_parallel_sparse_trie_always_parallel` test variant that exercises all tests with parallelism thresholds set to 1. Added `test_branch_collapse_multi_empty_subtries_blinded_remaining` to cover the case where removing multiple revealed leaves empties their subtries and leaves a single blinded sibling requiring a proof.

View File

@@ -1,5 +0,0 @@
---
reth-payload-builder: minor
---
Added observability metrics for payload resolve latency and new payload job creation latency to the payload builder service.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: patch
---
Fixed another branch collapse edge case where `check_subtrie_collapse_needs_proof` incorrectly compared removal count against total update count (including `Touched` entries), causing it to skip proof requests for blinded siblings and panic when the subtrie emptied. Added a regression test covering the removals + `Touched` + blinded sibling scenario.

View File

@@ -1,10 +0,0 @@
---
reth-chain-state: minor
reth-engine-primitives: minor
reth-engine-tree: minor
reth-node-core: minor
reth-node-events: minor
reth: patch
---
Added configurable slow block logging (`--engine.slow-block-threshold`) that emits a structured `warn!` log with detailed timing, state-operation counts, and cache hit-rate metrics for blocks whose total processing time exceeds the threshold. Introduced `ExecutionTimingStats`, `CacheStats`, `StateProviderStats`, and `SlowBlockInfo` types to carry execution statistics from block validation through persistence, and refactored `PersistenceResult` to carry commit duration alongside the last persisted block.

View File

@@ -1,6 +0,0 @@
---
reth-trie-db: minor
reth-engine-tree: minor
---
Added `PendingChangeset` and `PendingChangesetGuard` to `ChangesetCache` so concurrent readers wait for an in-progress computation instead of falling back to the expensive DB-based path. The guard automatically cancels the pending entry on drop (e.g. task panic), ensuring waiters always make progress.

View File

@@ -1,8 +0,0 @@
---
reth-engine-primitives: minor
reth-engine-tree: minor
reth-node-core: minor
reth-trie-parallel: minor
---
Added `--engine.proof-jitter` CLI option behind the `trie-debug` feature flag. When set, each proof worker sleeps for a random duration up to the specified value before starting proof computation, useful for stress-testing timing-sensitive proof logic.

View File

@@ -1,5 +0,0 @@
---
reth-rpc-convert: patch
---
Updated `alloy-evm` dependency to git revision `9bc2dba` and adapted `TxEnvConverter` impl to the updated `TryIntoTxEnv` trait signature that now includes a `Spec` generic parameter.

View File

@@ -1,6 +0,0 @@
---
reth-trie: patch
reth-trie-sparse: patch
---
Refactored test harness for sparse trie tests by extracting `TrieTestHarness` into a shared `reth-trie` test utility, replacing duplicated inline harness code across multiple test modules. Updated `proof_v2` return type to include an optional root hash, and converted `original_root` and `storage` from public fields to accessor methods.

View File

@@ -1,5 +0,0 @@
---
reth-trie: patch
---
Removed the local `increment_and_strip_trailing_zeros` function and `PATH_ALL_ZEROS` static in `proof_v2`, replacing them with the equivalent `Nibbles::next_without_prefix` and `Nibbles::is_zeroes` builtins. Also replaced manual `.get()` calls on `state_mask`/`hash_mask` with direct field access and switched to `Nibbles::unpack_array` over the unsafe `unpack_unchecked`.

View File

@@ -1,5 +0,0 @@
---
reth-engine-tree: patch
---
Added idle-time pre-computation of account trie upper hashes in the sparse trie payload processor when no pending proof results are available.

View File

@@ -1,7 +0,0 @@
---
reth-cli-commands: minor
reth-node-core: minor
reth: patch
---
Made v2 storage the default for all new databases, deprecating the `--storage.v2` flag to a hidden no-op kept for backwards compatibility. Updated CLI reference docs to remove the now-hidden flag from all command help pages.

View File

@@ -1,7 +0,0 @@
---
reth-engine-tree: patch
reth-trie-sparse: patch
reth-tasks: patch
---
Offloaded deallocation of expensive proof node buffers to a persistent background thread (`Runtime::spawn_drop`) to avoid blocking state root computation or lock-holding code.

View File

@@ -1,6 +0,0 @@
---
reth-trie-sparse: patch
reth-engine-tree: patch
---
Removed the `skip_proof_node_filtering` flag, `revealed_account_paths`/`revealed_paths` tracking, and the `filter_revealed_v2_proof_nodes` function from the sparse trie implementation. Also removed the corresponding skipped-nodes metrics, simplifying the proof node reveal path to always pass nodes directly to the sparse trie without pre-filtering.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: minor
---
Added a comprehensive generic `SparseTrie` test suite covering `set_root`, `reveal_nodes`, `update_leaves`, `root`, `take_updates`, `commit_updates`, `prune`, `wipe`/`clear`, `get_leaf_value`, `find_leaf`, `size_hint`, and integration lifecycle scenarios. Tests are stamped out for all concrete `SparseTrie` implementations via a macro.

View File

@@ -1,5 +0,0 @@
---
reth-engine-tree: patch
---
Downgraded per-transaction prewarm span from `debug_span!` to `trace_span!` to reduce noise in debug-level logging.

View File

@@ -1,7 +0,0 @@
---
reth-network-types: minor
reth-network: minor
reth-node-core: patch
---
Added `PersistedPeerInfo` struct to persist richer peer metadata (kind, fork ID, reputation) to disk. Updated `PeersConfig::with_basic_nodes_from_file` to support both the new `PersistedPeerInfo` format and the legacy `Vec<NodeRecord>` format with automatic conversion, and updated `write_peers_to_file` to exclude backed-off and banned peers.

View File

@@ -1,5 +0,0 @@
---
reth-cli-commands: minor
---
Added `reth_version` field to `SnapshotManifest` to record the Reth version that produced a snapshot. The field is optional and populated automatically during manifest generation.

View File

@@ -1,5 +0,0 @@
---
reth-network: minor
---
Added `fork_id` as a tiebreaker in peer selection when reputations are equal, preferring peers with a discovered `fork_id` as it indicates fork compatibility. Added a test to verify the tiebreaker behavior.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: patch
---
Fixed a bug where trie nodes could appear in both `updated_nodes` and `removed_nodes` simultaneously by removing entries from `removed_nodes` when a node is inserted as updated.

View File

@@ -1,8 +0,0 @@
---
reth-trie-common: minor
reth-trie: minor
reth-trie-parallel: minor
reth-engine-tree: patch
---
Moved `ProofV2Target`, `MultiProofTargetsV2`, and `ChunkedMultiProofTargetsV2` from `reth-trie-parallel::targets_v2` into a new `reth-trie-common::target_v2` module, making these types available at a lower level without pulling in the full parallel trie crate. Added a `multiproof_v2` method to `Proof` in `reth-trie` that generates a state multiproof using the V2 proof calculator with synchronous account value encoding.

View File

@@ -1,14 +0,0 @@
---
reth-engine-tree: patch
reth-evm-ethereum: patch
reth-evm: patch
reth-primitives-traits: patch
reth-revm: patch
reth-rpc-eth-api: patch
reth-rpc-eth-types: patch
reth-provider: patch
example-custom-evm: patch
example-precompile-cache: patch
---
Bumped revm to v35.0.0, revm-inspectors to 0.35.0, and alloy-evm to 0.29.0. Updated call sites throughout the codebase to align with the new APIs, including `ExecutionResult` field changes (`gas_used``gas.used()`/`gas.final_refunded()`), removal of `.without_state_clear()`, updated `EthPrecompiles::new(spec)` constructor, and updated `block_hashes.lowest()` access.

View File

@@ -1,9 +0,0 @@
---
reth-trie-sparse: minor
reth-engine-primitives: minor
reth-engine-tree: minor
reth-node-core: minor
reth-trie-common: patch
---
Added an arena-based sparse trie implementation (`ArenaParallelSparseTrie`) using `slotmap` arena allocation for node storage, enabling parallel subtrie mutation without per-node hashing overhead. Added `ConfigurableSparseTrie` enum to switch between the arena and hash-map implementations, and a `--engine.enable-arena-sparse-trie` CLI flag to opt in at runtime.

View File

@@ -1,7 +0,0 @@
---
reth-engine-primitives: patch
reth-engine-tree: patch
reth-node-core: patch
---
Removed `--engine.enable-arena-sparse-trie` CLI flag and made the arena-based sparse trie the default implementation. The hash-map-based `ParallelSparseTrie` variant is no longer selectable.

View File

@@ -1,7 +0,0 @@
---
reth-trie: major
reth-trie-db: major
reth-provider: minor
---
Added `MaskedTrieCursorFactory` and `MaskedTrieCursor` to handle prefix-set-based hash invalidation at the cursor layer, replacing the `DatabaseTrieWitness` trait abstraction. Removed `with_prefix_sets_mut` from `TrieWitness` and deleted `DatabaseTrieWitness` — callers should now wrap their cursor factory with `MaskedTrieCursorFactory` to apply prefix sets during witness/proof computation.

View File

@@ -1,5 +0,0 @@
---
reth-trie: patch
---
Fixed a potential panic in `ProofCalculator` by clearing internal computation state (`branch_stack`, `child_stack`, `branch_path`, etc.) after errors, preventing stale state from causing `usize` underflow panics when the calculator is reused. Added a test verifying correct behavior after simulated mid-computation errors.

View File

@@ -20,11 +20,6 @@
# include dist directory, where the reth binary is located after compilation
!/dist
# include PGO build helper used by Dockerfile.depot
!/.github
!/.github/scripts
!/.github/scripts/build_pgo_bolt.sh
# include licenses
!LICENSE-*

2
.github/CODEOWNERS vendored
View File

@@ -1,7 +1,7 @@
* @gakonst
crates/chain-state/ @fgimenez @mattsse
crates/chainspec/ @Rjected @joshieDo @mattsse
crates/cli/ @mattsse @Rjected
crates/cli/ @mattsse
crates/config/ @shekhirin @mattsse @Rjected
crates/consensus/ @mattsse @Rjected
crates/e2e-test-utils/ @mattsse @Rjected @klkvr @fgimenez

View File

@@ -5,4 +5,3 @@ self-hosted-runner:
- depot-ubuntu-latest-4
- depot-ubuntu-latest-8
- depot-ubuntu-latest-16
- available

View File

@@ -1,106 +0,0 @@
// Generates a rich GitHub Actions job summary for reth-bench results.
//
// Reads from environment:
// BENCH_WORK_DIR Directory containing summary.json
// BENCH_PR PR number (may be empty)
// BENCH_ACTOR GitHub user who triggered the bench
// BENCH_CORES CPU core limit (0 = all)
// BENCH_WARMUP_BLOCKS Number of warmup blocks
// BENCH_SAMPLY 'true' if samply profiling was enabled
// BENCH_ABBA 'true' if ABBA interleaved order was used
//
// Usage from actions/github-script:
// const jobSummary = require('./.github/scripts/bench-job-summary.js');
// await jobSummary({ core, context, chartSha, grafanaUrl, runId });
const fs = require('fs');
const { verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
module.exports = async function ({ core, context, chartSha, grafanaUrl, runId }) {
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const prNumber = process.env.BENCH_PR;
const actor = process.env.BENCH_ACTOR;
const commitUrl = `https://github.com/${repo}/commit`;
const { emoji, label } = verdict(summary.changes);
const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
// Header & metadata
const metaParts = [];
if (prNumber) metaParts.push(`**[PR #${prNumber}](https://github.com/${repo}/pull/${prNumber})**`);
metaParts.push(`triggered by @${actor}`);
let md = `# ${emoji} ${label}\n\n`;
md += metaParts.join(' · ') + '\n\n';
md += `**Baseline:** ${baselineLink}\n`;
md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
// Main comparison table
const rows = metricRows(summary);
md += `| Metric | Baseline | Feature | Change |\n`;
md += `|--------|----------|---------|--------|\n`;
for (const r of rows) {
md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
}
md += '\n';
// Wait time breakdown
const wtRows = waitTimeRows(summary);
if (wtRows.length > 0) {
md += `### Wait Time Breakdown\n\n`;
md += `| Metric | Baseline | Feature |\n`;
md += `|--------|----------|--------|\n`;
for (const r of wtRows) {
md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
}
md += '\n';
}
// Charts
if (chartSha) {
const prNum = prNumber || '0';
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`;
const charts = [
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
];
md += `### Charts\n\n`;
for (const chart of charts) {
md += `<details><summary>${chart.label}</summary>\n\n`;
md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
md += `</details>\n\n`;
}
}
// Samply profiles
const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
const samplyLinks = Object.entries(samplyUrls).map(([run, url]) => `- **${run}**: [Firefox Profiler](${url})`);
if (samplyLinks.length > 0) {
md += `### Samply Profiles\n\n${samplyLinks.join('\n')}\n\n`;
}
// Grafana
if (grafanaUrl) {
md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
}
// Node errors
try {
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
if (errors.trim()) md += '\n' + errors + '\n';
} catch {}
await core.summary.addRaw(md).write();
};

View File

@@ -1,276 +0,0 @@
#!/usr/bin/env python3
"""
Prometheus metrics proxy that fetches from a local reth node and
re-exposes with additional benchmark labels.
Reads labels from a JSON file (updated by local-reth-bench.sh between runs)
and injects them into every Prometheus metric line.
Returns empty 200 when reth is not running (clean Grafana gaps).
"""
import argparse
import ipaddress
import json
import subprocess
import sys
import time
from http.server import HTTPServer, BaseHTTPRequestHandler
from urllib.request import urlopen
from urllib.error import URLError
def read_labels(path):
try:
with open(path) as f:
return json.load(f)
except (FileNotFoundError, json.JSONDecodeError):
return {}
def inject_labels(metrics_bytes, label_str, label_names):
"""Inject labels into Prometheus text format.
Operates on bytes and uses simple string ops instead of regex
for speed on large payloads (reth exposes thousands of metrics).
Skips injecting into lines that already contain any of the label names
to avoid duplicate labels (which Prometheus rejects).
"""
if not label_str:
return metrics_bytes
label_bytes = label_str.encode("utf-8")
# Pre-encode label names for fast duplicate detection
label_name_bytes = [n.encode("utf-8") for n in label_names]
out = []
for line in metrics_bytes.split(b"\n"):
# Skip comments and blank lines
if line.startswith(b"#") or not line:
out.append(line)
continue
brace = line.find(b"{")
space = line.find(b" ")
if space == -1:
# Malformed, pass through
out.append(line)
elif brace != -1 and brace < space:
# Has labels: metric{existing="val"} 123
close = line.find(b"}", brace)
if close == -1:
out.append(line)
continue
# Filter out labels that already exist in this line
existing = line[brace + 1:close]
inject = label_bytes
if existing:
for name in label_name_bytes:
if name + b"=" in existing:
# Rebuild inject string excluding this label
inject = _remove_label(inject, name)
if not inject:
out.append(line)
continue
if close == brace + 1:
# Empty braces: metric{} 123
out.append(line[:close] + inject + line[close:])
else:
out.append(line[:close] + b"," + inject + line[close:])
else:
# No labels: metric 123
out.append(line[:space] + b"{" + label_bytes + b"}" + line[space:])
return b"\n".join(out)
def _remove_label(label_bytes, name):
"""Remove a single label (name=\"...\") from a comma-separated label string."""
parts = []
for part in label_bytes.split(b","):
if not part.startswith(name + b"="):
parts.append(part)
return b",".join(parts)
def build_label_str(labels):
"""Pre-format the label injection string: key1="val1",key2="val2" """
if not labels:
return ""
return ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
def build_elapsed_gauge(labels):
"""Build a bench_elapsed_seconds gauge from run_start_epoch in labels."""
start = labels.get("run_start_epoch")
if not start:
return b""
try:
elapsed = time.time() - float(start)
except (ValueError, TypeError):
return b""
# Build labels excluding internal keys
display = {k: v for k, v in labels.items()
if k not in ("run_start_epoch", "reference_epoch")}
lstr = build_label_str(display)
return (
f"# HELP bench_elapsed_seconds Seconds since benchmark run started\n"
f"# TYPE bench_elapsed_seconds gauge\n"
f"bench_elapsed_seconds{{{lstr}}} {elapsed:.1f}\n"
).encode("utf-8")
def compute_timestamp_ms(labels):
"""Compute a synthetic timestamp so all runs share a common time origin.
Returns the timestamp in milliseconds, or None if not enough info.
Uses: reference_epoch + (now - run_start_epoch) → all runs overlay at
the same Grafana time range.
"""
ref = labels.get("reference_epoch")
start = labels.get("run_start_epoch")
if not ref or not start:
return None
try:
elapsed = time.time() - float(start)
return int((float(ref) + elapsed) * 1000)
except (ValueError, TypeError):
return None
def inject_timestamps(metrics_bytes, timestamp_ms):
"""Append a Prometheus timestamp (ms) to every data line.
Prometheus text format: metric{labels} value [timestamp_ms]
Adding timestamps causes Prometheus to store all runs' samples
at the same relative time, enabling natural overlay in Grafana.
"""
if timestamp_ms is None:
return metrics_bytes
ts = str(timestamp_ms).encode("utf-8")
out = []
for line in metrics_bytes.split(b"\n"):
if line.startswith(b"#") or not line:
out.append(line)
else:
out.append(line + b" " + ts)
return b"\n".join(out)
class MetricsHandler(BaseHTTPRequestHandler):
# Use HTTP/1.1 so Content-Length is respected and Prometheus
# doesn't have to rely on connection close to detect end of body.
protocol_version = "HTTP/1.1"
def do_GET(self):
src = self.client_address[0]
try:
resp = urlopen(self.server.upstream, timeout=2)
metrics = resp.read()
except (URLError, ConnectionError, OSError):
# reth not running — return empty 200
self._send(b"")
#print(f" scrape from {src}: empty (reth not running)", flush=True)
return
all_labels = read_labels(self.server.labels_file)
# Internal keys — not injected as Prometheus labels
internal = ("run_start_epoch", "reference_epoch")
labels = {k: v for k, v in all_labels.items() if k not in internal}
label_str = build_label_str(labels)
label_names = sorted(labels.keys())
t0 = time.monotonic()
result = inject_labels(metrics, label_str, label_names)
result += build_elapsed_gauge(all_labels)
ts_ms = compute_timestamp_ms(all_labels)
result = inject_timestamps(result, ts_ms)
dt = time.monotonic() - t0
self._send(result)
print(f" scrape from {src}: {len(metrics)} -> {len(result)} bytes, "
f"inject {dt*1000:.1f}ms", flush=True)
def _send(self, body):
self.send_response(200)
self.send_header("Content-Type", "text/plain; version=0.0.4")
self.send_header("Content-Length", str(len(body)))
self.send_header("Connection", "close")
self.end_headers()
if body:
self.wfile.write(body)
def log_message(self, format, *args):
pass # suppress per-request logging
def resolve_bind_address(subnet_cidr):
"""Find the local IP address that belongs to the given subnet.
Uses ``ip -j addr show`` to enumerate interfaces and returns the first
address that falls within *subnet_cidr* (e.g. ``10.10.0.0/24``).
"""
network = ipaddress.ip_network(subnet_cidr, strict=False)
try:
result = subprocess.run(
["ip", "-j", "addr", "show"],
capture_output=True, text=True, check=True,
)
interfaces = json.loads(result.stdout)
except (subprocess.CalledProcessError, FileNotFoundError, json.JSONDecodeError) as exc:
print(f"Error: cannot enumerate interfaces: {exc}", file=sys.stderr)
sys.exit(1)
for iface in interfaces:
for addr_info in iface.get("addr_info", []):
try:
addr = ipaddress.ip_address(addr_info["local"])
except (KeyError, ValueError):
continue
if addr in network:
return str(addr)
print(f"Error: no interface address found in subnet {subnet_cidr}", file=sys.stderr)
sys.exit(1)
def main():
parser = argparse.ArgumentParser(description="Prometheus metrics proxy with label injection")
parser.add_argument("--labels", default="/tmp/bench-metrics-labels.json",
help="Path to JSON file with labels to inject (default: /tmp/bench-metrics-labels.json)")
parser.add_argument("--upstream", default="http://127.0.0.1:9100/",
help="Upstream reth metrics URL (default: http://127.0.0.1:9100/)")
bind_group = parser.add_mutually_exclusive_group()
bind_group.add_argument("--bind", default=None,
help="Address to bind the proxy (default: 0.0.0.0)")
bind_group.add_argument("--subnet", default=None,
help="Auto-detect bind address from a local interface in this subnet (e.g. 10.10.0.0/24)")
parser.add_argument("--port", type=int, default=9090,
help="Port to bind the proxy (default: 9090)")
args = parser.parse_args()
if args.subnet:
bind_addr = resolve_bind_address(args.subnet)
elif args.bind:
bind_addr = args.bind
else:
bind_addr = "0.0.0.0"
server = HTTPServer((bind_addr, args.port), MetricsHandler)
server.upstream = args.upstream
server.labels_file = args.labels
print(f"bench-metrics-proxy listening on {bind_addr}:{args.port}")
print(f" upstream: {args.upstream}")
print(f" labels: {args.labels}")
sys.stdout.flush()
server.serve_forever()
if __name__ == "__main__":
main()

View File

@@ -14,30 +14,14 @@
# baseline: <source-dir>/target/profiling/reth
# feature: <source-dir>/target/profiling/reth, reth-bench installed to cargo bin
#
# Required: mc (MinIO client) with a configured alias
# Required: mc (MinIO client) configured at /home/ubuntu/.mc
set -euo pipefail
MC="mc"
MC="mc --config-dir /home/ubuntu/.mc"
MODE="$1"
SOURCE_DIR="$2"
COMMIT="$3"
# Tracy support: when BENCH_TRACY is "on" or "full", add Tracy cargo features
# and frame pointers for accurate stack traces.
EXTRA_FEATURES=""
EXTRA_RUSTFLAGS=""
if [ "${BENCH_TRACY:-off}" != "off" ]; then
EXTRA_FEATURES="tracy,tracy-client/ondemand"
EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
fi
# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
else
BUILD_SUFFIX=""
fi
# Verify a cached reth binary was built from the expected commit.
# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
verify_binary() {
@@ -58,7 +42,7 @@ verify_binary() {
case "$MODE" in
baseline|main)
BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
BUCKET="minio/reth-binaries/${COMMIT}"
mkdir -p "${SOURCE_DIR}/target/profiling"
CACHE_VALID=false
@@ -75,23 +59,14 @@ case "$MODE" in
if [ "$CACHE_VALID" = false ]; then
echo "Building baseline (${COMMIT}) from source..."
cd "${SOURCE_DIR}"
FEATURES_ARG=""
WORKSPACE_ARG=""
if [ -n "$EXTRA_FEATURES" ]; then
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
FEATURES_ARG="--features ${EXTRA_FEATURES}"
WORKSPACE_ARG="--workspace"
fi
# shellcheck disable=SC2086
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
cargo build --profile profiling --bin reth $WORKSPACE_ARG $FEATURES_ARG
cargo build --profile profiling --bin reth
$MC cp target/profiling/reth "${BUCKET}/reth"
fi
;;
feature|branch)
BRANCH_SHA="${4:-$COMMIT}"
BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
BUCKET="minio/reth-binaries/${BRANCH_SHA}"
CACHE_VALID=false
if $MC stat "${BUCKET}/reth" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
@@ -110,14 +85,7 @@ case "$MODE" in
echo "Building feature (${COMMIT}) from source..."
cd "${SOURCE_DIR}"
rustup show active-toolchain || rustup default stable
if [ -n "$EXTRA_FEATURES" ]; then
# Can't use `make profiling` when adding features; build explicitly
# --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
cargo build --profile profiling --workspace --bin reth --features "${EXTRA_FEATURES}"
else
make profiling
fi
make profiling
make install-reth-bench
$MC cp target/profiling/reth "${BUCKET}/reth"
$MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"

View File

@@ -73,24 +73,22 @@ def plot_latency_and_throughput(
for r in baseline:
lat_s = r["new_payload_latency_us"] / 1_000_000
base_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
l, = ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
ax1.axhline(np.median(base_lat), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
l, = ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
ax2.axhline(np.median(base_ggas), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
l, = ax1.plot(feat_x, feat_lat, linewidth=0.8, label=feature_name)
ax1.axhline(np.median(feat_lat), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
ax1.plot(feat_x, feat_lat, linewidth=0.8, label=feature_name)
ax1.set_ylabel("Latency (ms)")
ax1.set_title("newPayload Latency per Block")
ax1.grid(True, alpha=0.3)
ax1.legend()
if baseline:
ax1.legend()
l, = ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=feature_name)
ax2.axhline(np.median(feat_ggas), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=feature_name)
ax2.set_ylabel("Ggas/s")
ax2.set_title("Execution Throughput per Block")
ax2.grid(True, alpha=0.3)
ax2.legend()
if baseline:
ax2.legend()
if baseline:
ax3 = axes[2]

View File

@@ -1,581 +0,0 @@
#!/usr/bin/env bash
#
# local-reth-bench.sh — Run the reth Engine API benchmark locally.
#
# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
# interleaved B-F-F-B execution, summary, charts) without any GitHub
# Actions glue (no PR comments, no artifact upload, no Slack).
#
# Usage:
# local-reth-bench.sh <baseline-ref> <feature-ref> [options]
#
# Options:
# --blocks N Number of blocks to benchmark (default: 500)
# --warmup N Number of warmup blocks (default: 100)
# --cores N Limit reth to N CPU cores, 0 = all available (default: 0)
# --samply Enable samply profiling
# --tracy MODE Tracy profiling: off, on, full (default: off)
# --tracy-filter F Tracy tracing filter (default: debug)
# --no-tune Skip system tuning (useful on dev machines / macOS)
#
# Requires: the reth repo at RETH_REPO (default: ~/reth)
#
# Dependencies (install before first run):
# mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
# make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
#
# The script delegates to the existing bench-reth-*.sh scripts in the reth
# repo for the actual build, snapshot, and run steps.
set -euo pipefail
# ── PATH ──────────────────────────────────────────────────────────────
# Ensure cargo and user-local bins (mc, uv) are visible
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
# ── Defaults ──────────────────────────────────────────────────────────
RETH_REPO="${RETH_REPO:-$HOME/reth}"
BLOCKS=500
WARMUP=100
CORES=0
SAMPLY=false
TRACY="off"
TRACY_FILTER="debug"
TUNE=true
BASELINE_REF=""
FEATURE_REF=""
# ── Parse arguments ──────────────────────────────────────────────────
usage() {
cat <<EOF
Usage: $(basename "$0") <baseline-ref> <feature-ref> [options]
Options:
--blocks N Number of blocks to benchmark (default: 500)
--warmup N Number of warmup blocks (default: 100)
--cores N Limit reth to N CPU cores (default: 0 = all)
--samply Enable samply profiling
--tracy MODE Tracy profiling: off, on, full (default: off)
on = tracing only (lower overhead)
full = tracing + CPU sampling (higher overhead)
--tracy-filter F Tracy tracing filter (default: debug)
--no-tune Skip system tuning
EOF
exit 1
}
while [[ $# -gt 0 ]]; do
case "$1" in
--blocks) BLOCKS="$2"; shift 2 ;;
--warmup) WARMUP="$2"; shift 2 ;;
--cores) CORES="$2"; shift 2 ;;
--samply) SAMPLY=true; shift ;;
--tracy) TRACY="$2"; shift 2 ;;
--tracy-filter) TRACY_FILTER="$2"; shift 2 ;;
--no-tune) TUNE=false; shift ;;
--help|-h) usage ;;
-*) echo "Unknown option: $1"; usage ;;
*)
if [ -z "$BASELINE_REF" ]; then
BASELINE_REF="$1"
elif [ -z "$FEATURE_REF" ]; then
FEATURE_REF="$1"
else
echo "Unexpected argument: $1"; usage
fi
shift
;;
esac
done
if [ -z "$BASELINE_REF" ] || [ -z "$FEATURE_REF" ]; then
echo "Error: both <baseline-ref> and <feature-ref> are required."
usage
fi
# Validate --tracy value
case "$TRACY" in
off|on|full) ;;
*) echo "Error: --tracy must be off, on, or full (got: $TRACY)"; usage ;;
esac
# Samply + tracy=full are mutually exclusive (both use perf sampling)
if [ "$SAMPLY" = "true" ] && [ "$TRACY" = "full" ]; then
echo "Warning: samply and tracy=full both use perf sampling; downgrading tracy to 'on'."
TRACY="on"
fi
# ── Check dependencies ───────────────────────────────────────────────
missing=()
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
echo "Error: missing required tools: ${missing[*]}"
echo "See the CI 'Install dependencies' step in .github/workflows/bench.yml for install instructions."
exit 1
fi
if [ "$TRACY" != "off" ]; then
if ! command -v tracy-capture &>/dev/null; then
echo "Error: tracy-capture is required for --tracy $TRACY"
exit 1
fi
fi
# Ensure tools that run via sudo are in a sudo-visible path.
# The bench scripts use `sudo schelk` / `sudo samply` but cargo installs
# them to ~/.cargo/bin which sudo's secure_path doesn't include.
for cmd in schelk samply; do
if command -v "$cmd" &>/dev/null && ! sudo sh -c "command -v $cmd" &>/dev/null; then
echo "Installing $cmd to /usr/local/bin (needed for sudo)..."
sudo install "$(command -v "$cmd")" /usr/local/bin/
fi
done
if [ ! -d "$RETH_REPO/.git" ]; then
echo "Error: RETH_REPO=$RETH_REPO is not a git repository."
echo "Set RETH_REPO or clone reth to ~/reth"
exit 1
fi
# ── Resolve paths ────────────────────────────────────────────────────
SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
SCRIPTS_DIR="${RETH_REPO}/.github/scripts"
BENCH_WORK_DIR="${RETH_REPO}/../bench-work-$(date +%Y%m%d-%H%M%S)"
BASELINE_SRC="${RETH_REPO}/../reth-baseline"
FEATURE_SRC="${RETH_REPO}/../reth-feature"
mkdir -p "$BENCH_WORK_DIR"
BENCH_WORK_DIR="$(cd "$BENCH_WORK_DIR" && pwd)"
# ── Global cleanup trap (restores system tuning on any exit) ─────────
TUNING_APPLIED=false
CSTATE_PID=
METRICS_PROXY_PID=
cleanup_global() {
[ -n "$METRICS_PROXY_PID" ] && kill "$METRICS_PROXY_PID" 2>/dev/null || true
if [ "$TUNING_APPLIED" = true ]; then
echo
echo "▸ Restoring system settings..."
[ -n "$CSTATE_PID" ] && kill "$CSTATE_PID" 2>/dev/null || true
sudo systemctl start irqbalance cron atd 2>/dev/null || true
echo " System settings restored."
fi
}
trap cleanup_global EXIT
echo "═══════════════════════════════════════════════════════════"
echo " reth local benchmark"
echo "═══════════════════════════════════════════════════════════"
echo " Baseline ref : $BASELINE_REF"
echo " Feature ref : $FEATURE_REF"
echo " Blocks : $BLOCKS"
echo " Warmup : $WARMUP"
echo " Cores : $CORES"
echo " Samply : $SAMPLY"
echo " Tracy : $TRACY"
echo " Tracy filter : $TRACY_FILTER"
echo " System tune : $TUNE"
echo " Work dir : $BENCH_WORK_DIR"
echo " Reth repo : $RETH_REPO"
echo "═══════════════════════════════════════════════════════════"
echo
# Enable sccache if available (matches CI's RUSTC_WRAPPER=sccache)
if command -v sccache &>/dev/null; then
export RUSTC_WRAPPER="sccache"
fi
# Export env vars expected by the bench-reth-*.sh scripts
export BENCH_BLOCKS="$BLOCKS"
export BENCH_WARMUP_BLOCKS="$WARMUP"
export BENCH_CORES="$CORES"
export BENCH_SAMPLY="$SAMPLY"
export BENCH_TRACY="$TRACY"
export BENCH_TRACY_FILTER="$TRACY_FILTER"
export BENCH_WORK_DIR
export SCHELK_MOUNT="${SCHELK_MOUNT:-/reth-bench}"
export BENCH_RPC_URL="${BENCH_RPC_URL:-https://ethereum.reth.rs/rpc}"
export BENCH_METRICS_ADDR="127.0.0.1:9100"
# ── Step 1: Resolve refs to full SHAs ────────────────────────────────
echo "▸ Resolving git refs..."
cd "$RETH_REPO"
resolve_ref() {
local ref="$1"
git fetch origin "$ref" --quiet 2>/dev/null || true
git rev-parse "$ref" 2>/dev/null \
|| git rev-parse "origin/$ref" 2>/dev/null \
|| { echo "Error: cannot resolve ref '$ref'"; exit 1; }
}
BASELINE_SHA="$(resolve_ref "$BASELINE_REF")"
FEATURE_SHA="$(resolve_ref "$FEATURE_REF")"
echo " Baseline SHA : $BASELINE_SHA"
echo " Feature SHA : $FEATURE_SHA"
echo
# ── Step 2: Prepare source directories ───────────────────────────────
echo "▸ Preparing source directories..."
prepare_source() {
local src_dir="$1" ref="$2"
if [ -d "$src_dir" ]; then
git -C "$src_dir" fetch origin "$ref" 2>/dev/null || true
else
git clone --recurse-submodules "$RETH_REPO" "$src_dir"
fi
git -C "$src_dir" checkout "$ref" --force
git -C "$src_dir" submodule update --init --recursive
}
prepare_source "$BASELINE_SRC" "$BASELINE_SHA"
prepare_source "$FEATURE_SRC" "$FEATURE_SHA"
BASELINE_SRC="$(cd "$BASELINE_SRC" && pwd)"
FEATURE_SRC="$(cd "$FEATURE_SRC" && pwd)"
echo " Baseline src : $BASELINE_SRC"
echo " Feature src : $FEATURE_SRC"
echo
# ── Step 3: Check / download snapshot ────────────────────────────────
echo "▸ Checking snapshot..."
cd "$RETH_REPO"
SNAPSHOT_NEEDED=false
if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
SNAPSHOT_NEEDED=true
echo " Snapshot needs update."
else
echo " Snapshot is up-to-date."
fi
echo
# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
echo "▸ Building binaries (parallel)..."
cd "$RETH_REPO"
FAIL=0
"${SCRIPTS_DIR}/bench-reth-build.sh" baseline "$BASELINE_SRC" "$BASELINE_SHA" &
PID_BASELINE=$!
"${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
PID_FEATURE=$!
PID_SNAPSHOT=
if [ "$SNAPSHOT_NEEDED" = "true" ]; then
echo " Also downloading snapshot in parallel..."
"${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
PID_SNAPSHOT=$!
fi
wait $PID_BASELINE || FAIL=1
wait $PID_FEATURE || FAIL=1
[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
if [ $FAIL -ne 0 ]; then
echo "Error: one or more parallel tasks failed (builds / snapshot)"
exit 1
fi
echo " Binaries built successfully."
echo
# ── Step 5: System tuning (optional) ────────────────────────────────
if [ "$TUNE" = "true" ]; then
echo "▸ Applying system tuning..."
sudo cpupower frequency-set -g performance 2>/dev/null || true
# Disable turbo boost (Intel + AMD)
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
sudo swapoff -a 2>/dev/null || true
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space 2>/dev/null || true
# Disable SMT (hyperthreading)
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
[ -f "$cpu" ] || continue
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
if [ "$current" != "$first" ]; then
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" 2>/dev/null || true
fi
done
echo " Online CPUs: $(nproc)"
# Disable transparent huge pages
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
if [ -d "$p" ]; then
echo never | sudo tee "$p/enabled" 2>/dev/null || true
echo never | sudo tee "$p/defrag" 2>/dev/null || true
break
fi
done
# Prevent deep C-states
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
CSTATE_PID=$!
# Pin IRQs to core 0
for irq in /proc/irq/*/smp_affinity_list; do
echo 0 | sudo tee "$irq" 2>/dev/null || true
done
# Stop noisy background services
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
TUNING_APPLIED=true
# Log environment for reproducibility (matches CI)
echo " === Benchmark environment ==="
echo " Kernel : $(uname -r)"
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA' | sed 's/^/ /'
echo " Governor : $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo unknown)"
echo " Freq : $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq 2>/dev/null || echo unknown)"
echo " THP : $(cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo unknown)"
free -h | sed 's/^/ /'
echo " System tuning applied."
echo
fi
# ── Step 5b: Tracefs mount (tracy=full only) ─────────────────────────
if [ "$TRACY" = "full" ] && [ "$(uname)" = "Linux" ]; then
echo "▸ Mounting tracefs for Tracy full mode..."
sudo mount -t tracefs tracefs /sys/kernel/tracing -o mode=755 2>/dev/null || true
fi
# ── Tracy upload & viewer helpers ────────────────────────────────────
TRACY_VIEWER_BASE="${TRACY_VIEWER_BASE:-}"
tracy_viewer_url() {
local profile_url="$1"
if [ -z "$TRACY_VIEWER_BASE" ]; then
echo ""
return
fi
local encoded
encoded=$(python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$profile_url")
echo "${TRACY_VIEWER_BASE}?profile_url=${encoded}"
}
upload_tracy() {
local label="$1" output_dir="$2" sha="$3"
local tracy_file="$output_dir/tracy-profile.tracy"
if [ ! -f "$tracy_file" ]; then
echo " Tracy: no profile found, skipping upload."
return
fi
local timestamp short_sha remote_name bucket mc_alias
timestamp=$(date +%Y%m%d-%H%M%S)
short_sha="${sha:0:7}"
remote_name="${label}-${short_sha}-${timestamp}.tracy"
bucket="${TRACY_BUCKET:-tracy-profiles}"
mc_alias="${MC_ALIAS:-minio}"
local minio_base="${TRACY_MINIO_URL:-http://minio.minio.svc.cluster.local:9000}"
echo " Tracy: uploading profile..."
if mc cp "$tracy_file" "${mc_alias}/${bucket}/${remote_name}"; then
local url="${minio_base}/${bucket}/${remote_name}"
echo "$url" > "$output_dir/tracy_url.txt"
local viewer
viewer=$(tracy_viewer_url "$url")
if [ -n "$viewer" ]; then
echo "$viewer" > "$output_dir/tracy_viewer_url.txt"
echo " Tracy: uploaded → $viewer"
else
echo " Tracy: uploaded → $url"
fi
else
echo " Tracy: upload failed (non-fatal)."
fi
# Delete large profile to free disk
rm -f "$tracy_file"
}
# ── Step 6: Pre-flight cleanup ───────────────────────────────────────
echo "▸ Pre-flight cleanup..."
pkill -f bench-metrics-proxy 2>/dev/null || true
sudo pkill -9 reth 2>/dev/null || true
sleep 1
if mountpoint -q "$SCHELK_MOUNT" 2>/dev/null; then
sudo umount -l "$SCHELK_MOUNT" 2>/dev/null || true
sudo schelk recover -y 2>/dev/null || true
fi
echo
# ── Step 7: Interleaved benchmark runs (B-F-F-B) ────────────────────
# This ordering reduces systematic bias from thermal drift and cache warming.
BASELINE_BIN="${BASELINE_SRC}/target/profiling/reth"
FEATURE_BIN="${FEATURE_SRC}/target/profiling/reth"
# Start metrics proxy (reth → label injection → Prometheus)
LABELS_FILE="/tmp/bench-metrics-labels.json"
echo '{}' > "$LABELS_FILE"
METRICS_SUBNET="${METRICS_SUBNET:-10.10.0.0/24}"
METRICS_PORT="${METRICS_PORT:-9090}"
python3 "${SELF_DIR}/bench-metrics-proxy.py" \
--labels "$LABELS_FILE" \
--upstream "http://${BENCH_METRICS_ADDR}/" \
--subnet "$METRICS_SUBNET" \
--port "$METRICS_PORT" &
METRICS_PROXY_PID=$!
echo "▸ Metrics proxy started (PID $METRICS_PROXY_PID) on subnet ${METRICS_SUBNET}, port ${METRICS_PORT}"
# Unique benchmark ID: local-<timestamp> for local runs, ci-<run_id> for CI
BENCH_ID="local-$(basename "$BENCH_WORK_DIR" | sed 's/bench-work-//')"
# Reference epoch: shared time origin so all runs overlay in Grafana.
# The proxy maps each run's elapsed time onto this common origin.
BENCH_REFERENCE_EPOCH=$(date +%s)
write_labels() {
local run_label="$1" run_type="$2" ref="$3" sha="$4"
LAST_RUN_START=$(date +%s)
cat > "$LABELS_FILE" <<-EOF
{"benchmark_run":"${run_label}","run_type":"${run_type}","git_ref":"${ref}","bench_sha":"${sha}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
EOF
}
run_bench() {
local label="$1" binary="$2" output_dir="$3"
echo "▸ Running benchmark: ${label}..."
cd "$RETH_REPO"
if command -v taskset &>/dev/null; then
taskset -c 0 "${SCRIPTS_DIR}/bench-reth-run.sh" "$label" "$binary" "$output_dir"
else
"${SCRIPTS_DIR}/bench-reth-run.sh" "$label" "$binary" "$output_dir"
fi
echo "${label} complete."
echo
}
write_labels "baseline-1" "baseline" "$BASELINE_REF" "$BASELINE_SHA"
run_bench "baseline-1" "$BASELINE_BIN" "$BENCH_WORK_DIR/baseline-1"
write_labels "feature-1" "feature" "$FEATURE_REF" "$FEATURE_SHA"
run_bench "feature-1" "$FEATURE_BIN" "$BENCH_WORK_DIR/feature-1"
write_labels "feature-2" "feature" "$FEATURE_REF" "$FEATURE_SHA"
run_bench "feature-2" "$FEATURE_BIN" "$BENCH_WORK_DIR/feature-2"
write_labels "baseline-2" "baseline" "$BASELINE_REF" "$BASELINE_SHA"
run_bench "baseline-2" "$BASELINE_BIN" "$BENCH_WORK_DIR/baseline-2"
# ── Compute Grafana URL ──────────────────────────────────────────────
GRAFANA_BASE_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr"
GRAFANA_DATASOURCE="ef57fux92e9z4e"
LAST_RUN_DURATION=$(( $(date +%s) - LAST_RUN_START ))
FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
GRAFANA_URL="${GRAFANA_BASE_URL}?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=${GRAFANA_DATASOURCE}&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
# ── Step 8: Scan logs for errors ─────────────────────────────────────
echo "▸ Scanning logs for errors..."
ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
found_errors=false
for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
LOG="$BENCH_WORK_DIR/$run_dir/node.log"
[ -f "$LOG" ] || continue
panics=$(grep -c -E 'panicked at' "$LOG" 2>/dev/null || true)
errors=$(grep -c ' ERROR ' "$LOG" 2>/dev/null || true)
if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
if [ "$found_errors" = false ]; then
printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
found_errors=true
fi
printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' \
"$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
if [ "$panics" -gt 0 ]; then
printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
if [ "$errors" -gt 0 ]; then
printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
printf '\n</details>\n\n' >> "$ERRORS_FILE"
fi
done
if [ "$found_errors" = true ]; then
echo " ⚠ Errors found — see $ERRORS_FILE"
else
echo " No errors found."
fi
echo
# ── Step 9: Parse results ───────────────────────────────────────────
echo "▸ Parsing results..."
cd "$RETH_REPO"
SUMMARY_ARGS=(
--output-summary "$BENCH_WORK_DIR/summary.json"
--output-markdown "$BENCH_WORK_DIR/comment.md"
--repo "paradigmxyz/reth"
--baseline-ref "$BASELINE_SHA"
--baseline-name "$BASELINE_REF"
--feature-name "$FEATURE_REF"
--feature-ref "$FEATURE_SHA"
--baseline-csv "$BENCH_WORK_DIR/baseline-1/combined_latency.csv" "$BENCH_WORK_DIR/baseline-2/combined_latency.csv"
--feature-csv "$BENCH_WORK_DIR/feature-1/combined_latency.csv" "$BENCH_WORK_DIR/feature-2/combined_latency.csv"
--gas-csv "$BENCH_WORK_DIR/feature-1/total_gas.csv"
--grafana-url "$GRAFANA_URL"
)
python3 "${SCRIPTS_DIR}/bench-reth-summary.py" "${SUMMARY_ARGS[@]}"
echo
# ── Step 10: Generate charts ─────────────────────────────────────────
echo "▸ Generating charts..."
CHART_ARGS=(
--output-dir "$BENCH_WORK_DIR/charts"
--feature "$BENCH_WORK_DIR/feature-1/combined_latency.csv" "$BENCH_WORK_DIR/feature-2/combined_latency.csv"
--baseline "$BENCH_WORK_DIR/baseline-1/combined_latency.csv" "$BENCH_WORK_DIR/baseline-2/combined_latency.csv"
--baseline-name "$BASELINE_REF"
--feature-name "$FEATURE_REF"
)
if python3 -c "import matplotlib" 2>/dev/null; then
python3 "${SCRIPTS_DIR}/bench-reth-charts.py" "${CHART_ARGS[@]}"
elif command -v uv &>/dev/null; then
uv run --with matplotlib python3 "${SCRIPTS_DIR}/bench-reth-charts.py" "${CHART_ARGS[@]}"
else
echo " Warning: matplotlib not available, skipping chart generation."
fi
echo
# ── Step 11: Upload Tracy profiles ────────────────────────────────────
if [ "$TRACY" != "off" ]; then
echo "▸ Uploading Tracy profiles..."
upload_tracy "baseline-1" "$BENCH_WORK_DIR/baseline-1" "$BASELINE_SHA"
upload_tracy "feature-1" "$BENCH_WORK_DIR/feature-1" "$FEATURE_SHA"
upload_tracy "feature-2" "$BENCH_WORK_DIR/feature-2" "$FEATURE_SHA"
upload_tracy "baseline-2" "$BENCH_WORK_DIR/baseline-2" "$BASELINE_SHA"
echo
fi
# ── Done (system restore happens via EXIT trap) ─────────────────────
echo "═══════════════════════════════════════════════════════════"
echo " Benchmark complete!"
echo "═══════════════════════════════════════════════════════════"
echo " Results : $BENCH_WORK_DIR/summary.json"
echo " Markdown : $BENCH_WORK_DIR/comment.md"
echo " Charts : $BENCH_WORK_DIR/charts/"
if [ -f "$ERRORS_FILE" ]; then
echo " Errors : $ERRORS_FILE"
fi
echo " Grafana : $GRAFANA_URL"
if [ "$TRACY" != "off" ]; then
echo " ─── Tracy Profiles ───"
for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
url_file="$BENCH_WORK_DIR/$run_dir/tracy_viewer_url.txt"
if [ -f "$url_file" ]; then
echo " $run_dir : $(cat "$url_file")"
fi
done
fi
echo "═══════════════════════════════════════════════════════════"

View File

@@ -6,13 +6,6 @@
# Usage: bench-reth-run.sh <label> <binary> <output-dir>
#
# Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
# Optional env: BENCH_BIG_BLOCKS (true/false), BENCH_WORK_DIR (for big blocks path)
# BENCH_RETH_NEW_PAYLOAD (true/false, default true)
# BENCH_WAIT_TIME (duration like 500ms, default empty)
# BENCH_BASELINE_ARGS (extra reth node args for baseline runs)
# BENCH_FEATURE_ARGS (extra reth node args for feature runs)
# BENCH_OTLP_TRACES_ENDPOINT (OTLP HTTP endpoint for traces, e.g. https://host/insert/opentelemetry/v1/traces)
# BENCH_OTLP_LOGS_ENDPOINT (OTLP HTTP endpoint for logs, e.g. https://host/insert/opentelemetry/v1/logs)
set -euo pipefail
LABEL="$1"
@@ -24,68 +17,23 @@ LOG="${OUTPUT_DIR}/node.log"
cleanup() {
kill "$TAIL_PID" 2>/dev/null || true
# Stop tracy-capture first (SIGINT makes it disconnect and flush to disk)
# Must happen before killing reth, otherwise reth keeps streaming data.
if [ -n "${TRACY_PID:-}" ] && kill -0 "$TRACY_PID" 2>/dev/null; then
echo "Stopping tracy-capture..."
kill -INT "$TRACY_PID" 2>/dev/null || true
if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
sudo kill "$RETH_PID"
for i in $(seq 1 30); do
kill -0 "$TRACY_PID" 2>/dev/null || break
if [ $((i % 10)) -eq 0 ]; then
echo "Waiting for tracy-capture to finish writing... (${i}s)"
fi
sudo kill -0 "$RETH_PID" 2>/dev/null || break
sleep 1
done
if kill -0 "$TRACY_PID" 2>/dev/null; then
echo "tracy-capture still running after 30s, killing..."
kill -9 "$TRACY_PID" 2>/dev/null || true
fi
wait "$TRACY_PID" 2>/dev/null || true
fi
if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
# Send SIGINT to the inner reth process by exact name (not -f which
# would also match samply's cmdline containing "reth"). Samply will
# capture reth's exit and save the profile.
sudo pkill -INT -x reth 2>/dev/null || true
# Wait for samply to finish writing the profile and exit
for i in $(seq 1 120); do
sudo pgrep -x samply > /dev/null 2>&1 || break
if [ $((i % 10)) -eq 0 ]; then
echo "Waiting for samply to finish writing profile... (${i}s)"
fi
sleep 1
done
if sudo pgrep -x samply > /dev/null 2>&1; then
echo "Samply still running after 120s, sending SIGTERM..."
sudo pkill -x samply 2>/dev/null || true
fi
else
sudo kill "$RETH_PID"
for i in $(seq 1 30); do
sudo kill -0 "$RETH_PID" 2>/dev/null || break
sleep 1
done
fi
sudo kill -9 "$RETH_PID" 2>/dev/null || true
sleep 1
fi
# Fix ownership of reth-created files (reth runs as root)
sudo chown -R "$(id -un):$(id -gn)" "$OUTPUT_DIR" 2>/dev/null || true
if mountpoint -q "$SCHELK_MOUNT"; then
sudo umount -l "$SCHELK_MOUNT" || true
sudo schelk recover -y || true
fi
}
TAIL_PID=
TRACY_PID=
trap cleanup EXIT
# Clean up stale schelk state from a previous cancelled run.
# If schelk thinks it's still mounted (e.g. a cancelled run skipped cleanup),
# recover first to reset state.
sudo schelk recover -y -k || true
# Mount
sudo schelk mount -y
sync
@@ -98,101 +46,18 @@ grep Cached /proc/meminfo
# CPU layout: core 0 = OS/IRQs/reth-bench/aux, cores 1+ = reth node
RETH_BENCH="$(which reth-bench)"
ONLINE=$(nproc --all)
MAX_RETH=$(( ONLINE - 1 ))
if [ "${BENCH_CORES:-0}" -gt 0 ] && [ "$BENCH_CORES" -lt "$MAX_RETH" ]; then
MAX_RETH=$BENCH_CORES
fi
RETH_CPUS="1-${MAX_RETH}"
BIG_BLOCKS="${BENCH_BIG_BLOCKS:-false}"
RETH_ARGS=(
node
--datadir "$DATADIR"
--log.file.directory "$OUTPUT_DIR/reth-logs"
--engine.accept-execution-requests-hash
--http
--http.port 8545
--ws
--ws.api all
--authrpc.port 8551
--disable-discovery
--no-persist-peers
)
# Gate flag on binary support (older baselines may not have it).
# Uses --help which exits immediately via clap without node init.
SYNC_STATE_IDLE=false
if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then
RETH_ARGS+=(--debug.startup-sync-state-idle)
SYNC_STATE_IDLE=true
fi
# Big blocks mode requires the testing API, skip-invalid-transactions, and
# skip-gas-limit-ramp-check + gas-limit override to avoid the 6800-block ramp.
if [ "$BIG_BLOCKS" = "true" ]; then
RETH_ARGS+=(--http.api eth,net,web3,reth,testing --rpc.max-request-size max --testing.skip-invalid-transactions --testing.skip-gas-limit-ramp-check --testing.gas-limit 1000000000)
fi
# Append per-label extra node args (baseline or feature)
EXTRA_NODE_ARGS=""
case "$LABEL" in
baseline*) EXTRA_NODE_ARGS="${BENCH_BASELINE_ARGS:-}" ;;
feature*) EXTRA_NODE_ARGS="${BENCH_FEATURE_ARGS:-}" ;;
esac
if [ -n "$EXTRA_NODE_ARGS" ]; then
# Word-split the string into individual args
# shellcheck disable=SC2206
RETH_ARGS+=($EXTRA_NODE_ARGS)
fi
if [ -n "${BENCH_METRICS_ADDR:-}" ]; then
RETH_ARGS+=(--metrics "$BENCH_METRICS_ADDR")
fi
# OTLP traces and logs export
if [ -n "${BENCH_OTLP_TRACES_ENDPOINT:-}" ]; then
RETH_ARGS+=(--tracing-otlp="${BENCH_OTLP_TRACES_ENDPOINT}" --tracing-otlp.service-name=reth-bench)
fi
if [ -n "${BENCH_OTLP_LOGS_ENDPOINT:-}" ]; then
RETH_ARGS+=(--logs-otlp="${BENCH_OTLP_LOGS_ENDPOINT}" --logs-otlp.filter=debug)
fi
# Tracy profiling: add --log.tracy flags and set environment
if [ "${BENCH_TRACY:-off}" != "off" ]; then
RETH_ARGS+=(--log.tracy --log.tracy.filter "${BENCH_TRACY_FILTER:-debug}")
if [ "${BENCH_TRACY}" = "on" ]; then
export TRACY_NO_SYS_TRACE=1
elif [ "${BENCH_TRACY}" = "full" ]; then
export TRACY_SAMPLING_HZ="${BENCH_TRACY_SAMPLING_HZ:-1}"
fi
fi
SUDO_ENV=()
if [ -n "${OTEL_RESOURCE_ATTRIBUTES:-}" ]; then
SUDO_ENV+=("OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES}")
SUDO_ENV+=("OTEL_BSP_MAX_QUEUE_SIZE=65536" "OTEL_BLRP_MAX_QUEUE_SIZE=65536")
fi
# Limit reth memory to 95% of available RAM to prevent OOM kills
TOTAL_MEM_KB=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
MEM_LIMIT=$(( TOTAL_MEM_KB * 95 / 100 * 1024 ))
echo "Memory limit: $(( MEM_LIMIT / 1024 / 1024 ))MB (95% of $(( TOTAL_MEM_KB / 1024 ))MB)"
if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
RETH_ARGS+=(--log.samply)
SAMPLY="$(which samply)"
sudo systemd-run --scope -p MemoryMax="$MEM_LIMIT" -p AllowedCPUs="$RETH_CPUS" \
env "${SUDO_ENV[@]}" nice -n -20 \
"$SAMPLY" record --save-only --presymbolicate --rate 10000 \
--output "$OUTPUT_DIR/samply-profile.json.gz" \
-- "$BINARY" "${RETH_ARGS[@]}" \
> "$LOG" 2>&1 &
else
sudo systemd-run --scope -p MemoryMax="$MEM_LIMIT" -p AllowedCPUs="$RETH_CPUS" \
env "${SUDO_ENV[@]}" nice -n -20 "$BINARY" "${RETH_ARGS[@]}" \
> "$LOG" 2>&1 &
fi
RETH_CPUS="1-$(( ONLINE - 1 ))"
sudo taskset -c "$RETH_CPUS" nice -n -20 "$BINARY" node \
--datadir "$DATADIR" \
--engine.accept-execution-requests-hash \
--http \
--http.port 8545 \
--ws \
--ws.api all \
--authrpc.port 8551 \
--disable-discovery \
--no-persist-peers \
> "$LOG" 2>&1 &
RETH_PID=$!
stdbuf -oL tail -f "$LOG" | sed -u "s/^/[reth] /" &
@@ -203,7 +68,7 @@ for i in $(seq 1 60); do
-H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
> /dev/null 2>&1; then
echo "reth (${LABEL}) RPC is up after ${i}s"
echo "reth (${LABEL}) is ready after ${i}s"
break
fi
if [ "$i" -eq 60 ]; then
@@ -214,87 +79,25 @@ for i in $(seq 1 60); do
sleep 1
done
# Wait for the pipeline to finish (eth_syncing returns false) so the
# engine is in live mode and can accept newPayload calls.
# Only possible when --debug.startup-sync-state-idle is supported.
if [ "$SYNC_STATE_IDLE" = "true" ]; then
for i in $(seq 1 300); do
SYNC_RESULT=$(curl -sf http://127.0.0.1:8545 -X POST \
-H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' 2>/dev/null || true)
if [ -n "$SYNC_RESULT" ] && jq -e '.result == false' <<< "$SYNC_RESULT" > /dev/null 2>&1; then
echo "reth (${LABEL}) pipeline finished after ${i}s, engine is live"
break
fi
if [ "$i" -eq 300 ]; then
echo "::error::reth (${LABEL}) pipeline did not finish within 300s"
cat "$LOG"
exit 1
fi
sleep 1
done
else
echo "reth (${LABEL}) binary does not support --debug.startup-sync-state-idle, skipping sync wait"
fi
# Run reth-bench with high priority but as the current user so output
# files are not root-owned (avoids EACCES on next checkout).
BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
# Build optional flags
EXTRA_BENCH_ARGS=()
if [ "${BENCH_RETH_NEW_PAYLOAD:-true}" != "false" ]; then
EXTRA_BENCH_ARGS+=(--reth-new-payload)
fi
if [ -n "${BENCH_WAIT_TIME:-}" ]; then
EXTRA_BENCH_ARGS+=(--wait-time "$BENCH_WAIT_TIME")
fi
# Warmup
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
--rpc-url "$BENCH_RPC_URL" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
--reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
if [ "$BIG_BLOCKS" = "true" ]; then
# Big blocks mode: replay pre-generated payloads
BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
# Start tracy-capture so profile only covers the benchmark
if [ "${BENCH_TRACY:-off}" != "off" ]; then
echo "Starting tracy-capture..."
tracy-capture -f -o "$OUTPUT_DIR/tracy-profile.tracy" &
TRACY_PID=$!
sleep 0.5 # give tracy-capture time to connect
fi
echo "Running big blocks benchmark (replay-payloads)..."
$BENCH_NICE "$RETH_BENCH" replay-payloads \
"${EXTRA_BENCH_ARGS[@]}" \
--payload-dir "$BIG_BLOCKS_DIR/payloads" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
else
# Standard mode: warmup + new-payload-fcu
# Warmup
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
--rpc-url "$BENCH_RPC_URL" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "${BENCH_WARMUP_BLOCKS:-50}" \
"${EXTRA_BENCH_ARGS[@]}" 2>&1 | sed -u "s/^/[bench] /"
# Start tracy-capture after warmup so profile only covers the benchmark
if [ "${BENCH_TRACY:-off}" != "off" ]; then
echo "Starting tracy-capture..."
tracy-capture -f -o "$OUTPUT_DIR/tracy-profile.tracy" &
TRACY_PID=$!
sleep 0.5 # give tracy-capture time to connect
fi
# Benchmark
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
--rpc-url "$BENCH_RPC_URL" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "$BENCH_BLOCKS" \
"${EXTRA_BENCH_ARGS[@]}" \
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
fi
# Benchmark
$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
--rpc-url "$BENCH_RPC_URL" \
--engine-rpc-url http://127.0.0.1:8551 \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "$BENCH_BLOCKS" \
--reth-new-payload \
--output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
# cleanup runs via trap

View File

@@ -1,17 +1,15 @@
#!/usr/bin/env bash
#
# Downloads the latest snapshot into the schelk volume using
# `reth download` with progress reporting to the GitHub PR comment.
# Downloads the latest nightly snapshot into the schelk volume with
# progress reporting to the GitHub PR comment.
#
# Skips the download if the manifest content hasn't changed since
# the last successful download (checked via SHA-256 of the manifest).
# Skips the download if the local ETag marker matches the remote one.
#
# Usage: bench-reth-snapshot.sh [--check]
# --check Only check if a download is needed; exits 0 if up-to-date, 10 if not.
# --check Only check if a download is needed; exits 0 if up-to-date, 1 if not.
#
# Required env:
# SCHELK_MOUNT schelk mount point (e.g. /reth-bench)
# BENCH_RETH_BINARY path to the reth binary
# GITHUB_TOKEN token for GitHub API calls (only for download)
# BENCH_COMMENT_ID PR comment ID to update (optional)
# BENCH_REPO owner/repo (e.g. paradigmxyz/reth)
@@ -20,65 +18,52 @@
# BENCH_CONFIG config summary line
set -euo pipefail
MC="mc"
BUCKET="minio/reth-snapshots"
MANIFEST_PATH="reth-1-minimal-stable/manifest.json"
BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous.tar.zst"
DATADIR="$SCHELK_MOUNT/datadir"
HASH_FILE="$HOME/.reth-bench-snapshot-hash"
ETAG_FILE="$HOME/.reth-bench-snapshot-etag"
# Fetch manifest and compute content hash for reliable freshness check
MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
exit 2
}
REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
# Get remote metadata via JSON for reliable parsing
MC_STAT=$(mc stat --json "$BUCKET" 2>/dev/null || true)
REMOTE_ETAG=$(echo "$MC_STAT" | jq -r '.etag // empty')
if [ -z "$REMOTE_ETAG" ]; then
echo "::warning::Failed to get ETag from mc stat, will re-download"
REMOTE_ETAG="unknown-$(date +%s)"
fi
LOCAL_HASH=""
[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
LOCAL_ETAG=""
[ -f "$ETAG_FILE" ] && LOCAL_ETAG=$(cat "$ETAG_FILE")
if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16})"
if [ "$REMOTE_ETAG" = "$LOCAL_ETAG" ]; then
echo "Snapshot is up-to-date (ETag: ${REMOTE_ETAG})"
if [ "${1:-}" = "--check" ]; then
exit 0
fi
exit 0
fi
echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16})"
echo "Snapshot needs update (local: ${LOCAL_ETAG:-<none>}, remote: ${REMOTE_ETAG})"
if [ "${1:-}" = "--check" ]; then
exit 10
fi
RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
if [ ! -x "$RETH" ]; then
echo "::error::reth binary not found or not executable at $RETH"
exit 1
fi
# Resolve the MinIO HTTP endpoint from the mc alias so reth can
# fetch archives over HTTP (the manifest's embedded base_url points
# to the cluster-internal address which is unreachable from runners).
MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
if [ -z "$MINIO_ENDPOINT" ]; then
echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
# Get compressed size for progress tracking
TOTAL_BYTES=$(echo "$MC_STAT" | jq -r '.size // empty')
if [ -z "$TOTAL_BYTES" ] || [ "$TOTAL_BYTES" = "0" ]; then
echo "::error::Failed to get snapshot size from mc stat"
exit 1
fi
BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/reth-1-minimal-stable"
# Rewrite manifest's base_url with the runner-reachable endpoint
MANIFEST_TMP=$(mktemp --suffix=.json)
trap 'rm -f -- "$MANIFEST_TMP"' EXIT
echo "$MANIFEST_CONTENT" \
| jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
echo "Snapshot size: $TOTAL_BYTES bytes ($(numfmt --to=iec "$TOTAL_BYTES"))"
# Prepare mount
mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
sudo schelk mount -y
sudo rm -rf "$DATADIR"
sudo mkdir -p "$DATADIR"
# reth download runs as current user (not root), needs write access
sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
update_comment() {
local status="$1"
local pct="$1"
[ -z "${BENCH_COMMENT_ID:-}" ] && return 0
local status="Building binaries & downloading snapshot… ${pct}%"
local body
body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
"$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
@@ -90,31 +75,53 @@ update_comment() {
> /dev/null 2>&1 || true
}
update_comment "Downloading snapshot…"
# Track compressed bytes flowing through the pipe
DL_BYTES_FILE=$(mktemp)
echo 0 > "$DL_BYTES_FILE"
# Download using reth download (manifest-path with rewritten base_url)
"$RETH" download \
--manifest-path "$MANIFEST_TMP" \
-y \
--minimal \
--datadir "$DATADIR"
# Start progress reporter in background
(
while true; do
sleep 10
CURRENT=$(cat "$DL_BYTES_FILE" 2>/dev/null || echo 0)
if [ "$TOTAL_BYTES" -gt 0 ]; then
PCT=$(( CURRENT * 100 / TOTAL_BYTES ))
[ "$PCT" -gt 100 ] && PCT=100
echo "Snapshot download: $(numfmt --to=iec "$CURRENT") / $(numfmt --to=iec "$TOTAL_BYTES") (${PCT}%)"
update_comment "$PCT"
fi
done
) &
PROGRESS_PID=$!
trap 'kill $PROGRESS_PID 2>/dev/null || true; rm -f "$DL_BYTES_FILE"' EXIT
update_comment "Downloading snapshot… done"
# Download and extract; python byte counter tracks compressed bytes received
mc cat "$BUCKET" | python3 -c "
import sys
count = 0
while True:
data = sys.stdin.buffer.read(1048576)
if not data:
break
count += len(data)
sys.stdout.buffer.write(data)
with open('$DL_BYTES_FILE', 'w') as f:
f.write(str(count))
" | pzstd -d -p 6 | sudo tar -xf - -C "$DATADIR"
# Stop progress reporter
kill $PROGRESS_PID 2>/dev/null || true
wait $PROGRESS_PID 2>/dev/null || true
update_comment "100"
echo "Snapshot download complete"
# Sanity check: verify expected directories exist
if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
ls -la "$DATADIR" || true
exit 1
fi
# Promote the new snapshot to become the schelk baseline (virgin volume).
# This copies changed blocks from scratch → virgin so that future
# `schelk recover` calls restore to this new state.
sync
sudo schelk promote -y
# Save manifest hash
echo "$REMOTE_HASH" > "$HASH_FILE"
echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16})"
# Save ETag marker
echo "$REMOTE_ETAG" > "$ETAG_FILE"
echo "Snapshot promoted to schelk baseline (ETag: ${REMOTE_ETAG})"

View File

@@ -107,10 +107,6 @@ def compute_stats(combined: list[dict]) -> dict:
mgas_s_values.append(r["gas_used"] / lat_s / 1_000_000)
mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0
total_latencies_ms = [r["total_latency_us"] / 1_000 for r in combined]
wall_clock_s = sum(total_latencies_ms) / 1_000
mean_total_lat_ms = sum(total_latencies_ms) / n
return {
"n": n,
"mean_ms": mean_lat,
@@ -119,8 +115,6 @@ def compute_stats(combined: list[dict]) -> dict:
"p90_ms": percentile(sorted_lat, 90),
"p99_ms": percentile(sorted_lat, 99),
"mean_mgas_s": mean_mgas_s,
"wall_clock_s": wall_clock_s,
"mean_total_lat_ms": mean_total_lat_ms,
}
@@ -145,14 +139,13 @@ def compute_wait_stats(combined: list[dict], field: str) -> dict:
def _paired_data(
baseline: list[dict], feature: list[dict]
) -> tuple[list[tuple[float, float]], list[float], list[float], list[float]]:
) -> tuple[list[tuple[float, float]], list[float], list[float]]:
"""Match blocks and return paired latencies and per-block diffs.
Returns:
pairs: list of (baseline_ms, feature_ms) tuples
lat_diffs_ms: list of feature baseline latency diffs in ms
mgas_diffs: list of feature baseline Mgas/s diffs
total_lat_diffs_ms: list of feature baseline total latency diffs in ms
"""
baseline_by_block = {r["block_number"]: r for r in baseline}
feature_by_block = {r["block_number"]: r for r in feature}
@@ -161,7 +154,6 @@ def _paired_data(
pairs = []
lat_diffs_ms = []
mgas_diffs = []
total_lat_diffs_ms = []
for bn in common_blocks:
b = baseline_by_block[bn]
f = feature_by_block[bn]
@@ -176,10 +168,7 @@ def _paired_data(
f["gas_used"] / f_lat_s / 1_000_000
- b["gas_used"] / b_lat_s / 1_000_000
)
total_lat_diffs_ms.append(
f["total_latency_us"] / 1_000 - b["total_latency_us"] / 1_000
)
return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms
return pairs, lat_diffs_ms, mgas_diffs
def compute_paired_stats(
@@ -194,14 +183,12 @@ def compute_paired_stats(
all_pairs = []
all_lat_diffs = []
all_mgas_diffs = []
all_total_lat_diffs = []
blocks_per_pair = []
for baseline, feature in zip(baseline_runs, feature_runs):
pairs, lat_diffs, mgas_diffs, total_lat_diffs = _paired_data(baseline, feature)
pairs, lat_diffs, mgas_diffs = _paired_data(baseline, feature)
all_pairs.extend(pairs)
all_lat_diffs.extend(lat_diffs)
all_mgas_diffs.extend(mgas_diffs)
all_total_lat_diffs.extend(total_lat_diffs)
blocks_per_pair.append(len(pairs))
if not all_lat_diffs:
@@ -240,11 +227,6 @@ def compute_paired_stats(
mgas_se = std_mgas_diff / math.sqrt(len(all_mgas_diffs)) if all_mgas_diffs else 0.0
mgas_ci = T_CRITICAL * mgas_se
mean_total_diff = sum(all_total_lat_diffs) / len(all_total_lat_diffs) if all_total_lat_diffs else 0.0
std_total_diff = stddev(all_total_lat_diffs, mean_total_diff) if len(all_total_lat_diffs) > 1 else 0.0
total_se = std_total_diff / math.sqrt(len(all_total_lat_diffs)) if all_total_lat_diffs else 0.0
wall_clock_ci_ms = T_CRITICAL * total_se
return {
"n": n,
"mean_diff_ms": mean_diff,
@@ -257,11 +239,17 @@ def compute_paired_stats(
"p99_ci_ms": (p99_boot[hi] - p99_boot[lo]) / 2,
"mean_mgas_diff": mean_mgas_diff,
"mgas_ci": mgas_ci,
"wall_clock_ci_ms": wall_clock_ci_ms,
"blocks": max(blocks_per_pair),
}
def compute_summary(combined: list[dict], gas: list[dict]) -> dict:
"""Compute aggregate metrics from parsed CSV data."""
blocks = len(combined)
return {
"blocks": blocks,
}
def format_duration(seconds: float) -> str:
if seconds >= 60:
@@ -286,61 +274,22 @@ def fmt_mgas(v: float) -> str:
return f"{v:.2f}"
def fmt_s(v: float) -> str:
return f"{v:.2f}s"
def significance(pct: float, ci_pct: float, lower_is_better: bool) -> str:
"""Return significance label: 'good', 'bad', or 'neutral'."""
significant = abs(pct) > ci_pct
if not significant:
return "neutral"
elif (pct < 0) == lower_is_better:
return "good"
else:
return "bad"
def change_str(pct: float, ci_pct: float, lower_is_better: bool) -> str:
"""Format change% with paired CI significance.
Significant if the CI doesn't cross zero (i.e. |pct| > ci_pct).
"""
sig = significance(pct, ci_pct, lower_is_better)
emoji = {"good": "", "bad": "", "neutral": ""}[sig]
significant = abs(pct) > ci_pct
if not significant:
emoji = ""
elif (pct < 0) == lower_is_better:
emoji = ""
else:
emoji = ""
return f"{pct:+.2f}% {emoji}{ci_pct:.2f}%)"
def compute_changes(
baseline_stats: dict, feature_stats: dict, paired_stats: dict
) -> dict:
"""Pre-compute change percentages and significance for each metric."""
def pct(base: float, feat: float) -> float:
return (feat - base) / base * 100.0 if base > 0 else 0.0
def ci_pct(ci_ms: float, base_ms: float) -> float:
return ci_ms / base_ms * 100.0 if base_ms > 0 else 0.0
metrics = [
("mean", "mean_ms", "ci_ms", "mean_ms", True),
("p50", "p50_ms", "p50_ci_ms", "p50_ms", True),
("p90", "p90_ms", "p90_ci_ms", "p90_ms", True),
("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True),
]
changes = {}
for name, stat_key, ci_key, base_key, lower_is_better in metrics:
p = pct(baseline_stats[stat_key], feature_stats[stat_key])
c = ci_pct(paired_stats[ci_key], baseline_stats[base_key])
changes[name] = {
"pct": round(p, 4),
"ci_pct": round(c, 4),
"sig": significance(p, c, lower_is_better),
}
return changes
def generate_comparison_table(
run1: dict,
run2: dict,
@@ -350,7 +299,6 @@ def generate_comparison_table(
baseline_name: str,
feature_name: str,
feature_sha: str,
big_blocks: bool = False,
) -> str:
"""Generate a markdown comparison table between baseline and feature."""
n = paired["blocks"]
@@ -360,7 +308,6 @@ def generate_comparison_table(
mean_pct = pct(run1["mean_ms"], run2["mean_ms"])
gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"])
wall_pct = pct(run1["wall_clock_s"], run2["wall_clock_s"])
p50_pct = pct(run1["p50_ms"], run2["p50_ms"])
p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
@@ -374,7 +321,6 @@ def generate_comparison_table(
# CI as a percentage of baseline mean
lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
wall_ci_pct = paired["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0
base_url = f"https://github.com/{repo}/commit"
baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
@@ -389,9 +335,8 @@ def generate_comparison_table(
f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, lower_is_better=True)} |",
f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, lower_is_better=True)} |",
"",
f"*{n} {'big blocks' if big_blocks else 'blocks'}*",
f"*{n} blocks*",
]
return "\n".join(lines)
@@ -422,7 +367,6 @@ def generate_markdown(
summary: dict, comparison_table: str,
wait_time_tables: list[str] | None = None,
behind_baseline: int = 0, repo: str = "", baseline_ref: str = "", baseline_name: str = "",
grafana_url: str | None = None,
) -> str:
"""Generate a markdown comment body."""
lines = ["## Benchmark Results", ""]
@@ -442,9 +386,6 @@ def generate_markdown(
lines.append(table)
lines.append("")
lines.append("</details>")
if grafana_url:
lines.append("")
lines.append(f"**[Grafana Dashboard]({grafana_url})**")
return "\n".join(lines)
@@ -471,8 +412,6 @@ def main():
parser.add_argument("--feature-name", "--branch-name", default=None, help="Feature branch name")
parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode")
parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL for this benchmark run")
args = parser.parse_args()
if len(args.baseline_csv) != len(args.feature_csv):
@@ -499,6 +438,11 @@ def main():
all_baseline = [r for run in baseline_runs for r in run]
all_feature = [r for run in feature_runs for r in run]
summary = compute_summary(all_feature, gas)
with open(args.output_summary, "w") as f:
json.dump(summary, f, indent=2)
print(f"Summary written to {args.output_summary}")
baseline_stats = compute_stats(all_baseline)
feature_stats = compute_stats(all_feature)
paired_stats = compute_paired_stats(baseline_runs, feature_runs)
@@ -521,7 +465,6 @@ def main():
baseline_name=baseline_name,
feature_name=feature_name,
feature_sha=feature_sha,
big_blocks=args.big_blocks,
)
print(f"Generated comparison ({paired_stats['n']} paired blocks, "
f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")
@@ -536,41 +479,13 @@ def main():
("execution_cache_wait_us", "Execution Cache Update Wait"),
]
wait_time_tables = []
wait_time_data = {}
for field, title in wait_fields:
b_stats = compute_wait_stats(all_baseline, field)
f_stats = compute_wait_stats(all_feature, field)
if b_stats and f_stats:
wait_time_data[field] = {
"title": title,
"baseline": b_stats,
"feature": f_stats,
}
table = generate_wait_time_table(title, b_stats, f_stats, baseline_label, feature_label)
if table:
wait_time_tables.append(table)
summary = {
"blocks": paired_stats["blocks"],
"big_blocks": args.big_blocks,
"baseline": {
"name": baseline_name,
"ref": baseline_ref,
"stats": baseline_stats,
},
"feature": {
"name": feature_name,
"ref": feature_sha,
"stats": feature_stats,
},
"paired": paired_stats,
"changes": compute_changes(baseline_stats, feature_stats, paired_stats),
"wait_times": wait_time_data,
}
with open(args.output_summary, "w") as f:
json.dump(summary, f, indent=2)
print(f"Summary written to {args.output_summary}")
markdown = generate_markdown(
summary, comparison_table,
wait_time_tables=wait_time_tables,
@@ -578,7 +493,6 @@ def main():
repo=args.repo,
baseline_ref=baseline_ref,
baseline_name=baseline_name,
grafana_url=args.grafana_url,
)
with open(args.output_markdown, "w") as f:

View File

@@ -1,132 +0,0 @@
#!/usr/bin/env bash
#
# Resolves baseline and feature refs for nightly regression benchmark runs.
#
# Queries the latest successful scheduled docker.yml run via GitHub API
# to find the commit that built the nightly Docker image. Compares with
# the last successful feature ref (from GH Actions cache) to determine
# baseline, detect staleness, and decide whether to skip.
#
# Usage: bench-nightly-refs.sh [--force]
#
# Outputs (via GITHUB_OUTPUT):
# baseline-ref — commit SHA for baseline
# feature-ref — commit SHA for feature (current nightly)
# should-skip — "true" if no new nightly since last run
# is-stale — "true" if latest nightly build is >24h old
# stale-age-hours — age of the nightly build in hours (only if stale)
# nightly-created — ISO timestamp of the nightly build
#
# Reads:
# .nightly-state/last-feature-ref (from GH Actions cache, may not exist)
#
# Requires: gh (GitHub CLI), jq, date
set -euo pipefail
FORCE="${1:-false}"
REPO="${GITHUB_REPOSITORY:-paradigmxyz/reth}"
# --- Step 1: Query latest successful scheduled docker.yml run ---
echo "::group::Querying latest nightly docker build"
RUNS_JSON=$(gh run list \
-R "$REPO" \
--workflow=docker.yml \
--event=schedule \
--status=completed \
--limit 5 \
--json headSha,createdAt,conclusion)
# Find the most recent successful run
LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
if [ -z "$LATEST" ]; then
echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
echo "Runs found: $RUNS_JSON"
exit 1
fi
FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt')
echo "Latest nightly commit: $FEATURE_REF"
echo "Built at: $CREATED_AT"
echo "::endgroup::"
# --- Step 2: Staleness check ---
echo "::group::Checking staleness"
NOW_EPOCH=$(date +%s)
# Handle both GNU date (-d) and BSD date (-j -f) for cross-platform compat
CREATED_EPOCH=$(date -d "$CREATED_AT" +%s 2>/dev/null || \
date -j -f "%Y-%m-%dT%H:%M:%SZ" "$CREATED_AT" +%s 2>/dev/null || \
date -j -f "%Y-%m-%dT%T%z" "$CREATED_AT" +%s 2>/dev/null || \
{ echo "::error::Cannot parse date: $CREATED_AT"; exit 1; })
AGE_SECONDS=$(( NOW_EPOCH - CREATED_EPOCH ))
AGE_HOURS=$(( AGE_SECONDS / 3600 ))
IS_STALE="false"
if [ "$AGE_HOURS" -gt 24 ]; then
IS_STALE="true"
echo "::warning::STALE NIGHTLY: Build is ${AGE_HOURS}h old (>24h threshold)"
echo "This indicates the nightly docker build failed — no new image was produced"
else
echo "Nightly build age: ${AGE_HOURS}h (within 24h threshold)"
fi
echo "::endgroup::"
# --- Step 3: Read last successful feature ref from cache ---
echo "::group::Reading cached state"
LAST_FEATURE_REF=""
STATE_FILE=".nightly-state/last-feature-ref"
if [ -f "$STATE_FILE" ]; then
LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
echo "Previous feature ref: $LAST_FEATURE_REF"
else
echo "No cached state found (first run)"
fi
echo "::endgroup::"
# --- Step 4: Determine baseline and skip logic ---
echo "::group::Resolving refs"
SHOULD_SKIP="false"
BASELINE_REF="$FEATURE_REF" # default for first run
if [ "$IS_STALE" = "true" ]; then
# Stale = error path, don't skip (will alert and fail downstream)
SHOULD_SKIP="false"
BASELINE_REF="${LAST_FEATURE_REF:-$FEATURE_REF}"
echo "Stale nightly detected — will alert and fail"
elif [ -z "$LAST_FEATURE_REF" ]; then
# First run: baseline = feature (self-comparison to establish baseline)
BASELINE_REF="$FEATURE_REF"
echo "First run — will benchmark nightly against itself to establish baseline"
elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
# No new nightly since last successful run
if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
echo "No new nightly, but force=true — running anyway"
BASELINE_REF="$LAST_FEATURE_REF"
else
SHOULD_SKIP="true"
echo "No new nightly since last run — will skip"
fi
else
# Normal case: new nightly available
BASELINE_REF="$LAST_FEATURE_REF"
echo "New nightly detected"
fi
echo "Baseline: $BASELINE_REF"
echo "Feature: $FEATURE_REF"
echo "Skip: $SHOULD_SKIP"
echo "Stale: $IS_STALE"
echo "::endgroup::"
# --- Step 5: Write outputs ---
{
echo "baseline-ref=$BASELINE_REF"
echo "feature-ref=$FEATURE_REF"
echo "should-skip=$SHOULD_SKIP"
echo "is-stale=$IS_STALE"
echo "stale-age-hours=$AGE_HOURS"
echo "nightly-created=$CREATED_AT"
} >> "$GITHUB_OUTPUT"

View File

@@ -1,308 +0,0 @@
// Sends Slack notifications for reth-bench results.
//
// Reads from environment:
// SLACK_BENCH_BOT_TOKEN Slack Bot User OAuth Token (xoxb-...)
// SLACK_BENCH_CHANNEL Public channel ID for significant improvements
// BENCH_WORK_DIR Directory containing summary.json
// BENCH_PR PR number (may be empty)
// BENCH_ACTOR GitHub user who triggered the bench
// BENCH_JOB_URL URL to the Actions job page
// BENCH_BASELINE_ARGS Extra CLI args for the baseline reth node
// BENCH_FEATURE_ARGS Extra CLI args for the feature reth node
// BENCH_SAMPLY 'true' if samply profiling was enabled
//
// Usage from actions/github-script:
// const notify = require('./.github/scripts/bench-slack-notify.js');
// await notify.success({ core, context });
// await notify.failure({ core, context, failedStep: '...' });
const fs = require('fs');
const path = require('path');
const { fmtChange, fmtMs, verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
const SLACK_API = 'https://slack.com/api/chat.postMessage';
function loadSlackUsers(repoRoot) {
try {
const raw = fs.readFileSync(path.join(repoRoot, '.github', 'scripts', 'bench-slack-users.json'), 'utf8');
const data = JSON.parse(raw);
// Filter out non-user-ID entries (like _comment)
const users = {};
for (const [k, v] of Object.entries(data)) {
if (!k.startsWith('_') && typeof v === 'string' && v.startsWith('U')) {
users[k] = v;
}
}
return users;
} catch {
return {};
}
}
async function postToSlack(token, channel, blocks, text, core, threadTs) {
const payload = { channel, blocks, text, unfurl_links: false };
if (threadTs) payload.thread_ts = threadTs;
const resp = await fetch(SLACK_API, {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify(payload),
});
const data = await resp.json();
if (!data.ok) {
core.warning(`Slack API error (channel ${channel}): ${JSON.stringify(data)}`);
}
return data;
}
function cell(text) {
const s = String(text);
return { type: 'raw_text', text: s || ' ' };
}
// Slack shortcodes for verdict (Block Kit header doesn't support unicode emoji)
const SLACK_VERDICT = {
'⚠️': ':warning:',
'❌': ':x:',
'✅': ':white_check_mark:',
'⚪': ':white_circle:',
};
function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls }) {
const { emoji, label } = verdict(summary.changes);
const headerEmoji = SLACK_VERDICT[emoji] || emoji;
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
const commitUrl = `https://github.com/${repo}/commit`;
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
// Meta line
const metaParts = [];
if (prNumber) metaParts.push(`*<${prUrl}|PR #${prNumber}>*`);
metaParts.push(`triggered by ${actorSlackId ? `<@${actorSlackId}>` : `@${actor}`}`);
// Baseline/feature lines with samply profile links
let baselineLine = `*Baseline:* ${baselineLink}`;
const bl1 = samplyUrls['baseline-1'];
const bl2 = samplyUrls['baseline-2'];
if (bl1) baselineLine += ` | <${bl1}|Samply 1>`;
if (bl2) baselineLine += ` | <${bl2}|Samply 2>`;
let featureLine = `*Feature:* ${featureLink}`;
const fl1 = samplyUrls['feature-1'];
const fl2 = samplyUrls['feature-2'];
if (fl1) featureLine += ` | <${fl1}|Samply 1>`;
if (fl2) featureLine += ` | <${fl2}|Samply 2>`;
const countsLine = blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | ');
const baselineArgs = process.env.BENCH_BASELINE_ARGS || '';
const featureArgs = process.env.BENCH_FEATURE_ARGS || '';
const argsLines = [];
if (baselineArgs) argsLines.push(`*Baseline Args:* \`${baselineArgs}\``);
if (featureArgs) argsLines.push(`*Feature Args:* \`${featureArgs}\``);
const sectionText = [metaParts.join(' | '), '', baselineLine, featureLine, ...argsLines, countsLine].join('\n');
// Action buttons
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
const buttons = [
{
type: 'button',
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
},
{
type: 'button',
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
url: diffUrl,
action_id: 'diff_button',
},
];
// Build table rows from shared metricRows
const rows = metricRows(summary);
const tableRows = [
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
];
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: `${headerEmoji} ${label}`, emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: sectionText },
},
{
type: 'table',
column_settings: [
{ align: 'left' },
{ align: 'right' },
{ align: 'right' },
{ align: 'right' },
],
rows: tableRows,
},
{
type: 'actions',
elements: buttons,
},
];
// Wait times as a separate table block (sent as threaded reply due to Slack one-table limit)
const threadBlocks = [];
const wtRows = waitTimeRows(summary);
if (wtRows.length > 0) {
const waitTableRows = [
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
];
threadBlocks.push({
type: 'table',
column_settings: [
{ align: 'left' },
{ align: 'right' },
{ align: 'right' },
],
rows: waitTableRows,
});
}
return { blocks, threadBlocks };
}
function buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep }) {
const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
const actorMention = actorSlackId ? `<@${actorSlackId}>` : `@${actor}`;
const parts = [
prNumber ? `*<${prUrl}|PR #${prNumber}>*` : '',
`by ${actorMention}`,
`failed while *${failedStep}*`,
].filter(Boolean);
const buttons = [
{
type: 'button',
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
},
];
return [
{
type: 'header',
text: { type: 'plain_text', text: ':rotating_light: Bench Failed', emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: parts.join(' | ') },
},
{
type: 'actions',
elements: buttons,
},
];
}
async function success({ core, context }) {
const token = process.env.SLACK_BENCH_BOT_TOKEN;
if (!token) {
core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
return;
}
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
core.warning('Could not read summary.json for Slack notification');
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const prNumber = process.env.BENCH_PR;
const actor = process.env.BENCH_ACTOR;
const jobUrl = process.env.BENCH_JOB_URL ||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
const actorSlackId = slackUsers[actor];
const { blocks, threadBlocks } = buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls });
const text = `Bench: ${summary.baseline.name} vs ${summary.feature.name}`;
async function sendWithThread(ch) {
const res = await postToSlack(token, ch, blocks, text, core);
if (res.ok && res.ts && threadBlocks.length > 0) {
for (const tb of threadBlocks) {
await postToSlack(token, ch, [tb], 'Wait time breakdown', core, res.ts);
}
}
}
// Post to public channel if any metric shows significant improvement or regression
const channel = process.env.SLACK_BENCH_CHANNEL;
let postedToChannel = false;
if (channel) {
const changes = summary.changes || {};
const hasImprovement = Object.values(changes).some(c => c.sig === 'good');
if (hasImprovement) {
await sendWithThread(channel);
postedToChannel = true;
} else {
core.info('No significant improvement, skipping public channel notification');
}
}
// DM the actor only when results were not posted to the public channel
if (!postedToChannel) {
if (actorSlackId) {
await sendWithThread(actorSlackId);
} else {
core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
}
} else {
core.info(`Results posted to channel, skipping DM to ${actor}`);
}
}
async function failure({ core, context, failedStep }) {
const token = process.env.SLACK_BENCH_BOT_TOKEN;
if (!token) {
core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const prNumber = process.env.BENCH_PR;
const actor = process.env.BENCH_ACTOR;
const jobUrl = process.env.BENCH_JOB_URL ||
`${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
const actorSlackId = slackUsers[actor];
const blocks = buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep });
const text = `Bench failed while ${failedStep}`;
// Always DM the actor
if (actorSlackId) {
await postToSlack(token, actorSlackId, blocks, text, core);
} else {
core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
}
// Only DM for failures, don't post to public channel
}
module.exports = { success, failure };

View File

@@ -1,24 +0,0 @@
{
"_comment": "Maps GitHub usernames to Slack user IDs. Find yours: Slack profile > ··· > Copy member ID.",
"shekhirin": "U09FAL2UMLJ",
"mattsse": "U09FQNPMRT3",
"klkvr": "U09FAK95FC2",
"joshieDo": "U09LHN6GYAU",
"mediocregopher": "U09FF75KMQU",
"yongkangc": "U09FB0ECTD4",
"gakonst": "U092SEPDM40",
"Rjected": "U09F6SCKRGT",
"DaniPopes": "U09FAT8EK2A",
"emmajam": "U0A34UN92HW",
"onbjerg": "U09FB0UK5AA",
"fgimenez": "U09G3GP7CSU",
"rakita": "U09FB3Z2M7Y",
"jxom": "U09F72MG083",
"tmm": "U0AD0U8E88N",
"pepyakin": "U0A7HKMGEHJ",
"grandizzy": "U09F8DBDDRT",
"SuperFluffy": "U095BKHB2Q4",
"kamsz": "U0A2563UBRD",
"zerosnacks": "U09FARPMN74",
"samczsun": "U096R14E4H3"
}

View File

@@ -1,95 +0,0 @@
// Shared utilities for reth-bench result rendering.
//
// Used by bench-job-summary.js and bench-slack-notify.js.
const fs = require('fs');
const path = require('path');
const SIG_EMOJI = { good: '✅', bad: '❌', neutral: '⚪' };
function fmtMs(v) { return v.toFixed(2) + 'ms'; }
function fmtMgas(v) { return v.toFixed(2); }
function fmtS(v) { return v.toFixed(2) + 's'; }
function fmtChange(ch) {
if (!ch || (!ch.pct && !ch.ci_pct)) return '';
const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
const ciStr = ch.ci_pct ? `${ch.ci_pct.toFixed(2)}%)` : '';
return `${pctStr}${ciStr} ${SIG_EMOJI[ch.sig]}`;
}
function verdict(changes) {
const vals = Object.values(changes);
const hasBad = vals.some(v => v.sig === 'bad');
const hasGood = vals.some(v => v.sig === 'good');
if (hasBad && hasGood) return { emoji: '⚠️', label: 'Mixed Results' };
if (hasBad) return { emoji: '❌', label: 'Regression' };
if (hasGood) return { emoji: '✅', label: 'Improvement' };
return { emoji: '⚪', label: 'No Difference' };
}
function loadSamplyUrls(workDir) {
const urls = {};
for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
try {
const url = fs.readFileSync(path.join(workDir, run, 'samply-profile-url.txt'), 'utf8').trim();
if (url) urls[run] = url;
} catch {}
}
return urls;
}
function blocksLabel(summary) {
const parts = [];
if (summary.big_blocks) {
parts.push({ key: 'Big Blocks', value: summary.blocks });
} else {
const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
if (warmup) parts.push({ key: 'Warmup', value: warmup });
parts.push({ key: 'Blocks', value: summary.blocks });
}
const cores = process.env.BENCH_CORES || '0';
if (cores !== '0') parts.push({ key: 'Cores', value: cores });
return parts;
}
// The 7 metric rows shared by all renderers.
// Returns an array of { label, baseline, feature, change } objects.
function metricRows(summary) {
const b = summary.baseline.stats;
const f = summary.feature.stats;
const c = summary.changes;
return [
{ label: 'Mean', baseline: fmtMs(b.mean_ms), feature: fmtMs(f.mean_ms), change: fmtChange(c.mean) },
{ label: 'StdDev', baseline: fmtMs(b.stddev_ms), feature: fmtMs(f.stddev_ms), change: '' },
{ label: 'P50', baseline: fmtMs(b.p50_ms), feature: fmtMs(f.p50_ms), change: fmtChange(c.p50) },
{ label: 'P90', baseline: fmtMs(b.p90_ms), feature: fmtMs(f.p90_ms), change: fmtChange(c.p90) },
{ label: 'P99', baseline: fmtMs(b.p99_ms), feature: fmtMs(f.p99_ms), change: fmtChange(c.p99) },
{ label: 'Mgas/s', baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
{ label: 'Wall Clock', baseline: fmtS(b.wall_clock_s), feature: fmtS(f.wall_clock_s), change: fmtChange(c.wall_clock) },
];
}
// Wait time rows: one row per metric showing mean values.
function waitTimeRows(summary) {
const waitTimes = summary.wait_times || {};
const rows = [];
for (const key of Object.keys(waitTimes)) {
const wt = waitTimes[key];
rows.push({ title: wt.title, baseline: fmtMs(wt.baseline.mean_ms), feature: fmtMs(wt.feature.mean_ms) });
}
return rows;
}
module.exports = {
SIG_EMOJI,
fmtMs,
fmtMgas,
fmtS,
fmtChange,
verdict,
loadSamplyUrls,
blocksLabel,
metricRows,
waitTimeRows,
};

View File

@@ -1,414 +0,0 @@
#!/usr/bin/env bash
#
# Full PGO+BOLT optimized build for reth using real reth-bench workloads.
#
# Phases:
# 1. Build PGO-instrumented reth, run reth-bench → collect PGO profiles
# 2. Build BOLT-instrumented reth (with PGO), run reth-bench → collect BOLT profiles
# 3. Build final PGO+BOLT optimized binary
#
# Required environment variables:
# DATADIR - Path to reth datadir (must already contain chain data)
# RPC_URL - Source RPC URL for reth-bench to fetch payloads from
#
# Optional environment variables:
# PGO_BLOCKS - Number of blocks for PGO profiling (default: 20)
# BOLT_BLOCKS - Number of blocks for BOLT profiling (default: 20)
# SKIP_BOLT - Temporarily skip BOLT phases (default: false)
# STRIP_SYMBOLS - Strip debug symbols from output binary (default: true)
# COLLECT_PGO_ONLY - Stop after producing merged.profdata (default: false)
# PGO_PROFDATA - Path to pre-collected merged.profdata (optional)
# PROFILE - Cargo profile (default: maxperf-symbols)
# FEATURES - Cargo features (default: jemalloc,asm-keccak,min-debug-logs)
# TARGET - Target triple (default: auto-detected)
# EXTRA_RUSTFLAGS - Additional RUSTFLAGS (e.g. -C target-cpu=x86-64-v3)
#
# Output:
# target/$PROFILE_DIR/reth — final optimized binary
set -euo pipefail
gha_section_start() {
local title="$1"
if [ -n "${GITHUB_ACTIONS:-}" ]; then
echo "::group::$title"
else
echo ""
echo "=== $title ==="
fi
}
gha_section_end() {
if [ -n "${GITHUB_ACTIONS:-}" ]; then
echo "::endgroup::"
fi
}
cd "$(dirname "$0")/../.."
# ── Configuration ──────────────────────────────────────────────────────────────
PGO_BLOCKS="${PGO_BLOCKS:-20}"
BOLT_BLOCKS="${BOLT_BLOCKS:-20}"
SKIP_BOLT="${SKIP_BOLT:-false}"
STRIP_SYMBOLS="${STRIP_SYMBOLS:-true}"
COLLECT_PGO_ONLY="${COLLECT_PGO_ONLY:-false}"
PROFILE="${PROFILE:-maxperf-symbols}"
FEATURES="${FEATURES:-jemalloc,asm-keccak,min-debug-logs}"
TARGET="${TARGET:-$(rustc -Vv | grep host | cut -d' ' -f2)}"
BASE_RUSTFLAGS="${RUSTFLAGS:-}"
EXTRA_RUSTFLAGS="${EXTRA_RUSTFLAGS:-}"
COMBINED_RUSTFLAGS="$BASE_RUSTFLAGS $EXTRA_RUSTFLAGS"
PGO_PROFDATA="${PGO_PROFDATA:-}"
DATADIR="${DATADIR:-}"
RPC_URL="${RPC_URL:-}"
SKIP_BOLT_BOOL=false
if [[ "${SKIP_BOLT,,}" == "true" || "$SKIP_BOLT" == "1" ]]; then
SKIP_BOLT_BOOL=true
fi
STRIP_SYMBOLS_BOOL=false
if [[ "${STRIP_SYMBOLS,,}" == "true" || "$STRIP_SYMBOLS" == "1" ]]; then
STRIP_SYMBOLS_BOOL=true
fi
COLLECT_PGO_ONLY_BOOL=false
if [[ "${COLLECT_PGO_ONLY,,}" == "true" || "$COLLECT_PGO_ONLY" == "1" ]]; then
COLLECT_PGO_ONLY_BOOL=true
fi
USE_PRECOLLECTED_PGO=false
if [ -n "$PGO_PROFDATA" ]; then
if [ ! -f "$PGO_PROFDATA" ]; then
echo "error: PGO_PROFDATA points to a missing file: $PGO_PROFDATA"
exit 1
fi
USE_PRECOLLECTED_PGO=true
fi
NEEDS_BENCH_WORKLOAD=true
if [ "$USE_PRECOLLECTED_PGO" = true ] && [ "$SKIP_BOLT_BOOL" = true ]; then
NEEDS_BENCH_WORKLOAD=false
fi
if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
: "${DATADIR:?DATADIR must be set to the reth data directory}"
: "${RPC_URL:?RPC_URL must be set}"
fi
if [[ "$PROFILE" == dev ]]; then
PROFILE_DIR=debug
else
PROFILE_DIR=$PROFILE
fi
MANIFEST_PATH="bin/reth"
LLVM_VERSION=$(rustc -Vv | grep -oP 'LLVM version: \K\d+')
PGO_DIR="$PWD/target/pgo-profiles"
BOLT_DIR="$PWD/target/bolt-profiles"
CARGO_ARGS=(--profile "$PROFILE" --features "$FEATURES" --manifest-path "$MANIFEST_PATH/Cargo.toml" --bin "reth" --locked)
# Enable debug symbols for BOLT (requires symbols to reorder code).
# Strip them at the end.
PROFILE_UPPER=$(echo "$PROFILE" | tr '[:lower:]-' '[:upper:]_')
export "CARGO_PROFILE_${PROFILE_UPPER}_STRIP=debuginfo"
gha_section_start "Full PGO+BOLT Build"
echo "Binary: reth"
echo "Manifest: $MANIFEST_PATH"
echo "Target: $TARGET"
echo "Profile: $PROFILE"
echo "Features: $FEATURES"
echo "LLVM: $LLVM_VERSION"
echo "PGO blocks: $PGO_BLOCKS"
echo "BOLT blocks: $BOLT_BLOCKS"
echo "Skip BOLT: $SKIP_BOLT"
echo "Strip symbols: $STRIP_SYMBOLS"
echo "Collect only: $COLLECT_PGO_ONLY"
echo "PGO profdata: ${PGO_PROFDATA:-<collect with reth-bench>}"
echo "RUSTFLAGS: ${BASE_RUSTFLAGS:-<unset>}"
echo "EXTRA_RUSTFLAGS: ${EXTRA_RUSTFLAGS:-<unset>}"
if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
echo "Datadir: $DATADIR"
echo "RPC URL: $RPC_URL"
else
echo "Datadir: <not required>"
echo "RPC URL: <not required>"
fi
gha_section_end
# ── Prerequisites ──────────────────────────────────────────────────────────────
gha_section_start "Installing prerequisites"
rustup component add llvm-tools-preview
LLVM_PROFDATA=$(find "$(rustc --print sysroot)" -name llvm-profdata -type f | head -1)
if [ -z "$LLVM_PROFDATA" ]; then
echo "error: llvm-profdata not found"
exit 1
fi
install_bolt() {
if command -v llvm-bolt &>/dev/null; then
echo "BOLT already installed"
return
fi
echo "Installing BOLT from apt.llvm.org..."
wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc >/dev/null
CODENAME=$(lsb_release -cs)
echo "deb http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-$LLVM_VERSION main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null
sudo apt-get update -qq
sudo apt-get install -y -qq "bolt-$LLVM_VERSION"
sudo ln -sf "/usr/bin/llvm-bolt-$LLVM_VERSION" /usr/local/bin/llvm-bolt
sudo ln -sf "/usr/bin/merge-fdata-$LLVM_VERSION" /usr/local/bin/merge-fdata
}
if [ "$SKIP_BOLT_BOOL" = true ]; then
echo "Skipping BOLT installation (SKIP_BOLT=$SKIP_BOLT)"
else
install_bolt
fi
gha_section_end
if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
# Build reth-bench once (non-instrumented) — reused for both phases.
gha_section_start "Building reth-bench"
RUSTFLAGS="$COMBINED_RUSTFLAGS" \
cargo build --profile "$PROFILE" --features "$FEATURES" \
--manifest-path bin/reth-bench/Cargo.toml --bin reth-bench --locked
RETH_BENCH_BIN="$(find target -name reth-bench -type f -executable | head -1)"
echo "reth-bench: $RETH_BENCH_BIN"
gha_section_end
else
gha_section_start "Building reth-bench"
echo "Skipping reth-bench build (pre-collected PGO with SKIP_BOLT=true)"
gha_section_end
fi
# ── Helpers ────────────────────────────────────────────────────────────────────
RETH_PID=
cleanup() {
if [ -n "${RETH_PID:-}" ] && kill -0 "$RETH_PID" 2>/dev/null; then
echo "Stopping reth (pid $RETH_PID)..."
sudo kill "$RETH_PID" 2>/dev/null || true
for i in $(seq 1 60); do
sudo kill -0 "$RETH_PID" 2>/dev/null || break
if [ $((i % 10)) -eq 0 ]; then
echo " waiting... (${i}s)"
fi
sleep 1
done
sudo kill -9 "$RETH_PID" 2>/dev/null || true
fi
}
trap cleanup EXIT
# Start reth, wait for RPC, run reth-bench, then stop reth.
# Arguments: $1 = reth binary path, $2 = number of blocks, $3 = log label
run_bench_workload() {
local reth_bin="$1" blocks="$2" label="$3"
local http_port=8545 authrpc_port=8551
echo "--- Starting reth ($label) ---"
sudo "$reth_bin" node \
--datadir "$DATADIR" \
--log.file.directory "/tmp/reth-${label}-logs" \
--engine.accept-execution-requests-hash \
--http --http.port "$http_port" \
--authrpc.port "$authrpc_port" \
--disable-discovery --no-persist-peers \
> "/tmp/reth-${label}.log" 2>&1 &
RETH_PID=$!
echo "Waiting for reth RPC..."
for i in $(seq 1 120); do
if curl -sf "http://127.0.0.1:$http_port" -X POST \
-H 'Content-Type: application/json' \
-d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
> /dev/null 2>&1; then
echo "reth is ready after ${i}s"
break
fi
if [ "$i" -eq 120 ]; then
echo "error: reth failed to start within 120s"
cat "/tmp/reth-${label}.log"
exit 1
fi
sleep 1
done
echo "Running reth-bench ($blocks blocks)..."
"$RETH_BENCH_BIN" new-payload-fcu \
--rpc-url "$RPC_URL" \
--engine-rpc-url "http://127.0.0.1:$authrpc_port" \
--jwt-secret "$DATADIR/jwt.hex" \
--advance "$blocks" \
--reth-new-payload 2>&1 | sed -u "s/^/[$label] /"
echo "Stopping reth ($label)..."
sudo kill "$RETH_PID" 2>/dev/null || true
for i in $(seq 1 60); do
sudo kill -0 "$RETH_PID" 2>/dev/null || break
sleep 1
done
sudo kill -9 "$RETH_PID" 2>/dev/null || true
RETH_PID=
}
publish_binary() {
local source_bin="$1"
for out in "target/$TARGET/$PROFILE_DIR" "target/$PROFILE_DIR"; do
local destination="$out/reth"
mkdir -p "$out"
# Skip copying when source and destination resolve to the same inode.
if [ -e "$destination" ] && [ "$source_bin" -ef "$destination" ]; then
continue
fi
cp "$source_bin" "$destination"
done
}
if [ "$USE_PRECOLLECTED_PGO" = true ]; then
gha_section_start "Phase 1: Using Pre-Collected PGO Profile"
rm -rf "$PGO_DIR"
mkdir -p "$PGO_DIR"
cp "$PGO_PROFDATA" "$PGO_DIR/merged.profdata"
echo "Using pre-collected profile: $PGO_PROFDATA"
echo "PGO profile: $PGO_DIR/merged.profdata ($(ls -lh "$PGO_DIR/merged.profdata" | awk '{print $5}'))"
gha_section_end
else
# ── Phase 1: PGO profile collection ───────────────────────────────────────
gha_section_start "Phase 1: PGO Profile Collection"
rm -rf "$PGO_DIR"
mkdir -p "$PGO_DIR"
echo "Building PGO-instrumented binary..."
RUSTFLAGS="-Cprofile-generate=$PGO_DIR -Crelocation-model=pic $COMBINED_RUSTFLAGS" \
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
PGO_RETH_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
echo "Instrumented binary: $PGO_RETH_BIN ($(ls -lh "$PGO_RETH_BIN" | awk '{print $5}'))"
run_bench_workload "$PGO_RETH_BIN" "$PGO_BLOCKS" "pgo"
# Fix ownership if reth ran as root.
sudo chown -R "$(id -un):$(id -gn)" "$PGO_DIR" 2>/dev/null || true
# Merge PGO profiles.
echo "Merging PGO profiles..."
PROFRAW_COUNT=$(find "$PGO_DIR" -name '*.profraw' | wc -l)
echo "Found $PROFRAW_COUNT .profraw files"
if [ "$PROFRAW_COUNT" -eq 0 ]; then
echo "error: no .profraw files — instrumented binary did not produce profiles"
exit 1
fi
"$LLVM_PROFDATA" merge -o "$PGO_DIR/merged.profdata" "$PGO_DIR"/*.profraw
echo "PGO profile: $PGO_DIR/merged.profdata ($(ls -lh "$PGO_DIR/merged.profdata" | awk '{print $5}'))"
gha_section_end
fi
if [ "$COLLECT_PGO_ONLY_BOOL" = true ]; then
gha_section_start "PGO Collection Complete"
echo "COLLECT_PGO_ONLY=true, skipping PGO/BOLT optimized binary build"
echo "Profile: $PGO_DIR/merged.profdata"
gha_section_end
exit 0
fi
if [ "$SKIP_BOLT_BOOL" = true ]; then
gha_section_start "BOLT Phase Skipped"
echo "SKIP_BOLT=$SKIP_BOLT, building PGO-only binary"
echo "Building PGO-optimized binary..."
RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata $COMBINED_RUSTFLAGS" \
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
if [ "$STRIP_SYMBOLS_BOOL" = true ]; then
echo "Stripping debug symbols..."
strip "$BUILT_BIN"
else
echo "Skipping strip (STRIP_SYMBOLS=$STRIP_SYMBOLS)"
fi
publish_binary "$BUILT_BIN"
gha_section_end
else
# ── Phase 2: BOLT profile collection (with PGO) ──────────────────────────
gha_section_start "Phase 2: BOLT Profile Collection (with PGO)"
rm -rf "$BOLT_DIR"
mkdir -p "$BOLT_DIR"
echo "Building BOLT-instrumented binary with PGO..."
# --emit-relocs preserves relocation entries in the binary, required by llvm-bolt -instrument
RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata -Clink-arg=-Wl,--emit-relocs $COMBINED_RUSTFLAGS" \
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
# Instrument with BOLT
BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
BOLT_INSTRUMENTED_BIN="$BUILT_BIN-bolt-instrumented"
echo "Instrumenting binary with BOLT..."
# --skip-funcs: skip compiler-generated drop_in_place functions that BOLT can't handle
# as split functions in relocation mode (triggered by --emit-relocs)
llvm-bolt "$BUILT_BIN" \
-instrument \
--instrumentation-file-append-pid \
--instrumentation-file="$BOLT_DIR/prof" \
--skip-funcs='.*drop_in_place.*' \
-o "$BOLT_INSTRUMENTED_BIN"
echo "BOLT-instrumented binary: $BOLT_INSTRUMENTED_BIN ($(ls -lh "$BOLT_INSTRUMENTED_BIN" | awk '{print $5}'))"
run_bench_workload "$BOLT_INSTRUMENTED_BIN" "$BOLT_BLOCKS" "bolt"
# Fix ownership for BOLT profiles
sudo chown -R "$(id -un):$(id -gn)" "$BOLT_DIR" 2>/dev/null || true
# Merge BOLT profiles
echo "Merging BOLT profiles..."
FDATA_COUNT=$(find "$BOLT_DIR" -name '*.fdata' | wc -l)
echo "Found $FDATA_COUNT .fdata files"
if [ "$FDATA_COUNT" -eq 0 ]; then
echo "error: no .fdata files — BOLT-instrumented binary did not produce profiles"
exit 1
fi
merge-fdata "$BOLT_DIR"/*.fdata > "$BOLT_DIR/merged.fdata"
echo "BOLT profile: $BOLT_DIR/merged.fdata ($(ls -lh "$BOLT_DIR/merged.fdata" | awk '{print $5}'))"
gha_section_end
# ── Phase 3: Final optimized build ───────────────────────────────────────
gha_section_start "Phase 3: Final PGO+BOLT Optimized Build"
echo "Building PGO-optimized binary..."
# --emit-relocs preserves relocation entries in the binary, required by llvm-bolt for code reordering
RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata -Clink-arg=-Wl,--emit-relocs $COMBINED_RUSTFLAGS" \
cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
OPTIMIZED_BIN="$BUILT_BIN-bolt-optimized"
echo "Optimizing with BOLT..."
llvm-bolt "$BUILT_BIN" \
-o "$OPTIMIZED_BIN" \
--data "$BOLT_DIR/merged.fdata" \
-reorder-blocks=ext-tsp \
-reorder-functions=cdsort \
-split-functions \
-split-all-cold \
-dyno-stats \
-icf=1 \
-use-gnu-stack \
--skip-funcs='.*drop_in_place.*'
if [ "$STRIP_SYMBOLS_BOOL" = true ]; then
echo "Stripping debug symbols..."
strip "$OPTIMIZED_BIN"
else
echo "Skipping strip (STRIP_SYMBOLS=$STRIP_SYMBOLS)"
fi
publish_binary "$OPTIMIZED_BIN"
gha_section_end
fi
gha_section_start "Build Complete"
ls -lh "target/$PROFILE_DIR/reth"
echo "Output: target/$PROFILE_DIR/reth"
gha_section_end

View File

@@ -3,7 +3,7 @@
#
# We'll use cargo-chef to speed up the build
#
FROM lukemathwalker/cargo-chef:latest-rust-1.94.0-trixie AS chef
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
WORKDIR /app
# Install system dependencies

View File

@@ -9,17 +9,10 @@ go build .
./hive -client reth # first builds and caches the client
# Run each hive command in the background for each simulator and wait
# Run each hive command in the background for each simulator and wait
echo "Building images"
# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
./hive -client reth --sim "ethereum/eels/consume-engine" \
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
--sim.buildarg branch=err-map-3 \
--sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/eels/consume-rlp" \
--sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
--sim.buildarg branch=err-map-3 \
--sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &

View File

@@ -16,19 +16,32 @@ rpc-compat:
# syncing mode, the test expects syncing to be false on start
- eth_syncing/check-syncing (reth)
engine-withdrawals: []
# no fix due to https://github.com/paradigmxyz/reth/issues/8732
engine-withdrawals:
- Withdrawals Fork On Genesis (Paris) (reth)
- Withdrawals Fork on Block 1 (Paris) (reth)
- Withdrawals Fork on Block 2 (Paris) (reth)
- Withdrawals Fork on Block 3 (Paris) (reth)
- Withdraw to a single account (Paris) (reth)
- Withdraw to two accounts (Paris) (reth)
- Withdraw many accounts (Paris) (reth)
- Withdraw zero amount (Paris) (reth)
- Empty Withdrawals (Paris) (reth)
- Corrupted Block Hash Payload (INVALID) (Paris) (reth)
- Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
engine-api: []
engine-api: [ ]
# no fix due to https://github.com/paradigmxyz/reth/issues/8732
engine-cancun:
- Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
# the test fails with older versions of the code for which it passed before, probably related to changes
# in hive or its dependencies
- Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
sync: []
sync: [ ]
engine-auth: []
engine-auth: [ ]
# EIP-7610 related tests (Revert creation in case of non-empty storage):
#
@@ -36,7 +49,7 @@ engine-auth: []
# The test artificially creates an empty account with storage, then tests EIP-7610's behavior.
# On mainnet, ~25 such accounts exist as contract addresses (derived from keccak(prefix, caller,
# nonce/salt), not from public keys). No private key exists for contract addresses. To trigger
# this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible.
# this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible.
#
# tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_*
# Requires hash collision on create2 address to target already deployed accounts with storage.
@@ -46,10 +59,6 @@ engine-auth: []
#
# System contract tests (already fixed and deployed):
#
# tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout and test_invalid_log_length
# System contract is already fixed and deployed; tests cover scenarios where contract is
# malformed which can't happen retroactively. No point in adding checks.
#
# tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment
# tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment
# Post-fork system contract deployment tests. Should fix for spec compliance but not realistic
@@ -58,44 +67,8 @@ eels/consume-engine:
- tests/prague/eip7702_set_code_tx/test_set_code_txs.py::test_set_code_to_non_empty_storage[fork_Prague-blockchain_test_engine-zero_nonce]-reth
- tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth
- tests/prague/eip7251_consolidations/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_index_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Prague-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
- tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-nonzero_balance]-reth
- tests/prague/eip7002_el_triggerable_withdrawals/test_contract_deployment.py::test_system_contract_deployment[fork_CancunToPragueAtTime15k-blockchain_test_engine-deploy_after_fork-zero_balance]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_False]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Prague-blockchain_test_engine-slice_bytes_True]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_index_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_True]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_False]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
- tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
@@ -146,20 +119,6 @@ eels/consume-engine:
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_engine_from_state_test]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-empty-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-sstore-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
# Blob limit tests:
#
@@ -254,17 +213,3 @@ eels/consume-rlp:
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-initcode-with-deploy]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-sstore-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_from_state_test]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
- tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
- tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-empty-initcode]-reth

View File

@@ -11,14 +11,19 @@
#
# When a test should no longer be ignored, remove it from this list.
# flaky
engine-withdrawals:
- Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
- Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
- Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
- Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
- Sync after 2 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts - No Transactions (Paris) (reth)
- Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
engine-cancun:
- Transaction Re-Org, New Payload on Revert Back (Cancun) (reth)
- Transaction Re-Org, Re-Org to Different Block (Cancun) (reth)
- Transaction Re-Org, Re-Org Out (Cancun) (reth)
- Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth)
# Hive test infra bug: geth sidecar switched to PathScheme for state storage, which has
# strict trie integrity requirements incompatible with inserting intentionally invalid blocks.
# Affects all clients, not just reth. Tracked: https://github.com/ethereum/hive/issues/1382
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
- Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
- Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Cancun) (reth)

View File

@@ -13,13 +13,7 @@ if [[ "${sim}" == *"eels"* ]]; then
fi
run_hive() {
hive \
--sim "${sim}" \
--sim.limit "${limit}" \
--sim.limit.exact=false \
--sim.parallelism "${parallelism}" \
--client reth \
2>&1 | tee /tmp/log || true
hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism "${parallelism}" --client reth 2>&1 | tee /tmp/log || true
}
check_log() {

View File

@@ -1,872 +0,0 @@
# Nightly regression benchmark.
#
# Compares the previous nightly build against the current nightly build to
# detect performance regressions. Runs daily after docker.yml produces a new
# nightly image at 01:00 UTC.
#
# State is persisted between runs via GitHub Actions cache: each successful
# run saves the feature commit SHA so the next run knows what to compare against.
on:
schedule:
- cron: "30 5 * * *" # 06:30 UTC daily
workflow_dispatch:
inputs:
force:
description: "Force run even if no new nightly (bypass skip logic)"
required: false
default: false
type: boolean
no_slack:
description: "Suppress Slack notifications"
required: false
default: true
type: boolean
env:
CARGO_TERM_COLOR: always
RUSTC_WRAPPER: "sccache"
name: bench-scheduled
permissions:
contents: read
actions: read
jobs:
# ---------------------------------------------------------------------------
# Job 1: Resolve nightly refs, check staleness, manage state
# ---------------------------------------------------------------------------
resolve-refs:
name: resolve-refs
runs-on: ubuntu-latest
outputs:
baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
feature-ref: ${{ steps.refs.outputs.feature-ref }}
should-skip: ${{ steps.refs.outputs.should-skip }}
is-stale: ${{ steps.refs.outputs.is-stale }}
stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
nightly-created: ${{ steps.refs.outputs.nightly-created }}
steps:
- uses: actions/checkout@v6
with:
sparse-checkout: .github/scripts
sparse-checkout-cone-mode: true
- name: Restore nightly state
id: state-cache
uses: actions/cache/restore@v4
with:
path: .nightly-state
key: bench-scheduled-state-dummy
restore-keys: |
bench-scheduled-state-
- name: Resolve nightly refs
id: refs
env:
GH_TOKEN: ${{ github.token }}
GITHUB_REPOSITORY: ${{ github.repository }}
run: |
FORCE="${{ inputs.force || 'false' }}"
.github/scripts/bench-scheduled-refs.sh "$FORCE"
- name: Alert on stale nightly
if: steps.refs.outputs.is-stale == 'true'
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const token = process.env.SLACK_BENCH_BOT_TOKEN;
const channel = process.env.SLACK_BENCH_CHANNEL;
if (!token || !channel) {
core.warning('Slack credentials not set, skipping stale nightly alert');
return;
}
const ageHours = '${{ steps.refs.outputs.stale-age-hours }}';
const created = '${{ steps.refs.outputs.nightly-created }}';
const featureRef = '${{ steps.refs.outputs.feature-ref }}';
const shortSha = featureRef.slice(0, 8);
const repo = '${{ github.repository }}';
const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: ':rotating_light: Nightly Regression: nightly build is stale', emoji: true },
},
{
type: 'section',
text: {
type: 'mrkdwn',
text: [
'*Nightly regression did not run* — nightly build is stale',
'',
`The latest nightly image was built from a commit that is *${ageHours}h old* (threshold: 24h).`,
`This means today's nightly docker build likely failed and no new image was produced.`,
'',
`Stale commit: \`${shortSha}\` (built at ${created})`,
'',
'*Action required:* Check the <https://github.com/' + repo + '/actions/workflows/docker.yml|docker.yml> workflow for failures.',
].join('\n'),
},
},
{
type: 'actions',
elements: [
{
type: 'button',
text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
url: runUrl,
action_id: 'ci_button',
},
],
},
];
const resp = await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
channel,
blocks,
text: 'Nightly regression: nightly build is stale',
unfurl_links: false,
}),
});
const data = await resp.json();
if (!data.ok) {
core.warning(`Slack API error: ${JSON.stringify(data)}`);
}
- name: Fail on stale nightly
if: steps.refs.outputs.is-stale == 'true'
run: |
echo "::error::Nightly build is stale (>24h old). Aborting."
exit 1
# ---------------------------------------------------------------------------
# Job 2: Run the benchmark
# ---------------------------------------------------------------------------
bench-scheduled:
needs: resolve-refs
if: |
needs.resolve-refs.outputs.should-skip != 'true' &&
needs.resolve-refs.outputs.is-stale != 'true'
name: bench-scheduled
runs-on: [self-hosted, Linux, X64, available]
timeout-minutes: 120
env:
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
SCHELK_MOUNT: /reth-bench
BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
BENCH_PR: ""
BENCH_ACTOR: "nightly-regression"
BENCH_BLOCKS: "2000"
BENCH_WARMUP_BLOCKS: "500"
BENCH_SAMPLY: "false"
BENCH_CORES: "0"
BENCH_BIG_BLOCKS: "false"
BENCH_RETH_NEW_PAYLOAD: "true"
BENCH_WAIT_TIME: ""
BENCH_BASELINE_ARGS: ""
BENCH_FEATURE_ARGS: ""
BENCH_ABBA: "true"
BENCH_COMMENT_ID: ""
BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
BENCH_METRICS_ADDR: "127.0.0.1:9100"
BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }}
steps:
- name: Clean up previous bench-work
run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
- uses: actions/checkout@v6
with:
submodules: true
fetch-depth: 0
ref: ${{ needs.resolve-refs.outputs.feature-ref }}
- name: Resolve job URL
id: job-url
uses: actions/github-script@v8
with:
script: |
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
owner: context.repo.owner,
repo: context.repo.repo,
run_id: context.runId,
});
const job = jobs.jobs.find(j => j.name === 'bench-scheduled');
const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
core.exportVariable('BENCH_JOB_URL', jobUrl);
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
continue-on-error: true
- name: Install dependencies
env:
DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
run: |
mkdir -p "$HOME/.local/bin"
# apt packages
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends \
python3 make jq zstd curl dmsetup \
linux-tools-"$(uname -r)" || \
sudo apt-get install -y --no-install-recommends linux-tools-generic
# mc (MinIO client)
if ! command -v mc &>/dev/null; then
curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
chmod +x "$HOME/.local/bin/mc"
fi
# uv (Python package manager)
if ! command -v uv &>/dev/null; then
curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
fi
# Configure git auth for private repos
git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
# thin-provisioning-tools (era_invalidate, required by schelk)
if ! command -v era_invalidate &>/dev/null; then
git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
sudo make -C /tmp/tpt install
rm -rf /tmp/tpt
fi
# schelk (snapshot rollback tool, invoked via sudo)
if ! sudo sh -c 'command -v schelk' &>/dev/null; then
cargo install --git https://github.com/tempoxyz/schelk --locked
sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
fi
- name: Check dependencies
run: |
export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
echo "$HOME/.local/bin" >> "$GITHUB_PATH"
echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
missing=()
for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
command -v "$cmd" &>/dev/null || missing+=("$cmd")
done
if [ ${#missing[@]} -gt 0 ]; then
echo "::error::Missing required tools: ${missing[*]}"
exit 1
fi
echo "All dependencies found"
- name: Resolve display names
id: refs
run: |
BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
echo "baseline-name=nightly-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
echo "feature-name=nightly-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
- name: Check if snapshot needs update
id: snapshot-check
run: |
set +e
.github/scripts/bench-reth-snapshot.sh --check
rc=$?
set -e
case "$rc" in
0) echo "needed=false" >> "$GITHUB_OUTPUT" ;;
10) echo "needed=true" >> "$GITHUB_OUTPUT" ;;
*) echo "::error::Snapshot check failed (exit $rc)"
exit "$rc" ;;
esac
- name: Prepare source dirs
run: |
if [ -d ../reth-baseline ]; then
git -C ../reth-baseline fetch origin "$BASELINE_REF"
else
git clone . ../reth-baseline
fi
git -C ../reth-baseline checkout "$BASELINE_REF"
if [ -d ../reth-feature ]; then
git -C ../reth-feature fetch origin "$FEATURE_REF"
else
git clone . ../reth-feature
fi
git -C ../reth-feature checkout "$FEATURE_REF"
- name: Build binaries
id: build
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BENCH_REPO: ${{ github.repository }}
run: |
BASELINE_DIR="$(cd ../reth-baseline && pwd)"
FEATURE_DIR="$(cd ../reth-feature && pwd)"
.github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "$BASELINE_REF" &
PID_BASELINE=$!
.github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "$FEATURE_REF" &
PID_FEATURE=$!
FAIL=0
wait $PID_BASELINE || FAIL=1
wait $PID_FEATURE || FAIL=1
if [ $FAIL -ne 0 ]; then
echo "::error::One or more build tasks failed"
exit 1
fi
- name: Download snapshot
id: snapshot-download
if: steps.snapshot-check.outputs.needed == 'true'
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BENCH_REPO: ${{ github.repository }}
BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
run: .github/scripts/bench-reth-snapshot.sh
# System tuning for reproducible benchmarks
- name: System setup
run: |
sudo cpupower frequency-set -g performance || true
# Disable turbo boost (Intel and AMD paths)
echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
sudo swapoff -a || true
echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
# Disable SMT (hyperthreading)
for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
if [ "$current" != "$first" ]; then
echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
fi
done
echo "Online CPUs: $(nproc)"
# Disable transparent huge pages
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
done || true
# Prevent deep C-states
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
# Move all IRQs to core 0
for irq in /proc/irq/*/smp_affinity_list; do
echo 0 | sudo tee "$irq" 2>/dev/null || true
done
# Stop noisy background services
sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
echo "=== Benchmark environment ==="
uname -r
lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
free -h
- name: Pre-flight cleanup
run: |
sudo pkill -9 reth || true
sleep 1
if mountpoint -q "$SCHELK_MOUNT"; then
sudo umount -l "$SCHELK_MOUNT" || true
sudo schelk recover -y || true
fi
rm -rf "$BENCH_WORK_DIR"
mkdir -p "$BENCH_WORK_DIR"
- name: Start metrics proxy
run: |
BENCH_ID="nightly-${{ github.run_id }}"
BENCH_REFERENCE_EPOCH=$(date +%s)
echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
LABELS_FILE="/tmp/bench-metrics-labels.json"
echo '{}' > "$LABELS_FILE"
echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
python3 .github/scripts/bench-metrics-proxy.py \
--labels "$LABELS_FILE" \
--upstream "http://${BENCH_METRICS_ADDR}/" \
--subnet 10.10.0.0/24 \
--port 9090 &
PROXY_PID=$!
echo "BENCH_METRICS_PROXY_PID=${PROXY_PID}" >> "$GITHUB_ENV"
echo "Metrics proxy started (PID $PROXY_PID)"
# Interleaved run order (B-F-F-B) to reduce systematic bias
- name: "Run benchmark: baseline (1/2)"
id: run-baseline-1
run: |
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
- name: "Run benchmark: feature (1/2)"
id: run-feature-1
run: |
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
- name: "Run benchmark: feature (2/2)"
id: run-feature-2
run: |
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
- name: "Run benchmark: baseline (2/2)"
id: run-baseline-2
run: |
LAST_RUN_START=$(date +%s)
echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
cat > "$BENCH_LABELS_FILE" <<LABELS
{"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
LABELS
taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
- name: Stop metrics proxy & generate Grafana URL
id: metrics
if: "!cancelled()"
run: |
kill "$BENCH_METRICS_PROXY_PID" 2>/dev/null || true
LAST_RUN_DURATION=$(( $(date +%s) - BENCH_LAST_RUN_START ))
FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=ef57fux92e9z4e&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
echo "Grafana URL: ${GRAFANA_URL}"
- name: Scan logs for errors
if: "!cancelled()"
run: |
ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
found=false
for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
LOG="$BENCH_WORK_DIR/$run_dir/node.log"
if [ ! -f "$LOG" ]; then continue; fi
panics=$(grep -c -E 'panicked at' "$LOG" || true)
errors=$(grep -c ' ERROR ' "$LOG" || true)
if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
if [ "$found" = false ]; then
printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
found=true
fi
printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
if [ "$panics" -gt 0 ]; then
printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
if [ "$errors" -gt 0 ]; then
printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
printf '```\n' >> "$ERRORS_FILE"
fi
printf '\n</details>\n\n' >> "$ERRORS_FILE"
fi
done
- name: Parse results
id: results
if: success()
env:
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
run: |
SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
if [ -n "$GRAFANA_URL" ]; then
SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
fi
# shellcheck disable=SC2086
python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
- name: Generate charts
if: success()
env:
BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
run: |
CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
# shellcheck disable=SC2086
uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
- name: Upload results
if: "!cancelled()"
uses: actions/upload-artifact@v7
with:
name: bench-scheduled-results
path: ${{ env.BENCH_WORK_DIR }}
- name: Push charts
id: push-charts
if: success()
run: |
RUN_ID=${{ github.run_id }}
CHART_DIR="nightly/${RUN_ID}"
CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
TMP_DIR=$(mktemp -d)
if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
true
else
git init "${TMP_DIR}"
git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
fi
mkdir -p "${TMP_DIR}/${CHART_DIR}"
cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
git -C "${TMP_DIR}" add "${CHART_DIR}"
git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
commit -m "nightly bench charts for run ${RUN_ID}"
git -C "${TMP_DIR}" push origin HEAD:main
echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
rm -rf "${TMP_DIR}"
- name: Write job summary
if: success()
uses: actions/github-script@v8
with:
script: |
const fs = require('fs');
const { verdict, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
return;
}
const repo = `${context.repo.owner}/${context.repo.repo}`;
const commitUrl = `https://github.com/${repo}/commit`;
const { emoji, label } = verdict(summary.changes);
const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
let md = `# ${emoji} Nightly Regression: ${label}\n\n`;
md += `**Baseline:** ${baselineLink}\n`;
md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
const rows = metricRows(summary);
md += `| Metric | Baseline | Feature | Change |\n`;
md += `|--------|----------|---------|--------|\n`;
for (const r of rows) {
md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
}
md += '\n';
const wtRows = waitTimeRows(summary);
if (wtRows.length > 0) {
md += `### Wait Time Breakdown\n\n`;
md += `| Metric | Baseline | Feature |\n`;
md += `|--------|----------|--------|\n`;
for (const r of wtRows) {
md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
}
md += '\n';
}
// Charts
const chartSha = '${{ steps.push-charts.outputs.sha }}';
if (chartSha) {
const runId = '${{ github.run_id }}';
const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`;
const charts = [
{ file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
{ file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
{ file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
];
md += `### Charts\n\n`;
for (const chart of charts) {
md += `<details><summary>${chart.label}</summary>\n\n`;
md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
md += `</details>\n\n`;
}
}
const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}';
if (grafanaUrl) {
md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
}
try {
const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
if (errors.trim()) md += '\n' + errors + '\n';
} catch {}
await core.summary.addRaw(md).write();
- name: Send Slack notification (success)
if: success() && env.BENCH_NO_SLACK != 'true'
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const fs = require('fs');
const { verdict, fmtChange, fmtMs, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
const token = process.env.SLACK_BENCH_BOT_TOKEN;
const channel = process.env.SLACK_BENCH_CHANNEL;
if (!token || !channel) {
core.info('Slack credentials not set, skipping notification');
return;
}
let summary;
try {
summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
} catch (e) {
core.warning('Could not read summary.json for Slack notification');
return;
}
// Only notify on significant changes (regression OR improvement)
const changes = summary.changes || {};
const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
if (!hasSignificant) {
core.info('No significant changes detected, skipping nightly Slack notification');
return;
}
const SLACK_VERDICT = {
'⚠️': ':warning:',
'❌': ':x:',
'✅': ':white_check_mark:',
'⚪': ':white_circle:',
};
const repo = `${context.repo.owner}/${context.repo.repo}`;
const { emoji, label } = verdict(changes);
const headerEmoji = SLACK_VERDICT[emoji] || emoji;
const commitUrl = `https://github.com/${repo}/commit`;
const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
const sectionText = [
'*Nightly Regression*',
'',
`*Baseline:* ${baselineLink}`,
`*Feature:* ${featureLink}`,
blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '),
].join('\n');
const rows = metricRows(summary);
const tableRows = [
[cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
];
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: `${headerEmoji} Nightly: ${label}`, emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: sectionText },
},
{
type: 'table',
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }, { align: 'right' }],
rows: tableRows,
},
{
type: 'actions',
elements: [
{
type: 'button',
text: { type: 'plain_text', text: 'CI :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
},
{
type: 'button',
text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
url: diffUrl,
action_id: 'diff_button',
},
],
},
];
const text = `Nightly Regression: ${summary.baseline.name} vs ${summary.feature.name}`;
const resp = await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({ channel, blocks, text, unfurl_links: false }),
});
const data = await resp.json();
if (!data.ok) {
core.warning(`Slack API error: ${JSON.stringify(data)}`);
return;
}
// Post wait time breakdown as threaded reply
const wtRows = waitTimeRows(summary);
if (data.ts && wtRows.length > 0) {
const waitTableRows = [
[cell('Wait Time'), cell('Baseline'), cell('Feature')],
...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
];
await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
channel,
thread_ts: data.ts,
blocks: [{
type: 'table',
column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }],
rows: waitTableRows,
}],
text: 'Wait time breakdown',
unfurl_links: false,
}),
});
}
- name: Send Slack notification (failure)
if: failure()
uses: actions/github-script@v8
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
with:
script: |
const token = process.env.SLACK_BENCH_BOT_TOKEN;
const channel = process.env.SLACK_BENCH_CHANNEL;
if (!token || !channel) return;
const steps_status = [
['building binaries', '${{ steps.build.outcome }}'],
['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
];
const failed = steps_status.find(([, o]) => o === 'failure');
const failedStep = failed ? failed[0] : 'unknown step';
const repo = `${context.repo.owner}/${context.repo.repo}`;
const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
const blocks = [
{
type: 'header',
text: { type: 'plain_text', text: ':rotating_light: Nightly Bench Failed', emoji: true },
},
{
type: 'section',
text: { type: 'mrkdwn', text: `*Nightly regression* failed while *${failedStep}*` },
},
{
type: 'actions',
elements: [{
type: 'button',
text: { type: 'plain_text', text: 'View Logs :github:', emoji: true },
url: jobUrl,
action_id: 'ci_button',
}],
},
];
await fetch('https://slack.com/api/chat.postMessage', {
method: 'POST',
headers: {
'Authorization': `Bearer ${token}`,
'Content-Type': 'application/json',
},
body: JSON.stringify({
channel,
blocks,
text: `Nightly bench failed while ${failedStep}`,
unfurl_links: false,
}),
});
- name: Restore system settings
if: always()
run: |
sudo systemctl start irqbalance cron atd 2>/dev/null || true
# ---------------------------------------------------------------------------
# Job 3: Save state on success
# ---------------------------------------------------------------------------
save-state:
needs: [resolve-refs, bench-scheduled]
if: success()
name: save-state
runs-on: ubuntu-latest
steps:
- name: Write state file
run: |
mkdir -p .nightly-state
echo "${{ needs.resolve-refs.outputs.feature-ref }}" > .nightly-state/last-feature-ref
- name: Save nightly state
uses: actions/cache/save@v4
with:
path: .nightly-state
key: bench-scheduled-state-${{ needs.resolve-refs.outputs.feature-ref }}

File diff suppressed because it is too large Load Diff

View File

@@ -6,7 +6,7 @@ on:
hive_target:
required: true
type: string
description: "Docker bake target to build (e.g. hive)"
description: "Docker bake target to build (e.g. hive-stable, hive-edge)"
artifact_name:
required: false
type: string
@@ -76,7 +76,7 @@ jobs:
*.dockerfile=Dockerfile
- name: Upload reth image
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./artifacts
path: ./artifacts

View File

@@ -28,30 +28,12 @@ on:
required: false
type: boolean
default: false
pgo:
description: "Enable PGO profiling"
required: false
type: boolean
default: false
pgo_blocks:
description: "Number of blocks to execute for PGO profiling"
required: false
type: string
default: "20"
jobs:
collect-pgo-profile:
if: github.repository == 'paradigmxyz/reth' && github.event_name == 'workflow_dispatch' && inputs.pgo
uses: ./.github/workflows/pgo-profile.yml
with:
pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
secrets: inherit
build:
if: github.repository == 'paradigmxyz/reth' && !failure() && !cancelled()
if: github.repository == 'paradigmxyz/reth'
name: Build Docker images
runs-on: ubuntu-24.04
needs: collect-pgo-profile
permissions:
packages: write
contents: read
@@ -63,7 +45,7 @@ jobs:
uses: depot/setup-action@v1
- name: Log in to GHCR
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}
@@ -76,30 +58,6 @@ jobs:
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
echo "dirty=false" >> "$GITHUB_OUTPUT"
- name: Download pre-collected PGO profile
if: ${{ github.event_name == 'workflow_dispatch' && inputs.pgo }}
uses: actions/download-artifact@v7
with:
name: pgo-profdata
path: dist
- name: Configure PGO build args
id: pgo
run: |
if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ "${{ inputs.pgo }}" == "true" ]]; then
if [ ! -f dist/merged.profdata ]; then
echo "::error::Expected dist/merged.profdata from collect-pgo-profile job"
exit 1
fi
echo "use_pgo_bolt=true" >> "$GITHUB_OUTPUT"
echo "pgo_profdata=dist/merged.profdata" >> "$GITHUB_OUTPUT"
echo "Using pre-collected PGO profile from collect-pgo-profile job"
else
echo "use_pgo_bolt=false" >> "$GITHUB_OUTPUT"
echo "pgo_profdata=" >> "$GITHUB_OUTPUT"
echo "PGO disabled"
fi
- name: Determine build parameters
id: params
run: |
@@ -149,9 +107,6 @@ jobs:
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
${{ steps.params.outputs.ethereum_set }}
*.args.USE_PGO_BOLT=${{ steps.pgo.outputs.use_pgo_bolt }}
*.args.PGO_PROFDATA=${{ steps.pgo.outputs.pgo_profdata }}
*.args.STRIP_SYMBOLS=false
- name: Verify image architectures
env:

View File

@@ -63,6 +63,6 @@ jobs:
run: |
cargo nextest run \
--no-fail-fast \
--locked \
--locked --features "edge" \
-p reth-e2e-test-utils \
-E 'binary(rocksdb)'

View File

@@ -5,10 +5,7 @@ name: hive
on:
workflow_dispatch:
schedule:
- cron: "0 */6 * * *"
pull_request:
branches:
- "**"
- cron: "0 0 * * *"
env:
CARGO_TERM_COLOR: always
@@ -18,26 +15,30 @@ concurrency:
cancel-in-progress: true
jobs:
prepare-reth-stable:
uses: ./.github/workflows/prepare-reth.yml
build-reth-stable:
uses: ./.github/workflows/docker-test.yml
with:
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak"
hive_target: hive-stable
artifact_name: "reth-stable"
secrets: inherit
build-reth-edge:
uses: ./.github/workflows/docker-test.yml
with:
hive_target: hive-edge
artifact_name: "reth-edge"
secrets: inherit
prepare-hive:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on:
group: Reth
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
steps:
- uses: actions/checkout@v6
- name: Checkout hive tests
uses: actions/checkout@v6
with:
repository: Soubhik-10/hive # rm later and use ethereum/hive
ref: master
repository: ethereum/hive
path: hivetests
- name: Get hive commit hash
@@ -74,7 +75,7 @@ jobs:
chmod +x hive
- name: Upload hive assets
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: hive_assets
path: ./hive_assets
@@ -83,7 +84,7 @@ jobs:
strategy:
fail-fast: false
matrix:
storage: [stable]
storage: [stable, edge]
# ethereum/rpc to be deprecated:
# https://github.com/ethereum/hive/pull/1117
scenario:
@@ -124,28 +125,26 @@ jobs:
- sim: ethereum/rpc-compat
include:
- eth_blockNumber
# - eth_call
# - eth_chainId
# - eth_createAccessList
# - eth_estimateGas
# - eth_feeHistory
# - eth_getBalance
# - eth_getBlockBy
# - eth_getBlockTransactionCountBy
# - eth_getCode
# - eth_getProof
# - eth_getStorage
# - eth_getTransactionBy
# - eth_getTransactionCount
# - eth_getTransactionReceipt
# - eth_sendRawTransaction
# - eth_syncing
# # debug_ rpc methods
# - debug_
- eth_call
- eth_chainId
- eth_createAccessList
- eth_estimateGas
- eth_feeHistory
- eth_getBalance
- eth_getBlockBy
- eth_getBlockTransactionCountBy
- eth_getCode
- eth_getProof
- eth_getStorage
- eth_getTransactionBy
- eth_getTransactionCount
- eth_getTransactionReceipt
- eth_sendRawTransaction
- eth_syncing
# debug_ rpc methods
- debug_
# consume-engine
- sim: ethereum/eels/consume-engine
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-engine
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-engine
@@ -166,8 +165,6 @@ jobs:
limit: .*tests/paris.*
# consume-rlp
- sim: ethereum/eels/consume-rlp
limit: .*tests/amsterdam.*
- sim: ethereum/eels/consume-rlp
limit: .*tests/osaka.*
- sim: ethereum/eels/consume-rlp
@@ -187,7 +184,8 @@ jobs:
- sim: ethereum/eels/consume-rlp
limit: .*tests/paris.*
needs:
- prepare-reth-stable
- build-reth-stable
- build-reth-edge
- prepare-hive
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
# Use larger runners for eels tests to avoid OOM runner crashes
@@ -200,13 +198,13 @@ jobs:
fetch-depth: 0
- name: Download hive assets
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
name: hive_assets
path: /tmp
- name: Download reth image
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
name: reth-${{ matrix.storage }}
path: /tmp
@@ -222,7 +220,7 @@ jobs:
- name: Checkout hive tests
uses: actions/checkout@v6
with:
repository: Soubhik-10/hive
repository: ethereum/hive
ref: master
path: hivetests

View File

@@ -22,7 +22,7 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.network }}
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
if: github.event_name != 'schedule'
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
@@ -30,6 +30,7 @@ jobs:
strategy:
matrix:
network: ["ethereum"]
storage: ["stable", "edge"]
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
@@ -46,7 +47,7 @@ jobs:
run: |
cargo nextest run \
--no-fail-fast \
--locked --features "asm-keccak ${{ matrix.network }}" \
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
--workspace --exclude ef-tests \
-E "kind(test) and not binary(e2e_testsuite)"

View File

@@ -40,7 +40,7 @@ jobs:
fetch-depth: 0
- name: Download reth image
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
name: artifacts
path: /tmp

View File

@@ -1,107 +0,0 @@
name: pgo-profile
on:
workflow_call:
inputs:
pgo_blocks:
description: "Number of blocks to execute for PGO profiling"
required: false
type: string
default: "20"
workflow_dispatch:
inputs:
pgo_blocks:
description: "Number of blocks to execute for PGO profiling"
required: false
type: string
default: "20"
jobs:
collect:
name: collect PGO profiles
runs-on: [self-hosted, Linux, X64]
timeout-minutes: 180
env:
SCHELK_MOUNT: /reth-bench
BENCH_RPC_URL: https://ethereum.reth.rs/rpc
RUSTC_WRAPPER: "sccache"
steps:
- uses: actions/checkout@v6
with:
submodules: true
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
with:
target: x86_64-unknown-linux-gnu
- uses: mozilla-actions/sccache-action@v0.0.9
continue-on-error: true
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install dependencies
run: |
sudo apt-get update -qq
sudo apt-get install -y --no-install-recommends \
dmsetup lsb-release wget linux-tools-"$(uname -r)" || \
sudo apt-get install -y --no-install-recommends linux-tools-generic
- name: Download snapshot if needed
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
BENCH_REPO: ${{ github.repository }}
run: |
if ! .github/scripts/bench-reth-snapshot.sh --check; then
echo "Snapshot outdated or missing, downloading..."
.github/scripts/bench-reth-snapshot.sh
fi
- name: Mount snapshot
run: |
sudo pkill -9 reth || true
sleep 1
if mountpoint -q "$SCHELK_MOUNT"; then
sudo umount -l "$SCHELK_MOUNT" || true
sudo schelk recover -y || true
fi
sudo schelk mount -y
sync
sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
- name: Collect PGO profile
run: |
DATADIR="$SCHELK_MOUNT/datadir" \
RPC_URL="$BENCH_RPC_URL" \
PGO_BLOCKS="${{ inputs.pgo_blocks || '20' }}" \
BOLT_BLOCKS="${{ inputs.pgo_blocks || '20' }}" \
COLLECT_PGO_ONLY=true \
SKIP_BOLT=true \
PROFILE=maxperf-symbols \
FEATURES="jemalloc,asm-keccak,min-debug-logs" \
TARGET=x86_64-unknown-linux-gnu \
EXTRA_RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" \
.github/scripts/build_pgo_bolt.sh
- name: Show PGO profile stats
run: |
LLVM_PROFDATA=$(find "$(rustc --print sysroot)" -name llvm-profdata -type f | head -1)
if [ -z "$LLVM_PROFDATA" ]; then
echo "::error::llvm-profdata not found in rust toolchain"
exit 1
fi
"$LLVM_PROFDATA" show --detailed-summary --topn=20 target/pgo-profiles/merged.profdata
- name: Upload PGO profile
uses: actions/upload-artifact@v7
with:
name: pgo-profdata
path: target/pgo-profiles/merged.profdata
retention-days: 1
- name: Recover snapshot
if: always()
run: |
if mountpoint -q "$SCHELK_MOUNT"; then
sudo umount -l "$SCHELK_MOUNT" || true
sudo schelk recover -y || true
fi

View File

@@ -1,30 +0,0 @@
name: Pull request audit
on:
pull_request:
types: [labeled]
jobs:
publish:
runs-on: ubuntu-latest
if: github.event.label.name == 'cyclops'
steps:
- name: Publish event
run: |
set -euo pipefail
echo "${{ secrets.EVENTS_KEY }}" > ${{ runner.temp }}/key
echo "${{ secrets.EVENTS_CERT }}" > ${{ runner.temp }}/cert
curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \
-H "Content-Type: application/json" \
--key ${{ runner.temp }}/key \
--cert ${{ runner.temp }}/cert \
-d '{
"repository": "${{ github.repository }}",
"event": "pr_audit",
"data": {
"pr_number": ${{ github.event.pull_request.number }},
"sha": "${{ github.event.pull_request.head.sha }}"
}
}'

View File

@@ -1,61 +0,0 @@
name: Prepare Reth Image
on:
workflow_call:
inputs:
image_tag:
required: true
type: string
description: "Docker image tag to use"
binary_name:
required: false
type: string
default: "reth"
description: "Binary name to build (reth or op-reth)"
cargo_features:
required: false
type: string
default: "asm-keccak"
description: "Cargo features to enable"
cargo_package:
required: false
type: string
description: "Optional cargo package path"
artifact_name:
required: false
type: string
default: "artifacts"
description: "Name for the uploaded artifact"
jobs:
prepare-reth:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: depot-ubuntu-latest-4
steps:
- uses: actions/checkout@v6
- run: mkdir artifacts
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and export reth image
uses: docker/build-push-action@v6
with:
context: .
file: .github/scripts/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |
CARGO_BIN=${{ inputs.binary_name }}
MANIFEST_PATH=${{ inputs.cargo_package }}
FEATURES=${{ inputs.cargo_features }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Upload reth image
id: upload
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./artifacts

View File

@@ -52,7 +52,7 @@ jobs:
uses: docker/setup-buildx-action@v3
- name: Log in to GitHub Container Registry
uses: docker/login-action@v4
uses: docker/login-action@v3
with:
registry: ghcr.io
username: ${{ github.actor }}

View File

@@ -13,14 +13,6 @@ on:
description: "Enable dry run mode (builds artifacts but skips uploads and release creation)"
type: boolean
default: false
pgo:
description: "Enable PGO profiling"
type: boolean
default: false
pgo_blocks:
description: "Number of blocks to execute for PGO profiling on self-hosted runner"
type: string
default: "20"
env:
REPO_NAME: ${{ github.repository_owner }}/reth
@@ -45,7 +37,7 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Extract version
run: echo "VERSION=${GITHUB_REF_NAME//\//-}" >> $GITHUB_OUTPUT
run: echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
id: extract_version
outputs:
VERSION: ${{ steps.extract_version.outputs.VERSION }}
@@ -77,22 +69,26 @@ jobs:
fail-fast: true
matrix:
configs:
- target: aarch64-unknown-linux-gnu
os: ubuntu-24.04-arm
- target: x86_64-unknown-linux-gnu
os: ubuntu-24.04
profile: maxperf
allow_fail: false
- target: aarch64-unknown-linux-gnu
os: ubuntu-24.04
profile: maxperf
allow_fail: false
rustflags: ""
native: true
- target: x86_64-apple-darwin
os: macos-14
profile: maxperf
allow_fail: false
rustflags: "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
- target: aarch64-apple-darwin
os: macos-14
profile: maxperf
allow_fail: false
rustflags: ""
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-24.04
profile: maxperf
allow_fail: true
build:
- command: build
binary: reth
@@ -104,7 +100,6 @@ jobs:
target: ${{ matrix.configs.target }}
- uses: mozilla-actions/sccache-action@v0.0.9
- name: Install cross main
if: ${{ !matrix.configs.native }}
id: cross_main
run: |
cargo install cross --locked --git https://github.com/cross-rs/cross
@@ -119,12 +114,7 @@ jobs:
echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
- name: Build Reth
run: |
if [ "${{ matrix.configs.native }}" = "true" ]; then
make PROFILE=${{ matrix.configs.profile }} EXTRA_RUSTFLAGS="${{ matrix.configs.rustflags }}" ${{ matrix.build.command }}-native-${{ matrix.configs.target }}
else
make PROFILE=${{ matrix.configs.profile }} EXTRA_RUSTFLAGS="${{ matrix.configs.rustflags }}" ${{ matrix.build.command }}-${{ matrix.configs.target }}
fi
run: make PROFILE=${{ matrix.configs.profile }} ${{ matrix.build.command }}-${{ matrix.configs.target }}
- name: Move binary
run: |
mkdir artifacts
@@ -145,105 +135,23 @@ jobs:
- name: Upload artifact
if: ${{ github.event.inputs.dry_run != 'true' }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
- name: Upload signature
if: ${{ github.event.inputs.dry_run != 'true' }}
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
collect-pgo-profile:
if: github.event_name == 'workflow_dispatch' && inputs.pgo
uses: ./.github/workflows/pgo-profile.yml
with:
pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
secrets: inherit
build-pgo:
if: github.event_name == 'workflow_dispatch' && inputs.pgo
name: build release (x86_64-linux PGO+BOLT)
runs-on: [self-hosted, Linux, X64]
needs: [extract-version, collect-pgo-profile]
timeout-minutes: 120
env:
RUSTC_WRAPPER: "sccache"
steps:
- uses: actions/checkout@v6
with:
submodules: true
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
with:
target: x86_64-unknown-linux-gnu
- uses: mozilla-actions/sccache-action@v0.0.9
continue-on-error: true
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Download pre-collected PGO profile
uses: actions/download-artifact@v7
with:
name: pgo-profdata
path: dist
- name: Verify PGO profile artifact
run: |
test -f dist/merged.profdata
ls -lh dist/merged.profdata
- name: Build Reth with PGO+BOLT
run: |
SKIP_BOLT=true \
PGO_PROFDATA="$PWD/dist/merged.profdata" \
PROFILE=maxperf-symbols \
FEATURES="jemalloc,asm-keccak,min-debug-logs" \
TARGET=x86_64-unknown-linux-gnu \
EXTRA_RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" \
.github/scripts/build_pgo_bolt.sh
- name: Move binary
run: |
mkdir artifacts
mv target/maxperf-symbols/reth ./artifacts
- name: Configure GPG and create artifacts
env:
GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
run: |
export GPG_TTY=$(tty)
echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import
cd artifacts
tar -czf reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz reth*
echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
mv *tar.gz* ..
shell: bash
- name: Upload artifact
if: ${{ github.event.inputs.dry_run != 'true' }}
uses: actions/upload-artifact@v7
with:
name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
- name: Upload signature
if: ${{ github.event.inputs.dry_run != 'true' }}
uses: actions/upload-artifact@v7
with:
name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
draft-release:
name: draft release
runs-on: ubuntu-latest
needs: [build, build-pgo, extract-version]
if: ${{ !failure() && !cancelled() && github.event.inputs.dry_run != 'true' }}
needs: [build, extract-version]
if: ${{ github.event.inputs.dry_run != 'true' }}
env:
VERSION: ${{ needs.extract-version.outputs.VERSION }}
permissions:
@@ -256,7 +164,7 @@ jobs:
with:
fetch-depth: 0
- name: Download artifacts
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
- name: Generate full changelog
id: changelog
run: |

View File

@@ -43,7 +43,7 @@ jobs:
echo "Binaries SHA256 on ${{ matrix.machine }}: $(cat checksum.sha256)"
- name: Upload the hash
uses: actions/upload-artifact@v7
uses: actions/upload-artifact@v6
with:
name: checksum-${{ matrix.machine }}
path: |
@@ -56,12 +56,12 @@ jobs:
runs-on: ubuntu-latest
steps:
- name: Download artifacts from machine-1
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
name: checksum-machine-1
path: machine-1/
- name: Download artifacts from machine-2
uses: actions/download-artifact@v8
uses: actions/download-artifact@v7
with:
name: checksum-machine-2
path: machine-2/

View File

@@ -51,12 +51,15 @@ jobs:
- name: Run execution stage
run: |
reth stage run execution --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
# NOTE: account-hashing, storage-hashing, and hashing stages are omitted.
# With storage v2 (now default), these stages are no-ops because the
# execution stage writes directly to HashedAccounts/HashedStorages.
# Running them here is harmful: `stage run` unwinds before executing,
# and the unwind reverts the hashed state that execution wrote, but
# the no-op execute never restores it — causing merkle to fail.
- name: Run account-hashing stage
run: |
reth stage run account-hashing --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
- name: Run storage hashing stage
run: |
reth stage run storage-hashing --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
- name: Run hashing stage
run: |
reth stage run hashing --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
- name: Run merkle stage
run: |
reth stage run merkle --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints

View File

@@ -19,13 +19,15 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.type }}
name: test / ${{ matrix.type }} / ${{ matrix.storage }}
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_BACKTRACE: 1
EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
strategy:
matrix:
type: [ethereum]
storage: [stable, edge]
include:
- type: ethereum
features: asm-keccak ethereum
@@ -48,14 +50,14 @@ jobs:
run: |
cargo nextest run \
--no-fail-fast \
--features "${{ matrix.features }}" --locked \
--features "${{ matrix.features }} $EDGE_FEATURES" --locked \
${{ matrix.exclude_args }} --workspace \
--exclude ef-tests --no-tests=warn \
-E "!kind(test) and not binary(e2e_testsuite)"
state:
name: Ethereum state tests
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
env:
RUST_LOG: info,sync=error
RUST_BACKTRACE: 1

View File

@@ -172,97 +172,10 @@ Before submitting changes, ensure:
2. **Clippy**: No warnings
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below)
6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### CLI Reference Docs (`book` CI Job)
The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless.
When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running:
```bash
make update-book-cli
```
This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes.
The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`.
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### Opening PRs against <https://github.com/paradigmxyz/reth>
#### Titles
Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope:
```
<type>(<scope>): <short description>
```
**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore`
**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net`
Examples:
- `fix(rpc): correct gas estimation for ERC-20 transfers`
- `perf: batch trie updates to reduce cursor overhead`
- `feat(engine): add new_payload_interval metric`
#### Descriptions
Keep it short. Say what changed and why — nothing more.
**Do:**
- Write 13 sentences summarizing the change
- Explain _why_ if the diff doesn't make it obvious
- Link related issues or EIPs
- Include benchmark numbers for perf changes
**Don't:**
- List every file changed — that's what the diff is for
- Repeat the title in the body
- Add "Files changed" or "Changes" sections
- Write walls of text that go stale when the diff is updated
- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage"
**Template:**
```
Closes #<issue>
<what changed, 1-3 sentences>
<why, if not obvious from the diff>
```
**Good example:**
```
Closes #16800
Adds fallback for external IP resolution so node startup doesn't fail
when STUN is unreachable. Falls back to the configured default.
```
**Bad example:**
```
## Summary
This PR introduces comprehensive improvements to the IP resolution system.
## Changes
- Modified `crates/net/discv4/src/lib.rs` to add fallback
- Modified `crates/net/discv4/src/config.rs` to add default IP
- Added tests in `crates/net/discv4/src/tests/ip.rs`
## Files Changed
- crates/net/discv4/src/lib.rs
- crates/net/discv4/src/config.rs
- crates/net/discv4/src/tests/ip.rs
```
#### Labels and CI
Label PRs appropriately, first check the available labels and then apply the relevant ones:
* when changes are RPC related, add A-rpc label
* when changes are docs related, add C-docs label
@@ -542,8 +455,5 @@ cargo build --release
cargo check --workspace --all-features
# Check documentation
cargo docs --document-private-items
# Regenerate CLI reference docs (after CLI changes)
make update-book-cli
cargo docs --document-private-items
```

1611
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "1.11.3"
version = "1.11.0"
edition = "2024"
rust-version = "1.93"
license = "MIT OR Apache-2.0"
@@ -10,6 +10,7 @@ exclude = [".github/"]
[workspace]
members = [
"bin/reth-bench/",
"bin/reth-bench-compare/",
"bin/reth/",
"crates/storage/rpc-provider/",
"crates/chain-state/",
@@ -168,7 +169,6 @@ rust.rust_2018_idioms = { level = "deny", priority = -1 }
rust.unreachable_pub = "warn"
rust.unused_must_use = "deny"
rust.rust_2024_incompatible_pat = "warn"
rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
rustdoc.all = "warn"
# rust.unnameable-types = "warn"
@@ -322,6 +322,7 @@ reth = { path = "bin/reth" }
reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" }
reth-basic-payload-builder = { path = "crates/payload/basic" }
reth-bench = { path = "bin/reth-bench" }
reth-bench-compare = { path = "bin/reth-bench-compare" }
reth-chain-state = { path = "crates/chain-state" }
reth-chainspec = { path = "crates/chainspec", default-features = false }
reth-cli = { path = "crates/cli/cli" }
@@ -396,7 +397,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
reth-payload-primitives = { path = "crates/payload/primitives" }
reth-payload-validator = { path = "crates/payload/validator" }
reth-payload-util = { path = "crates/payload/util" }
reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
reth-primitives = { path = "crates/primitives", default-features = false }
reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
reth-provider = { path = "crates/storage/provider" }
reth-prune = { path = "crates/prune/prune" }
@@ -434,24 +435,25 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-features = false }
# revm
revm = { version = "36.0.0", default-features = false }
revm-bytecode = { version = "9.0.0", default-features = false }
revm-database = { version = "12.0.0", default-features = false }
revm-state = { version = "10.0.0", default-features = false }
revm-primitives = { version = "22.1.0", default-features = false }
revm-interpreter = { version = "34.0.0", default-features = false }
revm-database-interface = { version = "10.0.0", default-features = false }
revm-inspectors = "0.36.0"
revm = { version = "34.0.0", default-features = false }
revm-bytecode = { version = "8.0.0", default-features = false }
revm-database = { version = "10.0.0", default-features = false }
revm-state = { version = "9.0.0", default-features = false }
revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.2"
# eth
alloy-dyn-abi = "1.5.0"
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-dyn-abi = "1.5.6"
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.6", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.3", default-features = false, features = ["rlp"] }
alloy-evm = { version = "0.29.2", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.4", default-features = false }
@@ -486,6 +488,7 @@ alloy-transport-ipc = { version = "1.7.3", default-features = false }
alloy-transport-ws = { version = "1.7.3", default-features = false }
# op
alloy-op-evm = { version = "0.27.2", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -504,7 +507,6 @@ bincode = "1.3"
bitflags = "2.4"
boyer-moore-magiclen = "0.2.16"
bytes = { version = "1.11.1", default-features = false }
blake3 = "1.8"
brotli = "8"
cfg-if = "1.0"
clap = "4"
@@ -520,7 +522,6 @@ humantime = "2.1"
humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
libc = "0.2"
lz4 = "1.28.1"
modular-bitfield = "0.13.1"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
@@ -539,13 +540,11 @@ serde_with = { version = "3", default-features = false, features = ["macros"] }
sha2 = { version = "0.10", default-features = false }
shellexpand = "3.0.0"
shlex = "1.3"
slotmap = "1"
smallvec = "1"
strum = { version = "0.27", default-features = false }
strum_macros = "0.27"
syn = "2.0"
thiserror = { version = "2.0.0", default-features = false }
thread-priority = "3.0.0"
tar = "0.4.44"
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
tracing-appender = "0.2"
@@ -665,7 +664,6 @@ cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossbeam-queue = "0.3"
crossbeam-utils = "0.8"
crossterm = "0.29.0"
csv = "1.3.0"
@@ -742,8 +740,10 @@ ipnet = "2.11"
# alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
# op-alloy-rpc-jsonrpsee = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
#
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "1207e33" }
#
@@ -753,71 +753,10 @@ ipnet = "2.11"
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "9bc2dba" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# =============================================================================
# BAL devnet patches (EIP-7778, EIP-7928 block access lists)
# =============================================================================
# revm staging patches
revm = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-database = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-state = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-context = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
op-revm = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", branch = "staging" }
# alloy-evm bal-devnet2 patches
alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "bal-devnet2" }
# alloy bal-devnet2 patches
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-consensus-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
alloy-tx-macros = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
# op-alloy bal-devnet2 patches
op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }

View File

@@ -1,10 +1,8 @@
# syntax=docker/dockerfile:1
# Dockerfile for reth, optimized for Depot builds
# Supports PGO+BOLT optimization for maximum performance
# Usage:
# reth: --build-arg BINARY=reth
# PGO+BOLT: --build-arg USE_PGO_BOLT=true (Linux x86_64/aarch64 only)
FROM rust:1.93 AS builder
WORKDIR /app
@@ -45,18 +43,6 @@ ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Enable PGO+BOLT optimization (Linux only)
ARG USE_PGO_BOLT=false
ENV USE_PGO_BOLT=$USE_PGO_BOLT
# Optional path to a pre-collected merged.profdata file in build context.
ARG PGO_PROFDATA=""
ENV PGO_PROFDATA=$PGO_PROFDATA
# Whether to strip debug symbols from PGO-built binaries.
ARG STRIP_SYMBOLS=true
ENV STRIP_SYMBOLS=$STRIP_SYMBOLS
# Build application
# Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
ARG TARGETPLATFORM
@@ -67,21 +53,12 @@ RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
export RUSTC_WRAPPER=sccache SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev SCCACHE_DIR=/sccache && \
sccache --start-server && \
if [ "$USE_PGO_BOLT" = "true" ] && [ "$TARGETPLATFORM" = "linux/amd64" ] && [ -n "$PGO_PROFDATA" ] && [ -f "$PGO_PROFDATA" ]; then \
apt-get update && apt-get install -y -qq lsb-release wget sudo && \
BINARY="$BINARY" PROFILE="$BUILD_PROFILE" FEATURES="$FEATURES" SKIP_BOLT=true STRIP_SYMBOLS="$STRIP_SYMBOLS" PGO_PROFDATA="$PGO_PROFDATA" \
./.github/scripts/build_pgo_bolt.sh; \
else \
if [ "$USE_PGO_BOLT" = "true" ]; then \
echo "PGO requested but pre-collected profile missing at '${PGO_PROFDATA:-<unset>}' - falling back to non-PGO build"; \
fi; \
if [ -n "$RUSTFLAGS" ]; then \
export RUSTFLAGS="$RUSTFLAGS"; \
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
fi && \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml; \
if [ -n "$RUSTFLAGS" ]; then \
export RUSTFLAGS="$RUSTFLAGS"; \
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
fi && \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml && \
sccache --show-stats
# Copy binary to a known location (ARG not resolved in COPY)

View File

@@ -17,9 +17,6 @@ FEATURES ?=
# Cargo profile for builds. Default is for local builds, CI uses an override.
PROFILE ?= release
# Extra RUSTFLAGS to append to build targets (e.g., "-C target-cpu=x86-64-v3")
EXTRA_RUSTFLAGS ?=
# Extra flags for Cargo
CARGO_INSTALL_EXTRA_FLAGS ?=
@@ -29,7 +26,7 @@ EF_TESTS_URL := https://github.com/ethereum/tests/archive/refs/tags/$(EF_TESTS_T
EF_TESTS_DIR := ./testing/ef-tests/ethereum-tests
# The release tag of https://github.com/ethereum/execution-spec-tests to use for EEST tests
EEST_TESTS_TAG := bal@v5.0.0
EEST_TESTS_TAG := v4.5.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
@@ -77,7 +74,7 @@ build-debug: ## Build the reth binary into `target/debug` directory.
cargo build --bin reth --features "$(FEATURES)"
# Builds the reth binary natively.
build-native-%:
$(if $(EXTRA_RUSTFLAGS),RUSTFLAGS="$(EXTRA_RUSTFLAGS)") cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
# The following commands use `cross` to build a cross-compile.
#
@@ -95,12 +92,11 @@ build-native-%:
# on other systems. JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to use 64-KiB
# pages. See: https://github.com/paradigmxyz/reth/issues/6742
build-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
build-native-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
# Note: The additional rustc compiler flags are for intrinsics needed by MDBX.
# See: https://github.com/cross-rs/cross/wiki/FAQ#undefined-reference-with-build-std
build-%:
RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc $(EXTRA_RUSTFLAGS)" \
RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \
cross build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
# Unfortunately we can't easily use cross to build for Darwin because of licensing issues.

View File

@@ -0,0 +1,102 @@
[package]
name = "reth-bench-compare"
version.workspace = true
edition.workspace = true
rust-version.workspace = true
license.workspace = true
homepage.workspace = true
repository.workspace = true
description = "Automated reth benchmark comparison between git references"
[lints]
workspace = true
[[bin]]
name = "reth-bench-compare"
path = "src/main.rs"
[dependencies]
# reth
reth-cli-runner.workspace = true
reth-cli-util.workspace = true
reth-node-core.workspace = true
reth-tracing.workspace = true
reth-chainspec.workspace = true
# alloy
alloy-provider = { workspace = true, features = ["reqwest-rustls-tls"], default-features = false }
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
alloy-rpc-types-eth.workspace = true
alloy-transport-ws.workspace = true
alloy-primitives.workspace = true
# CLI and argument parsing
clap = { workspace = true, features = ["derive", "env"] }
eyre.workspace = true
# Async runtime
tokio = { workspace = true, features = ["full"] }
tracing.workspace = true
# Serialization
serde = { workspace = true, features = ["derive"] }
serde_json.workspace = true
# Time handling
chrono = { workspace = true, features = ["serde"] }
# Path manipulation
shellexpand.workspace = true
# CSV handling
csv.workspace = true
# Process management
ctrlc.workspace = true
shlex.workspace = true
[target.'cfg(unix)'.dependencies]
nix = { version = "0.31", features = ["signal", "process"] }
[features]
default = ["jemalloc"]
asm-keccak = [
"reth-node-core/asm-keccak",
"alloy-primitives/asm-keccak",
]
jemalloc = [
"reth-cli-util/jemalloc",
"reth-node-core/jemalloc",
]
jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
tracy = [
"reth-node-core/tracy",
"reth-tracing/tracy",
]
min-error-logs = [
"tracing/release_max_level_error",
"reth-node-core/min-error-logs",
]
min-warn-logs = [
"tracing/release_max_level_warn",
"reth-node-core/min-warn-logs",
]
min-info-logs = [
"tracing/release_max_level_info",
"reth-node-core/min-info-logs",
]
min-debug-logs = [
"tracing/release_max_level_debug",
"reth-node-core/min-debug-logs",
]
min-trace-logs = [
"tracing/release_max_level_trace",
"reth-node-core/min-trace-logs",
]
# no-op feature flag for CI matrices
ethereum = []

View File

@@ -0,0 +1,50 @@
# reth-bench-compare
Compare reth performance between two git references.
## Usage
```bash
reth-bench-compare \
--baseline-ref main \
--feature-ref my-feature \
--blocks 100 \
--wait-for-persistence
```
## Arguments
| Argument | Description | Default | Required |
|----------|-------------|---------|----------|
| `--baseline-ref <REF>` | Git reference for baseline | - | Yes |
| `--feature-ref <REF>` | Git reference to compare | - | Yes |
| `--blocks <N>` | Number of blocks to benchmark | `100` | No |
| `--chain <CHAIN>` | Chain to benchmark | `mainnet` | No |
| `--datadir <PATH>` | Data directory path | OS-specific | No |
| `--rpc-url <URL>` | RPC endpoint for block data | Chain default | No |
| `--output-dir <PATH>` | Output directory | `./reth-bench-compare` | No |
| `--wait-for-persistence` | Wait for block persistence | `false` | No |
| `--persistence-threshold <N>` | Wait after every N+1 blocks | `2` | No |
| `--wait-time <DURATION>` | Fixed delay (legacy) | - | No |
| `--warmup-blocks <N>` | Cache warmup blocks | Same as `--blocks` | No |
| `--draw` | Generate charts (needs Python/uv) | `false` | No |
| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
| `-vvvv` | Debug logging | Info | No |
| `--features <FEATURES>` | Extra Rust features for both builds | - | No |
| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |
| `--baseline-rustflags <FLAGS>` | RUSTFLAGS for baseline only | Inherits `--rustflags` | No |
| `--feature-rustflags <FLAGS>` | RUSTFLAGS for feature only | Inherits `--rustflags` | No |
| `--baseline-args <ARGS>` | Extra args for baseline node | - | No |
| `--feature-args <ARGS>` | Extra args for feature node | - | No |
| `--metrics-port <PORT>` | Metrics endpoint port | `5005` | No |
| `--sudo` | Run with elevated privileges | `false` | No |
## Output
Results in `./reth-bench-compare/results/<timestamp>/`:
- `comparison_report.json` - Metrics comparison
- `per_block_comparison.csv` - Per-block statistics
- `baseline/` and `feature/` - Individual run results
- `latency_comparison.png` - Chart (if `--draw` used)

View File

@@ -0,0 +1,307 @@
//! Benchmark execution using reth-bench.
use crate::cli::Args;
use eyre::{eyre, Result, WrapErr};
use std::{
path::Path,
sync::{Arc, Mutex},
};
use tokio::{
fs::File as AsyncFile,
io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
process::Command,
};
use tracing::{debug, error, info, warn};
/// Manages benchmark execution using reth-bench
pub(crate) struct BenchmarkRunner {
rpc_url: String,
jwt_secret: String,
wait_time: Option<String>,
wait_for_persistence: bool,
persistence_threshold: Option<u64>,
warmup_blocks: u64,
}
impl BenchmarkRunner {
/// Create a new `BenchmarkRunner` from CLI arguments
pub(crate) fn new(args: &Args) -> Self {
Self {
rpc_url: args.get_rpc_url(),
jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(),
wait_time: args.wait_time.clone(),
wait_for_persistence: args.wait_for_persistence,
persistence_threshold: args.persistence_threshold,
warmup_blocks: args.get_warmup_blocks(),
}
}
/// Clear filesystem caches (page cache, dentries, and inodes)
pub(crate) async fn clear_fs_caches() -> Result<()> {
info!("Clearing filesystem caches...");
// First sync to ensure all pending writes are flushed
let sync_output =
Command::new("sync").output().await.wrap_err("Failed to execute sync command")?;
if !sync_output.status.success() {
return Err(eyre!("sync command failed"));
}
// Drop caches - requires sudo/root permissions
// 3 = drop pagecache, dentries, and inodes
let drop_caches_cmd = Command::new("sudo")
.args(["-n", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"])
.output()
.await;
match drop_caches_cmd {
Ok(output) if output.status.success() => {
info!("Successfully cleared filesystem caches");
Ok(())
}
Ok(output) => {
let stderr = String::from_utf8_lossy(&output.stderr);
if stderr.contains("sudo: a password is required") {
warn!("Unable to clear filesystem caches: sudo password required");
warn!(
"For optimal benchmarking, configure passwordless sudo for cache clearing:"
);
warn!(" echo '$USER ALL=(ALL) NOPASSWD: /bin/sh -c echo\\\\ [0-9]\\\\ \\\\>\\\\ /proc/sys/vm/drop_caches' | sudo tee /etc/sudoers.d/drop_caches");
Ok(())
} else {
Err(eyre!("Failed to clear filesystem caches: {}", stderr))
}
}
Err(e) => {
warn!("Unable to clear filesystem caches: {}", e);
Ok(())
}
}
}
/// Run a warmup benchmark for cache warming
pub(crate) async fn run_warmup(&self, from_block: u64) -> Result<()> {
let to_block = from_block + self.warmup_blocks;
info!(
"Running warmup benchmark from block {} to {} ({} blocks)",
from_block, to_block, self.warmup_blocks
);
// Build the reth-bench command for warmup (no output flag)
let mut cmd = Command::new("reth-bench");
cmd.args([
"new-payload-fcu",
"--rpc-url",
&self.rpc_url,
"--jwt-secret",
&self.jwt_secret,
"--from",
&from_block.to_string(),
"--to",
&to_block.to_string(),
"--wait-time=0ms", // Warmup should avoid persistence waits.
]);
cmd.env("RUST_LOG_STYLE", "never")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true);
// Set process group for consistent signal handling
#[cfg(unix)]
{
cmd.process_group(0);
}
debug!("Executing warmup reth-bench command: {:?}", cmd);
// Execute the warmup benchmark
let mut child = cmd.spawn().wrap_err("Failed to start warmup reth-bench process")?;
// Stream output at debug level
if let Some(stdout) = child.stdout.take() {
tokio::spawn(async move {
let reader = BufReader::new(stdout);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
debug!("[WARMUP] {}", line);
}
});
}
if let Some(stderr) = child.stderr.take() {
tokio::spawn(async move {
let reader = BufReader::new(stderr);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
debug!("[WARMUP] {}", line);
}
});
}
let status = child.wait().await.wrap_err("Failed to wait for warmup reth-bench")?;
if !status.success() {
return Err(eyre!("Warmup reth-bench failed with exit code: {:?}", status.code()));
}
info!("Warmup completed successfully");
Ok(())
}
/// Run a benchmark for the specified block range
pub(crate) async fn run_benchmark(
&self,
from_block: u64,
to_block: u64,
output_dir: &Path,
) -> Result<()> {
info!(
"Running benchmark from block {} to {} (output: {:?})",
from_block, to_block, output_dir
);
// Ensure output directory exists
std::fs::create_dir_all(output_dir)
.wrap_err_with(|| format!("Failed to create output directory: {output_dir:?}"))?;
// Create log file path for reth-bench output
let log_file_path = output_dir.join("reth_bench.log");
info!("reth-bench logs will be saved to: {:?}", log_file_path);
// Build the reth-bench command
let mut cmd = Command::new("reth-bench");
cmd.args([
"new-payload-fcu",
"--rpc-url",
&self.rpc_url,
"--jwt-secret",
&self.jwt_secret,
"--from",
&from_block.to_string(),
"--to",
&to_block.to_string(),
"--output",
&output_dir.to_string_lossy(),
]);
// Configure wait mode: both can be used together
// When both are set: wait at least wait_time, and also wait for persistence if needed
if let Some(ref wait_time) = self.wait_time {
cmd.args(["--wait-time", wait_time]);
}
if self.wait_for_persistence {
cmd.arg("--wait-for-persistence");
// Add persistence threshold if specified
if let Some(threshold) = self.persistence_threshold {
cmd.args(["--persistence-threshold", &threshold.to_string()]);
}
}
cmd.env("RUST_LOG_STYLE", "never")
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true);
// Set process group for consistent signal handling
#[cfg(unix)]
{
cmd.process_group(0);
}
// Debug log the command
debug!("Executing reth-bench command: {:?}", cmd);
// Execute the benchmark
let mut child = cmd.spawn().wrap_err("Failed to start reth-bench process")?;
// Capture stdout and stderr for error reporting
let stdout_lines = Arc::new(Mutex::new(Vec::new()));
let stderr_lines = Arc::new(Mutex::new(Vec::new()));
// Stream stdout with prefix at debug level, capture for error reporting, and write to log
// file
if let Some(stdout) = child.stdout.take() {
let stdout_lines_clone = stdout_lines.clone();
let log_file = AsyncFile::create(&log_file_path)
.await
.wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
tokio::spawn(async move {
let reader = BufReader::new(stdout);
let mut lines = reader.lines();
let mut log_file = log_file;
while let Ok(Some(line)) = lines.next_line().await {
debug!("[RETH-BENCH] {}", line);
if let Ok(mut captured) = stdout_lines_clone.lock() {
captured.push(line.clone());
}
// Write to log file (reth-bench output already has timestamps if needed)
let log_line = format!("{}\n", line);
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
debug!("Failed to write to log file: {}", e);
}
}
});
}
// Stream stderr with prefix at debug level, capture for error reporting, and write to log
// file
if let Some(stderr) = child.stderr.take() {
let stderr_lines_clone = stderr_lines.clone();
let log_file = AsyncFile::options()
.create(true)
.append(true)
.open(&log_file_path)
.await
.wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
tokio::spawn(async move {
let reader = BufReader::new(stderr);
let mut lines = reader.lines();
let mut log_file = log_file;
while let Ok(Some(line)) = lines.next_line().await {
debug!("[RETH-BENCH] {}", line);
if let Ok(mut captured) = stderr_lines_clone.lock() {
captured.push(line.clone());
}
// Write to log file (reth-bench output already has timestamps if needed)
let log_line = format!("{}\n", line);
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
debug!("Failed to write to log file: {}", e);
}
}
});
}
let status = child.wait().await.wrap_err("Failed to wait for reth-bench")?;
if !status.success() {
// Print all captured output when command fails
error!("reth-bench failed with exit code: {:?}", status.code());
if let Ok(stdout) = stdout_lines.lock() &&
!stdout.is_empty()
{
error!("reth-bench stdout:");
for line in stdout.iter() {
error!(" {}", line);
}
}
if let Ok(stderr) = stderr_lines.lock() &&
!stderr.is_empty()
{
error!("reth-bench stderr:");
for line in stderr.iter() {
error!(" {}", line);
}
}
return Err(eyre!("reth-bench failed with exit code: {:?}", status.code()));
}
info!("Benchmark completed");
Ok(())
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,723 @@
//! Results comparison and report generation.
use crate::cli::Args;
use chrono::{DateTime, Utc};
use csv::Reader;
use eyre::{eyre, Result, WrapErr};
use serde::{Deserialize, Serialize};
use std::{
cmp::Ordering,
collections::HashMap,
fs,
path::{Path, PathBuf},
};
use tracing::{info, warn};
/// Manages comparison between baseline and feature reference results
pub(crate) struct ComparisonGenerator {
output_dir: PathBuf,
timestamp: String,
baseline_ref_name: String,
feature_ref_name: String,
baseline_results: Option<BenchmarkResults>,
feature_results: Option<BenchmarkResults>,
baseline_command: Option<String>,
feature_command: Option<String>,
}
/// Represents the results from a single benchmark run
#[derive(Debug, Clone)]
pub(crate) struct BenchmarkResults {
pub ref_name: String,
pub combined_latency_data: Vec<CombinedLatencyRow>,
pub summary: BenchmarkSummary,
pub start_timestamp: Option<DateTime<Utc>>,
pub end_timestamp: Option<DateTime<Utc>>,
}
/// Combined latency CSV row structure
#[derive(Debug, Clone, Deserialize, Serialize)]
pub(crate) struct CombinedLatencyRow {
pub block_number: u64,
#[serde(default)]
pub transaction_count: Option<u64>,
pub gas_used: u64,
pub new_payload_latency: u128,
}
/// Total gas CSV row structure
#[derive(Debug, Clone, Deserialize, Serialize)]
pub(crate) struct TotalGasRow {
pub block_number: u64,
#[serde(default)]
pub transaction_count: Option<u64>,
pub gas_used: u64,
pub time: u128,
}
/// Summary statistics for a benchmark run.
///
/// Latencies are derived from per-block `engine_newPayload` timings (converted from µs to ms):
/// - `mean_new_payload_latency_ms`: arithmetic mean latency across blocks.
/// - `median_new_payload_latency_ms`: p50 latency across blocks.
/// - `p90_new_payload_latency_ms` / `p99_new_payload_latency_ms`: tail latencies across blocks.
#[derive(Debug, Clone, Serialize)]
pub(crate) struct BenchmarkSummary {
pub total_blocks: u64,
pub total_gas_used: u64,
pub total_duration_ms: u128,
pub mean_new_payload_latency_ms: f64,
pub median_new_payload_latency_ms: f64,
pub p90_new_payload_latency_ms: f64,
pub p99_new_payload_latency_ms: f64,
pub gas_per_second: f64,
pub blocks_per_second: f64,
pub min_block_number: u64,
pub max_block_number: u64,
}
/// Comparison report between two benchmark runs
#[derive(Debug, Serialize)]
pub(crate) struct ComparisonReport {
pub timestamp: String,
pub baseline: RefInfo,
pub feature: RefInfo,
pub comparison_summary: ComparisonSummary,
pub per_block_comparisons: Vec<BlockComparison>,
}
/// Information about a reference in the comparison
#[derive(Debug, Serialize)]
pub(crate) struct RefInfo {
pub ref_name: String,
pub summary: BenchmarkSummary,
pub start_timestamp: Option<DateTime<Utc>>,
pub end_timestamp: Option<DateTime<Utc>>,
pub reth_command: Option<String>,
}
/// Summary of the comparison between references.
///
/// Percent deltas are `(feature - baseline) / baseline * 100`:
/// - `new_payload_latency_mean_change_percent`: percent changes of the per-block means.
/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective
/// per-block percentiles.
/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the
/// mean and median of per-block percent deltas (feature vs baseline), capturing block-level
/// drift.
/// - `per_block_latency_change_std_dev_percent`: standard deviation of per-block percent changes,
/// measuring consistency of performance changes across blocks.
/// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time
/// across the run.
///
/// Positive means slower/higher; negative means faster/lower.
#[derive(Debug, Serialize)]
pub(crate) struct ComparisonSummary {
pub per_block_latency_change_mean_percent: f64,
pub per_block_latency_change_median_percent: f64,
pub per_block_latency_change_std_dev_percent: f64,
pub new_payload_total_latency_change_percent: f64,
pub new_payload_latency_mean_change_percent: f64,
pub new_payload_latency_p50_change_percent: f64,
pub new_payload_latency_p90_change_percent: f64,
pub new_payload_latency_p99_change_percent: f64,
pub gas_per_second_change_percent: f64,
pub blocks_per_second_change_percent: f64,
}
/// Per-block comparison data
#[derive(Debug, Serialize)]
pub(crate) struct BlockComparison {
pub block_number: u64,
#[serde(skip_serializing_if = "Option::is_none")]
pub transaction_count: Option<u64>,
pub gas_used: u64,
pub baseline_new_payload_latency: u128,
pub feature_new_payload_latency: u128,
pub new_payload_latency_change_percent: f64,
}
impl ComparisonGenerator {
/// Create a new comparison generator
pub(crate) fn new(args: &Args) -> Self {
let now: DateTime<Utc> = Utc::now();
let timestamp = now.format("%Y%m%d_%H%M%S").to_string();
Self {
output_dir: args.output_dir_path(),
timestamp,
baseline_ref_name: args.baseline_ref.clone(),
feature_ref_name: args.feature_ref.clone(),
baseline_results: None,
feature_results: None,
baseline_command: None,
feature_command: None,
}
}
/// Get the output directory for a specific reference
pub(crate) fn get_ref_output_dir(&self, ref_type: &str) -> PathBuf {
self.output_dir.join("results").join(&self.timestamp).join(ref_type)
}
/// Get the main output directory for this comparison run
pub(crate) fn get_output_dir(&self) -> PathBuf {
self.output_dir.join("results").join(&self.timestamp)
}
/// Add benchmark results for a reference
pub(crate) fn add_ref_results(&mut self, ref_type: &str, output_path: &Path) -> Result<()> {
let ref_name = match ref_type {
"baseline" => &self.baseline_ref_name,
"feature" => &self.feature_ref_name,
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
};
let results = self.load_benchmark_results(ref_name, output_path)?;
match ref_type {
"baseline" => self.baseline_results = Some(results),
"feature" => self.feature_results = Some(results),
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
}
info!("Loaded benchmark results for {} reference", ref_type);
Ok(())
}
/// Set the benchmark run timestamps for a reference
pub(crate) fn set_ref_timestamps(
&mut self,
ref_type: &str,
start: DateTime<Utc>,
end: DateTime<Utc>,
) -> Result<()> {
match ref_type {
"baseline" => {
if let Some(ref mut results) = self.baseline_results {
results.start_timestamp = Some(start);
results.end_timestamp = Some(end);
} else {
return Err(eyre!("Baseline results not loaded yet"));
}
}
"feature" => {
if let Some(ref mut results) = self.feature_results {
results.start_timestamp = Some(start);
results.end_timestamp = Some(end);
} else {
return Err(eyre!("Feature results not loaded yet"));
}
}
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
}
Ok(())
}
/// Set the reth command for a reference
pub(crate) fn set_ref_command(&mut self, ref_type: &str, command: String) -> Result<()> {
match ref_type {
"baseline" => {
self.baseline_command = Some(command);
}
"feature" => {
self.feature_command = Some(command);
}
_ => return Err(eyre!("Unknown reference type: {}", ref_type)),
}
Ok(())
}
/// Generate the final comparison report
pub(crate) async fn generate_comparison_report(&self) -> Result<()> {
info!("Generating comparison report...");
let baseline =
self.baseline_results.as_ref().ok_or_else(|| eyre!("Baseline results not loaded"))?;
let feature =
self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?;
let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?;
let comparison_summary = self.calculate_comparison_summary(
&baseline.summary,
&feature.summary,
&per_block_comparisons,
)?;
let report = ComparisonReport {
timestamp: self.timestamp.clone(),
baseline: RefInfo {
ref_name: baseline.ref_name.clone(),
summary: baseline.summary.clone(),
start_timestamp: baseline.start_timestamp,
end_timestamp: baseline.end_timestamp,
reth_command: self.baseline_command.clone(),
},
feature: RefInfo {
ref_name: feature.ref_name.clone(),
summary: feature.summary.clone(),
start_timestamp: feature.start_timestamp,
end_timestamp: feature.end_timestamp,
reth_command: self.feature_command.clone(),
},
comparison_summary,
per_block_comparisons,
};
// Write reports
self.write_comparison_reports(&report).await?;
// Print summary to console
self.print_comparison_summary(&report);
Ok(())
}
/// Load benchmark results from CSV files
fn load_benchmark_results(
&self,
ref_name: &str,
output_path: &Path,
) -> Result<BenchmarkResults> {
let combined_latency_path = output_path.join("combined_latency.csv");
let total_gas_path = output_path.join("total_gas.csv");
let combined_latency_data = self.load_combined_latency_csv(&combined_latency_path)?;
let total_gas_data = self.load_total_gas_csv(&total_gas_path)?;
let summary = self.calculate_summary(&combined_latency_data, &total_gas_data)?;
Ok(BenchmarkResults {
ref_name: ref_name.to_string(),
combined_latency_data,
summary,
start_timestamp: None,
end_timestamp: None,
})
}
/// Load combined latency CSV data
fn load_combined_latency_csv(&self, path: &Path) -> Result<Vec<CombinedLatencyRow>> {
let mut reader = Reader::from_path(path)
.wrap_err_with(|| format!("Failed to open combined latency CSV: {path:?}"))?;
let mut rows = Vec::new();
for result in reader.deserialize() {
let row: CombinedLatencyRow = result
.wrap_err_with(|| format!("Failed to parse combined latency row in {path:?}"))?;
rows.push(row);
}
if rows.is_empty() {
return Err(eyre!("No data found in combined latency CSV: {:?}", path));
}
Ok(rows)
}
/// Load total gas CSV data
fn load_total_gas_csv(&self, path: &Path) -> Result<Vec<TotalGasRow>> {
let mut reader = Reader::from_path(path)
.wrap_err_with(|| format!("Failed to open total gas CSV: {path:?}"))?;
let mut rows = Vec::new();
for result in reader.deserialize() {
let row: TotalGasRow =
result.wrap_err_with(|| format!("Failed to parse total gas row in {path:?}"))?;
rows.push(row);
}
if rows.is_empty() {
return Err(eyre!("No data found in total gas CSV: {:?}", path));
}
Ok(rows)
}
/// Calculate summary statistics for a benchmark run.
///
/// Computes latency statistics from per-block `new_payload_latency` values in `combined_data`
/// (converting from µs to ms), and throughput metrics using the total run duration from
/// `total_gas_data`. Percentiles (p50/p90/p99) use linear interpolation on sorted latencies.
fn calculate_summary(
&self,
combined_data: &[CombinedLatencyRow],
total_gas_data: &[TotalGasRow],
) -> Result<BenchmarkSummary> {
if combined_data.is_empty() || total_gas_data.is_empty() {
return Err(eyre!("Cannot calculate summary for empty data"));
}
let total_blocks = combined_data.len() as u64;
let total_gas_used: u64 = combined_data.iter().map(|r| r.gas_used).sum();
let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds
let latencies_ms: Vec<f64> =
combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).collect();
let mean_new_payload_latency_ms: f64 =
latencies_ms.iter().sum::<f64>() / total_blocks as f64;
let mut sorted_latencies_ms = latencies_ms;
sorted_latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
let median_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.5);
let p90_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.9);
let p99_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.99);
let total_duration_seconds = total_duration_ms as f64 / 1000.0;
let gas_per_second = if total_duration_seconds > f64::EPSILON {
total_gas_used as f64 / total_duration_seconds
} else {
0.0
};
let blocks_per_second = if total_duration_seconds > f64::EPSILON {
total_blocks as f64 / total_duration_seconds
} else {
0.0
};
let min_block_number = combined_data.first().unwrap().block_number;
let max_block_number = combined_data.last().unwrap().block_number;
Ok(BenchmarkSummary {
total_blocks,
total_gas_used,
total_duration_ms,
mean_new_payload_latency_ms,
median_new_payload_latency_ms,
p90_new_payload_latency_ms,
p99_new_payload_latency_ms,
gas_per_second,
blocks_per_second,
min_block_number,
max_block_number,
})
}
/// Calculate comparison summary between baseline and feature
fn calculate_comparison_summary(
&self,
baseline: &BenchmarkSummary,
feature: &BenchmarkSummary,
per_block_comparisons: &[BlockComparison],
) -> Result<ComparisonSummary> {
let calc_percent_change = |baseline: f64, feature: f64| -> f64 {
if baseline.abs() > f64::EPSILON {
((feature - baseline) / baseline) * 100.0
} else {
0.0
}
};
// Calculate per-block statistics. "Per-block" means: for each block, compute the percent
// change (feature - baseline) / baseline * 100, then calculate statistics across those
// per-block percent changes. This captures how consistently the feature performs relative
// to baseline across all blocks.
let per_block_percent_changes: Vec<f64> =
per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect();
let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() {
0.0
} else {
per_block_percent_changes.iter().sum::<f64>() / per_block_percent_changes.len() as f64
};
let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() {
0.0
} else {
let mut sorted = per_block_percent_changes.clone();
sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
percentile(&sorted, 0.5)
};
let per_block_latency_change_std_dev_percent =
calculate_std_dev(&per_block_percent_changes, per_block_latency_change_mean_percent);
let baseline_total_latency_ms =
baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64;
let feature_total_latency_ms =
feature.mean_new_payload_latency_ms * feature.total_blocks as f64;
let new_payload_total_latency_change_percent =
calc_percent_change(baseline_total_latency_ms, feature_total_latency_ms);
Ok(ComparisonSummary {
per_block_latency_change_mean_percent,
per_block_latency_change_median_percent,
per_block_latency_change_std_dev_percent,
new_payload_total_latency_change_percent,
new_payload_latency_mean_change_percent: calc_percent_change(
baseline.mean_new_payload_latency_ms,
feature.mean_new_payload_latency_ms,
),
new_payload_latency_p50_change_percent: calc_percent_change(
baseline.median_new_payload_latency_ms,
feature.median_new_payload_latency_ms,
),
new_payload_latency_p90_change_percent: calc_percent_change(
baseline.p90_new_payload_latency_ms,
feature.p90_new_payload_latency_ms,
),
new_payload_latency_p99_change_percent: calc_percent_change(
baseline.p99_new_payload_latency_ms,
feature.p99_new_payload_latency_ms,
),
gas_per_second_change_percent: calc_percent_change(
baseline.gas_per_second,
feature.gas_per_second,
),
blocks_per_second_change_percent: calc_percent_change(
baseline.blocks_per_second,
feature.blocks_per_second,
),
})
}
/// Calculate per-block comparisons
fn calculate_per_block_comparisons(
&self,
baseline: &BenchmarkResults,
feature: &BenchmarkResults,
) -> Result<Vec<BlockComparison>> {
let mut baseline_map: HashMap<u64, &CombinedLatencyRow> = HashMap::new();
for row in &baseline.combined_latency_data {
baseline_map.insert(row.block_number, row);
}
let mut comparisons = Vec::new();
for feature_row in &feature.combined_latency_data {
if let Some(baseline_row) = baseline_map.get(&feature_row.block_number) {
let calc_percent_change = |baseline: u128, feature: u128| -> f64 {
if baseline > 0 {
((feature as f64 - baseline as f64) / baseline as f64) * 100.0
} else {
0.0
}
};
let comparison = BlockComparison {
block_number: feature_row.block_number,
transaction_count: feature_row.transaction_count,
gas_used: feature_row.gas_used,
baseline_new_payload_latency: baseline_row.new_payload_latency,
feature_new_payload_latency: feature_row.new_payload_latency,
new_payload_latency_change_percent: calc_percent_change(
baseline_row.new_payload_latency,
feature_row.new_payload_latency,
),
};
comparisons.push(comparison);
} else {
warn!("Block {} not found in baseline data", feature_row.block_number);
}
}
Ok(comparisons)
}
/// Write comparison reports to files
async fn write_comparison_reports(&self, report: &ComparisonReport) -> Result<()> {
let report_dir = self.output_dir.join("results").join(&self.timestamp);
fs::create_dir_all(&report_dir)
.wrap_err_with(|| format!("Failed to create report directory: {report_dir:?}"))?;
// Write JSON report
let json_path = report_dir.join("comparison_report.json");
let json_content = serde_json::to_string_pretty(report)
.wrap_err("Failed to serialize comparison report to JSON")?;
fs::write(&json_path, json_content)
.wrap_err_with(|| format!("Failed to write JSON report: {json_path:?}"))?;
// Write CSV report for per-block comparisons
let csv_path = report_dir.join("per_block_comparison.csv");
let mut writer = csv::Writer::from_path(&csv_path)
.wrap_err_with(|| format!("Failed to create CSV writer: {csv_path:?}"))?;
for comparison in &report.per_block_comparisons {
writer.serialize(comparison).wrap_err("Failed to write comparison row to CSV")?;
}
writer.flush().wrap_err("Failed to flush CSV writer")?;
info!("Comparison reports written to: {:?}", report_dir);
Ok(())
}
/// Print comparison summary to console
fn print_comparison_summary(&self, report: &ComparisonReport) {
// Parse and format timestamp nicely
let formatted_timestamp = if let Ok(dt) = chrono::DateTime::parse_from_str(
&format!("{} +0000", report.timestamp.replace('_', " ")),
"%Y%m%d %H%M%S %z",
) {
dt.format("%Y-%m-%d %H:%M:%S UTC").to_string()
} else {
// Fallback to original if parsing fails
report.timestamp.clone()
};
println!("\n=== BENCHMARK COMPARISON SUMMARY ===");
println!("Timestamp: {formatted_timestamp}");
println!("Baseline: {}", report.baseline.ref_name);
println!("Feature: {}", report.feature.ref_name);
println!();
let summary = &report.comparison_summary;
println!("Performance Changes:");
println!(
" NewPayload Latency per-block mean change: {:+.2}%",
summary.per_block_latency_change_mean_percent
);
println!(
" NewPayload Latency per-block median change: {:+.2}%",
summary.per_block_latency_change_median_percent
);
println!(
" NewPayload Latency per-block std dev: {:.2}%",
summary.per_block_latency_change_std_dev_percent
);
println!(
" Total newPayload time change: {:+.2}%",
summary.new_payload_total_latency_change_percent
);
println!(
" NewPayload Latency mean: {:+.2}%",
summary.new_payload_latency_mean_change_percent
);
println!(
" NewPayload Latency p50: {:+.2}%",
summary.new_payload_latency_p50_change_percent
);
println!(
" NewPayload Latency p90: {:+.2}%",
summary.new_payload_latency_p90_change_percent
);
println!(
" NewPayload Latency p99: {:+.2}%",
summary.new_payload_latency_p99_change_percent
);
println!(
" Gas/Second: {:+.2}%",
summary.gas_per_second_change_percent
);
println!(
" Blocks/Second: {:+.2}%",
summary.blocks_per_second_change_percent
);
println!();
println!("Baseline Summary:");
let baseline = &report.baseline.summary;
println!(
" Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
baseline.total_blocks,
baseline.min_block_number,
baseline.max_block_number,
baseline.total_gas_used,
baseline.total_duration_ms as f64 / 1000.0
);
println!(" NewPayload latency (ms):");
println!(
" mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
baseline.mean_new_payload_latency_ms,
baseline.median_new_payload_latency_ms,
baseline.p90_new_payload_latency_ms,
baseline.p99_new_payload_latency_ms
);
if let (Some(start), Some(end)) =
(&report.baseline.start_timestamp, &report.baseline.end_timestamp)
{
println!(
" Started: {}, Ended: {}",
start.format("%Y-%m-%d %H:%M:%S UTC"),
end.format("%Y-%m-%d %H:%M:%S UTC")
);
}
if let Some(ref cmd) = report.baseline.reth_command {
println!(" Command: {}", cmd);
}
println!();
println!("Feature Summary:");
let feature = &report.feature.summary;
println!(
" Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
feature.total_blocks,
feature.min_block_number,
feature.max_block_number,
feature.total_gas_used,
feature.total_duration_ms as f64 / 1000.0
);
println!(" NewPayload latency (ms):");
println!(
" mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
feature.mean_new_payload_latency_ms,
feature.median_new_payload_latency_ms,
feature.p90_new_payload_latency_ms,
feature.p99_new_payload_latency_ms
);
if let (Some(start), Some(end)) =
(&report.feature.start_timestamp, &report.feature.end_timestamp)
{
println!(
" Started: {}, Ended: {}",
start.format("%Y-%m-%d %H:%M:%S UTC"),
end.format("%Y-%m-%d %H:%M:%S UTC")
);
}
if let Some(ref cmd) = report.feature.reth_command {
println!(" Command: {}", cmd);
}
println!();
}
}
/// Calculate standard deviation from a set of values and their mean.
///
/// Computes the population standard deviation using the formula:
/// `sqrt(sum((x - mean)²) / n)`
///
/// Returns 0.0 for empty input.
fn calculate_std_dev(values: &[f64], mean: f64) -> f64 {
if values.is_empty() {
return 0.0;
}
let variance = values
.iter()
.map(|x| {
let diff = x - mean;
diff * diff
})
.sum::<f64>() /
values.len() as f64;
variance.sqrt()
}
/// Calculate percentile using linear interpolation on a sorted slice.
///
/// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls
/// between two indices, linearly interpolates between those values. For example, with 100 values,
/// p90 computes rank = 0.9 × 99 = 89.1, then returns `values[89] × 0.9 + values[90] × 0.1`.
///
/// Returns 0.0 for empty input.
fn percentile(sorted_values: &[f64], percentile: f64) -> f64 {
if sorted_values.is_empty() {
return 0.0;
}
let clamped = percentile.clamp(0.0, 1.0);
let max_index = sorted_values.len() - 1;
let rank = clamped * max_index as f64;
let lower = rank.floor() as usize;
let upper = rank.ceil() as usize;
if lower == upper {
sorted_values[lower]
} else {
let weight = rank - lower as f64;
sorted_values[lower].mul_add(1.0 - weight, sorted_values[upper] * weight)
}
}

View File

@@ -0,0 +1,305 @@
//! Compilation operations for reth and reth-bench.
use crate::git::GitManager;
use eyre::{eyre, Result, WrapErr};
use std::{fs, path::PathBuf, process::Command};
use tracing::{debug, error, info, warn};
/// Manages compilation operations for reth components
#[derive(Debug)]
pub(crate) struct CompilationManager {
repo_root: String,
output_dir: PathBuf,
git_manager: GitManager,
}
impl CompilationManager {
/// Create a new `CompilationManager`
pub(crate) const fn new(
repo_root: String,
output_dir: PathBuf,
git_manager: GitManager,
) -> Result<Self> {
Ok(Self { repo_root, output_dir, git_manager })
}
/// Get the path to the cached binary using explicit commit hash
pub(crate) fn get_cached_binary_path_for_commit(&self, commit: &str) -> PathBuf {
let identifier = &commit[..8]; // Use first 8 chars of commit
self.output_dir.join("bin").join(format!("reth_{identifier}"))
}
/// Compile reth using cargo build and cache the binary
pub(crate) fn compile_reth(&self, commit: &str, features: &str, rustflags: &str) -> Result<()> {
// Validate that current git commit matches the expected commit
let current_commit = self.git_manager.get_current_commit()?;
if current_commit != commit {
return Err(eyre!(
"Git commit mismatch! Expected: {}, but currently at: {}",
&commit[..8],
&current_commit[..8]
));
}
let cached_path = self.get_cached_binary_path_for_commit(commit);
// Check if cached binary already exists (since path contains commit hash, it's valid)
if cached_path.exists() {
info!("Using cached binary (commit: {})", &commit[..8]);
return Ok(());
}
info!("No cached binary found, compiling (commit: {})...", &commit[..8]);
let binary_name = "reth";
info!(
"Compiling {} with profiling configuration (commit: {})...",
binary_name,
&commit[..8]
);
let mut cmd = Command::new("cargo");
cmd.arg("build").arg("--profile").arg("profiling");
cmd.arg("--features").arg(features);
info!("Using features: {features}");
cmd.current_dir(&self.repo_root);
// Set RUSTFLAGS
cmd.env("RUSTFLAGS", rustflags);
info!("Using RUSTFLAGS: {rustflags}");
info!("Compiling {binary_name} with {cmd:?}");
let output = cmd.output().wrap_err("Failed to execute cargo build command")?;
// Print stdout and stderr with prefixes at debug level
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stdout.lines() {
if !line.trim().is_empty() {
debug!("[CARGO] {}", line);
}
}
for line in stderr.lines() {
if !line.trim().is_empty() {
debug!("[CARGO] {}", line);
}
}
if !output.status.success() {
// Print all output when compilation fails
error!("Cargo build failed with exit code: {:?}", output.status.code());
if !stdout.trim().is_empty() {
error!("Cargo stdout:");
for line in stdout.lines() {
error!(" {}", line);
}
}
if !stderr.trim().is_empty() {
error!("Cargo stderr:");
for line in stderr.lines() {
error!(" {}", line);
}
}
return Err(eyre!("Compilation failed with exit code: {:?}", output.status.code()));
}
info!("{} compilation completed", binary_name);
// Copy the compiled binary to cache
let source_path =
PathBuf::from(&self.repo_root).join(format!("target/profiling/{}", binary_name));
if !source_path.exists() {
return Err(eyre!("Compiled binary not found at {:?}", source_path));
}
// Create bin directory if it doesn't exist
let bin_dir = self.output_dir.join("bin");
fs::create_dir_all(&bin_dir).wrap_err("Failed to create bin directory")?;
// Copy binary to cache
fs::copy(&source_path, &cached_path).wrap_err("Failed to copy binary to cache")?;
// Make the cached binary executable
#[cfg(unix)]
{
use std::os::unix::fs::PermissionsExt;
let mut perms = fs::metadata(&cached_path)?.permissions();
perms.set_mode(0o755);
fs::set_permissions(&cached_path, perms)?;
}
info!("Cached compiled binary at: {:?}", cached_path);
Ok(())
}
/// Check if reth-bench is available in PATH
pub(crate) fn is_reth_bench_available(&self) -> bool {
match Command::new("which").arg("reth-bench").output() {
Ok(output) => {
if output.status.success() {
let path = String::from_utf8_lossy(&output.stdout);
info!("Found reth-bench: {}", path.trim());
true
} else {
false
}
}
Err(_) => false,
}
}
/// Check if samply is available in PATH
pub(crate) fn is_samply_available(&self) -> bool {
match Command::new("which").arg("samply").output() {
Ok(output) => {
if output.status.success() {
let path = String::from_utf8_lossy(&output.stdout);
info!("Found samply: {}", path.trim());
true
} else {
false
}
}
Err(_) => false,
}
}
/// Install samply using cargo
pub(crate) fn install_samply(&self) -> Result<()> {
info!("Installing samply via cargo...");
let mut cmd = Command::new("cargo");
cmd.args(["install", "--locked", "samply"]);
info!("Installing samply with {cmd:?}");
let output = cmd.output().wrap_err("Failed to execute cargo install samply command")?;
// Print stdout and stderr with prefixes at debug level
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stdout.lines() {
if !line.trim().is_empty() {
debug!("[CARGO-SAMPLY] {}", line);
}
}
for line in stderr.lines() {
if !line.trim().is_empty() {
debug!("[CARGO-SAMPLY] {}", line);
}
}
if !output.status.success() {
// Print all output when installation fails
error!("Cargo install samply failed with exit code: {:?}", output.status.code());
if !stdout.trim().is_empty() {
error!("Cargo stdout:");
for line in stdout.lines() {
error!(" {}", line);
}
}
if !stderr.trim().is_empty() {
error!("Cargo stderr:");
for line in stderr.lines() {
error!(" {}", line);
}
}
return Err(eyre!(
"samply installation failed with exit code: {:?}",
output.status.code()
));
}
info!("Samply installation completed");
Ok(())
}
/// Ensure samply is available, installing if necessary
pub(crate) fn ensure_samply_available(&self) -> Result<()> {
if self.is_samply_available() {
Ok(())
} else {
warn!("samply not found in PATH, installing...");
self.install_samply()
}
}
/// Ensure reth-bench is available, compiling if necessary
pub(crate) fn ensure_reth_bench_available(&self) -> Result<()> {
if self.is_reth_bench_available() {
Ok(())
} else {
warn!("reth-bench not found in PATH, compiling and installing...");
self.compile_reth_bench()
}
}
/// Compile and install reth-bench using `make install-reth-bench`
pub(crate) fn compile_reth_bench(&self) -> Result<()> {
info!("Compiling and installing reth-bench...");
let mut cmd = Command::new("make");
cmd.arg("install-reth-bench").current_dir(&self.repo_root);
info!("Compiling reth-bench with {cmd:?}");
let output = cmd.output().wrap_err("Failed to execute make install-reth-bench command")?;
// Print stdout and stderr with prefixes at debug level
let stdout = String::from_utf8_lossy(&output.stdout);
let stderr = String::from_utf8_lossy(&output.stderr);
for line in stdout.lines() {
if !line.trim().is_empty() {
debug!("[MAKE-BENCH] {}", line);
}
}
for line in stderr.lines() {
if !line.trim().is_empty() {
debug!("[MAKE-BENCH] {}", line);
}
}
if !output.status.success() {
// Print all output when compilation fails
error!("Make install-reth-bench failed with exit code: {:?}", output.status.code());
if !stdout.trim().is_empty() {
error!("Make stdout:");
for line in stdout.lines() {
error!(" {}", line);
}
}
if !stderr.trim().is_empty() {
error!("Make stderr:");
for line in stderr.lines() {
error!(" {}", line);
}
}
return Err(eyre!(
"reth-bench compilation failed with exit code: {:?}",
output.status.code()
));
}
info!("Reth-bench compilation completed");
Ok(())
}
}

View File

@@ -0,0 +1,328 @@
//! Git operations for branch management.
use eyre::{eyre, Result, WrapErr};
use std::process::Command;
use tracing::{info, warn};
/// Manages git operations for branch switching
#[derive(Debug, Clone)]
pub(crate) struct GitManager {
repo_root: String,
}
impl GitManager {
/// Create a new `GitManager`, detecting the repository root
pub(crate) fn new() -> Result<Self> {
let output = Command::new("git")
.args(["rev-parse", "--show-toplevel"])
.output()
.wrap_err("Failed to execute git command - is git installed?")?;
if !output.status.success() {
return Err(eyre!("Not in a git repository or git command failed"));
}
let repo_root = String::from_utf8(output.stdout)
.wrap_err("Git output is not valid UTF-8")?
.trim()
.to_string();
let manager = Self { repo_root };
info!(
"Detected git repository at: {}, current reference: {}",
manager.repo_root(),
manager.get_current_ref()?
);
Ok(manager)
}
/// Get the current git branch name
pub(crate) fn get_current_branch(&self) -> Result<String> {
let output = Command::new("git")
.args(["branch", "--show-current"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to get current branch")?;
if !output.status.success() {
return Err(eyre!("Failed to determine current branch"));
}
let branch = String::from_utf8(output.stdout)
.wrap_err("Branch name is not valid UTF-8")?
.trim()
.to_string();
if branch.is_empty() {
return Err(eyre!("Not on a named branch (detached HEAD?)"));
}
Ok(branch)
}
/// Get the current git reference (branch name, tag, or commit hash)
pub(crate) fn get_current_ref(&self) -> Result<String> {
// First try to get branch name
if let Ok(branch) = self.get_current_branch() {
return Ok(branch);
}
// If not on a branch, check if we're on a tag
let tag_output = Command::new("git")
.args(["describe", "--exact-match", "--tags", "HEAD"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to check for tag")?;
if tag_output.status.success() {
let tag = String::from_utf8(tag_output.stdout)
.wrap_err("Tag name is not valid UTF-8")?
.trim()
.to_string();
return Ok(tag);
}
// If not on a branch or tag, return the commit hash
let commit_output = Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to get current commit")?;
if !commit_output.status.success() {
return Err(eyre!("Failed to get current commit hash"));
}
let commit_hash = String::from_utf8(commit_output.stdout)
.wrap_err("Commit hash is not valid UTF-8")?
.trim()
.to_string();
Ok(commit_hash)
}
/// Check if the git working directory has uncommitted changes to tracked files
pub(crate) fn validate_clean_state(&self) -> Result<()> {
let output = Command::new("git")
.args(["status", "--porcelain"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to check git status")?;
if !output.status.success() {
return Err(eyre!("Git status command failed"));
}
let status_output =
String::from_utf8(output.stdout).wrap_err("Git status output is not valid UTF-8")?;
// Check for uncommitted changes to tracked files
// Status codes: M = modified, A = added, D = deleted, R = renamed, C = copied, U = updated
// ?? = untracked files (we want to ignore these)
let has_uncommitted_changes = status_output.lines().any(|line| {
if line.len() >= 2 {
let status = &line[0..2];
// Ignore untracked files (??) and ignored files (!!)
!matches!(status, "??" | "!!")
} else {
false
}
});
if has_uncommitted_changes {
warn!("Git working directory has uncommitted changes to tracked files:");
for line in status_output.lines() {
if line.len() >= 2 && !matches!(&line[0..2], "??" | "!!") {
warn!(" {}", line);
}
}
return Err(eyre!(
"Git working directory has uncommitted changes to tracked files. Please commit or stash changes before running benchmark comparison."
));
}
// Check if there are untracked files and log them as info
let untracked_files: Vec<&str> =
status_output.lines().filter(|line| line.starts_with("??")).collect();
if !untracked_files.is_empty() {
info!(
"Git working directory has {} untracked files (this is OK)",
untracked_files.len()
);
}
info!("Git working directory is clean (no uncommitted changes to tracked files)");
Ok(())
}
/// Fetch all refs from remote to ensure we have latest branches and tags
pub(crate) fn fetch_all(&self) -> Result<()> {
let output = Command::new("git")
.args(["fetch", "--all", "--tags", "--quiet", "--force"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to fetch latest refs")?;
if output.status.success() {
info!("Fetched latest refs");
} else {
let stderr = String::from_utf8_lossy(&output.stderr);
// Only warn if there's actual error content, not just fetch progress
if !stderr.trim().is_empty() && !stderr.contains("-> origin/") {
warn!("Git fetch encountered issues (continuing anyway): {}", stderr);
}
}
Ok(())
}
/// Validate that the specified git references exist (branches, tags, or commits)
pub(crate) fn validate_refs(&self, refs: &[&str]) -> Result<()> {
for &git_ref in refs {
// Try to resolve the ref similar to `git checkout` by peeling to a commit.
// First try the ref as-is with ^{commit}, then fall back to origin/{ref}^{commit}.
let as_is = format!("{git_ref}^{{commit}}");
let ref_check = Command::new("git")
.args(["rev-parse", "--verify", &as_is])
.current_dir(&self.repo_root)
.output();
let found = if let Ok(output) = ref_check &&
output.status.success()
{
info!("Validated reference exists: {}", git_ref);
true
} else {
// Try remote-only branches via origin/{ref}
let origin_ref = format!("origin/{git_ref}^{{commit}}");
let origin_check = Command::new("git")
.args(["rev-parse", "--verify", &origin_ref])
.current_dir(&self.repo_root)
.output();
if let Ok(output) = origin_check &&
output.status.success()
{
info!("Validated remote reference exists: origin/{}", git_ref);
true
} else {
false
}
};
if !found {
return Err(eyre!(
"Git reference '{}' does not exist as branch, tag, or commit (tried '{}' and 'origin/{}^{{commit}}')",
git_ref,
format!("{git_ref}^{{commit}}"),
git_ref,
));
}
}
Ok(())
}
/// Switch to the specified git reference (branch, tag, or commit)
pub(crate) fn switch_ref(&self, git_ref: &str) -> Result<()> {
// First checkout the reference
let output = Command::new("git")
.args(["checkout", git_ref])
.current_dir(&self.repo_root)
.output()
.wrap_err_with(|| format!("Failed to switch to reference '{git_ref}'"))?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
return Err(eyre!("Failed to switch to reference '{}': {}", git_ref, stderr));
}
// Check if this is a branch that tracks a remote and pull latest changes
let is_branch = Command::new("git")
.args(["show-ref", "--verify", "--quiet", &format!("refs/heads/{git_ref}")])
.current_dir(&self.repo_root)
.status()
.map(|s| s.success())
.unwrap_or(false);
if is_branch {
// Check if the branch tracks a remote
let tracking_output = Command::new("git")
.args([
"rev-parse",
"--abbrev-ref",
"--symbolic-full-name",
&format!("{git_ref}@{{upstream}}"),
])
.current_dir(&self.repo_root)
.output();
if let Ok(output) = tracking_output &&
output.status.success()
{
let upstream = String::from_utf8_lossy(&output.stdout).trim().to_string();
if !upstream.is_empty() && upstream != format!("{git_ref}@{{upstream}}") {
// Branch tracks a remote, pull latest changes
info!("Pulling latest changes for branch: {}", git_ref);
let pull_output = Command::new("git")
.args(["pull", "--ff-only"])
.current_dir(&self.repo_root)
.output()
.wrap_err_with(|| {
format!("Failed to pull latest changes for branch '{git_ref}'")
})?;
if pull_output.status.success() {
info!("Successfully pulled latest changes for branch: {}", git_ref);
} else {
let stderr = String::from_utf8_lossy(&pull_output.stderr);
warn!("Failed to pull latest changes for branch '{}': {}", git_ref, stderr);
// Continue anyway, we'll use whatever version we have
}
}
}
}
// Verify the checkout succeeded by checking the current commit
let current_commit_output = Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to get current commit")?;
if !current_commit_output.status.success() {
return Err(eyre!("Failed to verify git checkout"));
}
info!("Switched to reference: {}", git_ref);
Ok(())
}
/// Get the current commit hash
pub(crate) fn get_current_commit(&self) -> Result<String> {
let output = Command::new("git")
.args(["rev-parse", "HEAD"])
.current_dir(&self.repo_root)
.output()
.wrap_err("Failed to get current commit")?;
if !output.status.success() {
return Err(eyre!("Failed to get current commit hash"));
}
let commit_hash = String::from_utf8(output.stdout)
.wrap_err("Commit hash is not valid UTF-8")?
.trim()
.to_string();
Ok(commit_hash)
}
/// Get the repository root path
pub(crate) fn repo_root(&self) -> &str {
&self.repo_root
}
}

View File

@@ -0,0 +1,47 @@
//! # reth-bench-compare
//!
//! Automated tool for comparing reth performance between two git branches.
//! This tool automates the complete workflow of compiling, running, and benchmarking
//! reth on different branches to provide meaningful performance comparisons.
#![doc(
html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
)]
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
#[global_allocator]
static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
use alloy_primitives as _;
mod benchmark;
mod cli;
mod comparison;
mod compilation;
mod git;
mod node;
use clap::Parser;
use cli::{run_comparison, Args};
use eyre::Result;
use reth_cli_runner::CliRunner;
fn main() -> Result<()> {
// Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
if std::env::var_os("RUST_BACKTRACE").is_none() {
unsafe {
std::env::set_var("RUST_BACKTRACE", "1");
}
}
let args = Args::parse();
// Initialize tracing
let _guard = args.init_tracing()?;
// Run until either exit or sigint or sigterm
let runner = CliRunner::try_default_runtime()?;
runner.run_command_until_exit(|ctx| run_comparison(args, ctx))
}

View File

@@ -0,0 +1,695 @@
//! Node management for starting, stopping, and controlling reth instances.
use crate::cli::Args;
use alloy_provider::{Provider, ProviderBuilder};
use alloy_rpc_client::RpcClient;
use alloy_rpc_types_eth::SyncStatus;
use alloy_transport_ws::WsConnect;
use eyre::{eyre, OptionExt, Result, WrapErr};
#[cfg(unix)]
use nix::sys::signal::{killpg, Signal};
#[cfg(unix)]
use nix::unistd::Pid;
use reth_chainspec::Chain;
use std::{fs, path::PathBuf, time::Duration};
use tokio::{
fs::File as AsyncFile,
io::{AsyncBufReadExt, AsyncWriteExt, BufReader as AsyncBufReader},
process::Command,
time::{sleep, timeout},
};
use tracing::{debug, info, warn};
/// Default websocket RPC port used by reth
const DEFAULT_WS_RPC_PORT: u16 = 8546;
/// Manages reth node lifecycle and operations
pub(crate) struct NodeManager {
datadir: Option<String>,
metrics_port: u16,
chain: Chain,
use_sudo: bool,
binary_path: Option<std::path::PathBuf>,
enable_profiling: bool,
output_dir: PathBuf,
additional_reth_args: Vec<String>,
comparison_dir: Option<PathBuf>,
tracing_endpoint: Option<String>,
otlp_max_queue_size: usize,
}
impl NodeManager {
/// Create a new `NodeManager` with configuration from CLI args
pub(crate) fn new(args: &Args) -> Self {
Self {
datadir: Some(args.datadir_path().to_string_lossy().to_string()),
metrics_port: args.metrics_port,
chain: args.chain,
use_sudo: args.sudo,
binary_path: None,
enable_profiling: args.profile,
output_dir: args.output_dir_path(),
// Filter out empty strings to prevent invalid arguments being passed to reth node
additional_reth_args: args
.reth_args
.iter()
.filter(|s| !s.is_empty())
.cloned()
.collect(),
comparison_dir: None,
tracing_endpoint: args.traces.otlp.as_ref().map(|u| u.to_string()),
otlp_max_queue_size: args.otlp_max_queue_size,
}
}
/// Set the comparison directory path for logging
pub(crate) fn set_comparison_dir(&mut self, dir: PathBuf) {
self.comparison_dir = Some(dir);
}
/// Get the log file path for a given reference type
fn get_log_file_path(&self, ref_type: &str) -> Result<PathBuf> {
let comparison_dir = self
.comparison_dir
.as_ref()
.ok_or_eyre("Comparison directory not set. Call set_comparison_dir first.")?;
// The comparison directory already contains the full path to results/<timestamp>
let log_dir = comparison_dir.join(ref_type);
// Create the directory if it doesn't exist
fs::create_dir_all(&log_dir)
.wrap_err(format!("Failed to create log directory: {:?}", log_dir))?;
let log_file = log_dir.join("reth_node.log");
Ok(log_file)
}
/// Get the perf event max sample rate from the system, capped at 10000
fn get_perf_sample_rate(&self) -> Option<String> {
let perf_rate_file = "/proc/sys/kernel/perf_event_max_sample_rate";
if let Ok(content) = fs::read_to_string(perf_rate_file) {
let rate_str = content.trim();
if !rate_str.is_empty() {
if let Ok(system_rate) = rate_str.parse::<u32>() {
let capped_rate = std::cmp::min(system_rate, 10000);
info!(
"Detected perf_event_max_sample_rate: {}, using: {}",
system_rate, capped_rate
);
return Some(capped_rate.to_string());
}
warn!("Failed to parse perf_event_max_sample_rate: {}", rate_str);
}
}
None
}
/// Get the absolute path to samply using 'which' command
async fn get_samply_path(&self) -> Result<String> {
let output = Command::new("which")
.arg("samply")
.output()
.await
.wrap_err("Failed to execute 'which samply' command")?;
if !output.status.success() {
return Err(eyre!("samply not found in PATH"));
}
let samply_path = String::from_utf8(output.stdout)
.wrap_err("samply path is not valid UTF-8")?
.trim()
.to_string();
if samply_path.is_empty() {
return Err(eyre!("which samply returned empty path"));
}
Ok(samply_path)
}
/// Build reth arguments as a vector of strings
fn build_reth_args(
&self,
binary_path_str: &str,
additional_args: &[String],
ref_type: &str,
) -> (Vec<String>, String) {
let mut reth_args = vec![binary_path_str.to_string(), "node".to_string()];
// Add chain argument (skip for mainnet as it's the default)
let chain_str = self.chain.to_string();
if chain_str != "mainnet" {
reth_args.extend_from_slice(&["--chain".to_string(), chain_str.clone()]);
}
// Add datadir if specified
if let Some(ref datadir) = self.datadir {
reth_args.extend_from_slice(&["--datadir".to_string(), datadir.clone()]);
}
// Add reth-specific arguments
let metrics_arg = format!("0.0.0.0:{}", self.metrics_port);
reth_args.extend_from_slice(&[
"--engine.accept-execution-requests-hash".to_string(),
"--metrics".to_string(),
metrics_arg,
"--http".to_string(),
"--http.api".to_string(),
"eth,reth".to_string(),
"--ws".to_string(),
"--ws.api".to_string(),
"eth,reth".to_string(),
"--disable-discovery".to_string(),
"--trusted-only".to_string(),
"--disable-tx-gossip".to_string(),
]);
// Add tracing arguments if OTLP endpoint is configured
if let Some(ref endpoint) = self.tracing_endpoint {
info!("Enabling OTLP tracing export to: {} (service: reth-{})", endpoint, ref_type);
// Endpoint requires equals per clap settings in reth
reth_args.push(format!("--tracing-otlp={}", endpoint));
}
// Add any additional arguments passed via command line (common to both baseline and
// feature)
reth_args.extend_from_slice(&self.additional_reth_args);
// Add reference-specific additional arguments
reth_args.extend_from_slice(additional_args);
(reth_args, chain_str)
}
/// Create a command for profiling mode
async fn create_profiling_command(
&self,
ref_type: &str,
reth_args: &[String],
) -> Result<Command> {
// Create profiles directory if it doesn't exist
let profile_dir = self.output_dir.join("profiles");
fs::create_dir_all(&profile_dir).wrap_err("Failed to create profiles directory")?;
let profile_path = profile_dir.join(format!("{}.json.gz", ref_type));
info!("Starting reth node with samply profiling...");
info!("Profile output: {:?}", profile_path);
// Get absolute path to samply
let samply_path = self.get_samply_path().await?;
let mut cmd = if self.use_sudo {
let mut sudo_cmd = Command::new("sudo");
sudo_cmd.arg(&samply_path);
sudo_cmd
} else {
Command::new(&samply_path)
};
// Add samply arguments
cmd.args(["record", "--save-only", "-o", &profile_path.to_string_lossy()]);
// Add rate argument if available
if let Some(rate) = self.get_perf_sample_rate() {
cmd.args(["--rate", &rate]);
}
// Add separator and complete reth command
cmd.arg("--");
cmd.args(reth_args);
// Enable tracing-samply
if supports_samply_flags(&reth_args[0]) {
cmd.arg("--log.samply");
}
// Set environment variable to disable log styling
cmd.env("RUST_LOG_STYLE", "never");
Ok(cmd)
}
/// Create a command for direct reth execution
fn create_direct_command(&self, reth_args: &[String]) -> Command {
let binary_path = &reth_args[0];
let mut cmd = if self.use_sudo {
info!("Starting reth node with sudo...");
let mut sudo_cmd = Command::new("sudo");
sudo_cmd.args(reth_args);
sudo_cmd
} else {
info!("Starting reth node...");
let mut reth_cmd = Command::new(binary_path);
reth_cmd.args(&reth_args[1..]); // Skip the binary path since it's the command
reth_cmd
};
// Set environment variable to disable log styling
cmd.env("RUST_LOG_STYLE", "never");
cmd
}
/// Start a reth node using the specified binary path and return the process handle
/// along with the formatted reth command string for reporting.
pub(crate) async fn start_node(
&mut self,
binary_path: &std::path::Path,
_git_ref: &str,
ref_type: &str,
additional_args: &[String],
) -> Result<(tokio::process::Child, String)> {
// Store the binary path for later use (e.g., in unwind_to_block)
self.binary_path = Some(binary_path.to_path_buf());
let binary_path_str = binary_path.to_string_lossy();
let (reth_args, _) = self.build_reth_args(&binary_path_str, additional_args, ref_type);
// Format the reth command string for reporting
let reth_command = shlex::try_join(reth_args.iter().map(|s| s.as_str()))
.wrap_err("Failed to format reth command string")?;
// Log additional arguments if any
if !self.additional_reth_args.is_empty() {
info!("Using common additional reth arguments: {:?}", self.additional_reth_args);
}
if !additional_args.is_empty() {
info!("Using reference-specific additional reth arguments: {:?}", additional_args);
}
let mut cmd = if self.enable_profiling {
self.create_profiling_command(ref_type, &reth_args).await?
} else {
self.create_direct_command(&reth_args)
};
// Set process group for better signal handling
#[cfg(unix)]
{
cmd.process_group(0);
}
// Set high queue size to prevent trace dropping during benchmarks
if self.tracing_endpoint.is_some() {
cmd.env("OTEL_BSP_MAX_QUEUE_SIZE", self.otlp_max_queue_size.to_string()); // Traces
cmd.env("OTEL_BLRP_MAX_QUEUE_SIZE", "10000"); // Logs
// Set service name to differentiate baseline vs feature runs in Jaeger
cmd.env("OTEL_SERVICE_NAME", format!("reth-{}", ref_type));
}
debug!("Executing reth command: {cmd:?}");
let mut child = cmd
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.kill_on_drop(true) // Kill on drop so that on Ctrl-C for parent process we stop all child processes
.spawn()
.wrap_err("Failed to start reth node")?;
info!(
"Reth node started with PID: {:?} (binary: {})",
child.id().ok_or_eyre("Reth node is not running")?,
binary_path_str
);
// Prepare log file path
let log_file_path = self.get_log_file_path(ref_type)?;
info!("Reth node logs will be saved to: {:?}", log_file_path);
// Stream stdout and stderr with prefixes at debug level and to log file
if let Some(stdout) = child.stdout.take() {
let log_file = AsyncFile::create(&log_file_path)
.await
.wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
tokio::spawn(async move {
let reader = AsyncBufReader::new(stdout);
let mut lines = reader.lines();
let mut log_file = log_file;
while let Ok(Some(line)) = lines.next_line().await {
debug!("[RETH] {}", line);
// Write to log file (reth already includes timestamps)
let log_line = format!("{}\n", line);
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
debug!("Failed to write to log file: {}", e);
}
}
});
}
if let Some(stderr) = child.stderr.take() {
let log_file = AsyncFile::options()
.create(true)
.append(true)
.open(&log_file_path)
.await
.wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
tokio::spawn(async move {
let reader = AsyncBufReader::new(stderr);
let mut lines = reader.lines();
let mut log_file = log_file;
while let Ok(Some(line)) = lines.next_line().await {
debug!("[RETH] {}", line);
// Write to log file (reth already includes timestamps)
let log_line = format!("{}\n", line);
if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
debug!("Failed to write to log file: {}", e);
}
}
});
}
// Give the node a moment to start up
sleep(Duration::from_secs(5)).await;
Ok((child, reth_command))
}
/// Wait for the node to be ready and return its current tip.
///
/// Fails early if the node process exits before becoming ready.
pub(crate) async fn wait_for_node_ready_and_get_tip(
&self,
child: &mut tokio::process::Child,
) -> Result<u64> {
info!("Waiting for node to be ready and synced...");
let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync
let check_interval = Duration::from_secs(2);
let rpc_url = "http://localhost:8545";
// Create Alloy provider
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
let provider = ProviderBuilder::new().connect_http(url);
let start_time = tokio::time::Instant::now();
let mut iteration = 0;
timeout(max_wait, async {
loop {
iteration += 1;
debug!(
"Readiness check iteration {} (elapsed: {:?})",
iteration,
start_time.elapsed()
);
// Check if the node process has exited.
if let Some(status) = child.try_wait()? {
return Err(eyre!("Node process exited unexpectedly with {status}"));
}
// First check if RPC is up and node is not syncing
match provider.syncing().await {
Ok(sync_result) => {
match sync_result {
SyncStatus::Info(sync_info) => {
debug!("Node is still syncing {sync_info:?}, waiting...");
}
_ => {
debug!("HTTP RPC is up and node is not syncing, checking block number...");
// Node is not syncing, now get the tip
match provider.get_block_number().await {
Ok(tip) => {
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
// Verify WebSocket RPC is ready (public endpoint, no JWT required)
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
debug!("Attempting WebSocket connection to {} (public endpoint)", ws_url);
let ws_connect = WsConnect::new(&ws_url);
match RpcClient::connect_pubsub(ws_connect).await
{
Ok(_) => {
info!(
"Node is ready (HTTP and WebSocket) at block: {} (took {:?}, {} iterations)",
tip, start_time.elapsed(), iteration
);
return Ok(tip);
}
Err(e) => {
debug!(
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
iteration, e
);
debug!("WebSocket error details: {}", e);
}
}
}
Err(e) => {
debug!("Failed to get block number (iteration {}): {:?}", iteration, e);
}
}
}
}
}
Err(e) => {
debug!("Node RPC not ready yet or failed to check sync status (iteration {}): {:?}", iteration, e);
}
}
debug!("Sleeping for {:?} before next check", check_interval);
sleep(check_interval).await;
}
})
.await
.wrap_err("Timed out waiting for node to be ready and synced")?
}
/// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
///
/// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
/// the node is still syncing.
pub(crate) async fn wait_for_rpc_and_get_tip(
&self,
child: &mut tokio::process::Child,
) -> Result<u64> {
info!("Waiting for node RPC to be ready (skipping sync wait)...");
let max_wait = Duration::from_secs(60);
let check_interval = Duration::from_secs(2);
let rpc_url = "http://localhost:8545";
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
let provider = ProviderBuilder::new().connect_http(url);
let start_time = tokio::time::Instant::now();
let mut iteration = 0;
timeout(max_wait, async {
loop {
iteration += 1;
debug!(
"RPC readiness check iteration {} (elapsed: {:?})",
iteration,
start_time.elapsed()
);
if let Some(status) = child.try_wait()? {
return Err(eyre!("Node process exited unexpectedly with {status}"));
}
match provider.get_block_number().await {
Ok(tip) => {
debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
let ws_connect = WsConnect::new(&ws_url);
match RpcClient::connect_pubsub(ws_connect).await {
Ok(_) => {
info!(
"Node RPC is ready at block: {} (took {:?}, {} iterations)",
tip,
start_time.elapsed(),
iteration
);
return Ok(tip);
}
Err(e) => {
debug!(
"HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
iteration, e
);
}
}
}
Err(e) => {
debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
}
}
sleep(check_interval).await;
}
})
.await
.wrap_err("Timed out waiting for node RPC to be ready")?
}
/// Stop the reth node gracefully
pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
let pid = child.id().ok_or_eyre("Child process ID should be available")?;
// Check if the process has already exited
match child.try_wait() {
Ok(Some(status)) => {
info!("Reth node (PID: {}) has already exited with status: {:?}", pid, status);
return Ok(());
}
Ok(None) => {
// Process is still running, proceed to stop it
info!("Stopping process gracefully with SIGINT (PID: {})...", pid);
}
Err(e) => {
return Err(eyre!("Failed to check process status: {}", e));
}
}
#[cfg(unix)]
{
// Send SIGINT to process group to mimic Ctrl-C behavior
let nix_pgid = Pid::from_raw(pid as i32);
match killpg(nix_pgid, Signal::SIGINT) {
Ok(()) => {}
Err(nix::errno::Errno::ESRCH) => {
info!("Process group {} has already exited", pid);
}
Err(e) => {
return Err(eyre!("Failed to send SIGINT to process group {}: {}", pid, e));
}
}
}
#[cfg(not(unix))]
{
// On non-Unix systems, fall back to using external kill command
let output = Command::new("taskkill")
.args(["/PID", &pid.to_string(), "/F"])
.output()
.await
.wrap_err("Failed to execute taskkill command")?;
if !output.status.success() {
let stderr = String::from_utf8_lossy(&output.stderr);
// Check if the error is because the process doesn't exist
if stderr.contains("not found") || stderr.contains("not exist") {
info!("Process {} has already exited", pid);
} else {
return Err(eyre!("Failed to kill process {}: {}", pid, stderr));
}
}
}
// Wait for the process to exit
match child.wait().await {
Ok(status) => {
info!("Reth node (PID: {}) exited with status: {:?}", pid, status);
}
Err(e) => {
// If we get an error here, it might be because the process already exited
debug!("Error waiting for process exit (may have already exited): {}", e);
}
}
Ok(())
}
/// Unwind the node to a specific block
pub(crate) async fn unwind_to_block(&self, block_number: u64) -> Result<()> {
if self.use_sudo {
info!("Unwinding node to block: {} (with sudo)", block_number);
} else {
info!("Unwinding node to block: {}", block_number);
}
// Use the binary path from the last start_node call, or fallback to default
let binary_path = self
.binary_path
.as_ref()
.map(|p| p.to_string_lossy().to_string())
.unwrap_or_else(|| "./target/profiling/reth".to_string());
let mut cmd = if self.use_sudo {
let mut sudo_cmd = Command::new("sudo");
sudo_cmd.args([&binary_path, "stage", "unwind"]);
sudo_cmd
} else {
let mut reth_cmd = Command::new(&binary_path);
reth_cmd.args(["stage", "unwind"]);
reth_cmd
};
// Add chain argument (skip for mainnet as it's the default)
let chain_str = self.chain.to_string();
if chain_str != "mainnet" {
cmd.args(["--chain", &chain_str]);
}
// Add datadir if specified
if let Some(ref datadir) = self.datadir {
cmd.args(["--datadir", datadir]);
}
cmd.args(["to-block", &block_number.to_string()]);
// Set environment variable to disable log styling
cmd.env("RUST_LOG_STYLE", "never");
// Debug log the command
debug!("Executing reth unwind command: {:?}", cmd);
let mut child = cmd
.stdout(std::process::Stdio::piped())
.stderr(std::process::Stdio::piped())
.spawn()
.wrap_err("Failed to start unwind command")?;
// Stream stdout and stderr with prefixes in real-time
if let Some(stdout) = child.stdout.take() {
tokio::spawn(async move {
let reader = AsyncBufReader::new(stdout);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
debug!("[RETH-UNWIND] {}", line);
}
});
}
if let Some(stderr) = child.stderr.take() {
tokio::spawn(async move {
let reader = AsyncBufReader::new(stderr);
let mut lines = reader.lines();
while let Ok(Some(line)) = lines.next_line().await {
debug!("[RETH-UNWIND] {}", line);
}
});
}
// Wait for the command to complete
let status = child.wait().await.wrap_err("Failed to wait for unwind command")?;
if !status.success() {
return Err(eyre!("Unwind command failed with exit code: {:?}", status.code()));
}
info!("Unwound to block: {}", block_number);
Ok(())
}
}
fn supports_samply_flags(bin: &str) -> bool {
let mut cmd = std::process::Command::new(bin);
// NOTE: The flag to check must come before --help.
// We pass --help as a shortcut to not execute any command.
cmd.args(["--log.samply", "--help"]);
debug!(?cmd, "Checking samply flags support");
let Ok(output) = cmd.output() else {
return false;
};
debug!(?output, "Samply flags support check");
output.status.success()
}

View File

@@ -17,6 +17,7 @@ workspace = true
reth-cli-runner.workspace = true
reth-cli-util.workspace = true
reth-engine-primitives.workspace = true
reth-ethereum-primitives.workspace = true
reth-fs-util.workspace = true
reth-node-api.workspace = true
reth-node-core.workspace = true
@@ -24,11 +25,13 @@ reth-primitives-traits.workspace = true
reth-rpc-api.workspace = true
reth-tracing.workspace = true
reth-chainspec.workspace = true
# alloy
alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true

View File

@@ -35,10 +35,6 @@ The `new-payload-fcu` command supports two optional waiting modes that can be us
- `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms` or `--wait-time 400` for 400ms)
- `--wait-for-persistence`: Waits for blocks to be persisted using the `reth_subscribePersistedBlock` subscription
Both `new-payload-fcu` and `new-payload-only` support `--rpc-block-fetch-retries <RETRIES>`
to control how many times block fetches are retried after an RPC failure. The default is `10`.
Use `--rpc-block-fetch-retries forever` to keep retrying indefinitely.
When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
By default, the WebSocket URL for persistence subscriptions is derived from `--engine-rpc-url` (converting to ws:// on port 8546). Use `--ws-rpc-url` to override this.

View File

@@ -7,7 +7,7 @@ use alloy_primitives::address;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::JwtSecret;
use alloy_transport::layers::{RateLimitRetryPolicy, RetryBackoffLayer};
use alloy_transport::layers::RetryBackoffLayer;
use reqwest::Url;
use reth_node_core::args::BenchmarkArgs;
use tracing::info;
@@ -31,12 +31,6 @@ pub(crate) struct BenchContext {
pub(crate) is_optimism: bool,
/// Whether to use `reth_newPayload` endpoint instead of `engine_newPayload*`.
pub(crate) use_reth_namespace: bool,
/// Whether to fetch and replay RLP-encoded blocks.
pub(crate) rlp_blocks: bool,
/// Whether to skip waiting for persistence (pass `wait_for_persistence: false`).
pub(crate) no_wait_for_persistence: bool,
/// Whether to skip waiting for caches (pass `wait_for_caches: false`).
pub(crate) no_wait_for_caches: bool,
}
impl BenchContext {
@@ -57,16 +51,9 @@ impl BenchContext {
}
}
// set up alloy client for blocks, retrying on 429/503 (default) and 502
let retry_policy =
RateLimitRetryPolicy::default().or(|err: &alloy_transport::TransportError| -> bool {
err.as_transport_err()
.and_then(|t| t.as_http_error())
.is_some_and(|e| e.status == 502)
});
let max_retries = bench_args.rpc_block_fetch_retries.as_max_retries();
// set up alloy client for blocks
let client = ClientBuilder::default()
.layer(RetryBackoffLayer::new_with_policy(max_retries, 800, u64::MAX, retry_policy))
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let block_provider = RootProvider::<AnyNetwork>::new(client);
@@ -99,11 +86,9 @@ impl BenchContext {
// Computes the block range for the benchmark.
//
// - If `--advance` is provided, fetches the latest block from the engine and sets:
// - If `--advance` is provided, fetches the latest block and sets:
// - `from = head + 1`
// - `to = head + advance`
// - If only `--to` is provided, fetches the latest block from the engine and sets:
// - `from = head`
// - Otherwise, uses the values from `--from` and `--to`.
let (from, to) = if let Some(advance) = bench_args.advance {
if advance == 0 {
@@ -116,14 +101,6 @@ impl BenchContext {
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block for --advance"))?;
let head_number = head_block.header.number;
(Some(head_number), Some(head_number + advance))
} else if bench_args.from.is_none() && bench_args.to.is_some() {
let head_block = auth_provider
.get_block_by_number(BlockNumberOrTag::Latest)
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block from engine"))?;
let head_number = head_block.header.number;
info!(target: "reth-bench", "No --from provided, derived from engine head: {}", head_number);
(Some(head_number), bench_args.to)
} else {
(bench_args.from, bench_args.to)
};
@@ -135,7 +112,7 @@ impl BenchContext {
.full()
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block from RPC"))?;
let mut benchmark_mode = BenchMode::new(from, to, latest_block.into_inner().number());
let mut benchmark_mode = BenchMode::new(from, to, latest_block.into_inner().number())?;
let first_block = match benchmark_mode {
BenchMode::Continuous(start) => {
@@ -165,10 +142,7 @@ impl BenchContext {
};
let next_block = first_block.header.number + 1;
let rlp_blocks = bench_args.rlp_blocks;
let use_reth_namespace = bench_args.reth_new_payload || rlp_blocks;
let no_wait_for_persistence = bench_args.no_wait_for_persistence;
let no_wait_for_caches = bench_args.no_wait_for_caches;
let use_reth_namespace = bench_args.reth_new_payload;
Ok(Self {
auth_provider,
block_provider,
@@ -176,9 +150,6 @@ impl BenchContext {
next_block,
is_optimism,
use_reth_namespace,
rlp_blocks,
no_wait_for_persistence,
no_wait_for_caches,
})
}
}

View File

@@ -0,0 +1,244 @@
//! Benchmarks empty block processing by ramping the block gas limit.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload_with_reth, payload_to_new_payload},
};
use alloy_eips::BlockNumberOrTag;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
use clap::Parser;
use reqwest::Url;
use reth_chainspec::ChainSpec;
use reth_cli_runner::CliContext;
use reth_ethereum_primitives::TransactionSigned;
use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
/// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
#[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
blocks: Option<u64>,
/// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
/// reaches or exceeds this value. Mutually exclusive with --blocks.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
/// billion).
#[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
target_gas_limit: Option<u64>,
/// The Engine API RPC URL.
#[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
engine_rpc_url: String,
/// Path to the JWT secret for Engine API authentication.
#[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
jwt_secret: PathBuf,
/// Output directory for benchmark results and generated payloads.
#[arg(long, value_name = "OUTPUT")]
output: PathBuf,
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
///
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
/// directly, waits for persistence and cache updates to complete before processing,
/// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
#[arg(long, default_value = "false", verbatim_doc_comment)]
reth_new_payload: bool,
}
/// Mode for determining when to stop ramping.
#[derive(Debug, Clone, Copy)]
enum RampMode {
/// Ramp for a fixed number of blocks.
Blocks(u64),
/// Ramp until reaching or exceeding target gas limit.
TargetGasLimit(u64),
}
impl Command {
/// Execute `benchmark gas-limit-ramp` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
let mode = match (self.blocks, self.target_gas_limit) {
(Some(blocks), None) => {
if blocks == 0 {
return Err(eyre::eyre!("--blocks must be greater than 0"));
}
RampMode::Blocks(blocks)
}
(None, Some(target)) => {
if target == 0 {
return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
}
RampMode::TargetGasLimit(target)
}
_ => {
return Err(eyre::eyre!(
"Exactly one of --blocks or --target-gas-limit must be specified"
));
}
};
// Ensure output directory exists
if self.output.is_file() {
return Err(eyre::eyre!("Output path must be a directory"));
}
if !self.output.exists() {
std::fs::create_dir_all(&self.output)?;
info!(target: "reth-bench", "Created output directory: {:?}", self.output);
}
// Set up authenticated provider (used for both Engine API and eth_ methods)
let jwt = std::fs::read_to_string(&self.jwt_secret)?;
let jwt = JwtSecret::from_hex(jwt)?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!(target: "reth-bench", "Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
let client = ClientBuilder::default().connect_with(auth_transport).await?;
let provider = RootProvider::<AnyNetwork>::new(client);
// Get chain spec - required for fork detection
let chain_id = provider.get_chain_id().await?;
let chain_spec = ChainSpec::from_chain_id(chain_id)
.ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
// Fetch the current head block as parent
let parent_block = provider
.get_block_by_number(BlockNumberOrTag::Latest)
.full()
.await?
.ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
let canonical_parent = parent_header.number;
let start_block = canonical_parent + 1;
match mode {
RampMode::Blocks(blocks) => {
info!(
target: "reth-bench",
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
"Starting gas limit ramp benchmark (block count mode)"
);
}
RampMode::TargetGasLimit(target) => {
info!(
target: "reth-bench",
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
target_gas_limit = target,
"Starting gas limit ramp benchmark (target gas limit mode)"
);
}
}
if self.reth_new_payload {
info!("Using reth_newPayload endpoint");
}
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
let timestamp = parent_header.timestamp.saturating_add(1);
let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
let new_payload_version = request.new_payload_version;
let (payload, sidecar) = build_payload(&provider, request).await?;
let mut block =
payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
let max_increase = max_gas_limit_increase(parent_header.gas_limit);
let gas_limit =
parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
block.header.gas_limit = gas_limit;
let block_hash = block.header.hash_slow();
// Regenerate the payload from the modified block, but keep the original sidecar
// which contains the actual execution requests data (not just the hash)
let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
let (version, params, execution_data) = payload_to_new_payload(
payload,
sidecar,
false,
block.header.withdrawals_root,
Some(new_payload_version),
)?;
// Save payload to file with version info for replay
let payload_path =
self.output.join(format!("payload_block_{}.json", block.header.number));
let file = GasRampPayloadFile {
version: version as u8,
block_hash,
params: params.clone(),
execution_data: Some(execution_data.clone()),
};
let payload_json = serde_json::to_string_pretty(&file)?;
std::fs::write(&payload_path, &payload_json)?;
info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
let reth_data = self.reth_new_payload.then_some(execution_data);
let _ = call_new_payload_with_reth(&provider, version, params, reth_data).await?;
let forkchoice_state = ForkchoiceState {
head_block_hash: block_hash,
safe_block_hash: block_hash,
finalized_block_hash: block_hash,
};
call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
parent_header = block.header;
parent_hash = block_hash;
blocks_processed += 1;
let progress = match mode {
RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
RampMode::TargetGasLimit(target) => {
let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
format!("{pct:.1}%")
}
};
info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
}
let final_gas_limit = parent_header.gas_limit;
info!(
target: "reth-bench",
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed,
final_gas_limit,
"Benchmark complete"
);
Ok(())
}
}
const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
(parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
}
const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
match mode {
RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}

View File

@@ -773,7 +773,6 @@ impl Command {
suggested_fee_recipient: alloy_primitives::Address::ZERO,
withdrawals: Some(vec![]),
parent_beacon_block_root: Some(B256::ZERO),
slot_number: None,
},
transactions: transactions.to_vec(),
extra_data: None,

View File

@@ -1,5 +1,6 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use eyre::Result;
use std::{
io::{BufReader, Read},
@@ -69,6 +70,180 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
}
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
PayloadAttributes, PayloadId,
};
use eyre::OptionExt;
use reth_chainspec::{ChainSpec, EthereumHardforks};
use reth_node_api::EngineApiMessageVersion;
use tracing::debug;
/// Prepared payload request data for triggering block building.
pub(crate) struct PayloadRequest {
/// The payload attributes for the new block.
pub(crate) attributes: PayloadAttributes,
/// The forkchoice state pointing to the parent block.
pub(crate) forkchoice_state: ForkchoiceState,
/// The engine API version for FCU calls.
pub(crate) fcu_version: EngineApiMessageVersion,
/// The getPayload version to use (1-5).
pub(crate) get_payload_version: u8,
/// The newPayload version to use.
pub(crate) new_payload_version: EngineApiMessageVersion,
}
/// Prepare payload attributes and forkchoice state for a new block.
pub(crate) fn prepare_payload_request(
chain_spec: &ChainSpec,
timestamp: u64,
parent_hash: B256,
) -> PayloadRequest {
let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
// FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
let fcu_version = if cancun_active {
EngineApiMessageVersion::V3
} else if shanghai_active {
EngineApiMessageVersion::V2
} else {
EngineApiMessageVersion::V1
};
// getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
// newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
let (get_payload_version, new_payload_version) = if osaka_active {
(5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
} else if prague_active {
(4, EngineApiMessageVersion::V4)
} else if cancun_active {
(3, EngineApiMessageVersion::V3)
} else if shanghai_active {
(2, EngineApiMessageVersion::V2)
} else {
(1, EngineApiMessageVersion::V1)
};
PayloadRequest {
attributes: PayloadAttributes {
timestamp,
prev_randao: B256::ZERO,
suggested_fee_recipient: Address::ZERO,
withdrawals: shanghai_active.then(Vec::new),
parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
},
forkchoice_state: ForkchoiceState {
head_block_hash: parent_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
},
fcu_version,
get_payload_version,
new_payload_version,
}
}
/// Trigger payload building via FCU and retrieve the built payload.
///
/// This sends a forkchoiceUpdated with payload attributes to start building,
/// then calls getPayload to retrieve the result.
pub(crate) async fn build_payload(
provider: &RootProvider<AnyNetwork>,
request: PayloadRequest,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
let fcu_result = call_forkchoice_updated(
provider,
request.fcu_version,
request.forkchoice_state,
Some(request.attributes.clone()),
)
.await?;
let payload_id =
fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
get_payload_with_sidecar(
provider,
request.get_payload_version,
payload_id,
request.attributes.parent_beacon_block_root,
)
.await
}
/// Convert an RPC block to a consensus header and block hash.
pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
let block_hash = block.header.hash;
let header = block.header.inner.clone().into_header_with_defaults();
(header, block_hash)
}
/// Compute versioned hashes from KZG commitments.
fn versioned_hashes_from_commitments(
commitments: &[alloy_primitives::FixedBytes<48>],
) -> Vec<B256> {
commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
}
/// Fetch an execution payload using the appropriate engine API version.
pub(crate) async fn get_payload_with_sidecar(
provider: &RootProvider<AnyNetwork>,
version: u8,
payload_id: PayloadId,
parent_beacon_block_root: Option<B256>,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
debug!(target: "reth-bench", get_payload_version = ?version, ?payload_id, "Sending getPayload");
match version {
1 => {
let payload = provider.get_payload_v1(payload_id).await?;
Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
}
2 => {
let envelope = provider.get_payload_v2(payload_id).await?;
let payload = match envelope.execution_payload {
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
};
Ok((payload, ExecutionPayloadSidecar::none()))
}
3 => {
let envelope = provider.get_payload_v3(payload_id).await?;
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V3")?,
versioned_hashes,
};
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v3(cancun_fields),
))
}
4 => {
let envelope = provider.get_payload_v4(payload_id).await?;
Ok(envelope.into_payload_and_sidecar(
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V4")?,
))
}
5 => {
let envelope = provider.get_payload_v5(payload_id).await?;
Ok(envelope.into_payload_and_sidecar(
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V5")?,
))
}
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -6,6 +6,7 @@ use reth_node_core::args::LogArgs;
use reth_tracing::FileWorkerGuard;
mod context;
mod gas_limit_ramp;
mod generate_big_block;
pub(crate) mod helpers;
pub use generate_big_block::{
@@ -15,6 +16,7 @@ pub(crate) mod metrics_scraper;
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod persistence_waiter;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
@@ -35,6 +37,9 @@ pub enum Subcommands {
/// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
NewPayloadFcu(new_payload_fcu::Command),
/// Benchmark which builds empty blocks with a ramped gas limit.
GasLimitRamp(gas_limit_ramp::Command),
/// Benchmark which only calls subsequent `newPayload` calls.
NewPayloadOnly(new_payload_only::Command),
@@ -94,6 +99,7 @@ impl BenchmarkCommand {
match self.command {
Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,

View File

@@ -1,5 +1,13 @@
//! Runs the `reth bench` command, calling first newPayload for each block, then calling
//! forkchoiceUpdated.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
bench::{
@@ -9,16 +17,16 @@ use crate::{
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
},
},
valid_payload::{
block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload_with_reth},
};
use alloy_provider::{ext::DebugApi, Provider};
use alloy_provider::Provider;
use alloy_rpc_types_engine::ForkchoiceState;
use clap::Parser;
use eyre::{Context, OptionExt};
use futures::{stream, StreamExt, TryStreamExt};
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
@@ -39,6 +47,16 @@ pub struct Command {
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
@@ -86,17 +104,54 @@ impl Command {
if let Some(duration) = self.wait_time {
info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
target: "reth-bench",
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options
// When both are set: wait at least wait_time, and also wait for persistence if needed
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), true) => {
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
Some(PersistenceWaiter::with_duration_and_subscription(
duration,
sub,
self.persistence_threshold,
self.persistence_timeout,
))
}
(Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
self.persistence_timeout,
))
}
(None, false) => None,
};
let BenchContext {
benchmark_mode,
block_provider,
auth_provider,
next_block,
mut next_block,
is_optimism,
use_reth_namespace,
rlp_blocks,
no_wait_for_persistence,
no_wait_for_caches,
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
@@ -104,85 +159,69 @@ impl Command {
let mut metrics_scraper = MetricsScraper::maybe_new(self.benchmark.metrics_url.clone());
if use_reth_namespace {
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
info!("Using reth_newPayload endpoint");
}
let buffer_size = self.rpc_block_buffer_size;
let mut blocks = Box::pin(
stream::iter((next_block..)
.take_while(|next_block| {
benchmark_mode.contains(*next_block)
}))
.map(|next_block| {
let block_provider = block_provider.clone();
async move {
let block_res = block_provider
.get_block_by_number(next_block.into())
.full()
.await
.wrap_err_with(|| {
format!("Failed to fetch block by number {next_block}")
});
let block =
match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
Ok(block) => block,
Err(e) => {
tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
return Err(e)
}
};
// Use a oneshot channel to propagate errors from the spawned task
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
let rlp = if rlp_blocks {
let rlp = match block_provider
.debug_get_raw_block(next_block.into())
.await
{
Ok(rlp) => rlp,
Err(e) => {
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
return Err(e.into())
}
};
Some(rlp)
} else {
None
};
let head_block_hash = block.header.hash;
let safe_block_hash = block_provider
.get_block_by_number(block.header.number.saturating_sub(32).into());
let finalized_block_hash = block_provider
.get_block_by_number(block.header.number.saturating_sub(64).into());
let (safe, finalized) =
tokio::join!(safe_block_hash, finalized_block_hash);
let safe_block_hash = match safe {
Ok(Some(block)) => block.header.hash,
Ok(None) | Err(_) => head_block_hash,
};
let finalized_block_hash = match finalized {
Ok(Some(block)) => block.header.hash,
Ok(None) | Err(_) => head_block_hash,
};
Ok((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
tokio::task::spawn(async move {
while benchmark_mode.contains(next_block) {
let block_res = block_provider
.get_block_by_number(next_block.into())
.full()
.await
.wrap_err_with(|| format!("Failed to fetch block by number {next_block}"));
let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
Ok(block) => block,
Err(e) => {
tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
let _ = error_sender.send(e);
break;
}
})
.buffered(buffer_size),
);
};
let head_block_hash = block.header.hash;
let safe_block_hash = block_provider
.get_block_by_number(block.header.number.saturating_sub(32).into());
let finalized_block_hash = block_provider
.get_block_by_number(block.header.number.saturating_sub(64).into());
let (safe, finalized) = tokio::join!(safe_block_hash, finalized_block_hash,);
let safe_block_hash = match safe {
Ok(Some(block)) => block.header.hash,
Ok(None) | Err(_) => head_block_hash,
};
let finalized_block_hash = match finalized {
Ok(Some(block)) => block.header.hash,
Ok(None) | Err(_) => head_block_hash,
};
next_block += 1;
if let Err(e) = sender
.send((block, head_block_hash, safe_block_hash, finalized_block_hash))
.await
{
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
break;
}
}
});
let mut results = Vec::new();
let mut blocks_processed = 0u64;
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
while let Some((block, head, safe, finalized, rlp)) = {
while let Some((block, head, safe, finalized)) = {
let wait_start = Instant::now();
let result = blocks.try_next().await?;
let result = receiver.recv().await;
total_wait_time += wait_start.elapsed();
result
} {
@@ -199,17 +238,11 @@ impl Command {
finalized_block_hash: finalized,
};
let (version, params) = block_to_new_payload(
block,
is_optimism,
rlp,
use_reth_namespace,
no_wait_for_persistence,
no_wait_for_caches,
)?;
let (version, params, execution_data) = block_to_new_payload(block, is_optimism)?;
let start = Instant::now();
let reth_data = use_reth_namespace.then_some(execution_data);
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
call_new_payload_with_reth(&auth_provider, version, params, reth_data).await?;
let np_latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
@@ -228,7 +261,7 @@ impl Command {
};
let fcu_start = Instant::now();
call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?;
let fcu_latency = fcu_start.elapsed();
let total_latency = if server_timings.is_some() {
@@ -264,8 +297,8 @@ impl Command {
warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
}
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
let gas_row =
@@ -273,6 +306,15 @@ impl Command {
results.push((gas_row, combined_result));
}
// Check if the spawned task encountered an error
if let Ok(error) = error_receiver.try_recv() {
return Err(error);
}
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();

View File

@@ -11,7 +11,7 @@ use crate::{
},
valid_payload::{block_to_new_payload, call_new_payload_with_reth},
};
use alloy_provider::{ext::DebugApi, Provider};
use alloy_provider::Provider;
use clap::Parser;
use csv::Writer;
use eyre::{Context, OptionExt};
@@ -51,9 +51,6 @@ impl Command {
mut next_block,
is_optimism,
use_reth_namespace,
rlp_blocks,
no_wait_for_persistence,
no_wait_for_caches,
} = BenchContext::new(&self.benchmark, self.rpc_url).await?;
let total_blocks = benchmark_mode.total_blocks();
@@ -86,21 +83,8 @@ impl Command {
}
};
let rlp = if rlp_blocks {
let Ok(rlp) = block_provider.debug_get_raw_block(next_block.into()).await
else {
tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}");
let _ = error_sender
.send(eyre::eyre!("Failed to fetch raw block {next_block}"));
break;
};
Some(rlp)
} else {
None
};
next_block += 1;
if let Err(e) = sender.send((block, rlp)).await {
if let Err(e) = sender.send(block).await {
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
break;
}
@@ -112,7 +96,7 @@ impl Command {
let total_benchmark_duration = Instant::now();
let mut total_wait_time = Duration::ZERO;
while let Some((block, rlp)) = {
while let Some(block) = {
let wait_start = Instant::now();
let result = receiver.recv().await;
total_wait_time += wait_start.elapsed();
@@ -124,18 +108,12 @@ impl Command {
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
let (version, params) = block_to_new_payload(
block,
is_optimism,
rlp,
use_reth_namespace,
no_wait_for_persistence,
no_wait_for_caches,
)?;
let (version, params, execution_data) = block_to_new_payload(block, is_optimism)?;
let start = Instant::now();
let reth_data = use_reth_namespace.then_some(execution_data);
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
call_new_payload_with_reth(&auth_provider, version, params, reth_data).await?;
let latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());

View File

@@ -1,10 +1,11 @@
//! Contains various benchmark output formats, either for logging or for
//! serialization to / from files.
use alloy_primitives::B256;
use csv::Writer;
use eyre::OptionExt;
use reth_primitives_traits::constants::GIGAGAS;
use serde::{ser::SerializeStruct, Serialize};
use serde::{ser::SerializeStruct, Deserialize, Serialize};
use std::{fs, path::Path, time::Duration};
use tracing::info;
@@ -17,6 +18,20 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
/// This is the suffix for new payload output csv files.
pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
/// Serialized format for gas ramp payloads on disk.
#[derive(Debug, Serialize, Deserialize)]
pub(crate) struct GasRampPayloadFile {
/// Engine API version (1-5).
pub(crate) version: u8,
/// The block hash for FCU.
pub(crate) block_hash: B256,
/// The params to pass to newPayload.
pub(crate) params: serde_json::Value,
/// The execution data for `reth_newPayload`.
#[serde(skip_serializing_if = "Option::is_none", default)]
pub(crate) execution_data: Option<alloy_rpc_types_engine::ExecutionData>,
}
/// This represents the results of a single `newPayload` call in the benchmark, containing the gas
/// used and the `newPayload` latency.
#[derive(Debug)]
@@ -99,27 +114,16 @@ impl CombinedResult {
impl std::fmt::Display for CombinedResult {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
let np = &self.new_payload_result;
write!(
f,
"Block {} processed at {:.4} Ggas/s, used {} total gas. Combined: {:.4} Ggas/s. fcu: {:?}, newPayload: {:?}",
self.block_number,
np.gas_per_second() / GIGAGAS as f64,
np.gas_used,
self.new_payload_result.gas_per_second() / GIGAGAS as f64,
self.new_payload_result.gas_used,
self.combined_gas_per_second() / GIGAGAS as f64,
self.fcu_latency,
np.latency,
)?;
if !np.execution_cache_wait.is_zero() {
write!(f, ", execution cache wait: {:?}", np.execution_cache_wait)?;
}
if !np.sparse_trie_wait.is_zero() {
write!(f, ", trie cache wait: {:?}", np.sparse_trie_wait)?;
}
if let Some(d) = np.persistence_wait {
write!(f, ", persistence wait: {d:?}")?;
}
Ok(())
self.new_payload_result.latency
)
}
}

View File

@@ -0,0 +1,334 @@
//! Persistence waiting utilities for benchmarks.
//!
//! Provides waiting behavior to control benchmark pacing:
//! - **Fixed duration waits**: Sleep for a fixed time between blocks
//! - **Persistence-based waits**: Wait for blocks to be persisted using
//! `reth_subscribePersistedBlock` subscription
//! - **Combined mode**: Wait at least the fixed duration, and also wait for persistence if the
//! block hasn't been persisted yet (whichever takes longer)
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_transport_ws::WsConnect;
use eyre::Context;
use futures::StreamExt;
use std::time::Duration;
use tracing::{debug, info};
/// Default `WebSocket` RPC port for reth.
const DEFAULT_WS_RPC_PORT: u16 = 8546;
use url::Url;
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `ws_rpc_url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since we may only have the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
pub(crate) fn derive_ws_rpc_url(
ws_rpc_url: Option<&str>,
engine_rpc_url: &str,
) -> eyre::Result<Url> {
if let Some(ws_url) = ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url
.set_port(Some(DEFAULT_WS_RPC_PORT))
.map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
pub(crate) struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
///
/// The `keepalive_interval` is set to match `persistence_timeout` so that the `WebSocket`
/// connection is not dropped during long MDBX commits that block the server from responding
/// to pings.
pub(crate) async fn setup_persistence_subscription(
ws_url: Url,
persistence_timeout: Duration,
) -> eyre::Result<PersistenceSubscription> {
info!(target: "reth-bench", "Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect =
WsConnect::new(ws_url.to_string()).with_keepalive_interval(persistence_timeout);
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!(target: "reth-bench", "Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
pub(crate) struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
pub(crate) const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
pub(crate) const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Creates a waiter that combines both duration and persistence waiting.
///
/// Waits at least `wait_time` between blocks, and also waits for persistence
/// if the block hasn't been persisted yet (whichever takes longer).
pub(crate) const fn with_duration_and_subscription(
wait_time: Duration,
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: Some(wait_time),
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
///
/// When both `wait_time` and `subscription` are set (combined mode):
/// - Always waits at least `wait_time`
/// - Additionally waits for persistence if we're at a persistence checkpoint
#[allow(clippy::manual_is_multiple_of)]
pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
// Always wait for the fixed duration if configured
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
}
// Check persistence if subscription is configured
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
use std::time::Instant;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -1,4 +1,15 @@
//! Command for replaying pre-generated payloads from disk.
//!
//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
//!
//! Supports configurable waiting behavior:
//! - **`--wait-time`**: Fixed sleep interval between blocks.
//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
//! `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
//! threshold. This ensures the benchmark doesn't outpace persistence.
//!
//! Both options can be used together or independently.
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
@@ -6,13 +17,17 @@ use crate::{
helpers::parse_duration,
metrics_scraper::MetricsScraper,
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
},
},
valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
valid_payload::{call_forkchoice_updated, call_new_payload_with_reth},
};
use alloy_primitives::B256;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV4, ExecutionPayloadSidecar,
@@ -21,13 +36,13 @@ use alloy_rpc_types_engine::{
use clap::Parser;
use eyre::Context;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_api::EngineApiMessageVersion;
use reth_rpc_api::RethNewPayloadInput;
use std::{
path::PathBuf,
time::{Duration, Instant},
};
use tracing::{debug, info, warn};
use tracing::{debug, info};
use url::Url;
/// `reth bench replay-payloads` command
@@ -57,9 +72,9 @@ pub struct Command {
#[arg(long, value_name = "SKIP", default_value = "0")]
skip: usize,
/// Deprecated: gas ramp is no longer needed. Use `--testing.skip-gas-limit-ramp-check`
/// and `--testing.gas-limit` on the reth node instead. This flag is accepted but ignored.
#[arg(long, value_name = "GAS_RAMP_DIR", hide = true)]
/// Optional directory containing gas ramp payloads to replay first.
/// These are replayed before the main payloads to warm up the gas limit.
#[arg(long, value_name = "GAS_RAMP_DIR")]
gas_ramp_dir: Option<PathBuf>,
/// Optional output directory for benchmark results (CSV files).
@@ -73,6 +88,47 @@ pub struct Command {
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
/// Wait for blocks to be persisted before sending the next batch.
///
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// The subscription uses the regular RPC websocket endpoint (no JWT required).
#[arg(long, default_value = "false", verbatim_doc_comment)]
wait_for_persistence: bool,
/// Engine persistence threshold used for deciding when to wait for persistence.
///
/// The benchmark waits after every `(threshold + 1)` blocks. By default this
/// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
/// at blocks 3, 6, 9, etc.
#[arg(
long = "persistence-threshold",
value_name = "PERSISTENCE_THRESHOLD",
default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
verbatim_doc_comment
)]
persistence_threshold: u64,
/// Timeout for waiting on persistence at each checkpoint.
///
/// Must be long enough to account for the persistence thread being blocked
/// by pruning after the previous save.
#[arg(
long = "persistence-timeout",
value_name = "PERSISTENCE_TIMEOUT",
value_parser = parse_duration,
default_value = "120s",
verbatim_doc_comment
)]
persistence_timeout: Duration,
/// Optional `WebSocket` RPC URL for persistence subscription.
/// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
#[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
ws_rpc_url: Option<String>,
/// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
///
/// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
@@ -81,20 +137,6 @@ pub struct Command {
#[arg(long, default_value = "false", verbatim_doc_comment)]
reth_new_payload: bool,
/// Skip waiting for in-flight persistence before processing.
///
/// Only works with `--reth-new-payload`. When set, passes `wait_for_persistence: false`
/// to the `reth_newPayload` endpoint.
#[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
no_wait_for_persistence: bool,
/// Skip waiting for execution cache and sparse trie locks before processing.
///
/// Only works with `--reth-new-payload`. When set, passes `wait_for_caches: false`
/// to the `reth_newPayload` endpoint.
#[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
no_wait_for_caches: bool,
/// Optional Prometheus metrics endpoint to scrape after each block.
///
/// When provided, reth-bench will fetch metrics from this URL after each
@@ -114,6 +156,16 @@ struct LoadedPayload {
block_hash: B256,
}
/// A gas ramp payload loaded from disk.
struct GasRampPayload {
/// Block number from filename.
block_number: u64,
/// Engine API version for newPayload.
version: EngineApiMessageVersion,
/// The file contents.
file: GasRampPayloadFile,
}
impl Command {
/// Execute the `replay-payloads` command.
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
@@ -123,9 +175,43 @@ impl Command {
if let Some(duration) = self.wait_time {
info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.reth_new_payload {
info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
if self.wait_for_persistence {
info!(
target: "reth-bench",
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
if self.reth_new_payload {
info!("Using reth_newPayload endpoint");
}
// Set up waiter based on configured options
// When both are set: wait at least wait_time, and also wait for persistence if needed
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), true) => {
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
Some(PersistenceWaiter::with_duration_and_subscription(
duration,
sub,
self.persistence_threshold,
self.persistence_timeout,
))
}
(Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
self.persistence_timeout,
))
}
(None, false) => None,
};
let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
@@ -156,16 +242,18 @@ impl Command {
"Using initial parent block"
);
// Warn if deprecated --gas-ramp-dir is passed
if self.gas_ramp_dir.is_some() {
warn!(
target: "reth-bench",
"--gas-ramp-dir is deprecated and ignored. Use --testing.skip-gas-limit-ramp-check \
and --testing.gas-limit on the reth node instead."
);
}
// Load all payloads upfront to avoid I/O delays between phases
let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
if payloads.is_empty() {
return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
}
info!(target: "reth-bench", count = payloads.len(), "Loaded gas ramp payloads from disk");
payloads
} else {
Vec::new()
};
let payloads = self.load_payloads()?;
if payloads.is_empty() {
return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
@@ -174,6 +262,47 @@ impl Command {
let mut parent_hash = initial_parent_hash;
// Replay gas ramp payloads first
for (i, payload) in gas_ramp_payloads.iter().enumerate() {
info!(
target: "reth-bench",
gas_ramp_payload = i + 1,
total = gas_ramp_payloads.len(),
block_number = payload.block_number,
block_hash = %payload.file.block_hash,
"Executing gas ramp payload (newPayload + FCU)"
);
let reth_data =
if self.reth_new_payload { payload.file.execution_data.clone() } else { None };
let _ = call_new_payload_with_reth(
&auth_provider,
payload.version,
payload.file.params.clone(),
reth_data,
)
.await?;
let fcu_state = ForkchoiceState {
head_block_hash: payload.file.block_hash,
safe_block_hash: parent_hash,
finalized_block_hash: parent_hash,
};
call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
info!(target: "reth-bench", gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
if let Some(w) = &mut waiter {
w.on_block(payload.block_number).await?;
}
parent_hash = payload.file.block_hash;
}
if !gas_ramp_payloads.is_empty() {
info!(target: "reth-bench", count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
}
let mut results = Vec::new();
let total_benchmark_duration = Instant::now();
@@ -207,41 +336,31 @@ impl Command {
"Sending newPayload"
);
let (version, params) = if self.reth_new_payload {
let reth_data = ExecutionData {
payload: execution_payload.clone().into(),
sidecar: ExecutionPayloadSidecar::v4(
CancunPayloadFields {
versioned_hashes: Vec::new(),
parent_beacon_block_root: B256::ZERO,
},
PraguePayloadFields {
requests: envelope.execution_requests.clone().into(),
},
),
};
(
None,
serde_json::to_value((
RethNewPayloadInput::ExecutionData(reth_data),
self.no_wait_for_persistence.then_some(false),
self.no_wait_for_caches.then_some(false),
))?,
)
} else {
(
Some(EngineApiMessageVersion::V4),
serde_json::to_value((
execution_payload.clone(),
Vec::<B256>::new(),
B256::ZERO,
envelope.execution_requests.to_vec(),
))?,
)
};
let params = serde_json::to_value((
execution_payload.clone(),
Vec::<B256>::new(),
B256::ZERO,
envelope.execution_requests.to_vec(),
))?;
let server_timings =
call_new_payload_with_reth(&auth_provider, version, params).await?;
let reth_data = self.reth_new_payload.then(|| ExecutionData {
payload: execution_payload.clone().into(),
sidecar: ExecutionPayloadSidecar::v4(
CancunPayloadFields {
versioned_hashes: Vec::new(),
parent_beacon_block_root: B256::ZERO,
},
PraguePayloadFields { requests: envelope.execution_requests.clone().into() },
),
});
let server_timings = call_new_payload_with_reth(
&auth_provider,
EngineApiMessageVersion::V4,
params,
reth_data,
)
.await?;
let np_latency =
server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
@@ -265,8 +384,10 @@ impl Command {
finalized_block_hash: parent_hash,
};
debug!(target: "reth-bench", method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
let fcu_start = Instant::now();
call_forkchoice_updated_with_reth(&auth_provider, version, fcu_state).await?;
let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
let fcu_latency = fcu_start.elapsed();
let total_latency =
@@ -291,13 +412,22 @@ impl Command {
tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
}
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
}
let gas_row =
TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
results.push((gas_row, combined_result));
debug!(target: "reth-bench", ?fcu_result, "Payload executed successfully");
parent_hash = block_hash;
}
// Drop waiter - we don't need to wait for final blocks to persist
// since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
drop(waiter);
let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
results.into_iter().unzip();
@@ -384,4 +514,60 @@ impl Command {
Ok(payloads)
}
/// Load and parse gas ramp payload files from a directory.
fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
let mut payloads = Vec::new();
let entries: Vec<_> = std::fs::read_dir(dir)
.wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
.filter_map(|e| e.ok())
.filter(|e| {
e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
e.file_name().to_string_lossy().starts_with("payload_block_")
})
.collect();
// Parse filenames to get block numbers and sort
let mut indexed_paths: Vec<(u64, PathBuf)> = entries
.into_iter()
.filter_map(|e| {
let name = e.file_name();
let name_str = name.to_string_lossy();
// Extract block number from "payload_block_NNN.json"
let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
let block_number: u64 = block_str.parse().ok()?;
Some((block_number, e.path()))
})
.collect();
indexed_paths.sort_by_key(|(num, _)| *num);
for (block_number, path) in indexed_paths {
let content = std::fs::read_to_string(&path)
.wrap_err_with(|| format!("Failed to read {:?}", path))?;
let file: GasRampPayloadFile = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let version = match file.version {
1 => EngineApiMessageVersion::V1,
2 => EngineApiMessageVersion::V2,
3 => EngineApiMessageVersion::V3,
4 => EngineApiMessageVersion::V4,
5 => EngineApiMessageVersion::V5,
v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
};
info!(
block_number,
block_hash = %file.block_hash,
path = %path.display(),
"Loaded gas ramp payload"
);
payloads.push(GasRampPayload { block_number, version, file });
}
Ok(payloads)
}
}

View File

@@ -240,7 +240,6 @@ impl Command {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
@@ -255,9 +254,6 @@ impl Command {
ExecutionPayload::V1(p) => p.block_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
ExecutionPayload::V4(p) => {
p.payload_inner.payload_inner.payload_inner.block_hash
}
}
}
};
@@ -266,9 +262,6 @@ impl Command {
ExecutionPayload::V1(p) => p.block_hash = new_hash,
ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
ExecutionPayload::V4(p) => {
p.payload_inner.payload_inner.payload_inner.block_hash = new_hash
}
}
}

View File

@@ -34,16 +34,17 @@ impl BenchMode {
}
/// Create a [`BenchMode`] from optional `from` and `to` fields.
///
/// If only `--to` is provided, `from` is derived as `latest_block + 1`.
pub const fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Self {
pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
// If neither `--from` nor `--to` are provided, we will run the benchmark continuously,
// starting at the latest block.
match (from, to) {
(Some(from), Some(to)) => Self::Range(from..=to),
(None, None) => Self::Continuous(latest_block),
(Some(start), None) => Self::Continuous(start),
(None, Some(to)) => Self::Range(latest_block + 1..=to),
(Some(from), Some(to)) => Ok(Self::Range(from..=to)),
(None, None) => Ok(Self::Continuous(latest_block)),
(Some(start), None) => Ok(Self::Continuous(start)),
_ => {
// both or neither are allowed, everything else is ambiguous
Err(eyre::eyre!("`from` and `to` must be provided together, or not at all."))
}
}
}
}

View File

@@ -3,7 +3,7 @@
//! before sending additional calls.
use alloy_eips::eip7685::Requests;
use alloy_primitives::{Bytes, B256};
use alloy_primitives::B256;
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
use alloy_rpc_types_engine::{
ExecutionData, ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar,
@@ -12,7 +12,6 @@ use alloy_rpc_types_engine::{
use alloy_transport::TransportResult;
use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
use reth_node_api::EngineApiMessageVersion;
use reth_rpc_api::RethNewPayloadInput;
use serde::Deserialize;
use std::time::Duration;
use tracing::{debug, error};
@@ -167,27 +166,10 @@ where
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
///
/// Returns `(version, versioned_params, execution_data)`.
///
/// When `no_wait_for_persistence` or `no_wait_for_caches` is `true` and using `reth_newPayload`,
/// passes the corresponding `wait_for_*: false` to skip that wait.
pub(crate) fn block_to_new_payload(
block: AnyRpcBlock,
is_optimism: bool,
rlp: Option<Bytes>,
reth_new_payload: bool,
no_wait_for_persistence: bool,
no_wait_for_caches: bool,
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
if let Some(rlp) = rlp {
return Ok((
None,
serde_json::to_value((
RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),
no_wait_for_persistence.then_some(false),
no_wait_for_caches.then_some(false),
))?,
));
}
) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value, ExecutionData)> {
let block = block
.into_inner()
.map_header(|header| header.map(|h| h.into_header_with_defaults()))
@@ -199,21 +181,7 @@ pub(crate) fn block_to_new_payload(
// Convert to execution payload
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
let (version, params, execution_data) =
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)?;
if reth_new_payload {
Ok((
None,
serde_json::to_value((
RethNewPayloadInput::ExecutionData(execution_data),
no_wait_for_persistence.then_some(false),
no_wait_for_caches.then_some(false),
))?,
))
} else {
Ok((Some(version), params))
}
payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
}
/// Converts an execution payload and sidecar into versioned engine API params and an
@@ -230,20 +198,6 @@ pub(crate) fn payload_to_new_payload(
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
let (version, params) = match payload {
ExecutionPayload::V4(payload) => {
let cancun = sidecar.cancun().unwrap();
let prague = sidecar.prague().unwrap();
let requests = prague.requests.requests_hash();
(
EngineApiMessageVersion::V5,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
requests,
))?,
)
}
ExecutionPayload::V3(payload) => {
let cancun = sidecar.cancun().unwrap();
@@ -265,8 +219,8 @@ pub(crate) fn payload_to_new_payload(
))?,
)
} else {
// Preserve the original RequestsOrHash payload for engine_newPayloadV4.
let requests = prague.requests.clone();
// Extract actual Requests from RequestsOrHash
let requests = prague.requests.requests_hash();
(
version,
serde_json::to_value((
@@ -312,15 +266,17 @@ pub(crate) fn payload_to_new_payload(
#[allow(dead_code)]
pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
provider: P,
version: Option<EngineApiMessageVersion>,
version: EngineApiMessageVersion,
params: serde_json::Value,
) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
call_new_payload_with_reth(provider, version, params).await
) -> TransportResult<Option<NewPayloadTimingBreakdown>> {
call_new_payload_with_reth(provider, version, params, None).await
}
/// Response from `reth_newPayload` endpoint, which includes server-measured latency.
#[derive(Debug, Deserialize)]
struct RethPayloadStatus {
#[serde(flatten)]
status: PayloadStatus,
latency_us: u64,
#[serde(default)]
persistence_wait_us: Option<u64>,
@@ -344,50 +300,72 @@ pub(crate) struct NewPayloadTimingBreakdown {
}
/// Calls either `engine_newPayload*` or `reth_newPayload` depending on whether
/// `version` is provided.
/// `reth_execution_data` is provided.
///
/// When `version` is `None`, uses `reth_newPayload` endpoint with provided params.
/// When `reth_execution_data` is `Some`, uses the `reth_newPayload` endpoint which takes
/// `ExecutionData` directly and waits for persistence and cache updates to complete.
///
/// Returns the server-reported timing breakdown when using the reth namespace, or `None` for
/// the standard engine namespace.
pub(crate) async fn call_new_payload_with_reth<N: Network, P: Provider<N>>(
provider: P,
version: Option<EngineApiMessageVersion>,
version: EngineApiMessageVersion,
params: serde_json::Value,
) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
let method = version.map(|v| v.method_name()).unwrap_or("reth_newPayload");
reth_execution_data: Option<ExecutionData>,
) -> TransportResult<Option<NewPayloadTimingBreakdown>> {
if let Some(execution_data) = reth_execution_data {
let method = "reth_newPayload";
let reth_params = serde_json::to_value((execution_data.clone(),))
.expect("ExecutionData serialization cannot fail");
debug!(target: "reth-bench", method, "Sending newPayload");
debug!(target: "reth-bench", method, "Sending newPayload");
let resp = loop {
let resp: serde_json::Value = provider.client().request(method, &params).await?;
let status = PayloadStatus::deserialize(&resp)?;
let mut resp: RethPayloadStatus = provider.client().request(method, &reth_params).await?;
if status.is_valid() {
break resp;
while !resp.status.is_valid() {
if resp.status.is_invalid() {
error!(target: "reth-bench", status=?resp.status, "Invalid {method}");
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
std::io::Error::other(format!("Invalid {method}: {:?}", resp.status)),
)))
}
if resp.status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
"invalid range: no canonical state found for parent of requested block",
))
}
resp = provider.client().request(method, &reth_params).await?;
}
if status.is_invalid() {
return Err(eyre::eyre!("Invalid {method}: {status:?}"));
}
if status.is_syncing() {
return Err(eyre::eyre!(
"invalid range: no canonical state found for parent of requested block"
));
}
};
if version.is_some() {
return Ok(None);
Ok(Some(NewPayloadTimingBreakdown {
latency: Duration::from_micros(resp.latency_us),
persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
}))
} else {
let method = version.method_name();
debug!(target: "reth-bench", method, "Sending newPayload");
let mut status: PayloadStatus = provider.client().request(method, &params).await?;
while !status.is_valid() {
if status.is_invalid() {
error!(target: "reth-bench", ?status, ?params, "Invalid {method}",);
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
std::io::Error::other(format!("Invalid {method}: {status:?}")),
)))
}
if status.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
"invalid range: no canonical state found for parent of requested block",
))
}
status = provider.client().request(method, &params).await?;
}
Ok(None)
}
let resp: RethPayloadStatus = serde_json::from_value(resp)?;
Ok(Some(NewPayloadTimingBreakdown {
latency: Duration::from_micros(resp.latency_us),
persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
}))
}
/// Calls the correct `engine_forkchoiceUpdated` method depending on the given
@@ -401,12 +379,9 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
forkchoice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for Cancun, Prague, and Amsterdam (there is no FCU V4-V6)
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 |
EngineApiMessageVersion::V6 => {
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V2 => {
@@ -417,47 +392,3 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
}
}
}
/// Calls either `reth_forkchoiceUpdated` or the standard `engine_forkchoiceUpdated*` depending
/// on `use_reth`.
///
/// When `use_reth` is true, uses the `reth_forkchoiceUpdated` endpoint which sends a regular FCU
/// with no payload attributes.
pub(crate) async fn call_forkchoice_updated_with_reth<
N: Network,
P: Provider<N> + EngineApiValidWaitExt<N>,
>(
provider: P,
message_version: Option<EngineApiMessageVersion>,
forkchoice_state: ForkchoiceState,
) -> TransportResult<ForkchoiceUpdated> {
if let Some(message_version) = message_version {
call_forkchoice_updated(provider, message_version, forkchoice_state, None).await
} else {
let method = "reth_forkchoiceUpdated";
let reth_params = serde_json::to_value((forkchoice_state,))
.expect("ForkchoiceState serialization cannot fail");
debug!(target: "reth-bench", method, "Sending forkchoiceUpdated");
loop {
let resp: ForkchoiceUpdated = provider.client().request(method, &reth_params).await?;
if resp.is_valid() {
break Ok(resp)
}
if resp.is_invalid() {
error!(target: "reth-bench", ?resp, "Invalid {method}");
return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
std::io::Error::other(format!("Invalid {method}: {resp:?}")),
)))
}
if resp.is_syncing() {
return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
"invalid range: no canonical state found for parent of requested block",
))
}
}
}
}

View File

@@ -30,8 +30,7 @@ workspace = true
# reth
reth-ethereum-cli.workspace = true
reth-chainspec.workspace = true
reth-primitives-traits.workspace = true
reth-ethereum-primitives.workspace = true
reth-primitives.workspace = true
reth-db = { workspace = true, features = ["mdbx"] }
reth-provider.workspace = true
reth-revm.workspace = true
@@ -70,7 +69,7 @@ aquamarine.workspace = true
clap = { workspace = true, features = ["derive", "env"] }
[dev-dependencies]
alloy-node-bindings = "1.5.2"
alloy-node-bindings = "1.6.3"
alloy-provider = { workspace = true, features = ["reqwest"] }
alloy-rpc-types-eth.workspace = true
backon.workspace = true
@@ -89,6 +88,7 @@ default = [
"keccak-cache-global",
"asm-keccak",
"min-debug-logs",
"rocksdb",
]
otlp = [
@@ -110,6 +110,7 @@ dev = ["reth-ethereum-cli/dev"]
asm-keccak = [
"reth-node-core/asm-keccak",
"reth-primitives/asm-keccak",
"reth-ethereum-cli/asm-keccak",
"reth-node-ethereum/asm-keccak",
"alloy-primitives/asm-keccak",
@@ -124,7 +125,6 @@ jemalloc = [
"reth-node-core/jemalloc",
"reth-node-metrics/jemalloc",
"reth-ethereum-cli/jemalloc",
"reth-provider/jemalloc",
]
jemalloc-prof = [
"reth-cli-util/jemalloc",
@@ -191,6 +191,8 @@ min-trace-logs = [
]
trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"]
rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
edge = ["rocksdb"]
[[bin]]
name = "reth"

View File

@@ -124,11 +124,9 @@ pub mod providers {
pub use reth_provider::*;
}
/// Re-exported primitives.
#[allow(ambiguous_glob_reexports)]
/// Re-exported from `reth_primitives`.
pub mod primitives {
pub use reth_ethereum_primitives::*;
pub use reth_primitives_traits::*;
pub use reth_primitives::*;
}
/// Re-exported from `reth_ethereum_consensus`.

View File

@@ -56,6 +56,7 @@ alloy-signer.workspace = true
alloy-signer-local.workspace = true
rand.workspace = true
revm-state.workspace = true
criterion.workspace = true
[features]
serde = [
@@ -85,3 +86,8 @@ test-utils = [
"reth-ethereum-primitives/test-utils",
]
rayon = ["dep:rayon"]
[[bench]]
name = "canonical_hashes_range"
harness = false
required-features = ["test-utils"]

Some files were not shown because too many files have changed in this diff Show More