chore: fix bal devnet 2 (#23141)

Co-authored-by: Soubhik Singha Mahapatra <soubhiksmp2004@gmail.com>
Co-authored-by: stevencartavia <112043913+stevencartavia@users.noreply.github.com>
Co-authored-by: Emma Jamieson-Hoare <ejamieson19@gmail.com>
Co-authored-by: Amp <amp@ampcode.com>

.changelog/cool-owls-rest.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Fixed a bug in `merge_subtrie_updates` where source insertions did not cancel destination removals (and vice versa), causing inconsistent trie updates accumulated across multiple `root()` calls without intermediate `take_updates()`. Added a test covering the cross-cancellation behavior.

.changelog/cool-suns-rest.md

@@ -0,0 +1,5 @@
+---
+reth-transaction-pool: minor
+---
+
+Added support for optional custom stateless and stateful validation hooks in `EthTransactionValidator` via `set_additional_stateless_validation` and `set_additional_stateful_validation` methods. Also implemented a manual `Debug` impl to handle the non-`Debug` function pointer fields.

.changelog/dark-ants-write.md

@@ -0,0 +1,5 @@
+---
+reth-tasks: patch
+---
+
+Added panic handler to all rayon thread pools that logs panics via `tracing::error` instead of aborting the process.

.changelog/dry-koalas-wink.md

@@ -0,0 +1,5 @@
+---
+reth-network-types: patch
+---
+
+Increased default maximum concurrent outbound dials from 15 to 30.

.changelog/dry-lakes-sleep.md

@@ -0,0 +1,6 @@
+---
+reth-trie-common: minor
+reth-trie: minor
+---
+
+Added `contains_range` method to `PrefixSet` for checking if any key falls within a half-open range. Added prefix set support to `ProofCalculator` via `with_prefix_set`, enabling stale cached hash invalidation and branch collapse detection when keys are inserted or removed; propagated storage prefix sets through `SyncAccountValueEncoder`.

.changelog/dull-clams-pack.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Refactored arena trie internals by adding a `BranchChildIdx::sibling()` helper, deduplicating `Index`/`NodeArena` type aliases, and replacing `is_empty()` with a `drop_root()` method. Fixed a bug where `cursor.pop()` was called before checking if the leaf was the root node, which could cause incorrect dirty-state propagation.

.changelog/dull-seals-laugh.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: minor
+---
+
+Fixed a bug in `ArenaParallelSparseTrie` where subtrie updates that would completely empty a subtrie were incorrectly dispatched to parallel workers instead of being processed inline, preventing correct branch collapse detection when blinded siblings are present. Refactored the `SparseTrie` test suite to accept a `fn() -> T` factory instead of requiring `T: Default`, enabling a new `arena_parallel_sparse_trie_always_parallel` test variant that exercises all tests with parallelism thresholds set to 1. Added `test_branch_collapse_multi_empty_subtries_blinded_remaining` to cover the case where removing multiple revealed leaves empties their subtries and leaves a single blinded sibling requiring a proof.

.changelog/fair-wolves-smile.md

@@ -0,0 +1,5 @@
+---
+reth-payload-builder: minor
+---
+
+Added observability metrics for payload resolve latency and new payload job creation latency to the payload builder service.

.changelog/fine-horses-whisper.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Fixed another branch collapse edge case where `check_subtrie_collapse_needs_proof` incorrectly compared removal count against total update count (including `Touched` entries), causing it to skip proof requests for blinded siblings and panic when the subtrie emptied. Added a regression test covering the removals + `Touched` + blinded sibling scenario.

.changelog/fine-koalas-shout.md

@@ -0,0 +1,10 @@
+---
+reth-chain-state: minor
+reth-engine-primitives: minor
+reth-engine-tree: minor
+reth-node-core: minor
+reth-node-events: minor
+reth: patch
+---
+
+Added configurable slow block logging (`--engine.slow-block-threshold`) that emits a structured `warn!` log with detailed timing, state-operation counts, and cache hit-rate metrics for blocks whose total processing time exceeds the threshold. Introduced `ExecutionTimingStats`, `CacheStats`, `StateProviderStats`, and `SlowBlockInfo` types to carry execution statistics from block validation through persistence, and refactored `PersistenceResult` to carry commit duration alongside the last persisted block.

.changelog/icy-tigers-buzz.md

@@ -0,0 +1,6 @@
+---
+reth-trie-db: minor
+reth-engine-tree: minor
+---
+
+Added `PendingChangeset` and `PendingChangesetGuard` to `ChangesetCache` so concurrent readers wait for an in-progress computation instead of falling back to the expensive DB-based path. The guard automatically cancels the pending entry on drop (e.g. task panic), ensuring waiters always make progress.

.changelog/keen-geese-bake.md

@@ -0,0 +1,5 @@
+---
+reth-engine-tree: patch
+---
+
+Added sub-phase timing histograms to the sparse trie event loop, tracking channel wait, proof coalescing, multiproof reveal, and trie update durations separately.

.changelog/merry-bees-break.md

@@ -0,0 +1,8 @@
+---
+reth-engine-primitives: minor
+reth-engine-tree: minor
+reth-node-core: minor
+reth-trie-parallel: minor
+---
+
+Added `--engine.proof-jitter` CLI option behind the `trie-debug` feature flag. When set, each proof worker sleeps for a random duration up to the specified value before starting proof computation, useful for stress-testing timing-sensitive proof logic.

.changelog/mild-foxes-cook.md

@@ -0,0 +1,5 @@
+---
+reth-rpc-convert: patch
+---
+
+Updated `alloy-evm` dependency to git revision `9bc2dba` and adapted `TxEnvConverter` impl to the updated `TryIntoTxEnv` trait signature that now includes a `Spec` generic parameter.

.changelog/neat-ducks-whisper.md

@@ -0,0 +1,6 @@
+---
+reth-trie: patch
+reth-trie-sparse: patch
+---
+
+Refactored test harness for sparse trie tests by extracting `TrieTestHarness` into a shared `reth-trie` test utility, replacing duplicated inline harness code across multiple test modules. Updated `proof_v2` return type to include an optional root hash, and converted `original_root` and `storage` from public fields to accessor methods.

.changelog/nice-bears-cook.md

@@ -0,0 +1,5 @@
+---
+reth-trie: patch
+---
+
+Removed the local `increment_and_strip_trailing_zeros` function and `PATH_ALL_ZEROS` static in `proof_v2`, replacing them with the equivalent `Nibbles::next_without_prefix` and `Nibbles::is_zeroes` builtins. Also replaced manual `.get()` calls on `state_mask`/`hash_mask` with direct field access and switched to `Nibbles::unpack_array` over the unsafe `unpack_unchecked`.

.changelog/nice-whales-buzz.md

@@ -0,0 +1,5 @@
+---
+reth-engine-tree: patch
+---
+
+Added idle-time pre-computation of account trie upper hashes in the sparse trie payload processor when no pending proof results are available.

.changelog/odd-frogs-break.md

@@ -0,0 +1,7 @@
+---
+reth-cli-commands: minor
+reth-node-core: minor
+reth: patch
+---
+
+Made v2 storage the default for all new databases, deprecating the `--storage.v2` flag to a hidden no-op kept for backwards compatibility. Updated CLI reference docs to remove the now-hidden flag from all command help pages.

.changelog/proud-slugs-fly.md

@@ -0,0 +1,7 @@
+---
+reth-engine-tree: patch
+reth-trie-sparse: patch
+reth-tasks: patch
+---
+
+Offloaded deallocation of expensive proof node buffers to a persistent background thread (`Runtime::spawn_drop`) to avoid blocking state root computation or lock-holding code.

.changelog/rare-tigers-nod.md

@@ -0,0 +1,6 @@
+---
+reth-trie-sparse: patch
+reth-engine-tree: patch
+---
+
+Removed the `skip_proof_node_filtering` flag, `revealed_account_paths`/`revealed_paths` tracking, and the `filter_revealed_v2_proof_nodes` function from the sparse trie implementation. Also removed the corresponding skipped-nodes metrics, simplifying the proof node reveal path to always pass nodes directly to the sparse trie without pre-filtering.

.changelog/rare-whales-pack.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: minor
+---
+
+Added a comprehensive generic `SparseTrie` test suite covering `set_root`, `reveal_nodes`, `update_leaves`, `root`, `take_updates`, `commit_updates`, `prune`, `wipe`/`clear`, `get_leaf_value`, `find_leaf`, `size_hint`, and integration lifecycle scenarios. Tests are stamped out for all concrete `SparseTrie` implementations via a macro.

.changelog/safe-waves-cook.md

@@ -0,0 +1,5 @@
+---
+reth-engine-tree: patch
+---
+
+Downgraded per-transaction prewarm span from `debug_span!` to `trace_span!` to reduce noise in debug-level logging.

.changelog/safe-waves-read.md

@@ -0,0 +1,7 @@
+---
+reth-network-types: minor
+reth-network: minor
+reth-node-core: patch
+---
+
+Added `PersistedPeerInfo` struct to persist richer peer metadata (kind, fork ID, reputation) to disk. Updated `PeersConfig::with_basic_nodes_from_file` to support both the new `PersistedPeerInfo` format and the legacy `Vec<NodeRecord>` format with automatic conversion, and updated `write_peers_to_file` to exclude backed-off and banned peers.

.changelog/tall-fish-climb.md

@@ -0,0 +1,5 @@
+---
+reth-cli-commands: minor
+---
+
+Added `reth_version` field to `SnapshotManifest` to record the Reth version that produced a snapshot. The field is optional and populated automatically during manifest generation.

.changelog/tall-stars-shout.md

@@ -0,0 +1,5 @@
+---
+reth-network: minor
+---
+
+Added `fork_id` as a tiebreaker in peer selection when reputations are equal, preferring peers with a discovered `fork_id` as it indicates fork compatibility. Added a test to verify the tiebreaker behavior.

.changelog/tidy-stars-cry.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Fixed a bug where trie nodes could appear in both `updated_nodes` and `removed_nodes` simultaneously by removing entries from `removed_nodes` when a node is inserted as updated.

.changelog/vain-bats-bark.md

@@ -0,0 +1,8 @@
+---
+reth-trie-common: minor
+reth-trie: minor
+reth-trie-parallel: minor
+reth-engine-tree: patch
+---
+
+Moved `ProofV2Target`, `MultiProofTargetsV2`, and `ChunkedMultiProofTargetsV2` from `reth-trie-parallel::targets_v2` into a new `reth-trie-common::target_v2` module, making these types available at a lower level without pulling in the full parallel trie crate. Added a `multiproof_v2` method to `Proof` in `reth-trie` that generates a state multiproof using the V2 proof calculator with synchronous account value encoding.

.changelog/vain-birds-laugh.md

@@ -0,0 +1,14 @@
+---
+reth-engine-tree: patch
+reth-evm-ethereum: patch
+reth-evm: patch
+reth-primitives-traits: patch
+reth-revm: patch
+reth-rpc-eth-api: patch
+reth-rpc-eth-types: patch
+reth-provider: patch
+example-custom-evm: patch
+example-precompile-cache: patch
+---
+
+Bumped revm to v35.0.0, revm-inspectors to 0.35.0, and alloy-evm to 0.29.0. Updated call sites throughout the codebase to align with the new APIs, including `ExecutionResult` field changes (`gas_used` → `gas.used()`/`gas.final_refunded()`), removal of `.without_state_clear()`, updated `EthPrecompiles::new(spec)` constructor, and updated `block_hashes.lowest()` access.

.changelog/vain-goats-cook.md

@@ -0,0 +1,9 @@
+---
+reth-trie-sparse: minor
+reth-engine-primitives: minor
+reth-engine-tree: minor
+reth-node-core: minor
+reth-trie-common: patch
+---
+
+Added an arena-based sparse trie implementation (`ArenaParallelSparseTrie`) using `slotmap` arena allocation for node storage, enabling parallel subtrie mutation without per-node hashing overhead. Added `ConfigurableSparseTrie` enum to switch between the arena and hash-map implementations, and a `--engine.enable-arena-sparse-trie` CLI flag to opt in at runtime.

.changelog/warm-trees-grow.md

@@ -0,0 +1,7 @@
+---
+reth-engine-primitives: patch
+reth-engine-tree: patch
+reth-node-core: patch
+---
+
+Removed `--engine.enable-arena-sparse-trie` CLI flag and made the arena-based sparse trie the default implementation. The hash-map-based `ParallelSparseTrie` variant is no longer selectable.

.changelog/young-horses-play.md

@@ -0,0 +1,7 @@
+---
+reth-trie: major
+reth-trie-db: major
+reth-provider: minor
+---
+
+Added `MaskedTrieCursorFactory` and `MaskedTrieCursor` to handle prefix-set-based hash invalidation at the cursor layer, replacing the `DatabaseTrieWitness` trait abstraction. Removed `with_prefix_sets_mut` from `TrieWitness` and deleted `DatabaseTrieWitness` — callers should now wrap their cursor factory with `MaskedTrieCursorFactory` to apply prefix sets during witness/proof computation.

.changelog/zesty-clouds-wave.md

@@ -0,0 +1,5 @@
+---
+reth-trie: patch
+---
+
+Fixed a potential panic in `ProofCalculator` by clearing internal computation state (`branch_stack`, `child_stack`, `branch_path`, etc.) after errors, preventing stale state from causing `usize` underflow panics when the calculator is reused. Added a test verifying correct behavior after simulated mid-computation errors.

.dockerignore

@@ -20,6 +20,11 @@
 # include dist directory, where the reth binary is located after compilation
 !/dist
 
+# include PGO build helper used by Dockerfile.depot
+!/.github
+!/.github/scripts
+!/.github/scripts/build_pgo_bolt.sh
+
 # include licenses
 !LICENSE-*
 

.github/CODEOWNERS

@@ -1,7 +1,7 @@
 *                           @gakonst
 crates/chain-state/         @fgimenez @mattsse
 crates/chainspec/           @Rjected @joshieDo @mattsse
-crates/cli/                 @mattsse
+crates/cli/                 @mattsse @Rjected
 crates/config/              @shekhirin @mattsse @Rjected 
 crates/consensus/           @mattsse @Rjected
 crates/e2e-test-utils/      @mattsse @Rjected @klkvr @fgimenez

.github/actionlint.yaml

@@ -5,3 +5,4 @@ self-hosted-runner:
     - depot-ubuntu-latest-4
     - depot-ubuntu-latest-8
     - depot-ubuntu-latest-16
+    - available

.github/scripts/bench-job-summary.js

@@ -0,0 +1,106 @@
+// Generates a rich GitHub Actions job summary for reth-bench results.
+//
+// Reads from environment:
+//   BENCH_WORK_DIR       – Directory containing summary.json
+//   BENCH_PR             – PR number (may be empty)
+//   BENCH_ACTOR          – GitHub user who triggered the bench
+//   BENCH_CORES          – CPU core limit (0 = all)
+//   BENCH_WARMUP_BLOCKS  – Number of warmup blocks
+//   BENCH_SAMPLY         – 'true' if samply profiling was enabled
+//   BENCH_ABBA           – 'true' if ABBA interleaved order was used
+//
+// Usage from actions/github-script:
+//   const jobSummary = require('./.github/scripts/bench-job-summary.js');
+//   await jobSummary({ core, context, chartSha, grafanaUrl, runId });
+
+const fs = require('fs');
+const { verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
+
+module.exports = async function ({ core, context, chartSha, grafanaUrl, runId }) {
+  let summary;
+  try {
+    summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+  } catch (e) {
+    await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
+    return;
+  }
+
+  const repo = `${context.repo.owner}/${context.repo.repo}`;
+  const prNumber = process.env.BENCH_PR;
+  const actor = process.env.BENCH_ACTOR;
+  const commitUrl = `https://github.com/${repo}/commit`;
+
+  const { emoji, label } = verdict(summary.changes);
+  const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
+  const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
+  const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+
+  // Header & metadata
+  const metaParts = [];
+  if (prNumber) metaParts.push(`**[PR #${prNumber}](https://github.com/${repo}/pull/${prNumber})**`);
+  metaParts.push(`triggered by @${actor}`);
+
+  let md = `# ${emoji} ${label}\n\n`;
+  md += metaParts.join(' · ') + '\n\n';
+  md += `**Baseline:** ${baselineLink}\n`;
+  md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
+  md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
+
+  // Main comparison table
+  const rows = metricRows(summary);
+  md += `| Metric | Baseline | Feature | Change |\n`;
+  md += `|--------|----------|---------|--------|\n`;
+  for (const r of rows) {
+    md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
+  }
+  md += '\n';
+
+  // Wait time breakdown
+  const wtRows = waitTimeRows(summary);
+  if (wtRows.length > 0) {
+    md += `### Wait Time Breakdown\n\n`;
+    md += `| Metric | Baseline | Feature |\n`;
+    md += `|--------|----------|--------|\n`;
+    for (const r of wtRows) {
+      md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
+    }
+    md += '\n';
+  }
+
+  // Charts
+  if (chartSha) {
+    const prNum = prNumber || '0';
+    const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`;
+    const charts = [
+      { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
+      { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
+      { file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
+    ];
+    md += `### Charts\n\n`;
+    for (const chart of charts) {
+      md += `<details><summary>${chart.label}</summary>\n\n`;
+      md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
+      md += `</details>\n\n`;
+    }
+  }
+
+  // Samply profiles
+  const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
+  const samplyLinks = Object.entries(samplyUrls).map(([run, url]) => `- **${run}**: [Firefox Profiler](${url})`);
+  if (samplyLinks.length > 0) {
+    md += `### Samply Profiles\n\n${samplyLinks.join('\n')}\n\n`;
+  }
+
+  // Grafana
+  if (grafanaUrl) {
+    md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
+  }
+
+  // Node errors
+  try {
+    const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
+    if (errors.trim()) md += '\n' + errors + '\n';
+  } catch {}
+
+  await core.summary.addRaw(md).write();
+};

.github/scripts/bench-metrics-proxy.py

@@ -0,0 +1,276 @@
+#!/usr/bin/env python3
+"""
+Prometheus metrics proxy that fetches from a local reth node and
+re-exposes with additional benchmark labels.
+
+Reads labels from a JSON file (updated by local-reth-bench.sh between runs)
+and injects them into every Prometheus metric line.
+
+Returns empty 200 when reth is not running (clean Grafana gaps).
+"""
+import argparse
+import ipaddress
+import json
+import subprocess
+import sys
+import time
+from http.server import HTTPServer, BaseHTTPRequestHandler
+from urllib.request import urlopen
+from urllib.error import URLError
+
+
+def read_labels(path):
+    try:
+        with open(path) as f:
+            return json.load(f)
+    except (FileNotFoundError, json.JSONDecodeError):
+        return {}
+
+
+def inject_labels(metrics_bytes, label_str, label_names):
+    """Inject labels into Prometheus text format.
+
+    Operates on bytes and uses simple string ops instead of regex
+    for speed on large payloads (reth exposes thousands of metrics).
+
+    Skips injecting into lines that already contain any of the label names
+    to avoid duplicate labels (which Prometheus rejects).
+    """
+    if not label_str:
+        return metrics_bytes
+
+    label_bytes = label_str.encode("utf-8")
+    # Pre-encode label names for fast duplicate detection
+    label_name_bytes = [n.encode("utf-8") for n in label_names]
+    out = []
+    for line in metrics_bytes.split(b"\n"):
+        # Skip comments and blank lines
+        if line.startswith(b"#") or not line:
+            out.append(line)
+            continue
+
+        brace = line.find(b"{")
+        space = line.find(b" ")
+
+        if space == -1:
+            # Malformed, pass through
+            out.append(line)
+        elif brace != -1 and brace < space:
+            # Has labels: metric{existing="val"} 123
+            close = line.find(b"}", brace)
+            if close == -1:
+                out.append(line)
+                continue
+
+            # Filter out labels that already exist in this line
+            existing = line[brace + 1:close]
+            inject = label_bytes
+            if existing:
+                for name in label_name_bytes:
+                    if name + b"=" in existing:
+                        # Rebuild inject string excluding this label
+                        inject = _remove_label(inject, name)
+                if not inject:
+                    out.append(line)
+                    continue
+
+            if close == brace + 1:
+                # Empty braces: metric{} 123
+                out.append(line[:close] + inject + line[close:])
+            else:
+                out.append(line[:close] + b"," + inject + line[close:])
+        else:
+            # No labels: metric 123
+            out.append(line[:space] + b"{" + label_bytes + b"}" + line[space:])
+
+    return b"\n".join(out)
+
+
+def _remove_label(label_bytes, name):
+    """Remove a single label (name=\"...\") from a comma-separated label string."""
+    parts = []
+    for part in label_bytes.split(b","):
+        if not part.startswith(name + b"="):
+            parts.append(part)
+    return b",".join(parts)
+
+
+def build_label_str(labels):
+    """Pre-format the label injection string: key1="val1",key2="val2" """
+    if not labels:
+        return ""
+    return ",".join(f'{k}="{v}"' for k, v in sorted(labels.items()))
+
+
+def build_elapsed_gauge(labels):
+    """Build a bench_elapsed_seconds gauge from run_start_epoch in labels."""
+    start = labels.get("run_start_epoch")
+    if not start:
+        return b""
+    try:
+        elapsed = time.time() - float(start)
+    except (ValueError, TypeError):
+        return b""
+    # Build labels excluding internal keys
+    display = {k: v for k, v in labels.items()
+               if k not in ("run_start_epoch", "reference_epoch")}
+    lstr = build_label_str(display)
+    return (
+        f"# HELP bench_elapsed_seconds Seconds since benchmark run started\n"
+        f"# TYPE bench_elapsed_seconds gauge\n"
+        f"bench_elapsed_seconds{{{lstr}}} {elapsed:.1f}\n"
+    ).encode("utf-8")
+
+
+def compute_timestamp_ms(labels):
+    """Compute a synthetic timestamp so all runs share a common time origin.
+
+    Returns the timestamp in milliseconds, or None if not enough info.
+    Uses: reference_epoch + (now - run_start_epoch) → all runs overlay at
+    the same Grafana time range.
+    """
+    ref = labels.get("reference_epoch")
+    start = labels.get("run_start_epoch")
+    if not ref or not start:
+        return None
+    try:
+        elapsed = time.time() - float(start)
+        return int((float(ref) + elapsed) * 1000)
+    except (ValueError, TypeError):
+        return None
+
+
+def inject_timestamps(metrics_bytes, timestamp_ms):
+    """Append a Prometheus timestamp (ms) to every data line.
+
+    Prometheus text format: metric{labels} value [timestamp_ms]
+    Adding timestamps causes Prometheus to store all runs' samples
+    at the same relative time, enabling natural overlay in Grafana.
+    """
+    if timestamp_ms is None:
+        return metrics_bytes
+
+    ts = str(timestamp_ms).encode("utf-8")
+    out = []
+    for line in metrics_bytes.split(b"\n"):
+        if line.startswith(b"#") or not line:
+            out.append(line)
+        else:
+            out.append(line + b" " + ts)
+    return b"\n".join(out)
+
+
+class MetricsHandler(BaseHTTPRequestHandler):
+    # Use HTTP/1.1 so Content-Length is respected and Prometheus
+    # doesn't have to rely on connection close to detect end of body.
+    protocol_version = "HTTP/1.1"
+
+    def do_GET(self):
+        src = self.client_address[0]
+        try:
+            resp = urlopen(self.server.upstream, timeout=2)
+            metrics = resp.read()
+        except (URLError, ConnectionError, OSError):
+            # reth not running — return empty 200
+            self._send(b"")
+            #print(f"  scrape from {src}: empty (reth not running)", flush=True)
+            return
+
+        all_labels = read_labels(self.server.labels_file)
+        # Internal keys — not injected as Prometheus labels
+        internal = ("run_start_epoch", "reference_epoch")
+        labels = {k: v for k, v in all_labels.items() if k not in internal}
+        label_str = build_label_str(labels)
+        label_names = sorted(labels.keys())
+
+        t0 = time.monotonic()
+        result = inject_labels(metrics, label_str, label_names)
+        result += build_elapsed_gauge(all_labels)
+        ts_ms = compute_timestamp_ms(all_labels)
+        result = inject_timestamps(result, ts_ms)
+        dt = time.monotonic() - t0
+
+        self._send(result)
+        print(f"  scrape from {src}: {len(metrics)} -> {len(result)} bytes, "
+              f"inject {dt*1000:.1f}ms", flush=True)
+
+    def _send(self, body):
+        self.send_response(200)
+        self.send_header("Content-Type", "text/plain; version=0.0.4")
+        self.send_header("Content-Length", str(len(body)))
+        self.send_header("Connection", "close")
+        self.end_headers()
+        if body:
+            self.wfile.write(body)
+
+    def log_message(self, format, *args):
+        pass  # suppress per-request logging
+
+
+def resolve_bind_address(subnet_cidr):
+    """Find the local IP address that belongs to the given subnet.
+
+    Uses ``ip -j addr show`` to enumerate interfaces and returns the first
+    address that falls within *subnet_cidr* (e.g. ``10.10.0.0/24``).
+    """
+    network = ipaddress.ip_network(subnet_cidr, strict=False)
+    try:
+        result = subprocess.run(
+            ["ip", "-j", "addr", "show"],
+            capture_output=True, text=True, check=True,
+        )
+        interfaces = json.loads(result.stdout)
+    except (subprocess.CalledProcessError, FileNotFoundError, json.JSONDecodeError) as exc:
+        print(f"Error: cannot enumerate interfaces: {exc}", file=sys.stderr)
+        sys.exit(1)
+
+    for iface in interfaces:
+        for addr_info in iface.get("addr_info", []):
+            try:
+                addr = ipaddress.ip_address(addr_info["local"])
+            except (KeyError, ValueError):
+                continue
+            if addr in network:
+                return str(addr)
+
+    print(f"Error: no interface address found in subnet {subnet_cidr}", file=sys.stderr)
+    sys.exit(1)
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Prometheus metrics proxy with label injection")
+    parser.add_argument("--labels", default="/tmp/bench-metrics-labels.json",
+                        help="Path to JSON file with labels to inject (default: /tmp/bench-metrics-labels.json)")
+    parser.add_argument("--upstream", default="http://127.0.0.1:9100/",
+                        help="Upstream reth metrics URL (default: http://127.0.0.1:9100/)")
+
+    bind_group = parser.add_mutually_exclusive_group()
+    bind_group.add_argument("--bind", default=None,
+                            help="Address to bind the proxy (default: 0.0.0.0)")
+    bind_group.add_argument("--subnet", default=None,
+                            help="Auto-detect bind address from a local interface in this subnet (e.g. 10.10.0.0/24)")
+
+    parser.add_argument("--port", type=int, default=9090,
+                        help="Port to bind the proxy (default: 9090)")
+    args = parser.parse_args()
+
+    if args.subnet:
+        bind_addr = resolve_bind_address(args.subnet)
+    elif args.bind:
+        bind_addr = args.bind
+    else:
+        bind_addr = "0.0.0.0"
+
+    server = HTTPServer((bind_addr, args.port), MetricsHandler)
+    server.upstream = args.upstream
+    server.labels_file = args.labels
+
+    print(f"bench-metrics-proxy listening on {bind_addr}:{args.port}")
+    print(f"  upstream: {args.upstream}")
+    print(f"  labels:   {args.labels}")
+    sys.stdout.flush()
+    server.serve_forever()
+
+
+if __name__ == "__main__":
+    main()

.github/scripts/bench-reth-build.sh

@@ -14,46 +14,110 @@
 #   baseline: <source-dir>/target/profiling/reth
 #   feature:  <source-dir>/target/profiling/reth, reth-bench installed to cargo bin
 #
-# Required: mc (MinIO client) configured at /home/ubuntu/.mc
+# Required: mc (MinIO client) with a configured alias
 set -euo pipefail
 
-MC="mc --config-dir /home/ubuntu/.mc"
+MC="mc"
 MODE="$1"
 SOURCE_DIR="$2"
 COMMIT="$3"
 
+# Tracy support: when BENCH_TRACY is "on" or "full", add Tracy cargo features
+# and frame pointers for accurate stack traces.
+EXTRA_FEATURES=""
+EXTRA_RUSTFLAGS=""
+if [ "${BENCH_TRACY:-off}" != "off" ]; then
+  EXTRA_FEATURES="tracy,tracy-client/ondemand"
+  EXTRA_RUSTFLAGS=" -C force-frame-pointers=yes"
+fi
+
+# Cache suffix: hash of features+rustflags so different build configs get separate cache entries
+if [ -n "$EXTRA_FEATURES" ] || [ -n "$EXTRA_RUSTFLAGS" ]; then
+  BUILD_SUFFIX="-$(echo "${EXTRA_FEATURES}${EXTRA_RUSTFLAGS}" | sha256sum | cut -c1-12)"
+else
+  BUILD_SUFFIX=""
+fi
+
+# Verify a cached reth binary was built from the expected commit.
+# `reth --version` outputs "Commit SHA: <full-sha>" on its own line.
+verify_binary() {
+  local binary="$1" expected_commit="$2"
+  local version binary_sha
+  version=$("$binary" --version 2>/dev/null) || return 1
+  binary_sha=$(echo "$version" | sed -n 's/^Commit SHA: *//p')
+  if [ -z "$binary_sha" ]; then
+    echo "Warning: could not extract commit SHA from version output"
+    return 1
+  fi
+  if [ "$binary_sha" = "$expected_commit" ]; then
+    return 0
+  fi
+  echo "Cache mismatch: binary built from ${binary_sha} but expected ${expected_commit}"
+  return 1
+}
+
 case "$MODE" in
   baseline|main)
-    BUCKET="minio/reth-binaries/${COMMIT}"
+    BUCKET="minio/reth-binaries/${COMMIT}${BUILD_SUFFIX}"
     mkdir -p "${SOURCE_DIR}/target/profiling"
 
+    CACHE_VALID=false
     if $MC stat "${BUCKET}/reth" &>/dev/null; then
       echo "Cache hit for baseline (${COMMIT}), downloading binary..."
       $MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
       chmod +x "${SOURCE_DIR}/target/profiling/reth"
-    else
-      echo "Cache miss for baseline (${COMMIT}), building from source..."
+      if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
+        CACHE_VALID=true
+      else
+        echo "Cached baseline binary is stale, rebuilding..."
+      fi
+    fi
+    if [ "$CACHE_VALID" = false ]; then
+      echo "Building baseline (${COMMIT}) from source..."
       cd "${SOURCE_DIR}"
-      cargo build --profile profiling --bin reth
+      FEATURES_ARG=""
+      WORKSPACE_ARG=""
+      if [ -n "$EXTRA_FEATURES" ]; then
+        # --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
+        FEATURES_ARG="--features ${EXTRA_FEATURES}"
+        WORKSPACE_ARG="--workspace"
+      fi
+      # shellcheck disable=SC2086
+      RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
+        cargo build --profile profiling --bin reth $WORKSPACE_ARG $FEATURES_ARG
       $MC cp target/profiling/reth "${BUCKET}/reth"
     fi
     ;;
 
   feature|branch)
     BRANCH_SHA="${4:-$COMMIT}"
-    BUCKET="minio/reth-binaries/${BRANCH_SHA}"
+    BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
 
+    CACHE_VALID=false
     if $MC stat "${BUCKET}/reth" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
       echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
       mkdir -p "${SOURCE_DIR}/target/profiling"
       $MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
       $MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench
       chmod +x "${SOURCE_DIR}/target/profiling/reth" /home/ubuntu/.cargo/bin/reth-bench
-    else
-      echo "Cache miss for ${BRANCH_SHA}, building from source..."
+      if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
+        CACHE_VALID=true
+      else
+        echo "Cached feature binary is stale, rebuilding..."
+      fi
+    fi
+    if [ "$CACHE_VALID" = false ]; then
+      echo "Building feature (${COMMIT}) from source..."
       cd "${SOURCE_DIR}"
       rustup show active-toolchain || rustup default stable
-      make profiling
+      if [ -n "$EXTRA_FEATURES" ]; then
+        # Can't use `make profiling` when adding features; build explicitly
+        # --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
+        RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
+          cargo build --profile profiling --workspace --bin reth --features "${EXTRA_FEATURES}"
+      else
+        make profiling
+      fi
       make install-reth-bench
       $MC cp target/profiling/reth "${BUCKET}/reth"
       $MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"

.github/scripts/bench-reth-charts.py

@@ -73,22 +73,24 @@ def plot_latency_and_throughput(
         for r in baseline:
             lat_s = r["new_payload_latency_us"] / 1_000_000
             base_ggas.append(r["gas_used"] / lat_s / GIGAGAS if lat_s > 0 else 0)
-        ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
-        ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
+        l, = ax1.plot(base_x, base_lat, linewidth=0.8, label=baseline_name, alpha=0.7)
+        ax1.axhline(np.median(base_lat), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
+        l, = ax2.plot(base_x, base_ggas, linewidth=0.8, label=baseline_name, alpha=0.7)
+        ax2.axhline(np.median(base_ggas), color=l.get_color(), linestyle="--", linewidth=1, alpha=0.7, label=f"{baseline_name} median")
 
-    ax1.plot(feat_x, feat_lat, linewidth=0.8, label=feature_name)
+    l, = ax1.plot(feat_x, feat_lat, linewidth=0.8, label=feature_name)
+    ax1.axhline(np.median(feat_lat), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
     ax1.set_ylabel("Latency (ms)")
     ax1.set_title("newPayload Latency per Block")
     ax1.grid(True, alpha=0.3)
-    if baseline:
-        ax1.legend()
+    ax1.legend()
 
-    ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=feature_name)
+    l, = ax2.plot(feat_x, feat_ggas, linewidth=0.8, label=feature_name)
+    ax2.axhline(np.median(feat_ggas), color=l.get_color(), linestyle="--", linewidth=1, label=f"{feature_name} median")
     ax2.set_ylabel("Ggas/s")
     ax2.set_title("Execution Throughput per Block")
     ax2.grid(True, alpha=0.3)
-    if baseline:
-        ax2.legend()
+    ax2.legend()
 
     if baseline:
         ax3 = axes[2]

.github/scripts/bench-reth-local.sh

@@ -0,0 +1,581 @@
+#!/usr/bin/env bash
+#
+# local-reth-bench.sh — Run the reth Engine API benchmark locally.
+#
+# Replicates the CI bench.yml workflow (build, snapshot, system tuning,
+# interleaved B-F-F-B execution, summary, charts) without any GitHub
+# Actions glue (no PR comments, no artifact upload, no Slack).
+#
+# Usage:
+#   local-reth-bench.sh <baseline-ref> <feature-ref> [options]
+#
+# Options:
+#   --blocks N      Number of blocks to benchmark (default: 500)
+#   --warmup N      Number of warmup blocks (default: 100)
+#   --cores N       Limit reth to N CPU cores, 0 = all available (default: 0)
+#   --samply        Enable samply profiling
+#   --tracy MODE    Tracy profiling: off, on, full (default: off)
+#   --tracy-filter F Tracy tracing filter (default: debug)
+#   --no-tune       Skip system tuning (useful on dev machines / macOS)
+#
+# Requires: the reth repo at RETH_REPO (default: ~/reth)
+#
+# Dependencies (install before first run):
+#   mc (MinIO client), schelk, cpupower, taskset, stdbuf, python3, curl,
+#   make, uv, pzstd, jq, Rust toolchain (cargo/rustup)
+#
+# The script delegates to the existing bench-reth-*.sh scripts in the reth
+# repo for the actual build, snapshot, and run steps.
+set -euo pipefail
+
+# ── PATH ──────────────────────────────────────────────────────────────
+# Ensure cargo and user-local bins (mc, uv) are visible
+export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+
+# ── Defaults ──────────────────────────────────────────────────────────
+RETH_REPO="${RETH_REPO:-$HOME/reth}"
+BLOCKS=500
+WARMUP=100
+CORES=0
+SAMPLY=false
+TRACY="off"
+TRACY_FILTER="debug"
+TUNE=true
+BASELINE_REF=""
+FEATURE_REF=""
+
+# ── Parse arguments ──────────────────────────────────────────────────
+usage() {
+  cat <<EOF
+Usage: $(basename "$0") <baseline-ref> <feature-ref> [options]
+
+Options:
+  --blocks N         Number of blocks to benchmark (default: 500)
+  --warmup N         Number of warmup blocks (default: 100)
+  --cores N          Limit reth to N CPU cores (default: 0 = all)
+  --samply           Enable samply profiling
+  --tracy MODE       Tracy profiling: off, on, full (default: off)
+                       on   = tracing only (lower overhead)
+                       full = tracing + CPU sampling (higher overhead)
+  --tracy-filter F   Tracy tracing filter (default: debug)
+  --no-tune          Skip system tuning
+EOF
+  exit 1
+}
+
+while [[ $# -gt 0 ]]; do
+  case "$1" in
+    --blocks)       BLOCKS="$2"; shift 2 ;;
+    --warmup)       WARMUP="$2"; shift 2 ;;
+    --cores)        CORES="$2"; shift 2 ;;
+    --samply)       SAMPLY=true; shift ;;
+    --tracy)        TRACY="$2"; shift 2 ;;
+    --tracy-filter) TRACY_FILTER="$2"; shift 2 ;;
+    --no-tune)      TUNE=false; shift ;;
+    --help|-h)  usage ;;
+    -*)         echo "Unknown option: $1"; usage ;;
+    *)
+      if [ -z "$BASELINE_REF" ]; then
+        BASELINE_REF="$1"
+      elif [ -z "$FEATURE_REF" ]; then
+        FEATURE_REF="$1"
+      else
+        echo "Unexpected argument: $1"; usage
+      fi
+      shift
+      ;;
+  esac
+done
+
+if [ -z "$BASELINE_REF" ] || [ -z "$FEATURE_REF" ]; then
+  echo "Error: both <baseline-ref> and <feature-ref> are required."
+  usage
+fi
+
+# Validate --tracy value
+case "$TRACY" in
+  off|on|full) ;;
+  *) echo "Error: --tracy must be off, on, or full (got: $TRACY)"; usage ;;
+esac
+
+# Samply + tracy=full are mutually exclusive (both use perf sampling)
+if [ "$SAMPLY" = "true" ] && [ "$TRACY" = "full" ]; then
+  echo "Warning: samply and tracy=full both use perf sampling; downgrading tracy to 'on'."
+  TRACY="on"
+fi
+
+# ── Check dependencies ───────────────────────────────────────────────
+missing=()
+for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq cargo; do
+  command -v "$cmd" &>/dev/null || missing+=("$cmd")
+done
+if [ ${#missing[@]} -gt 0 ]; then
+  echo "Error: missing required tools: ${missing[*]}"
+  echo "See the CI 'Install dependencies' step in .github/workflows/bench.yml for install instructions."
+  exit 1
+fi
+
+if [ "$TRACY" != "off" ]; then
+  if ! command -v tracy-capture &>/dev/null; then
+    echo "Error: tracy-capture is required for --tracy $TRACY"
+    exit 1
+  fi
+fi
+
+# Ensure tools that run via sudo are in a sudo-visible path.
+# The bench scripts use `sudo schelk` / `sudo samply` but cargo installs
+# them to ~/.cargo/bin which sudo's secure_path doesn't include.
+for cmd in schelk samply; do
+  if command -v "$cmd" &>/dev/null && ! sudo sh -c "command -v $cmd" &>/dev/null; then
+    echo "Installing $cmd to /usr/local/bin (needed for sudo)..."
+    sudo install "$(command -v "$cmd")" /usr/local/bin/
+  fi
+done
+
+if [ ! -d "$RETH_REPO/.git" ]; then
+  echo "Error: RETH_REPO=$RETH_REPO is not a git repository."
+  echo "Set RETH_REPO or clone reth to ~/reth"
+  exit 1
+fi
+
+# ── Resolve paths ────────────────────────────────────────────────────
+SELF_DIR="$(cd "$(dirname "$0")" && pwd)"
+SCRIPTS_DIR="${RETH_REPO}/.github/scripts"
+BENCH_WORK_DIR="${RETH_REPO}/../bench-work-$(date +%Y%m%d-%H%M%S)"
+BASELINE_SRC="${RETH_REPO}/../reth-baseline"
+FEATURE_SRC="${RETH_REPO}/../reth-feature"
+
+mkdir -p "$BENCH_WORK_DIR"
+BENCH_WORK_DIR="$(cd "$BENCH_WORK_DIR" && pwd)"
+
+# ── Global cleanup trap (restores system tuning on any exit) ─────────
+TUNING_APPLIED=false
+CSTATE_PID=
+METRICS_PROXY_PID=
+cleanup_global() {
+  [ -n "$METRICS_PROXY_PID" ] && kill "$METRICS_PROXY_PID" 2>/dev/null || true
+  if [ "$TUNING_APPLIED" = true ]; then
+    echo
+    echo "▸ Restoring system settings..."
+    [ -n "$CSTATE_PID" ] && kill "$CSTATE_PID" 2>/dev/null || true
+    sudo systemctl start irqbalance cron atd 2>/dev/null || true
+    echo "  System settings restored."
+  fi
+}
+trap cleanup_global EXIT
+
+echo "═══════════════════════════════════════════════════════════"
+echo "  reth local benchmark"
+echo "═══════════════════════════════════════════════════════════"
+echo "  Baseline ref : $BASELINE_REF"
+echo "  Feature ref  : $FEATURE_REF"
+echo "  Blocks       : $BLOCKS"
+echo "  Warmup       : $WARMUP"
+echo "  Cores        : $CORES"
+echo "  Samply       : $SAMPLY"
+echo "  Tracy        : $TRACY"
+echo "  Tracy filter : $TRACY_FILTER"
+echo "  System tune  : $TUNE"
+echo "  Work dir     : $BENCH_WORK_DIR"
+echo "  Reth repo    : $RETH_REPO"
+echo "═══════════════════════════════════════════════════════════"
+echo
+
+# Enable sccache if available (matches CI's RUSTC_WRAPPER=sccache)
+if command -v sccache &>/dev/null; then
+  export RUSTC_WRAPPER="sccache"
+fi
+
+# Export env vars expected by the bench-reth-*.sh scripts
+export BENCH_BLOCKS="$BLOCKS"
+export BENCH_WARMUP_BLOCKS="$WARMUP"
+export BENCH_CORES="$CORES"
+export BENCH_SAMPLY="$SAMPLY"
+export BENCH_TRACY="$TRACY"
+export BENCH_TRACY_FILTER="$TRACY_FILTER"
+export BENCH_WORK_DIR
+export SCHELK_MOUNT="${SCHELK_MOUNT:-/reth-bench}"
+export BENCH_RPC_URL="${BENCH_RPC_URL:-https://ethereum.reth.rs/rpc}"
+export BENCH_METRICS_ADDR="127.0.0.1:9100"
+
+# ── Step 1: Resolve refs to full SHAs ────────────────────────────────
+echo "▸ Resolving git refs..."
+cd "$RETH_REPO"
+
+resolve_ref() {
+  local ref="$1"
+  git fetch origin "$ref" --quiet 2>/dev/null || true
+  git rev-parse "$ref" 2>/dev/null \
+    || git rev-parse "origin/$ref" 2>/dev/null \
+    || { echo "Error: cannot resolve ref '$ref'"; exit 1; }
+}
+
+BASELINE_SHA="$(resolve_ref "$BASELINE_REF")"
+FEATURE_SHA="$(resolve_ref "$FEATURE_REF")"
+echo "  Baseline SHA : $BASELINE_SHA"
+echo "  Feature SHA  : $FEATURE_SHA"
+echo
+
+# ── Step 2: Prepare source directories ───────────────────────────────
+echo "▸ Preparing source directories..."
+
+prepare_source() {
+  local src_dir="$1" ref="$2"
+  if [ -d "$src_dir" ]; then
+    git -C "$src_dir" fetch origin "$ref" 2>/dev/null || true
+  else
+    git clone --recurse-submodules "$RETH_REPO" "$src_dir"
+  fi
+  git -C "$src_dir" checkout "$ref" --force
+  git -C "$src_dir" submodule update --init --recursive
+}
+
+prepare_source "$BASELINE_SRC" "$BASELINE_SHA"
+prepare_source "$FEATURE_SRC" "$FEATURE_SHA"
+BASELINE_SRC="$(cd "$BASELINE_SRC" && pwd)"
+FEATURE_SRC="$(cd "$FEATURE_SRC" && pwd)"
+echo "  Baseline src : $BASELINE_SRC"
+echo "  Feature src  : $FEATURE_SRC"
+echo
+
+# ── Step 3: Check / download snapshot ────────────────────────────────
+echo "▸ Checking snapshot..."
+cd "$RETH_REPO"
+SNAPSHOT_NEEDED=false
+if ! "${SCRIPTS_DIR}/bench-reth-snapshot.sh" --check; then
+  SNAPSHOT_NEEDED=true
+  echo "  Snapshot needs update."
+else
+  echo "  Snapshot is up-to-date."
+fi
+echo
+
+# ── Step 4: Build binaries (+ snapshot download) in parallel ─────────
+echo "▸ Building binaries (parallel)..."
+cd "$RETH_REPO"
+
+FAIL=0
+
+"${SCRIPTS_DIR}/bench-reth-build.sh" baseline "$BASELINE_SRC" "$BASELINE_SHA" &
+PID_BASELINE=$!
+
+"${SCRIPTS_DIR}/bench-reth-build.sh" feature "$FEATURE_SRC" "$FEATURE_SHA" &
+PID_FEATURE=$!
+
+PID_SNAPSHOT=
+if [ "$SNAPSHOT_NEEDED" = "true" ]; then
+  echo "  Also downloading snapshot in parallel..."
+  "${SCRIPTS_DIR}/bench-reth-snapshot.sh" &
+  PID_SNAPSHOT=$!
+fi
+
+wait $PID_BASELINE || FAIL=1
+wait $PID_FEATURE  || FAIL=1
+[ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
+
+if [ $FAIL -ne 0 ]; then
+  echo "Error: one or more parallel tasks failed (builds / snapshot)"
+  exit 1
+fi
+echo "  Binaries built successfully."
+echo
+
+# ── Step 5: System tuning (optional) ────────────────────────────────
+if [ "$TUNE" = "true" ]; then
+  echo "▸ Applying system tuning..."
+
+  sudo cpupower frequency-set -g performance 2>/dev/null || true
+
+  # Disable turbo boost (Intel + AMD)
+  echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
+  echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
+
+  sudo swapoff -a 2>/dev/null || true
+  echo 0 | sudo tee /proc/sys/kernel/randomize_va_space 2>/dev/null || true
+
+  # Disable SMT (hyperthreading)
+  for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
+    [ -f "$cpu" ] || continue
+    first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
+    current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
+    if [ "$current" != "$first" ]; then
+      echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" 2>/dev/null || true
+    fi
+  done
+  echo "  Online CPUs: $(nproc)"
+
+  # Disable transparent huge pages
+  for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
+    if [ -d "$p" ]; then
+      echo never | sudo tee "$p/enabled" 2>/dev/null || true
+      echo never | sudo tee "$p/defrag" 2>/dev/null || true
+      break
+    fi
+  done
+
+  # Prevent deep C-states
+  sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
+  CSTATE_PID=$!
+
+  # Pin IRQs to core 0
+  for irq in /proc/irq/*/smp_affinity_list; do
+    echo 0 | sudo tee "$irq" 2>/dev/null || true
+  done
+
+  # Stop noisy background services
+  sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
+
+  TUNING_APPLIED=true
+
+  # Log environment for reproducibility (matches CI)
+  echo "  === Benchmark environment ==="
+  echo "  Kernel : $(uname -r)"
+  lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA' | sed 's/^/  /'
+  echo "  Governor : $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor 2>/dev/null || echo unknown)"
+  echo "  Freq     : $(cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq 2>/dev/null || echo unknown)"
+  echo "  THP      : $(cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo unknown)"
+  free -h | sed 's/^/  /'
+  echo "  System tuning applied."
+  echo
+fi
+
+# ── Step 5b: Tracefs mount (tracy=full only) ─────────────────────────
+if [ "$TRACY" = "full" ] && [ "$(uname)" = "Linux" ]; then
+  echo "▸ Mounting tracefs for Tracy full mode..."
+  sudo mount -t tracefs tracefs /sys/kernel/tracing -o mode=755 2>/dev/null || true
+fi
+
+# ── Tracy upload & viewer helpers ────────────────────────────────────
+TRACY_VIEWER_BASE="${TRACY_VIEWER_BASE:-}"
+
+tracy_viewer_url() {
+  local profile_url="$1"
+  if [ -z "$TRACY_VIEWER_BASE" ]; then
+    echo ""
+    return
+  fi
+  local encoded
+  encoded=$(python3 -c "import urllib.parse, sys; print(urllib.parse.quote(sys.argv[1], safe=''))" "$profile_url")
+  echo "${TRACY_VIEWER_BASE}?profile_url=${encoded}"
+}
+
+upload_tracy() {
+  local label="$1" output_dir="$2" sha="$3"
+  local tracy_file="$output_dir/tracy-profile.tracy"
+
+  if [ ! -f "$tracy_file" ]; then
+    echo "  Tracy: no profile found, skipping upload."
+    return
+  fi
+
+  local timestamp short_sha remote_name bucket mc_alias
+  timestamp=$(date +%Y%m%d-%H%M%S)
+  short_sha="${sha:0:7}"
+  remote_name="${label}-${short_sha}-${timestamp}.tracy"
+  bucket="${TRACY_BUCKET:-tracy-profiles}"
+  mc_alias="${MC_ALIAS:-minio}"
+  local minio_base="${TRACY_MINIO_URL:-http://minio.minio.svc.cluster.local:9000}"
+
+  echo "  Tracy: uploading profile..."
+  if mc cp "$tracy_file" "${mc_alias}/${bucket}/${remote_name}"; then
+    local url="${minio_base}/${bucket}/${remote_name}"
+    echo "$url" > "$output_dir/tracy_url.txt"
+    local viewer
+    viewer=$(tracy_viewer_url "$url")
+    if [ -n "$viewer" ]; then
+      echo "$viewer" > "$output_dir/tracy_viewer_url.txt"
+      echo "  Tracy: uploaded → $viewer"
+    else
+      echo "  Tracy: uploaded → $url"
+    fi
+  else
+    echo "  Tracy: upload failed (non-fatal)."
+  fi
+
+  # Delete large profile to free disk
+  rm -f "$tracy_file"
+}
+
+# ── Step 6: Pre-flight cleanup ───────────────────────────────────────
+echo "▸ Pre-flight cleanup..."
+pkill -f bench-metrics-proxy 2>/dev/null || true
+sudo pkill -9 reth 2>/dev/null || true
+sleep 1
+if mountpoint -q "$SCHELK_MOUNT" 2>/dev/null; then
+  sudo umount -l "$SCHELK_MOUNT" 2>/dev/null || true
+  sudo schelk recover -y 2>/dev/null || true
+fi
+echo
+
+# ── Step 7: Interleaved benchmark runs (B-F-F-B) ────────────────────
+# This ordering reduces systematic bias from thermal drift and cache warming.
+BASELINE_BIN="${BASELINE_SRC}/target/profiling/reth"
+FEATURE_BIN="${FEATURE_SRC}/target/profiling/reth"
+
+# Start metrics proxy (reth → label injection → Prometheus)
+LABELS_FILE="/tmp/bench-metrics-labels.json"
+echo '{}' > "$LABELS_FILE"
+METRICS_SUBNET="${METRICS_SUBNET:-10.10.0.0/24}"
+METRICS_PORT="${METRICS_PORT:-9090}"
+python3 "${SELF_DIR}/bench-metrics-proxy.py" \
+  --labels "$LABELS_FILE" \
+  --upstream "http://${BENCH_METRICS_ADDR}/" \
+  --subnet "$METRICS_SUBNET" \
+  --port "$METRICS_PORT" &
+METRICS_PROXY_PID=$!
+echo "▸ Metrics proxy started (PID $METRICS_PROXY_PID) on subnet ${METRICS_SUBNET}, port ${METRICS_PORT}"
+
+# Unique benchmark ID: local-<timestamp> for local runs, ci-<run_id> for CI
+BENCH_ID="local-$(basename "$BENCH_WORK_DIR" | sed 's/bench-work-//')"
+# Reference epoch: shared time origin so all runs overlay in Grafana.
+# The proxy maps each run's elapsed time onto this common origin.
+BENCH_REFERENCE_EPOCH=$(date +%s)
+
+write_labels() {
+  local run_label="$1" run_type="$2" ref="$3" sha="$4"
+  LAST_RUN_START=$(date +%s)
+  cat > "$LABELS_FILE" <<-EOF
+	{"benchmark_run":"${run_label}","run_type":"${run_type}","git_ref":"${ref}","bench_sha":"${sha}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+	EOF
+}
+
+run_bench() {
+  local label="$1" binary="$2" output_dir="$3"
+  echo "▸ Running benchmark: ${label}..."
+  cd "$RETH_REPO"
+  if command -v taskset &>/dev/null; then
+    taskset -c 0 "${SCRIPTS_DIR}/bench-reth-run.sh" "$label" "$binary" "$output_dir"
+  else
+    "${SCRIPTS_DIR}/bench-reth-run.sh" "$label" "$binary" "$output_dir"
+  fi
+  echo "  ✓ ${label} complete."
+  echo
+}
+
+write_labels "baseline-1" "baseline" "$BASELINE_REF" "$BASELINE_SHA"
+run_bench "baseline-1" "$BASELINE_BIN" "$BENCH_WORK_DIR/baseline-1"
+
+write_labels "feature-1" "feature" "$FEATURE_REF" "$FEATURE_SHA"
+run_bench "feature-1"  "$FEATURE_BIN"  "$BENCH_WORK_DIR/feature-1"
+
+write_labels "feature-2" "feature" "$FEATURE_REF" "$FEATURE_SHA"
+run_bench "feature-2"  "$FEATURE_BIN"  "$BENCH_WORK_DIR/feature-2"
+
+write_labels "baseline-2" "baseline" "$BASELINE_REF" "$BASELINE_SHA"
+run_bench "baseline-2" "$BASELINE_BIN" "$BENCH_WORK_DIR/baseline-2"
+
+# ── Compute Grafana URL ──────────────────────────────────────────────
+GRAFANA_BASE_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr"
+GRAFANA_DATASOURCE="ef57fux92e9z4e"
+LAST_RUN_DURATION=$(( $(date +%s) - LAST_RUN_START ))
+FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
+TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
+GRAFANA_URL="${GRAFANA_BASE_URL}?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=${GRAFANA_DATASOURCE}&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
+
+# ── Step 8: Scan logs for errors ─────────────────────────────────────
+echo "▸ Scanning logs for errors..."
+ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
+found_errors=false
+for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
+  LOG="$BENCH_WORK_DIR/$run_dir/node.log"
+  [ -f "$LOG" ] || continue
+  panics=$(grep -c -E 'panicked at' "$LOG" 2>/dev/null || true)
+  errors=$(grep -c ' ERROR ' "$LOG" 2>/dev/null || true)
+  if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
+    if [ "$found_errors" = false ]; then
+      printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
+      found_errors=true
+    fi
+    printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' \
+      "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
+    if [ "$panics" -gt 0 ]; then
+      printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
+      grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
+      printf '```\n' >> "$ERRORS_FILE"
+    fi
+    if [ "$errors" -gt 0 ]; then
+      printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
+      grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
+      printf '```\n' >> "$ERRORS_FILE"
+    fi
+    printf '\n</details>\n\n' >> "$ERRORS_FILE"
+  fi
+done
+if [ "$found_errors" = true ]; then
+  echo "  ⚠ Errors found — see $ERRORS_FILE"
+else
+  echo "  No errors found."
+fi
+echo
+
+# ── Step 9: Parse results ───────────────────────────────────────────
+echo "▸ Parsing results..."
+cd "$RETH_REPO"
+
+SUMMARY_ARGS=(
+  --output-summary "$BENCH_WORK_DIR/summary.json"
+  --output-markdown "$BENCH_WORK_DIR/comment.md"
+  --repo "paradigmxyz/reth"
+  --baseline-ref "$BASELINE_SHA"
+  --baseline-name "$BASELINE_REF"
+  --feature-name "$FEATURE_REF"
+  --feature-ref "$FEATURE_SHA"
+  --baseline-csv "$BENCH_WORK_DIR/baseline-1/combined_latency.csv" "$BENCH_WORK_DIR/baseline-2/combined_latency.csv"
+  --feature-csv "$BENCH_WORK_DIR/feature-1/combined_latency.csv" "$BENCH_WORK_DIR/feature-2/combined_latency.csv"
+  --gas-csv "$BENCH_WORK_DIR/feature-1/total_gas.csv"
+  --grafana-url "$GRAFANA_URL"
+)
+
+python3 "${SCRIPTS_DIR}/bench-reth-summary.py" "${SUMMARY_ARGS[@]}"
+echo
+
+# ── Step 10: Generate charts ─────────────────────────────────────────
+echo "▸ Generating charts..."
+CHART_ARGS=(
+  --output-dir "$BENCH_WORK_DIR/charts"
+  --feature "$BENCH_WORK_DIR/feature-1/combined_latency.csv" "$BENCH_WORK_DIR/feature-2/combined_latency.csv"
+  --baseline "$BENCH_WORK_DIR/baseline-1/combined_latency.csv" "$BENCH_WORK_DIR/baseline-2/combined_latency.csv"
+  --baseline-name "$BASELINE_REF"
+  --feature-name "$FEATURE_REF"
+)
+
+if python3 -c "import matplotlib" 2>/dev/null; then
+  python3 "${SCRIPTS_DIR}/bench-reth-charts.py" "${CHART_ARGS[@]}"
+elif command -v uv &>/dev/null; then
+  uv run --with matplotlib python3 "${SCRIPTS_DIR}/bench-reth-charts.py" "${CHART_ARGS[@]}"
+else
+  echo "  Warning: matplotlib not available, skipping chart generation."
+fi
+echo
+
+# ── Step 11: Upload Tracy profiles ────────────────────────────────────
+if [ "$TRACY" != "off" ]; then
+  echo "▸ Uploading Tracy profiles..."
+  upload_tracy "baseline-1" "$BENCH_WORK_DIR/baseline-1" "$BASELINE_SHA"
+  upload_tracy "feature-1"  "$BENCH_WORK_DIR/feature-1"  "$FEATURE_SHA"
+  upload_tracy "feature-2"  "$BENCH_WORK_DIR/feature-2"  "$FEATURE_SHA"
+  upload_tracy "baseline-2" "$BENCH_WORK_DIR/baseline-2" "$BASELINE_SHA"
+  echo
+fi
+
+# ── Done (system restore happens via EXIT trap) ─────────────────────
+echo "═══════════════════════════════════════════════════════════"
+echo "  Benchmark complete!"
+echo "═══════════════════════════════════════════════════════════"
+echo "  Results  : $BENCH_WORK_DIR/summary.json"
+echo "  Markdown : $BENCH_WORK_DIR/comment.md"
+echo "  Charts   : $BENCH_WORK_DIR/charts/"
+if [ -f "$ERRORS_FILE" ]; then
+  echo "  Errors   : $ERRORS_FILE"
+fi
+echo "  Grafana  : $GRAFANA_URL"
+if [ "$TRACY" != "off" ]; then
+  echo "  ─── Tracy Profiles ───"
+  for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
+    url_file="$BENCH_WORK_DIR/$run_dir/tracy_viewer_url.txt"
+    if [ -f "$url_file" ]; then
+      echo "  $run_dir : $(cat "$url_file")"
+    fi
+  done
+fi
+echo "═══════════════════════════════════════════════════════════"

.github/scripts/bench-reth-run.sh

@@ -6,29 +6,86 @@
 # Usage: bench-reth-run.sh <label> <binary> <output-dir>
 #
 # Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
+# Optional env: BENCH_BIG_BLOCKS (true/false), BENCH_WORK_DIR (for big blocks path)
+#               BENCH_RETH_NEW_PAYLOAD (true/false, default true)
+#               BENCH_WAIT_TIME (duration like 500ms, default empty)
+#               BENCH_BASELINE_ARGS (extra reth node args for baseline runs)
+#               BENCH_FEATURE_ARGS (extra reth node args for feature runs)
+#               BENCH_OTLP_TRACES_ENDPOINT (OTLP HTTP endpoint for traces, e.g. https://host/insert/opentelemetry/v1/traces)
+#               BENCH_OTLP_LOGS_ENDPOINT (OTLP HTTP endpoint for logs, e.g. https://host/insert/opentelemetry/v1/logs)
 set -euo pipefail
 
 LABEL="$1"
 BINARY="$2"
 OUTPUT_DIR="$3"
 DATADIR="$SCHELK_MOUNT/datadir"
-LOG="/tmp/reth-bench-node-${LABEL}.log"
+mkdir -p "$OUTPUT_DIR"
+LOG="${OUTPUT_DIR}/node.log"
 
 cleanup() {
   kill "$TAIL_PID" 2>/dev/null || true
-  if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
-    sudo kill "$RETH_PID"
+  # Stop tracy-capture first (SIGINT makes it disconnect and flush to disk)
+  # Must happen before killing reth, otherwise reth keeps streaming data.
+  if [ -n "${TRACY_PID:-}" ] && kill -0 "$TRACY_PID" 2>/dev/null; then
+    echo "Stopping tracy-capture..."
+    kill -INT "$TRACY_PID" 2>/dev/null || true
     for i in $(seq 1 30); do
-      sudo kill -0 "$RETH_PID" 2>/dev/null || break
+      kill -0 "$TRACY_PID" 2>/dev/null || break
+      if [ $((i % 10)) -eq 0 ]; then
+        echo "Waiting for tracy-capture to finish writing... (${i}s)"
+      fi
       sleep 1
     done
-    sudo kill -9 "$RETH_PID" 2>/dev/null || true
+    if kill -0 "$TRACY_PID" 2>/dev/null; then
+      echo "tracy-capture still running after 30s, killing..."
+      kill -9 "$TRACY_PID" 2>/dev/null || true
+    fi
+    wait "$TRACY_PID" 2>/dev/null || true
+  fi
+  if [ -n "${RETH_PID:-}" ] && sudo kill -0 "$RETH_PID" 2>/dev/null; then
+    if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
+      # Send SIGINT to the inner reth process by exact name (not -f which
+      # would also match samply's cmdline containing "reth"). Samply will
+      # capture reth's exit and save the profile.
+      sudo pkill -INT -x reth 2>/dev/null || true
+      # Wait for samply to finish writing the profile and exit
+      for i in $(seq 1 120); do
+        sudo pgrep -x samply > /dev/null 2>&1 || break
+        if [ $((i % 10)) -eq 0 ]; then
+          echo "Waiting for samply to finish writing profile... (${i}s)"
+        fi
+        sleep 1
+      done
+      if sudo pgrep -x samply > /dev/null 2>&1; then
+        echo "Samply still running after 120s, sending SIGTERM..."
+        sudo pkill -x samply 2>/dev/null || true
+      fi
+    else
+      sudo kill "$RETH_PID"
+      for i in $(seq 1 30); do
+        sudo kill -0 "$RETH_PID" 2>/dev/null || break
+        sleep 1
+      done
+    fi
+    sudo kill -9 "$RETH_PID" 2>/dev/null || true
+    sleep 1
+  fi
+  # Fix ownership of reth-created files (reth runs as root)
+  sudo chown -R "$(id -un):$(id -gn)" "$OUTPUT_DIR" 2>/dev/null || true
+  if mountpoint -q "$SCHELK_MOUNT"; then
+    sudo umount -l "$SCHELK_MOUNT" || true
+    sudo schelk recover -y || true
   fi
-  mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
 }
 TAIL_PID=
+TRACY_PID=
 trap cleanup EXIT
 
+# Clean up stale schelk state from a previous cancelled run.
+# If schelk thinks it's still mounted (e.g. a cancelled run skipped cleanup),
+# recover first to reset state.
+sudo schelk recover -y -k || true
+
 # Mount
 sudo schelk mount -y
 sync
@@ -41,18 +98,101 @@ grep Cached /proc/meminfo
 # CPU layout: core 0 = OS/IRQs/reth-bench/aux, cores 1+ = reth node
 RETH_BENCH="$(which reth-bench)"
 ONLINE=$(nproc --all)
-RETH_CPUS="1-$(( ONLINE - 1 ))"
-sudo taskset -c "$RETH_CPUS" nice -n -20 "$BINARY" node \
-  --datadir "$DATADIR" \
-  --engine.accept-execution-requests-hash \
-  --http \
-  --http.port 8545 \
-  --ws \
-  --ws.api all \
-  --authrpc.port 8551 \
-  --disable-discovery \
-  --no-persist-peers \
-  > "$LOG" 2>&1 &
+MAX_RETH=$(( ONLINE - 1 ))
+if [ "${BENCH_CORES:-0}" -gt 0 ] && [ "$BENCH_CORES" -lt "$MAX_RETH" ]; then
+  MAX_RETH=$BENCH_CORES
+fi
+RETH_CPUS="1-${MAX_RETH}"
+
+BIG_BLOCKS="${BENCH_BIG_BLOCKS:-false}"
+
+RETH_ARGS=(
+  node
+  --datadir "$DATADIR"
+  --log.file.directory "$OUTPUT_DIR/reth-logs"
+  --engine.accept-execution-requests-hash
+  --http
+  --http.port 8545
+  --ws
+  --ws.api all
+  --authrpc.port 8551
+  --disable-discovery
+  --no-persist-peers
+)
+
+# Gate flag on binary support (older baselines may not have it).
+# Uses --help which exits immediately via clap without node init.
+SYNC_STATE_IDLE=false
+if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then
+  RETH_ARGS+=(--debug.startup-sync-state-idle)
+  SYNC_STATE_IDLE=true
+fi
+
+# Big blocks mode requires the testing API, skip-invalid-transactions, and
+# skip-gas-limit-ramp-check + gas-limit override to avoid the 6800-block ramp.
+if [ "$BIG_BLOCKS" = "true" ]; then
+  RETH_ARGS+=(--http.api eth,net,web3,reth,testing --rpc.max-request-size max --testing.skip-invalid-transactions --testing.skip-gas-limit-ramp-check --testing.gas-limit 1000000000)
+fi
+
+# Append per-label extra node args (baseline or feature)
+EXTRA_NODE_ARGS=""
+case "$LABEL" in
+  baseline*) EXTRA_NODE_ARGS="${BENCH_BASELINE_ARGS:-}" ;;
+  feature*)  EXTRA_NODE_ARGS="${BENCH_FEATURE_ARGS:-}" ;;
+esac
+if [ -n "$EXTRA_NODE_ARGS" ]; then
+  # Word-split the string into individual args
+  # shellcheck disable=SC2206
+  RETH_ARGS+=($EXTRA_NODE_ARGS)
+fi
+
+if [ -n "${BENCH_METRICS_ADDR:-}" ]; then
+  RETH_ARGS+=(--metrics "$BENCH_METRICS_ADDR")
+fi
+
+# OTLP traces and logs export
+if [ -n "${BENCH_OTLP_TRACES_ENDPOINT:-}" ]; then
+  RETH_ARGS+=(--tracing-otlp="${BENCH_OTLP_TRACES_ENDPOINT}" --tracing-otlp.service-name=reth-bench)
+fi
+if [ -n "${BENCH_OTLP_LOGS_ENDPOINT:-}" ]; then
+  RETH_ARGS+=(--logs-otlp="${BENCH_OTLP_LOGS_ENDPOINT}" --logs-otlp.filter=debug)
+fi
+
+# Tracy profiling: add --log.tracy flags and set environment
+if [ "${BENCH_TRACY:-off}" != "off" ]; then
+  RETH_ARGS+=(--log.tracy --log.tracy.filter "${BENCH_TRACY_FILTER:-debug}")
+  if [ "${BENCH_TRACY}" = "on" ]; then
+    export TRACY_NO_SYS_TRACE=1
+  elif [ "${BENCH_TRACY}" = "full" ]; then
+    export TRACY_SAMPLING_HZ="${BENCH_TRACY_SAMPLING_HZ:-1}"
+  fi
+fi
+
+SUDO_ENV=()
+if [ -n "${OTEL_RESOURCE_ATTRIBUTES:-}" ]; then
+  SUDO_ENV+=("OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES}")
+  SUDO_ENV+=("OTEL_BSP_MAX_QUEUE_SIZE=65536" "OTEL_BLRP_MAX_QUEUE_SIZE=65536")
+fi
+
+# Limit reth memory to 95% of available RAM to prevent OOM kills
+TOTAL_MEM_KB=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
+MEM_LIMIT=$(( TOTAL_MEM_KB * 95 / 100 * 1024 ))
+echo "Memory limit: $(( MEM_LIMIT / 1024 / 1024 ))MB (95% of $(( TOTAL_MEM_KB / 1024 ))MB)"
+
+if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
+  RETH_ARGS+=(--log.samply)
+  SAMPLY="$(which samply)"
+  sudo systemd-run --scope -p MemoryMax="$MEM_LIMIT" -p AllowedCPUs="$RETH_CPUS" \
+    env "${SUDO_ENV[@]}" nice -n -20 \
+    "$SAMPLY" record --save-only --presymbolicate --rate 10000 \
+    --output "$OUTPUT_DIR/samply-profile.json.gz" \
+    -- "$BINARY" "${RETH_ARGS[@]}" \
+    > "$LOG" 2>&1 &
+else
+  sudo systemd-run --scope -p MemoryMax="$MEM_LIMIT" -p AllowedCPUs="$RETH_CPUS" \
+    env "${SUDO_ENV[@]}" nice -n -20 "$BINARY" "${RETH_ARGS[@]}" \
+    > "$LOG" 2>&1 &
+fi
 
 RETH_PID=$!
 stdbuf -oL tail -f "$LOG" | sed -u "s/^/[reth] /" &
@@ -63,7 +203,7 @@ for i in $(seq 1 60); do
     -H 'Content-Type: application/json' \
     -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
     > /dev/null 2>&1; then
-    echo "reth (${LABEL}) is ready after ${i}s"
+    echo "reth (${LABEL}) RPC is up after ${i}s"
     break
   fi
   if [ "$i" -eq 60 ]; then
@@ -74,21 +214,87 @@ for i in $(seq 1 60); do
   sleep 1
 done
 
-# Warmup
-sudo nice -n -20 "$RETH_BENCH" new-payload-fcu \
-  --rpc-url "$BENCH_RPC_URL" \
-  --engine-rpc-url http://127.0.0.1:8551 \
-  --jwt-secret "$DATADIR/jwt.hex" \
-  --advance "${BENCH_WARMUP_BLOCKS:-50}" \
-  --reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
+# Wait for the pipeline to finish (eth_syncing returns false) so the
+# engine is in live mode and can accept newPayload calls.
+# Only possible when --debug.startup-sync-state-idle is supported.
+if [ "$SYNC_STATE_IDLE" = "true" ]; then
+  for i in $(seq 1 300); do
+    SYNC_RESULT=$(curl -sf http://127.0.0.1:8545 -X POST \
+      -H 'Content-Type: application/json' \
+      -d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' 2>/dev/null || true)
+    if [ -n "$SYNC_RESULT" ] && jq -e '.result == false' <<< "$SYNC_RESULT" > /dev/null 2>&1; then
+      echo "reth (${LABEL}) pipeline finished after ${i}s, engine is live"
+      break
+    fi
+    if [ "$i" -eq 300 ]; then
+      echo "::error::reth (${LABEL}) pipeline did not finish within 300s"
+      cat "$LOG"
+      exit 1
+    fi
+    sleep 1
+  done
+else
+  echo "reth (${LABEL}) binary does not support --debug.startup-sync-state-idle, skipping sync wait"
+fi
 
-# Benchmark
-sudo nice -n -20 "$RETH_BENCH" new-payload-fcu \
-  --rpc-url "$BENCH_RPC_URL" \
-  --engine-rpc-url http://127.0.0.1:8551 \
-  --jwt-secret "$DATADIR/jwt.hex" \
-  --advance "$BENCH_BLOCKS" \
-  --reth-new-payload \
-  --output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
+# Run reth-bench with high priority but as the current user so output
+# files are not root-owned (avoids EACCES on next checkout).
+BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
+
+# Build optional flags
+EXTRA_BENCH_ARGS=()
+if [ "${BENCH_RETH_NEW_PAYLOAD:-true}" != "false" ]; then
+  EXTRA_BENCH_ARGS+=(--reth-new-payload)
+fi
+if [ -n "${BENCH_WAIT_TIME:-}" ]; then
+  EXTRA_BENCH_ARGS+=(--wait-time "$BENCH_WAIT_TIME")
+fi
+
+if [ "$BIG_BLOCKS" = "true" ]; then
+  # Big blocks mode: replay pre-generated payloads
+  BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
+
+  # Start tracy-capture so profile only covers the benchmark
+  if [ "${BENCH_TRACY:-off}" != "off" ]; then
+    echo "Starting tracy-capture..."
+    tracy-capture -f -o "$OUTPUT_DIR/tracy-profile.tracy" &
+    TRACY_PID=$!
+    sleep 0.5  # give tracy-capture time to connect
+  fi
+
+  echo "Running big blocks benchmark (replay-payloads)..."
+  $BENCH_NICE "$RETH_BENCH" replay-payloads \
+    "${EXTRA_BENCH_ARGS[@]}" \
+    --payload-dir "$BIG_BLOCKS_DIR/payloads" \
+    --engine-rpc-url http://127.0.0.1:8551 \
+    --jwt-secret "$DATADIR/jwt.hex" \
+    --output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
+else
+  # Standard mode: warmup + new-payload-fcu
+  # Warmup
+  $BENCH_NICE "$RETH_BENCH" new-payload-fcu \
+    --rpc-url "$BENCH_RPC_URL" \
+    --engine-rpc-url http://127.0.0.1:8551 \
+    --jwt-secret "$DATADIR/jwt.hex" \
+    --advance "${BENCH_WARMUP_BLOCKS:-50}" \
+    "${EXTRA_BENCH_ARGS[@]}" 2>&1 | sed -u "s/^/[bench] /"
+
+  # Start tracy-capture after warmup so profile only covers the benchmark
+  if [ "${BENCH_TRACY:-off}" != "off" ]; then
+    echo "Starting tracy-capture..."
+    tracy-capture -f -o "$OUTPUT_DIR/tracy-profile.tracy" &
+    TRACY_PID=$!
+    sleep 0.5  # give tracy-capture time to connect
+  fi
+
+  # Benchmark
+  $BENCH_NICE "$RETH_BENCH" new-payload-fcu \
+    --rpc-url "$BENCH_RPC_URL" \
+    --engine-rpc-url http://127.0.0.1:8551 \
+    --jwt-secret "$DATADIR/jwt.hex" \
+    --advance "$BENCH_BLOCKS" \
+    "${EXTRA_BENCH_ARGS[@]}" \
+    --output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
+fi
 
 # cleanup runs via trap

.github/scripts/bench-reth-snapshot.sh

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 #
-# Downloads the latest nightly snapshot into the schelk volume with
-# progress reporting to the GitHub PR comment.
+# Downloads the latest snapshot into the schelk volume using
+# `reth download` with progress reporting to the GitHub PR comment.
 #
-# Skips the download if the local ETag marker matches the remote one.
+# Skips the download if the manifest content hasn't changed since
+# the last successful download (checked via SHA-256 of the manifest).
 #
 # Usage: bench-reth-snapshot.sh [--check]
-#   --check   Only check if a download is needed; exits 0 if up-to-date, 1 if not.
+#   --check   Only check if a download is needed; exits 0 if up-to-date, 10 if not.
 #
 # Required env:
 #   SCHELK_MOUNT       – schelk mount point (e.g. /reth-bench)
+#   BENCH_RETH_BINARY  – path to the reth binary
 #   GITHUB_TOKEN       – token for GitHub API calls (only for download)
 #   BENCH_COMMENT_ID   – PR comment ID to update (optional)
 #   BENCH_REPO         – owner/repo (e.g. paradigmxyz/reth)
@@ -18,52 +20,65 @@
 #   BENCH_CONFIG       – config summary line
 set -euo pipefail
 
-BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous.tar.zst"
+MC="mc"
+BUCKET="minio/reth-snapshots"
+MANIFEST_PATH="reth-1-minimal-stable/manifest.json"
 DATADIR="$SCHELK_MOUNT/datadir"
-ETAG_FILE="$HOME/.reth-bench-snapshot-etag"
+HASH_FILE="$HOME/.reth-bench-snapshot-hash"
 
-# Get remote metadata via JSON for reliable parsing
-MC_STAT=$(mc stat --json "$BUCKET" 2>/dev/null || true)
-REMOTE_ETAG=$(echo "$MC_STAT" | jq -r '.etag // empty')
-if [ -z "$REMOTE_ETAG" ]; then
-  echo "::warning::Failed to get ETag from mc stat, will re-download"
-  REMOTE_ETAG="unknown-$(date +%s)"
-fi
+# Fetch manifest and compute content hash for reliable freshness check
+MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
+  echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
+  exit 2
+}
+REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
 
-LOCAL_ETAG=""
-[ -f "$ETAG_FILE" ] && LOCAL_ETAG=$(cat "$ETAG_FILE")
+LOCAL_HASH=""
+[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
 
-if [ "$REMOTE_ETAG" = "$LOCAL_ETAG" ]; then
-  echo "Snapshot is up-to-date (ETag: ${REMOTE_ETAG})"
-  if [ "${1:-}" = "--check" ]; then
-    exit 0
-  fi
+if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
+  echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
   exit 0
 fi
 
-echo "Snapshot needs update (local: ${LOCAL_ETAG:-<none>}, remote: ${REMOTE_ETAG})"
+echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
 if [ "${1:-}" = "--check" ]; then
+  exit 10
+fi
+
+RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
+if [ ! -x "$RETH" ]; then
+  echo "::error::reth binary not found or not executable at $RETH"
   exit 1
 fi
 
-# Get compressed size for progress tracking
-TOTAL_BYTES=$(echo "$MC_STAT" | jq -r '.size // empty')
-if [ -z "$TOTAL_BYTES" ] || [ "$TOTAL_BYTES" = "0" ]; then
-  echo "::error::Failed to get snapshot size from mc stat"
+# Resolve the MinIO HTTP endpoint from the mc alias so reth can
+# fetch archives over HTTP (the manifest's embedded base_url points
+# to the cluster-internal address which is unreachable from runners).
+MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
+if [ -z "$MINIO_ENDPOINT" ]; then
+  echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
   exit 1
 fi
-echo "Snapshot size: $TOTAL_BYTES bytes ($(numfmt --to=iec "$TOTAL_BYTES"))"
+BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/reth-1-minimal-stable"
+
+# Rewrite manifest's base_url with the runner-reachable endpoint
+MANIFEST_TMP=$(mktemp --suffix=.json)
+trap 'rm -f -- "$MANIFEST_TMP"' EXIT
+echo "$MANIFEST_CONTENT" \
+  | jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
 
 # Prepare mount
 mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
 sudo schelk mount -y
 sudo rm -rf "$DATADIR"
 sudo mkdir -p "$DATADIR"
+# reth download runs as current user (not root), needs write access
+sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
 
 update_comment() {
-  local pct="$1"
+  local status="$1"
   [ -z "${BENCH_COMMENT_ID:-}" ] && return 0
-  local status="Building binaries & downloading snapshot… ${pct}%"
   local body
   body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
     "$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
@@ -75,51 +90,31 @@ update_comment() {
     > /dev/null 2>&1 || true
 }
 
-# Track compressed bytes flowing through the pipe
-DL_BYTES_FILE=$(mktemp)
-echo 0 > "$DL_BYTES_FILE"
+update_comment "Downloading snapshot…"
 
-# Start progress reporter in background
-(
-  while true; do
-    sleep 10
-    CURRENT=$(cat "$DL_BYTES_FILE" 2>/dev/null || echo 0)
-    if [ "$TOTAL_BYTES" -gt 0 ]; then
-      PCT=$(( CURRENT * 100 / TOTAL_BYTES ))
-      [ "$PCT" -gt 100 ] && PCT=100
-      echo "Snapshot download: $(numfmt --to=iec "$CURRENT") / $(numfmt --to=iec "$TOTAL_BYTES") (${PCT}%)"
-      update_comment "$PCT"
-    fi
-  done
-) &
-PROGRESS_PID=$!
-trap 'kill $PROGRESS_PID 2>/dev/null || true; rm -f "$DL_BYTES_FILE"' EXIT
+# Download using reth download (manifest-path with rewritten base_url)
+"$RETH" download \
+  --manifest-path "$MANIFEST_TMP" \
+  -y \
+  --minimal \
+  --datadir "$DATADIR"
 
-# Download and extract; python byte counter tracks compressed bytes received
-mc cat "$BUCKET" | python3 -c "
-import sys
-count = 0
-while True:
-    data = sys.stdin.buffer.read(1048576)
-    if not data:
-        break
-    count += len(data)
-    sys.stdout.buffer.write(data)
-    with open('$DL_BYTES_FILE', 'w') as f:
-        f.write(str(count))
-" | pzstd -d -p 6 | sudo tar -xf - -C "$DATADIR"
-
-# Stop progress reporter
-kill $PROGRESS_PID 2>/dev/null || true
-wait $PROGRESS_PID 2>/dev/null || true
-
-update_comment "100"
+update_comment "Downloading snapshot… done"
 echo "Snapshot download complete"
 
-# Sync the new snapshot as the schelk baseline
-sync
-sudo schelk recover -y
+# Sanity check: verify expected directories exist
+if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
+  echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
+  ls -la "$DATADIR" || true
+  exit 1
+fi
 
-# Save ETag marker
-echo "$REMOTE_ETAG" > "$ETAG_FILE"
-echo "Snapshot synced to schelk (ETag: ${REMOTE_ETAG})"
+# Promote the new snapshot to become the schelk baseline (virgin volume).
+# This copies changed blocks from scratch → virgin so that future
+# `schelk recover` calls restore to this new state.
+sync
+sudo schelk promote -y
+
+# Save manifest hash
+echo "$REMOTE_HASH" > "$HASH_FILE"
+echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"

.github/scripts/bench-reth-summary.py

@@ -107,6 +107,10 @@ def compute_stats(combined: list[dict]) -> dict:
             mgas_s_values.append(r["gas_used"] / lat_s / 1_000_000)
     mean_mgas_s = sum(mgas_s_values) / len(mgas_s_values) if mgas_s_values else 0
 
+    total_latencies_ms = [r["total_latency_us"] / 1_000 for r in combined]
+    wall_clock_s = sum(total_latencies_ms) / 1_000
+    mean_total_lat_ms = sum(total_latencies_ms) / n
+
     return {
         "n": n,
         "mean_ms": mean_lat,
@@ -115,6 +119,8 @@ def compute_stats(combined: list[dict]) -> dict:
         "p90_ms": percentile(sorted_lat, 90),
         "p99_ms": percentile(sorted_lat, 99),
         "mean_mgas_s": mean_mgas_s,
+        "wall_clock_s": wall_clock_s,
+        "mean_total_lat_ms": mean_total_lat_ms,
     }
 
 
@@ -139,13 +145,14 @@ def compute_wait_stats(combined: list[dict], field: str) -> dict:
 
 def _paired_data(
     baseline: list[dict], feature: list[dict]
-) -> tuple[list[tuple[float, float]], list[float], list[float]]:
+) -> tuple[list[tuple[float, float]], list[float], list[float], list[float]]:
     """Match blocks and return paired latencies and per-block diffs.
 
     Returns:
         pairs: list of (baseline_ms, feature_ms) tuples
         lat_diffs_ms: list of feature − baseline latency diffs in ms
         mgas_diffs: list of feature − baseline Mgas/s diffs
+        total_lat_diffs_ms: list of feature − baseline total latency diffs in ms
     """
     baseline_by_block = {r["block_number"]: r for r in baseline}
     feature_by_block = {r["block_number"]: r for r in feature}
@@ -154,6 +161,7 @@ def _paired_data(
     pairs = []
     lat_diffs_ms = []
     mgas_diffs = []
+    total_lat_diffs_ms = []
     for bn in common_blocks:
         b = baseline_by_block[bn]
         f = feature_by_block[bn]
@@ -168,7 +176,10 @@ def _paired_data(
                 f["gas_used"] / f_lat_s / 1_000_000
                 - b["gas_used"] / b_lat_s / 1_000_000
             )
-    return pairs, lat_diffs_ms, mgas_diffs
+        total_lat_diffs_ms.append(
+            f["total_latency_us"] / 1_000 - b["total_latency_us"] / 1_000
+        )
+    return pairs, lat_diffs_ms, mgas_diffs, total_lat_diffs_ms
 
 
 def compute_paired_stats(
@@ -183,12 +194,14 @@ def compute_paired_stats(
     all_pairs = []
     all_lat_diffs = []
     all_mgas_diffs = []
+    all_total_lat_diffs = []
     blocks_per_pair = []
     for baseline, feature in zip(baseline_runs, feature_runs):
-        pairs, lat_diffs, mgas_diffs = _paired_data(baseline, feature)
+        pairs, lat_diffs, mgas_diffs, total_lat_diffs = _paired_data(baseline, feature)
         all_pairs.extend(pairs)
         all_lat_diffs.extend(lat_diffs)
         all_mgas_diffs.extend(mgas_diffs)
+        all_total_lat_diffs.extend(total_lat_diffs)
         blocks_per_pair.append(len(pairs))
 
     if not all_lat_diffs:
@@ -227,6 +240,11 @@ def compute_paired_stats(
     mgas_se = std_mgas_diff / math.sqrt(len(all_mgas_diffs)) if all_mgas_diffs else 0.0
     mgas_ci = T_CRITICAL * mgas_se
 
+    mean_total_diff = sum(all_total_lat_diffs) / len(all_total_lat_diffs) if all_total_lat_diffs else 0.0
+    std_total_diff = stddev(all_total_lat_diffs, mean_total_diff) if len(all_total_lat_diffs) > 1 else 0.0
+    total_se = std_total_diff / math.sqrt(len(all_total_lat_diffs)) if all_total_lat_diffs else 0.0
+    wall_clock_ci_ms = T_CRITICAL * total_se
+
     return {
         "n": n,
         "mean_diff_ms": mean_diff,
@@ -239,17 +257,11 @@ def compute_paired_stats(
         "p99_ci_ms": (p99_boot[hi] - p99_boot[lo]) / 2,
         "mean_mgas_diff": mean_mgas_diff,
         "mgas_ci": mgas_ci,
+        "wall_clock_ci_ms": wall_clock_ci_ms,
         "blocks": max(blocks_per_pair),
     }
 
 
-def compute_summary(combined: list[dict], gas: list[dict]) -> dict:
-    """Compute aggregate metrics from parsed CSV data."""
-    blocks = len(combined)
-    return {
-        "blocks": blocks,
-    }
-
 
 def format_duration(seconds: float) -> str:
     if seconds >= 60:
@@ -274,22 +286,61 @@ def fmt_mgas(v: float) -> str:
     return f"{v:.2f}"
 
 
+def fmt_s(v: float) -> str:
+    return f"{v:.2f}s"
+
+
+def significance(pct: float, ci_pct: float, lower_is_better: bool) -> str:
+    """Return significance label: 'good', 'bad', or 'neutral'."""
+    significant = abs(pct) > ci_pct
+    if not significant:
+        return "neutral"
+    elif (pct < 0) == lower_is_better:
+        return "good"
+    else:
+        return "bad"
+
+
 def change_str(pct: float, ci_pct: float, lower_is_better: bool) -> str:
     """Format change% with paired CI significance.
 
     Significant if the CI doesn't cross zero (i.e. |pct| > ci_pct).
     """
-    significant = abs(pct) > ci_pct
-    if not significant:
-        emoji = "⚪"
-    elif (pct < 0) == lower_is_better:
-        emoji = "✅"
-    else:
-        emoji = "❌"
-
+    sig = significance(pct, ci_pct, lower_is_better)
+    emoji = {"good": "✅", "bad": "❌", "neutral": "⚪"}[sig]
     return f"{pct:+.2f}% {emoji} (±{ci_pct:.2f}%)"
 
 
+def compute_changes(
+    baseline_stats: dict, feature_stats: dict, paired_stats: dict
+) -> dict:
+    """Pre-compute change percentages and significance for each metric."""
+    def pct(base: float, feat: float) -> float:
+        return (feat - base) / base * 100.0 if base > 0 else 0.0
+
+    def ci_pct(ci_ms: float, base_ms: float) -> float:
+        return ci_ms / base_ms * 100.0 if base_ms > 0 else 0.0
+
+    metrics = [
+        ("mean", "mean_ms", "ci_ms", "mean_ms", True),
+        ("p50", "p50_ms", "p50_ci_ms", "p50_ms", True),
+        ("p90", "p90_ms", "p90_ci_ms", "p90_ms", True),
+        ("p99", "p99_ms", "p99_ci_ms", "p99_ms", True),
+        ("mgas_s", "mean_mgas_s", "mgas_ci", "mean_mgas_s", False),
+        ("wall_clock", "wall_clock_s", "wall_clock_ci_ms", "mean_total_lat_ms", True),
+    ]
+    changes = {}
+    for name, stat_key, ci_key, base_key, lower_is_better in metrics:
+        p = pct(baseline_stats[stat_key], feature_stats[stat_key])
+        c = ci_pct(paired_stats[ci_key], baseline_stats[base_key])
+        changes[name] = {
+            "pct": round(p, 4),
+            "ci_pct": round(c, 4),
+            "sig": significance(p, c, lower_is_better),
+        }
+    return changes
+
+
 def generate_comparison_table(
     run1: dict,
     run2: dict,
@@ -299,6 +350,7 @@ def generate_comparison_table(
     baseline_name: str,
     feature_name: str,
     feature_sha: str,
+    big_blocks: bool = False,
 ) -> str:
     """Generate a markdown comparison table between baseline and feature."""
     n = paired["blocks"]
@@ -308,6 +360,7 @@ def generate_comparison_table(
 
     mean_pct = pct(run1["mean_ms"], run2["mean_ms"])
     gas_pct = pct(run1["mean_mgas_s"], run2["mean_mgas_s"])
+    wall_pct = pct(run1["wall_clock_s"], run2["wall_clock_s"])
 
     p50_pct = pct(run1["p50_ms"], run2["p50_ms"])
     p90_pct = pct(run1["p90_ms"], run2["p90_ms"])
@@ -321,6 +374,7 @@ def generate_comparison_table(
     # CI as a percentage of baseline mean
     lat_ci_pct = paired["ci_ms"] / run1["mean_ms"] * 100.0 if run1["mean_ms"] > 0 else 0.0
     mgas_ci_pct = paired["mgas_ci"] / run1["mean_mgas_s"] * 100.0 if run1["mean_mgas_s"] > 0 else 0.0
+    wall_ci_pct = paired["wall_clock_ci_ms"] / run1["mean_total_lat_ms"] * 100.0 if run1["mean_total_lat_ms"] > 0 else 0.0
 
     base_url = f"https://github.com/{repo}/commit"
     baseline_label = f"[`{baseline_name}`]({base_url}/{baseline_ref})"
@@ -335,8 +389,9 @@ def generate_comparison_table(
         f"| P90 | {fmt_ms(run1['p90_ms'])} | {fmt_ms(run2['p90_ms'])} | {change_str(p90_pct, p90_ci_pct, lower_is_better=True)} |",
         f"| P99 | {fmt_ms(run1['p99_ms'])} | {fmt_ms(run2['p99_ms'])} | {change_str(p99_pct, p99_ci_pct, lower_is_better=True)} |",
         f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
+        f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, lower_is_better=True)} |",
         "",
-        f"*{n} blocks*",
+        f"*{n} {'big blocks' if big_blocks else 'blocks'}*",
     ]
     return "\n".join(lines)
 
@@ -367,6 +422,7 @@ def generate_markdown(
     summary: dict, comparison_table: str,
     wait_time_tables: list[str] | None = None,
     behind_baseline: int = 0, repo: str = "", baseline_ref: str = "", baseline_name: str = "",
+    grafana_url: str | None = None,
 ) -> str:
     """Generate a markdown comment body."""
     lines = ["## Benchmark Results", ""]
@@ -386,6 +442,9 @@ def generate_markdown(
                 lines.append(table)
                 lines.append("")
         lines.append("</details>")
+    if grafana_url:
+        lines.append("")
+        lines.append(f"**[Grafana Dashboard]({grafana_url})**")
     return "\n".join(lines)
 
 
@@ -412,6 +471,8 @@ def main():
     parser.add_argument("--feature-name", "--branch-name", default=None, help="Feature branch name")
     parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
     parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
+    parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode")
+    parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL for this benchmark run")
     args = parser.parse_args()
 
     if len(args.baseline_csv) != len(args.feature_csv):
@@ -438,11 +499,6 @@ def main():
     all_baseline = [r for run in baseline_runs for r in run]
     all_feature = [r for run in feature_runs for r in run]
 
-    summary = compute_summary(all_feature, gas)
-    with open(args.output_summary, "w") as f:
-        json.dump(summary, f, indent=2)
-    print(f"Summary written to {args.output_summary}")
-
     baseline_stats = compute_stats(all_baseline)
     feature_stats = compute_stats(all_feature)
     paired_stats = compute_paired_stats(baseline_runs, feature_runs)
@@ -465,6 +521,7 @@ def main():
         baseline_name=baseline_name,
         feature_name=feature_name,
         feature_sha=feature_sha,
+        big_blocks=args.big_blocks,
     )
     print(f"Generated comparison ({paired_stats['n']} paired blocks, "
           f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")
@@ -479,13 +536,41 @@ def main():
         ("execution_cache_wait_us", "Execution Cache Update Wait"),
     ]
     wait_time_tables = []
+    wait_time_data = {}
     for field, title in wait_fields:
         b_stats = compute_wait_stats(all_baseline, field)
         f_stats = compute_wait_stats(all_feature, field)
+        if b_stats and f_stats:
+            wait_time_data[field] = {
+                "title": title,
+                "baseline": b_stats,
+                "feature": f_stats,
+            }
         table = generate_wait_time_table(title, b_stats, f_stats, baseline_label, feature_label)
         if table:
             wait_time_tables.append(table)
 
+    summary = {
+        "blocks": paired_stats["blocks"],
+        "big_blocks": args.big_blocks,
+        "baseline": {
+            "name": baseline_name,
+            "ref": baseline_ref,
+            "stats": baseline_stats,
+        },
+        "feature": {
+            "name": feature_name,
+            "ref": feature_sha,
+            "stats": feature_stats,
+        },
+        "paired": paired_stats,
+        "changes": compute_changes(baseline_stats, feature_stats, paired_stats),
+        "wait_times": wait_time_data,
+    }
+    with open(args.output_summary, "w") as f:
+        json.dump(summary, f, indent=2)
+    print(f"Summary written to {args.output_summary}")
+
     markdown = generate_markdown(
         summary, comparison_table,
         wait_time_tables=wait_time_tables,
@@ -493,6 +578,7 @@ def main():
         repo=args.repo,
         baseline_ref=baseline_ref,
         baseline_name=baseline_name,
+        grafana_url=args.grafana_url,
     )
 
     with open(args.output_markdown, "w") as f:

.github/scripts/bench-scheduled-refs.sh

@@ -0,0 +1,132 @@
+#!/usr/bin/env bash
+#
+# Resolves baseline and feature refs for nightly regression benchmark runs.
+#
+# Queries the latest successful scheduled docker.yml run via GitHub API
+# to find the commit that built the nightly Docker image. Compares with
+# the last successful feature ref (from GH Actions cache) to determine
+# baseline, detect staleness, and decide whether to skip.
+#
+# Usage: bench-nightly-refs.sh [--force]
+#
+# Outputs (via GITHUB_OUTPUT):
+#   baseline-ref    — commit SHA for baseline
+#   feature-ref     — commit SHA for feature (current nightly)
+#   should-skip     — "true" if no new nightly since last run
+#   is-stale        — "true" if latest nightly build is >24h old
+#   stale-age-hours — age of the nightly build in hours (only if stale)
+#   nightly-created — ISO timestamp of the nightly build
+#
+# Reads:
+#   .nightly-state/last-feature-ref (from GH Actions cache, may not exist)
+#
+# Requires: gh (GitHub CLI), jq, date
+set -euo pipefail
+
+FORCE="${1:-false}"
+REPO="${GITHUB_REPOSITORY:-paradigmxyz/reth}"
+
+# --- Step 1: Query latest successful scheduled docker.yml run ---
+echo "::group::Querying latest nightly docker build"
+
+RUNS_JSON=$(gh run list \
+  -R "$REPO" \
+  --workflow=docker.yml \
+  --event=schedule \
+  --status=completed \
+  --limit 5 \
+  --json headSha,createdAt,conclusion)
+
+# Find the most recent successful run
+LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
+
+if [ -z "$LATEST" ]; then
+  echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
+  echo "Runs found: $RUNS_JSON"
+  exit 1
+fi
+
+FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
+CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt')
+echo "Latest nightly commit: $FEATURE_REF"
+echo "Built at: $CREATED_AT"
+echo "::endgroup::"
+
+# --- Step 2: Staleness check ---
+echo "::group::Checking staleness"
+NOW_EPOCH=$(date +%s)
+# Handle both GNU date (-d) and BSD date (-j -f) for cross-platform compat
+CREATED_EPOCH=$(date -d "$CREATED_AT" +%s 2>/dev/null || \
+  date -j -f "%Y-%m-%dT%H:%M:%SZ" "$CREATED_AT" +%s 2>/dev/null || \
+  date -j -f "%Y-%m-%dT%T%z" "$CREATED_AT" +%s 2>/dev/null || \
+  { echo "::error::Cannot parse date: $CREATED_AT"; exit 1; })
+
+AGE_SECONDS=$(( NOW_EPOCH - CREATED_EPOCH ))
+AGE_HOURS=$(( AGE_SECONDS / 3600 ))
+IS_STALE="false"
+
+if [ "$AGE_HOURS" -gt 24 ]; then
+  IS_STALE="true"
+  echo "::warning::STALE NIGHTLY: Build is ${AGE_HOURS}h old (>24h threshold)"
+  echo "This indicates the nightly docker build failed — no new image was produced"
+else
+  echo "Nightly build age: ${AGE_HOURS}h (within 24h threshold)"
+fi
+echo "::endgroup::"
+
+# --- Step 3: Read last successful feature ref from cache ---
+echo "::group::Reading cached state"
+LAST_FEATURE_REF=""
+STATE_FILE=".nightly-state/last-feature-ref"
+if [ -f "$STATE_FILE" ]; then
+  LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
+  echo "Previous feature ref: $LAST_FEATURE_REF"
+else
+  echo "No cached state found (first run)"
+fi
+echo "::endgroup::"
+
+# --- Step 4: Determine baseline and skip logic ---
+echo "::group::Resolving refs"
+SHOULD_SKIP="false"
+BASELINE_REF="$FEATURE_REF"  # default for first run
+
+if [ "$IS_STALE" = "true" ]; then
+  # Stale = error path, don't skip (will alert and fail downstream)
+  SHOULD_SKIP="false"
+  BASELINE_REF="${LAST_FEATURE_REF:-$FEATURE_REF}"
+  echo "Stale nightly detected — will alert and fail"
+elif [ -z "$LAST_FEATURE_REF" ]; then
+  # First run: baseline = feature (self-comparison to establish baseline)
+  BASELINE_REF="$FEATURE_REF"
+  echo "First run — will benchmark nightly against itself to establish baseline"
+elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
+  # No new nightly since last successful run
+  if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
+    echo "No new nightly, but force=true — running anyway"
+    BASELINE_REF="$LAST_FEATURE_REF"
+  else
+    SHOULD_SKIP="true"
+    echo "No new nightly since last run — will skip"
+  fi
+else
+  # Normal case: new nightly available
+  BASELINE_REF="$LAST_FEATURE_REF"
+  echo "New nightly detected"
+fi
+
+echo "Baseline: $BASELINE_REF"
+echo "Feature:  $FEATURE_REF"
+echo "Skip:     $SHOULD_SKIP"
+echo "Stale:    $IS_STALE"
+echo "::endgroup::"
+
+# --- Step 5: Write outputs ---
+{
+  echo "baseline-ref=$BASELINE_REF"
+  echo "feature-ref=$FEATURE_REF"
+  echo "should-skip=$SHOULD_SKIP"
+  echo "is-stale=$IS_STALE"
+  echo "stale-age-hours=$AGE_HOURS"
+  echo "nightly-created=$CREATED_AT"
+} >> "$GITHUB_OUTPUT"

.github/scripts/bench-slack-notify.js

@@ -0,0 +1,308 @@
+// Sends Slack notifications for reth-bench results.
+//
+// Reads from environment:
+//   SLACK_BENCH_BOT_TOKEN  – Slack Bot User OAuth Token (xoxb-...)
+//   SLACK_BENCH_CHANNEL    – Public channel ID for significant improvements
+//   BENCH_WORK_DIR         – Directory containing summary.json
+//   BENCH_PR               – PR number (may be empty)
+//   BENCH_ACTOR            – GitHub user who triggered the bench
+//   BENCH_JOB_URL          – URL to the Actions job page
+//   BENCH_BASELINE_ARGS    – Extra CLI args for the baseline reth node
+//   BENCH_FEATURE_ARGS     – Extra CLI args for the feature reth node
+//   BENCH_SAMPLY           – 'true' if samply profiling was enabled
+//
+// Usage from actions/github-script:
+//   const notify = require('./.github/scripts/bench-slack-notify.js');
+//   await notify.success({ core, context });
+//   await notify.failure({ core, context, failedStep: '...' });
+
+const fs = require('fs');
+const path = require('path');
+const { fmtChange, fmtMs, verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
+
+const SLACK_API = 'https://slack.com/api/chat.postMessage';
+
+function loadSlackUsers(repoRoot) {
+  try {
+    const raw = fs.readFileSync(path.join(repoRoot, '.github', 'scripts', 'bench-slack-users.json'), 'utf8');
+    const data = JSON.parse(raw);
+    // Filter out non-user-ID entries (like _comment)
+    const users = {};
+    for (const [k, v] of Object.entries(data)) {
+      if (!k.startsWith('_') && typeof v === 'string' && v.startsWith('U')) {
+        users[k] = v;
+      }
+    }
+    return users;
+  } catch {
+    return {};
+  }
+}
+
+async function postToSlack(token, channel, blocks, text, core, threadTs) {
+  const payload = { channel, blocks, text, unfurl_links: false };
+  if (threadTs) payload.thread_ts = threadTs;
+  const resp = await fetch(SLACK_API, {
+    method: 'POST',
+    headers: {
+      'Authorization': `Bearer ${token}`,
+      'Content-Type': 'application/json',
+    },
+    body: JSON.stringify(payload),
+  });
+  const data = await resp.json();
+  if (!data.ok) {
+    core.warning(`Slack API error (channel ${channel}): ${JSON.stringify(data)}`);
+  }
+  return data;
+}
+
+function cell(text) {
+  const s = String(text);
+  return { type: 'raw_text', text: s || ' ' };
+}
+
+// Slack shortcodes for verdict (Block Kit header doesn't support unicode emoji)
+const SLACK_VERDICT = {
+  '⚠️': ':warning:',
+  '❌': ':x:',
+  '✅': ':white_check_mark:',
+  '⚪': ':white_circle:',
+};
+
+function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls }) {
+  const { emoji, label } = verdict(summary.changes);
+  const headerEmoji = SLACK_VERDICT[emoji] || emoji;
+
+  const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
+  const commitUrl = `https://github.com/${repo}/commit`;
+  const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
+  const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
+
+  // Meta line
+  const metaParts = [];
+  if (prNumber) metaParts.push(`*<${prUrl}|PR #${prNumber}>*`);
+  metaParts.push(`triggered by ${actorSlackId ? `<@${actorSlackId}>` : `@${actor}`}`);
+
+  // Baseline/feature lines with samply profile links
+  let baselineLine = `*Baseline:* ${baselineLink}`;
+  const bl1 = samplyUrls['baseline-1'];
+  const bl2 = samplyUrls['baseline-2'];
+  if (bl1) baselineLine += ` | <${bl1}|Samply 1>`;
+  if (bl2) baselineLine += ` | <${bl2}|Samply 2>`;
+
+  let featureLine = `*Feature:* ${featureLink}`;
+  const fl1 = samplyUrls['feature-1'];
+  const fl2 = samplyUrls['feature-2'];
+  if (fl1) featureLine += ` | <${fl1}|Samply 1>`;
+  if (fl2) featureLine += ` | <${fl2}|Samply 2>`;
+
+  const countsLine = blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | ');
+
+  const baselineArgs = process.env.BENCH_BASELINE_ARGS || '';
+  const featureArgs = process.env.BENCH_FEATURE_ARGS || '';
+  const argsLines = [];
+  if (baselineArgs) argsLines.push(`*Baseline Args:* \`${baselineArgs}\``);
+  if (featureArgs) argsLines.push(`*Feature Args:* \`${featureArgs}\``);
+
+  const sectionText = [metaParts.join(' | '), '', baselineLine, featureLine, ...argsLines, countsLine].join('\n');
+
+  // Action buttons
+  const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+  const buttons = [
+    {
+      type: 'button',
+      text: { type: 'plain_text', text: 'CI :github:', emoji: true },
+      url: jobUrl,
+      action_id: 'ci_button',
+    },
+    {
+      type: 'button',
+      text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
+      url: diffUrl,
+      action_id: 'diff_button',
+    },
+  ];
+
+  // Build table rows from shared metricRows
+  const rows = metricRows(summary);
+  const tableRows = [
+    [cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
+    ...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
+  ];
+
+  const blocks = [
+    {
+      type: 'header',
+      text: { type: 'plain_text', text: `${headerEmoji} ${label}`, emoji: true },
+    },
+    {
+      type: 'section',
+      text: { type: 'mrkdwn', text: sectionText },
+    },
+    {
+      type: 'table',
+      column_settings: [
+        { align: 'left' },
+        { align: 'right' },
+        { align: 'right' },
+        { align: 'right' },
+      ],
+      rows: tableRows,
+    },
+    {
+      type: 'actions',
+      elements: buttons,
+    },
+  ];
+
+  // Wait times as a separate table block (sent as threaded reply due to Slack one-table limit)
+  const threadBlocks = [];
+  const wtRows = waitTimeRows(summary);
+  if (wtRows.length > 0) {
+    const waitTableRows = [
+      [cell('Wait Time'), cell('Baseline'), cell('Feature')],
+      ...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
+    ];
+    threadBlocks.push({
+      type: 'table',
+      column_settings: [
+        { align: 'left' },
+        { align: 'right' },
+        { align: 'right' },
+      ],
+      rows: waitTableRows,
+    });
+  }
+
+  return { blocks, threadBlocks };
+}
+
+function buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep }) {
+  const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
+  const actorMention = actorSlackId ? `<@${actorSlackId}>` : `@${actor}`;
+  const parts = [
+    prNumber ? `*<${prUrl}|PR #${prNumber}>*` : '',
+    `by ${actorMention}`,
+    `failed while *${failedStep}*`,
+  ].filter(Boolean);
+
+  const buttons = [
+    {
+      type: 'button',
+      text: { type: 'plain_text', text: 'CI :github:', emoji: true },
+      url: jobUrl,
+      action_id: 'ci_button',
+    },
+  ];
+
+  return [
+    {
+      type: 'header',
+      text: { type: 'plain_text', text: ':rotating_light: Bench Failed', emoji: true },
+    },
+    {
+      type: 'section',
+      text: { type: 'mrkdwn', text: parts.join(' | ') },
+    },
+    {
+      type: 'actions',
+      elements: buttons,
+    },
+  ];
+}
+
+async function success({ core, context }) {
+  const token = process.env.SLACK_BENCH_BOT_TOKEN;
+  if (!token) {
+    core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
+    return;
+  }
+
+  let summary;
+  try {
+    summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+  } catch (e) {
+    core.warning('Could not read summary.json for Slack notification');
+    return;
+  }
+
+  const repo = `${context.repo.owner}/${context.repo.repo}`;
+  const prNumber = process.env.BENCH_PR;
+  const actor = process.env.BENCH_ACTOR;
+  const jobUrl = process.env.BENCH_JOB_URL ||
+    `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+  const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
+
+  const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
+  const actorSlackId = slackUsers[actor];
+
+  const { blocks, threadBlocks } = buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls });
+  const text = `Bench: ${summary.baseline.name} vs ${summary.feature.name}`;
+
+  async function sendWithThread(ch) {
+    const res = await postToSlack(token, ch, blocks, text, core);
+    if (res.ok && res.ts && threadBlocks.length > 0) {
+      for (const tb of threadBlocks) {
+        await postToSlack(token, ch, [tb], 'Wait time breakdown', core, res.ts);
+      }
+    }
+  }
+
+  // Post to public channel if any metric shows significant improvement or regression
+  const channel = process.env.SLACK_BENCH_CHANNEL;
+  let postedToChannel = false;
+  if (channel) {
+    const changes = summary.changes || {};
+    const hasImprovement = Object.values(changes).some(c => c.sig === 'good');
+    if (hasImprovement) {
+      await sendWithThread(channel);
+      postedToChannel = true;
+    } else {
+      core.info('No significant improvement, skipping public channel notification');
+    }
+  }
+
+  // DM the actor only when results were not posted to the public channel
+  if (!postedToChannel) {
+    if (actorSlackId) {
+      await sendWithThread(actorSlackId);
+    } else {
+      core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
+    }
+  } else {
+    core.info(`Results posted to channel, skipping DM to ${actor}`);
+  }
+}
+
+async function failure({ core, context, failedStep }) {
+  const token = process.env.SLACK_BENCH_BOT_TOKEN;
+  if (!token) {
+    core.info('SLACK_BENCH_BOT_TOKEN not set, skipping Slack notification');
+    return;
+  }
+
+  const repo = `${context.repo.owner}/${context.repo.repo}`;
+  const prNumber = process.env.BENCH_PR;
+  const actor = process.env.BENCH_ACTOR;
+  const jobUrl = process.env.BENCH_JOB_URL ||
+    `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+
+  const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
+  const actorSlackId = slackUsers[actor];
+
+  const blocks = buildFailureBlocks({ prNumber, actor, actorSlackId, jobUrl, repo, failedStep });
+  const text = `Bench failed while ${failedStep}`;
+
+  // Always DM the actor
+  if (actorSlackId) {
+    await postToSlack(token, actorSlackId, blocks, text, core);
+  } else {
+    core.info(`No Slack user mapping for GitHub user '${actor}', skipping DM`);
+  }
+
+  // Only DM for failures, don't post to public channel
+}
+
+module.exports = { success, failure };

.github/scripts/bench-slack-users.json

@@ -0,0 +1,24 @@
+{
+  "_comment": "Maps GitHub usernames to Slack user IDs. Find yours: Slack profile > ··· > Copy member ID.",
+  "shekhirin": "U09FAL2UMLJ",
+  "mattsse": "U09FQNPMRT3",
+  "klkvr": "U09FAK95FC2",
+  "joshieDo": "U09LHN6GYAU",
+  "mediocregopher": "U09FF75KMQU",
+  "yongkangc": "U09FB0ECTD4",
+  "gakonst": "U092SEPDM40",
+  "Rjected": "U09F6SCKRGT",
+  "DaniPopes": "U09FAT8EK2A",
+  "emmajam": "U0A34UN92HW",
+  "onbjerg": "U09FB0UK5AA",
+  "fgimenez": "U09G3GP7CSU",
+  "rakita": "U09FB3Z2M7Y",
+  "jxom": "U09F72MG083",
+  "tmm": "U0AD0U8E88N",
+  "pepyakin": "U0A7HKMGEHJ",
+  "grandizzy": "U09F8DBDDRT",
+  "SuperFluffy": "U095BKHB2Q4",
+  "kamsz": "U0A2563UBRD",
+  "zerosnacks": "U09FARPMN74",
+  "samczsun": "U096R14E4H3"
+}

.github/scripts/bench-utils.js

@@ -0,0 +1,95 @@
+// Shared utilities for reth-bench result rendering.
+//
+// Used by bench-job-summary.js and bench-slack-notify.js.
+
+const fs = require('fs');
+const path = require('path');
+
+const SIG_EMOJI = { good: '✅', bad: '❌', neutral: '⚪' };
+
+function fmtMs(v) { return v.toFixed(2) + 'ms'; }
+function fmtMgas(v) { return v.toFixed(2); }
+function fmtS(v) { return v.toFixed(2) + 's'; }
+
+function fmtChange(ch) {
+  if (!ch || (!ch.pct && !ch.ci_pct)) return '';
+  const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
+  const ciStr = ch.ci_pct ? ` (±${ch.ci_pct.toFixed(2)}%)` : '';
+  return `${pctStr}${ciStr} ${SIG_EMOJI[ch.sig]}`;
+}
+
+function verdict(changes) {
+  const vals = Object.values(changes);
+  const hasBad = vals.some(v => v.sig === 'bad');
+  const hasGood = vals.some(v => v.sig === 'good');
+  if (hasBad && hasGood) return { emoji: '⚠️', label: 'Mixed Results' };
+  if (hasBad) return { emoji: '❌', label: 'Regression' };
+  if (hasGood) return { emoji: '✅', label: 'Improvement' };
+  return { emoji: '⚪', label: 'No Difference' };
+}
+
+function loadSamplyUrls(workDir) {
+  const urls = {};
+  for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
+    try {
+      const url = fs.readFileSync(path.join(workDir, run, 'samply-profile-url.txt'), 'utf8').trim();
+      if (url) urls[run] = url;
+    } catch {}
+  }
+  return urls;
+}
+
+function blocksLabel(summary) {
+  const parts = [];
+  if (summary.big_blocks) {
+    parts.push({ key: 'Big Blocks', value: summary.blocks });
+  } else {
+    const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
+    if (warmup) parts.push({ key: 'Warmup', value: warmup });
+    parts.push({ key: 'Blocks', value: summary.blocks });
+  }
+  const cores = process.env.BENCH_CORES || '0';
+  if (cores !== '0') parts.push({ key: 'Cores', value: cores });
+  return parts;
+}
+
+// The 7 metric rows shared by all renderers.
+// Returns an array of { label, baseline, feature, change } objects.
+function metricRows(summary) {
+  const b = summary.baseline.stats;
+  const f = summary.feature.stats;
+  const c = summary.changes;
+  return [
+    { label: 'Mean',       baseline: fmtMs(b.mean_ms),       feature: fmtMs(f.mean_ms),       change: fmtChange(c.mean) },
+    { label: 'StdDev',     baseline: fmtMs(b.stddev_ms),     feature: fmtMs(f.stddev_ms),     change: '' },
+    { label: 'P50',        baseline: fmtMs(b.p50_ms),        feature: fmtMs(f.p50_ms),        change: fmtChange(c.p50) },
+    { label: 'P90',        baseline: fmtMs(b.p90_ms),        feature: fmtMs(f.p90_ms),        change: fmtChange(c.p90) },
+    { label: 'P99',        baseline: fmtMs(b.p99_ms),        feature: fmtMs(f.p99_ms),        change: fmtChange(c.p99) },
+    { label: 'Mgas/s',     baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
+    { label: 'Wall Clock', baseline: fmtS(b.wall_clock_s),   feature: fmtS(f.wall_clock_s),   change: fmtChange(c.wall_clock) },
+  ];
+}
+
+// Wait time rows: one row per metric showing mean values.
+function waitTimeRows(summary) {
+  const waitTimes = summary.wait_times || {};
+  const rows = [];
+  for (const key of Object.keys(waitTimes)) {
+    const wt = waitTimes[key];
+    rows.push({ title: wt.title, baseline: fmtMs(wt.baseline.mean_ms), feature: fmtMs(wt.feature.mean_ms) });
+  }
+  return rows;
+}
+
+module.exports = {
+  SIG_EMOJI,
+  fmtMs,
+  fmtMgas,
+  fmtS,
+  fmtChange,
+  verdict,
+  loadSamplyUrls,
+  blocksLabel,
+  metricRows,
+  waitTimeRows,
+};

.github/scripts/build_pgo_bolt.sh

@@ -0,0 +1,414 @@
+#!/usr/bin/env bash
+#
+# Full PGO+BOLT optimized build for reth using real reth-bench workloads.
+#
+# Phases:
+#   1. Build PGO-instrumented reth, run reth-bench → collect PGO profiles
+#   2. Build BOLT-instrumented reth (with PGO), run reth-bench → collect BOLT profiles
+#   3. Build final PGO+BOLT optimized binary
+#
+# Required environment variables:
+#   DATADIR    - Path to reth datadir (must already contain chain data)
+#   RPC_URL    - Source RPC URL for reth-bench to fetch payloads from
+#
+# Optional environment variables:
+#   PGO_BLOCKS      - Number of blocks for PGO profiling (default: 20)
+#   BOLT_BLOCKS     - Number of blocks for BOLT profiling (default: 20)
+#   SKIP_BOLT       - Temporarily skip BOLT phases (default: false)
+#   STRIP_SYMBOLS   - Strip debug symbols from output binary (default: true)
+#   COLLECT_PGO_ONLY - Stop after producing merged.profdata (default: false)
+#   PGO_PROFDATA    - Path to pre-collected merged.profdata (optional)
+#   PROFILE         - Cargo profile (default: maxperf-symbols)
+#   FEATURES        - Cargo features (default: jemalloc,asm-keccak,min-debug-logs)
+#   TARGET          - Target triple (default: auto-detected)
+#   EXTRA_RUSTFLAGS - Additional RUSTFLAGS (e.g. -C target-cpu=x86-64-v3)
+#
+# Output:
+#   target/$PROFILE_DIR/reth  — final optimized binary
+set -euo pipefail
+
+gha_section_start() {
+    local title="$1"
+    if [ -n "${GITHUB_ACTIONS:-}" ]; then
+        echo "::group::$title"
+    else
+        echo ""
+        echo "=== $title ==="
+    fi
+}
+
+gha_section_end() {
+    if [ -n "${GITHUB_ACTIONS:-}" ]; then
+        echo "::endgroup::"
+    fi
+}
+
+cd "$(dirname "$0")/../.."
+
+# ── Configuration ──────────────────────────────────────────────────────────────
+PGO_BLOCKS="${PGO_BLOCKS:-20}"
+BOLT_BLOCKS="${BOLT_BLOCKS:-20}"
+SKIP_BOLT="${SKIP_BOLT:-false}"
+STRIP_SYMBOLS="${STRIP_SYMBOLS:-true}"
+COLLECT_PGO_ONLY="${COLLECT_PGO_ONLY:-false}"
+PROFILE="${PROFILE:-maxperf-symbols}"
+FEATURES="${FEATURES:-jemalloc,asm-keccak,min-debug-logs}"
+TARGET="${TARGET:-$(rustc -Vv | grep host | cut -d' ' -f2)}"
+BASE_RUSTFLAGS="${RUSTFLAGS:-}"
+EXTRA_RUSTFLAGS="${EXTRA_RUSTFLAGS:-}"
+COMBINED_RUSTFLAGS="$BASE_RUSTFLAGS $EXTRA_RUSTFLAGS"
+PGO_PROFDATA="${PGO_PROFDATA:-}"
+DATADIR="${DATADIR:-}"
+RPC_URL="${RPC_URL:-}"
+
+SKIP_BOLT_BOOL=false
+if [[ "${SKIP_BOLT,,}" == "true" || "$SKIP_BOLT" == "1" ]]; then
+    SKIP_BOLT_BOOL=true
+fi
+
+STRIP_SYMBOLS_BOOL=false
+if [[ "${STRIP_SYMBOLS,,}" == "true" || "$STRIP_SYMBOLS" == "1" ]]; then
+    STRIP_SYMBOLS_BOOL=true
+fi
+
+COLLECT_PGO_ONLY_BOOL=false
+if [[ "${COLLECT_PGO_ONLY,,}" == "true" || "$COLLECT_PGO_ONLY" == "1" ]]; then
+    COLLECT_PGO_ONLY_BOOL=true
+fi
+
+USE_PRECOLLECTED_PGO=false
+if [ -n "$PGO_PROFDATA" ]; then
+    if [ ! -f "$PGO_PROFDATA" ]; then
+        echo "error: PGO_PROFDATA points to a missing file: $PGO_PROFDATA"
+        exit 1
+    fi
+    USE_PRECOLLECTED_PGO=true
+fi
+
+NEEDS_BENCH_WORKLOAD=true
+if [ "$USE_PRECOLLECTED_PGO" = true ] && [ "$SKIP_BOLT_BOOL" = true ]; then
+    NEEDS_BENCH_WORKLOAD=false
+fi
+
+if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
+    : "${DATADIR:?DATADIR must be set to the reth data directory}"
+    : "${RPC_URL:?RPC_URL must be set}"
+fi
+
+if [[ "$PROFILE" == dev ]]; then
+    PROFILE_DIR=debug
+else
+    PROFILE_DIR=$PROFILE
+fi
+
+MANIFEST_PATH="bin/reth"
+
+LLVM_VERSION=$(rustc -Vv | grep -oP 'LLVM version: \K\d+')
+PGO_DIR="$PWD/target/pgo-profiles"
+BOLT_DIR="$PWD/target/bolt-profiles"
+CARGO_ARGS=(--profile "$PROFILE" --features "$FEATURES" --manifest-path "$MANIFEST_PATH/Cargo.toml" --bin "reth" --locked)
+
+# Enable debug symbols for BOLT (requires symbols to reorder code).
+# Strip them at the end.
+PROFILE_UPPER=$(echo "$PROFILE" | tr '[:lower:]-' '[:upper:]_')
+export "CARGO_PROFILE_${PROFILE_UPPER}_STRIP=debuginfo"
+
+gha_section_start "Full PGO+BOLT Build"
+echo "Binary:      reth"
+echo "Manifest:    $MANIFEST_PATH"
+echo "Target:      $TARGET"
+echo "Profile:     $PROFILE"
+echo "Features:    $FEATURES"
+echo "LLVM:        $LLVM_VERSION"
+echo "PGO blocks:  $PGO_BLOCKS"
+echo "BOLT blocks: $BOLT_BLOCKS"
+echo "Skip BOLT:   $SKIP_BOLT"
+echo "Strip symbols: $STRIP_SYMBOLS"
+echo "Collect only: $COLLECT_PGO_ONLY"
+echo "PGO profdata: ${PGO_PROFDATA:-<collect with reth-bench>}"
+echo "RUSTFLAGS:   ${BASE_RUSTFLAGS:-<unset>}"
+echo "EXTRA_RUSTFLAGS: ${EXTRA_RUSTFLAGS:-<unset>}"
+if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
+    echo "Datadir:     $DATADIR"
+    echo "RPC URL:     $RPC_URL"
+else
+    echo "Datadir:     <not required>"
+    echo "RPC URL:     <not required>"
+fi
+gha_section_end
+
+# ── Prerequisites ──────────────────────────────────────────────────────────────
+gha_section_start "Installing prerequisites"
+rustup component add llvm-tools-preview
+
+LLVM_PROFDATA=$(find "$(rustc --print sysroot)" -name llvm-profdata -type f | head -1)
+if [ -z "$LLVM_PROFDATA" ]; then
+    echo "error: llvm-profdata not found"
+    exit 1
+fi
+
+install_bolt() {
+    if command -v llvm-bolt &>/dev/null; then
+        echo "BOLT already installed"
+        return
+    fi
+    echo "Installing BOLT from apt.llvm.org..."
+    wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc >/dev/null
+    CODENAME=$(lsb_release -cs)
+    echo "deb http://apt.llvm.org/$CODENAME/ llvm-toolchain-$CODENAME-$LLVM_VERSION main" | sudo tee /etc/apt/sources.list.d/llvm.list >/dev/null
+    sudo apt-get update -qq
+    sudo apt-get install -y -qq "bolt-$LLVM_VERSION"
+    sudo ln -sf "/usr/bin/llvm-bolt-$LLVM_VERSION" /usr/local/bin/llvm-bolt
+    sudo ln -sf "/usr/bin/merge-fdata-$LLVM_VERSION" /usr/local/bin/merge-fdata
+}
+if [ "$SKIP_BOLT_BOOL" = true ]; then
+    echo "Skipping BOLT installation (SKIP_BOLT=$SKIP_BOLT)"
+else
+    install_bolt
+fi
+gha_section_end
+
+if [ "$NEEDS_BENCH_WORKLOAD" = true ]; then
+    # Build reth-bench once (non-instrumented) — reused for both phases.
+    gha_section_start "Building reth-bench"
+    RUSTFLAGS="$COMBINED_RUSTFLAGS" \
+        cargo build --profile "$PROFILE" --features "$FEATURES" \
+        --manifest-path bin/reth-bench/Cargo.toml --bin reth-bench --locked
+    RETH_BENCH_BIN="$(find target -name reth-bench -type f -executable | head -1)"
+    echo "reth-bench: $RETH_BENCH_BIN"
+    gha_section_end
+else
+    gha_section_start "Building reth-bench"
+    echo "Skipping reth-bench build (pre-collected PGO with SKIP_BOLT=true)"
+    gha_section_end
+fi
+
+# ── Helpers ────────────────────────────────────────────────────────────────────
+RETH_PID=
+cleanup() {
+    if [ -n "${RETH_PID:-}" ] && kill -0 "$RETH_PID" 2>/dev/null; then
+        echo "Stopping reth (pid $RETH_PID)..."
+        sudo kill "$RETH_PID" 2>/dev/null || true
+        for i in $(seq 1 60); do
+            sudo kill -0 "$RETH_PID" 2>/dev/null || break
+            if [ $((i % 10)) -eq 0 ]; then
+                echo "  waiting... (${i}s)"
+            fi
+            sleep 1
+        done
+        sudo kill -9 "$RETH_PID" 2>/dev/null || true
+    fi
+}
+trap cleanup EXIT
+
+# Start reth, wait for RPC, run reth-bench, then stop reth.
+# Arguments: $1 = reth binary path, $2 = number of blocks, $3 = log label
+run_bench_workload() {
+    local reth_bin="$1" blocks="$2" label="$3"
+    local http_port=8545 authrpc_port=8551
+
+    echo "--- Starting reth ($label) ---"
+    sudo "$reth_bin" node \
+        --datadir "$DATADIR" \
+        --log.file.directory "/tmp/reth-${label}-logs" \
+        --engine.accept-execution-requests-hash \
+        --http --http.port "$http_port" \
+        --authrpc.port "$authrpc_port" \
+        --disable-discovery --no-persist-peers \
+        > "/tmp/reth-${label}.log" 2>&1 &
+    RETH_PID=$!
+
+    echo "Waiting for reth RPC..."
+    for i in $(seq 1 120); do
+        if curl -sf "http://127.0.0.1:$http_port" -X POST \
+            -H 'Content-Type: application/json' \
+            -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
+            > /dev/null 2>&1; then
+            echo "reth is ready after ${i}s"
+            break
+        fi
+        if [ "$i" -eq 120 ]; then
+            echo "error: reth failed to start within 120s"
+            cat "/tmp/reth-${label}.log"
+            exit 1
+        fi
+        sleep 1
+    done
+
+    echo "Running reth-bench ($blocks blocks)..."
+    "$RETH_BENCH_BIN" new-payload-fcu \
+        --rpc-url "$RPC_URL" \
+        --engine-rpc-url "http://127.0.0.1:$authrpc_port" \
+        --jwt-secret "$DATADIR/jwt.hex" \
+        --advance "$blocks" \
+        --reth-new-payload 2>&1 | sed -u "s/^/[$label] /"
+
+    echo "Stopping reth ($label)..."
+    sudo kill "$RETH_PID" 2>/dev/null || true
+    for i in $(seq 1 60); do
+        sudo kill -0 "$RETH_PID" 2>/dev/null || break
+        sleep 1
+    done
+    sudo kill -9 "$RETH_PID" 2>/dev/null || true
+    RETH_PID=
+}
+
+publish_binary() {
+    local source_bin="$1"
+    for out in "target/$TARGET/$PROFILE_DIR" "target/$PROFILE_DIR"; do
+        local destination="$out/reth"
+        mkdir -p "$out"
+        # Skip copying when source and destination resolve to the same inode.
+        if [ -e "$destination" ] && [ "$source_bin" -ef "$destination" ]; then
+            continue
+        fi
+        cp "$source_bin" "$destination"
+    done
+}
+
+if [ "$USE_PRECOLLECTED_PGO" = true ]; then
+    gha_section_start "Phase 1: Using Pre-Collected PGO Profile"
+    rm -rf "$PGO_DIR"
+    mkdir -p "$PGO_DIR"
+    cp "$PGO_PROFDATA" "$PGO_DIR/merged.profdata"
+    echo "Using pre-collected profile: $PGO_PROFDATA"
+    echo "PGO profile: $PGO_DIR/merged.profdata ($(ls -lh "$PGO_DIR/merged.profdata" | awk '{print $5}'))"
+    gha_section_end
+else
+    # ── Phase 1: PGO profile collection ───────────────────────────────────────
+    gha_section_start "Phase 1: PGO Profile Collection"
+
+    rm -rf "$PGO_DIR"
+    mkdir -p "$PGO_DIR"
+
+    echo "Building PGO-instrumented binary..."
+    RUSTFLAGS="-Cprofile-generate=$PGO_DIR -Crelocation-model=pic $COMBINED_RUSTFLAGS" \
+        cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
+
+    PGO_RETH_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
+    echo "Instrumented binary: $PGO_RETH_BIN ($(ls -lh "$PGO_RETH_BIN" | awk '{print $5}'))"
+
+    run_bench_workload "$PGO_RETH_BIN" "$PGO_BLOCKS" "pgo"
+
+    # Fix ownership if reth ran as root.
+    sudo chown -R "$(id -un):$(id -gn)" "$PGO_DIR" 2>/dev/null || true
+
+    # Merge PGO profiles.
+    echo "Merging PGO profiles..."
+    PROFRAW_COUNT=$(find "$PGO_DIR" -name '*.profraw' | wc -l)
+    echo "Found $PROFRAW_COUNT .profraw files"
+    if [ "$PROFRAW_COUNT" -eq 0 ]; then
+        echo "error: no .profraw files — instrumented binary did not produce profiles"
+        exit 1
+    fi
+    "$LLVM_PROFDATA" merge -o "$PGO_DIR/merged.profdata" "$PGO_DIR"/*.profraw
+    echo "PGO profile: $PGO_DIR/merged.profdata ($(ls -lh "$PGO_DIR/merged.profdata" | awk '{print $5}'))"
+    gha_section_end
+fi
+
+if [ "$COLLECT_PGO_ONLY_BOOL" = true ]; then
+    gha_section_start "PGO Collection Complete"
+    echo "COLLECT_PGO_ONLY=true, skipping PGO/BOLT optimized binary build"
+    echo "Profile: $PGO_DIR/merged.profdata"
+    gha_section_end
+    exit 0
+fi
+
+if [ "$SKIP_BOLT_BOOL" = true ]; then
+    gha_section_start "BOLT Phase Skipped"
+    echo "SKIP_BOLT=$SKIP_BOLT, building PGO-only binary"
+    echo "Building PGO-optimized binary..."
+    RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata $COMBINED_RUSTFLAGS" \
+        cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
+
+    BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
+    if [ "$STRIP_SYMBOLS_BOOL" = true ]; then
+        echo "Stripping debug symbols..."
+        strip "$BUILT_BIN"
+    else
+        echo "Skipping strip (STRIP_SYMBOLS=$STRIP_SYMBOLS)"
+    fi
+    publish_binary "$BUILT_BIN"
+    gha_section_end
+else
+    # ── Phase 2: BOLT profile collection (with PGO) ──────────────────────────
+    gha_section_start "Phase 2: BOLT Profile Collection (with PGO)"
+
+    rm -rf "$BOLT_DIR"
+    mkdir -p "$BOLT_DIR"
+
+    echo "Building BOLT-instrumented binary with PGO..."
+    # --emit-relocs preserves relocation entries in the binary, required by llvm-bolt -instrument
+    RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata -Clink-arg=-Wl,--emit-relocs $COMBINED_RUSTFLAGS" \
+        cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
+
+    # Instrument with BOLT
+    BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
+    BOLT_INSTRUMENTED_BIN="$BUILT_BIN-bolt-instrumented"
+
+    echo "Instrumenting binary with BOLT..."
+    # --skip-funcs: skip compiler-generated drop_in_place functions that BOLT can't handle
+    # as split functions in relocation mode (triggered by --emit-relocs)
+    llvm-bolt "$BUILT_BIN" \
+        -instrument \
+        --instrumentation-file-append-pid \
+        --instrumentation-file="$BOLT_DIR/prof" \
+        --skip-funcs='.*drop_in_place.*' \
+        -o "$BOLT_INSTRUMENTED_BIN"
+    echo "BOLT-instrumented binary: $BOLT_INSTRUMENTED_BIN ($(ls -lh "$BOLT_INSTRUMENTED_BIN" | awk '{print $5}'))"
+
+    run_bench_workload "$BOLT_INSTRUMENTED_BIN" "$BOLT_BLOCKS" "bolt"
+
+    # Fix ownership for BOLT profiles
+    sudo chown -R "$(id -un):$(id -gn)" "$BOLT_DIR" 2>/dev/null || true
+
+    # Merge BOLT profiles
+    echo "Merging BOLT profiles..."
+    FDATA_COUNT=$(find "$BOLT_DIR" -name '*.fdata' | wc -l)
+    echo "Found $FDATA_COUNT .fdata files"
+    if [ "$FDATA_COUNT" -eq 0 ]; then
+        echo "error: no .fdata files — BOLT-instrumented binary did not produce profiles"
+        exit 1
+    fi
+    merge-fdata "$BOLT_DIR"/*.fdata > "$BOLT_DIR/merged.fdata"
+    echo "BOLT profile: $BOLT_DIR/merged.fdata ($(ls -lh "$BOLT_DIR/merged.fdata" | awk '{print $5}'))"
+    gha_section_end
+
+    # ── Phase 3: Final optimized build ───────────────────────────────────────
+    gha_section_start "Phase 3: Final PGO+BOLT Optimized Build"
+
+    echo "Building PGO-optimized binary..."
+    # --emit-relocs preserves relocation entries in the binary, required by llvm-bolt for code reordering
+    RUSTFLAGS="-Cprofile-use=$PGO_DIR/merged.profdata -Clink-arg=-Wl,--emit-relocs $COMBINED_RUSTFLAGS" \
+        cargo build "${CARGO_ARGS[@]}" --target "$TARGET"
+
+    BUILT_BIN="$PWD/target/$TARGET/$PROFILE_DIR/reth"
+    OPTIMIZED_BIN="$BUILT_BIN-bolt-optimized"
+
+    echo "Optimizing with BOLT..."
+    llvm-bolt "$BUILT_BIN" \
+        -o "$OPTIMIZED_BIN" \
+        --data "$BOLT_DIR/merged.fdata" \
+        -reorder-blocks=ext-tsp \
+        -reorder-functions=cdsort \
+        -split-functions \
+        -split-all-cold \
+        -dyno-stats \
+        -icf=1 \
+        -use-gnu-stack \
+        --skip-funcs='.*drop_in_place.*'
+
+    if [ "$STRIP_SYMBOLS_BOOL" = true ]; then
+        echo "Stripping debug symbols..."
+        strip "$OPTIMIZED_BIN"
+    else
+        echo "Skipping strip (STRIP_SYMBOLS=$STRIP_SYMBOLS)"
+    fi
+    publish_binary "$OPTIMIZED_BIN"
+    gha_section_end
+fi
+
+gha_section_start "Build Complete"
+ls -lh "target/$PROFILE_DIR/reth"
+echo "Output: target/$PROFILE_DIR/reth"
+gha_section_end

.github/scripts/hive/Dockerfile

@@ -3,7 +3,7 @@
 # 
 # We'll use cargo-chef to speed up the build
 # 
-FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
+FROM lukemathwalker/cargo-chef:latest-rust-1.94.0-trixie AS chef
 WORKDIR /app
 
 # Install system dependencies

.github/scripts/hive/build_simulators.sh

@@ -9,10 +9,17 @@ go build .
 
 ./hive -client reth # first builds and caches the client
 
-# Run each hive command in the background for each simulator and wait
+# Run each hive command in the background for each simulator and wait 
 echo "Building images"
 # TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs  we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
-./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/eels/consume-engine" \
+    --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
+    --sim.buildarg branch=err-map-3 \
+    --sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/eels/consume-rlp" \
+    --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/bal@v5.1.0/fixtures_bal.tar.gz \
+    --sim.buildarg branch=err-map-3 \
+    --sim.timelimit 1s || true &
 ./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
 ./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
 ./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &

.github/scripts/hive/expected_failures.yaml

@@ -16,32 +16,19 @@ rpc-compat:
   # syncing mode, the test expects syncing to be false on start
   - eth_syncing/check-syncing (reth)
 
-# no fix due to https://github.com/paradigmxyz/reth/issues/8732
-engine-withdrawals:
-  - Withdrawals Fork On Genesis (Paris) (reth)
-  - Withdrawals Fork on Block 1 (Paris) (reth)
-  - Withdrawals Fork on Block 2 (Paris) (reth)
-  - Withdrawals Fork on Block 3 (Paris) (reth)
-  - Withdraw to a single account (Paris) (reth)
-  - Withdraw to two accounts (Paris) (reth)
-  - Withdraw many accounts (Paris) (reth)
-  - Withdraw zero amount (Paris) (reth)
-  - Empty Withdrawals (Paris) (reth)
-  - Corrupted Block Hash Payload (INVALID) (Paris) (reth)
-  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
+engine-withdrawals: []
 
-engine-api: [ ]
+engine-api: []
 
-# no fix due to https://github.com/paradigmxyz/reth/issues/8732
 engine-cancun:
   - Invalid PayloadAttributes, Missing BeaconRoot, Syncing=True (Cancun) (reth)
   # the test fails with older versions of the code for which it passed before, probably related to changes
   # in hive or its dependencies
   - Blob Transaction Ordering, Multiple Clients (Cancun) (reth)
 
-sync: [ ]
+sync: []
 
-engine-auth: [ ]
+engine-auth: []
 
 # EIP-7610 related tests (Revert creation in case of non-empty storage):
 #
@@ -49,7 +36,7 @@ engine-auth: [ ]
 #   The test artificially creates an empty account with storage, then tests EIP-7610's behavior.
 #   On mainnet, ~25 such accounts exist as contract addresses (derived from keccak(prefix, caller,
 #   nonce/salt), not from public keys). No private key exists for contract addresses. To trigger
-#   this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible. 
+#   this with EIP-7702, you'd need to recover a private key from one of the already deployed contract addresses - mathematically impossible.
 #
 # tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_*
 #   Requires hash collision on create2 address to target already deployed accounts with storage.
@@ -97,6 +84,18 @@ eels/consume-engine:
   - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Osaka-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
   - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_False]-reth
   - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Osaka-blockchain_test_engine-slice_bytes_True]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_size-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_index_offset-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_True]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_log_length[fork_Amsterdam-blockchain_test_engine-slice_bytes_False]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_size-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_pubkey_offset-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_size-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_withdrawal_credentials_offset-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_size-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_amount_offset-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_size-value_zero]-reth
+  - tests/prague/eip6110_deposits/test_modified_contract.py::test_invalid_layout[fork_Amsterdam-blockchain_test_engine-log_argument_signature_offset-value_zero]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Osaka-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Paris-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
@@ -147,6 +146,20 @@ eels/consume-engine:
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_engine_from_state_test]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-initcode-with-deploy]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-empty-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_engine_from_state_test-sstore-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_engine_from_state_test-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_engine_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
 
 # Blob limit tests:
 #
@@ -241,3 +254,17 @@ eels/consume-rlp:
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Prague-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
   - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Shanghai-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-initcode-with-deploy]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-sstore-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_2-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_collision_with_create2_revert_in_initcode[fork_Amsterdam-blockchain_test_from_state_test]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_1-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_tx[fork_Amsterdam-tx_type_0-blockchain_test_from_state_test-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-correct-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_initcollision.py::test_init_collision_create_opcode[fork_Amsterdam-blockchain_test_from_state_test-opcode_CREATE2-non-empty-balance-revert-initcode]-reth
+  - tests/paris/eip7610_create_collision/test_revert_in_create.py::test_create2_collision_storage[fork_Amsterdam-blockchain_test_from_state_test-empty-initcode]-reth

.github/scripts/hive/ignored_tests.yaml

@@ -11,17 +11,16 @@
 #
 # When a test should no longer be ignored, remove it from this list.
 
-# flaky
-engine-withdrawals:
-  - Withdrawals Fork on Block 1 - 8 Block Re-Org NewPayload (Paris) (reth)
-  - Withdrawals Fork on Block 8 - 10 Block Re-Org NewPayload (Paris) (reth)
-  - Withdrawals Fork on Canonical Block 8 / Side Block 7 - 10 Block Re-Org (Paris) (reth)
-  - Sync after 128 blocks - Withdrawals on Block 2 - Multiple Withdrawal Accounts (Paris) (reth)
 engine-cancun:
   - Transaction Re-Org, New Payload on Revert Back (Cancun) (reth)
   - Transaction Re-Org, Re-Org to Different Block (Cancun) (reth)
   - Transaction Re-Org, Re-Org Out (Cancun) (reth)
   - Invalid Missing Ancestor ReOrg, StateRoot, EmptyTxs=False, Invalid P9 (Cancun) (reth)
+  # Hive test infra bug: geth sidecar switched to PathScheme for state storage, which has
+  # strict trie integrity requirements incompatible with inserting intentionally invalid blocks.
+  # Affects all clients, not just reth. Tracked: https://github.com/ethereum/hive/issues/1382
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=False, Invalid P8 (Cancun) (reth)
+  - Invalid Missing Ancestor Syncing ReOrg, Timestamp, EmptyTxs=False, CanonicalReOrg=True, Invalid P8 (Cancun) (reth)
   - Multiple New Payloads Extending Canonical Chain, Wait for Canonical Payload (Cancun) (reth)
 engine-api:
   - Transaction Re-Org, Re-Org Out (Paris) (reth)

.github/scripts/hive/run_simulator.sh

@@ -6,8 +6,20 @@ cd hivetests/
 sim="${1}"
 limit="${2}"
 
+# Use lower parallelism for eels tests to avoid OOM-killing the runner
+parallelism=16
+if [[ "${sim}" == *"eels"* ]]; then
+    parallelism=4
+fi
+
 run_hive() {
-    hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism 16 --client reth 2>&1 | tee /tmp/log || true
+    hive \
+  --sim "${sim}" \
+  --sim.limit "${limit}" \
+  --sim.limit.exact=false \
+  --sim.parallelism "${parallelism}" \
+  --client reth \
+  2>&1 | tee /tmp/log || true
 }
 
 check_log() {

.github/workflows/bench-scheduled.yml

@@ -0,0 +1,872 @@
+# Nightly regression benchmark.
+#
+# Compares the previous nightly build against the current nightly build to
+# detect performance regressions. Runs daily after docker.yml produces a new
+# nightly image at 01:00 UTC.
+#
+# State is persisted between runs via GitHub Actions cache: each successful
+# run saves the feature commit SHA so the next run knows what to compare against.
+
+on:
+  schedule:
+    - cron: "30 5 * * *" # 06:30 UTC daily
+  workflow_dispatch:
+    inputs:
+      force:
+        description: "Force run even if no new nightly (bypass skip logic)"
+        required: false
+        default: false
+        type: boolean
+      no_slack:
+        description: "Suppress Slack notifications"
+        required: false
+        default: true
+        type: boolean
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTC_WRAPPER: "sccache"
+
+name: bench-scheduled
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Job 1: Resolve nightly refs, check staleness, manage state
+  # ---------------------------------------------------------------------------
+  resolve-refs:
+    name: resolve-refs
+    runs-on: ubuntu-latest
+    outputs:
+      baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
+      feature-ref: ${{ steps.refs.outputs.feature-ref }}
+      should-skip: ${{ steps.refs.outputs.should-skip }}
+      is-stale: ${{ steps.refs.outputs.is-stale }}
+      stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
+      nightly-created: ${{ steps.refs.outputs.nightly-created }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/scripts
+          sparse-checkout-cone-mode: true
+
+      - name: Restore nightly state
+        id: state-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: .nightly-state
+          key: bench-scheduled-state-dummy
+          restore-keys: |
+            bench-scheduled-state-
+
+      - name: Resolve nightly refs
+        id: refs
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+        run: |
+          FORCE="${{ inputs.force || 'false' }}"
+          .github/scripts/bench-scheduled-refs.sh "$FORCE"
+
+      - name: Alert on stale nightly
+        if: steps.refs.outputs.is-stale == 'true'
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) {
+              core.warning('Slack credentials not set, skipping stale nightly alert');
+              return;
+            }
+
+            const ageHours = '${{ steps.refs.outputs.stale-age-hours }}';
+            const created = '${{ steps.refs.outputs.nightly-created }}';
+            const featureRef = '${{ steps.refs.outputs.feature-ref }}';
+            const shortSha = featureRef.slice(0, 8);
+            const repo = '${{ github.repository }}';
+            const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: ':rotating_light: Nightly Regression: nightly build is stale', emoji: true },
+              },
+              {
+                type: 'section',
+                text: {
+                  type: 'mrkdwn',
+                  text: [
+                    '*Nightly regression did not run* — nightly build is stale',
+                    '',
+                    `The latest nightly image was built from a commit that is *${ageHours}h old* (threshold: 24h).`,
+                    `This means today's nightly docker build likely failed and no new image was produced.`,
+                    '',
+                    `Stale commit: \`${shortSha}\` (built at ${created})`,
+                    '',
+                    '*Action required:* Check the <https://github.com/' + repo + '/actions/workflows/docker.yml|docker.yml> workflow for failures.',
+                  ].join('\n'),
+                },
+              },
+              {
+                type: 'actions',
+                elements: [
+                  {
+                    type: 'button',
+                    text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
+                    url: runUrl,
+                    action_id: 'ci_button',
+                  },
+                ],
+              },
+            ];
+
+            const resp = await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: {
+                'Authorization': `Bearer ${token}`,
+                'Content-Type': 'application/json',
+              },
+              body: JSON.stringify({
+                channel,
+                blocks,
+                text: 'Nightly regression: nightly build is stale',
+                unfurl_links: false,
+              }),
+            });
+            const data = await resp.json();
+            if (!data.ok) {
+              core.warning(`Slack API error: ${JSON.stringify(data)}`);
+            }
+
+      - name: Fail on stale nightly
+        if: steps.refs.outputs.is-stale == 'true'
+        run: |
+          echo "::error::Nightly build is stale (>24h old). Aborting."
+          exit 1
+
+  # ---------------------------------------------------------------------------
+  # Job 2: Run the benchmark
+  # ---------------------------------------------------------------------------
+  bench-scheduled:
+    needs: resolve-refs
+    if: |
+      needs.resolve-refs.outputs.should-skip != 'true' &&
+      needs.resolve-refs.outputs.is-stale != 'true'
+    name: bench-scheduled
+    runs-on: [self-hosted, Linux, X64, available]
+    timeout-minutes: 120
+    env:
+      BENCH_RPC_URL: https://ethereum.reth.rs/rpc
+      SCHELK_MOUNT: /reth-bench
+      BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
+      BENCH_PR: ""
+      BENCH_ACTOR: "nightly-regression"
+      BENCH_BLOCKS: "2000"
+      BENCH_WARMUP_BLOCKS: "500"
+      BENCH_SAMPLY: "false"
+      BENCH_CORES: "0"
+      BENCH_BIG_BLOCKS: "false"
+      BENCH_RETH_NEW_PAYLOAD: "true"
+      BENCH_WAIT_TIME: ""
+      BENCH_BASELINE_ARGS: ""
+      BENCH_FEATURE_ARGS: ""
+      BENCH_ABBA: "true"
+      BENCH_COMMENT_ID: ""
+      BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
+      BENCH_METRICS_ADDR: "127.0.0.1:9100"
+      BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
+      BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
+      BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
+      FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }}
+    steps:
+      - name: Clean up previous bench-work
+        run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
+
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
+          fetch-depth: 0
+          ref: ${{ needs.resolve-refs.outputs.feature-ref }}
+
+      - name: Resolve job URL
+        id: job-url
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+            });
+            const job = jobs.jobs.find(j => j.name === 'bench-scheduled');
+            const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            core.exportVariable('BENCH_JOB_URL', jobUrl);
+
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: mozilla-actions/sccache-action@v0.0.9
+        continue-on-error: true
+
+      - name: Install dependencies
+        env:
+          DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
+        run: |
+          mkdir -p "$HOME/.local/bin"
+
+          # apt packages
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends \
+            python3 make jq zstd curl dmsetup \
+            linux-tools-"$(uname -r)" || \
+            sudo apt-get install -y --no-install-recommends linux-tools-generic
+
+          # mc (MinIO client)
+          if ! command -v mc &>/dev/null; then
+            curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
+            chmod +x "$HOME/.local/bin/mc"
+          fi
+
+          # uv (Python package manager)
+          if ! command -v uv &>/dev/null; then
+            curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
+          fi
+
+          # Configure git auth for private repos
+          git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
+
+          # thin-provisioning-tools (era_invalidate, required by schelk)
+          if ! command -v era_invalidate &>/dev/null; then
+            git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
+            sudo make -C /tmp/tpt install
+            rm -rf /tmp/tpt
+          fi
+
+          # schelk (snapshot rollback tool, invoked via sudo)
+          if ! sudo sh -c 'command -v schelk' &>/dev/null; then
+            cargo install --git https://github.com/tempoxyz/schelk --locked
+            sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
+          fi
+
+      - name: Check dependencies
+        run: |
+          export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+          missing=()
+          for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
+            command -v "$cmd" &>/dev/null || missing+=("$cmd")
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            echo "::error::Missing required tools: ${missing[*]}"
+            exit 1
+          fi
+          echo "All dependencies found"
+
+      - name: Resolve display names
+        id: refs
+        run: |
+          BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
+          FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
+          echo "baseline-name=nightly-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
+          echo "feature-name=nightly-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
+          echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
+          echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
+
+      - name: Check if snapshot needs update
+        id: snapshot-check
+        run: |
+          set +e
+          .github/scripts/bench-reth-snapshot.sh --check
+          rc=$?
+          set -e
+          case "$rc" in
+            0)  echo "needed=false" >> "$GITHUB_OUTPUT" ;;
+            10) echo "needed=true"  >> "$GITHUB_OUTPUT" ;;
+            *)  echo "::error::Snapshot check failed (exit $rc)"
+                exit "$rc" ;;
+          esac
+
+      - name: Prepare source dirs
+        run: |
+          if [ -d ../reth-baseline ]; then
+            git -C ../reth-baseline fetch origin "$BASELINE_REF"
+          else
+            git clone . ../reth-baseline
+          fi
+          git -C ../reth-baseline checkout "$BASELINE_REF"
+
+          if [ -d ../reth-feature ]; then
+            git -C ../reth-feature fetch origin "$FEATURE_REF"
+          else
+            git clone . ../reth-feature
+          fi
+          git -C ../reth-feature checkout "$FEATURE_REF"
+
+      - name: Build binaries
+        id: build
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCH_REPO: ${{ github.repository }}
+        run: |
+          BASELINE_DIR="$(cd ../reth-baseline && pwd)"
+          FEATURE_DIR="$(cd ../reth-feature && pwd)"
+
+          .github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "$BASELINE_REF" &
+          PID_BASELINE=$!
+          .github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "$FEATURE_REF" &
+          PID_FEATURE=$!
+
+          FAIL=0
+          wait $PID_BASELINE || FAIL=1
+          wait $PID_FEATURE || FAIL=1
+          if [ $FAIL -ne 0 ]; then
+            echo "::error::One or more build tasks failed"
+            exit 1
+          fi
+
+      - name: Download snapshot
+        id: snapshot-download
+        if: steps.snapshot-check.outputs.needed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCH_REPO: ${{ github.repository }}
+          BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
+        run: .github/scripts/bench-reth-snapshot.sh
+
+      # System tuning for reproducible benchmarks
+      - name: System setup
+        run: |
+          sudo cpupower frequency-set -g performance || true
+          # Disable turbo boost (Intel and AMD paths)
+          echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
+          echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
+          sudo swapoff -a || true
+          echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
+          # Disable SMT (hyperthreading)
+          for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
+            first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
+            current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
+            if [ "$current" != "$first" ]; then
+              echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
+            fi
+          done
+          echo "Online CPUs: $(nproc)"
+          # Disable transparent huge pages
+          for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
+            [ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
+          done || true
+          # Prevent deep C-states
+          sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
+          # Move all IRQs to core 0
+          for irq in /proc/irq/*/smp_affinity_list; do
+            echo 0 | sudo tee "$irq" 2>/dev/null || true
+          done
+          # Stop noisy background services
+          sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
+          echo "=== Benchmark environment ==="
+          uname -r
+          lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
+          cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
+          cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
+          cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
+          free -h
+
+      - name: Pre-flight cleanup
+        run: |
+          sudo pkill -9 reth || true
+          sleep 1
+          if mountpoint -q "$SCHELK_MOUNT"; then
+            sudo umount -l "$SCHELK_MOUNT" || true
+            sudo schelk recover -y || true
+          fi
+          rm -rf "$BENCH_WORK_DIR"
+          mkdir -p "$BENCH_WORK_DIR"
+
+      - name: Start metrics proxy
+        run: |
+          BENCH_ID="nightly-${{ github.run_id }}"
+          BENCH_REFERENCE_EPOCH=$(date +%s)
+          echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
+          echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
+
+          LABELS_FILE="/tmp/bench-metrics-labels.json"
+          echo '{}' > "$LABELS_FILE"
+          echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
+
+          python3 .github/scripts/bench-metrics-proxy.py \
+            --labels "$LABELS_FILE" \
+            --upstream "http://${BENCH_METRICS_ADDR}/" \
+            --subnet 10.10.0.0/24 \
+            --port 9090 &
+          PROXY_PID=$!
+          echo "BENCH_METRICS_PROXY_PID=${PROXY_PID}" >> "$GITHUB_ENV"
+          echo "Metrics proxy started (PID $PROXY_PID)"
+
+      # Interleaved run order (B-F-F-B) to reduce systematic bias
+      - name: "Run benchmark: baseline (1/2)"
+        id: run-baseline-1
+        run: |
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
+
+      - name: "Run benchmark: feature (1/2)"
+        id: run-feature-1
+        run: |
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
+
+      - name: "Run benchmark: feature (2/2)"
+        id: run-feature-2
+        run: |
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
+
+      - name: "Run benchmark: baseline (2/2)"
+        id: run-baseline-2
+        run: |
+          LAST_RUN_START=$(date +%s)
+          echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
+
+      - name: Stop metrics proxy & generate Grafana URL
+        id: metrics
+        if: "!cancelled()"
+        run: |
+          kill "$BENCH_METRICS_PROXY_PID" 2>/dev/null || true
+
+          LAST_RUN_DURATION=$(( $(date +%s) - BENCH_LAST_RUN_START ))
+          FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
+          TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
+          GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=ef57fux92e9z4e&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
+          echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
+          echo "Grafana URL: ${GRAFANA_URL}"
+
+      - name: Scan logs for errors
+        if: "!cancelled()"
+        run: |
+          ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
+          found=false
+          for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
+            LOG="$BENCH_WORK_DIR/$run_dir/node.log"
+            if [ ! -f "$LOG" ]; then continue; fi
+
+            panics=$(grep -c -E 'panicked at' "$LOG" || true)
+            errors=$(grep -c ' ERROR ' "$LOG" || true)
+
+            if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
+              if [ "$found" = false ]; then
+                printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
+                found=true
+              fi
+              printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
+              if [ "$panics" -gt 0 ]; then
+                printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
+                grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
+                printf '```\n' >> "$ERRORS_FILE"
+              fi
+              if [ "$errors" -gt 0 ]; then
+                printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
+                grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
+                printf '```\n' >> "$ERRORS_FILE"
+              fi
+              printf '\n</details>\n\n' >> "$ERRORS_FILE"
+            fi
+          done
+
+      - name: Parse results
+        id: results
+        if: success()
+        env:
+          BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
+          FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
+        run: |
+          SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
+          SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
+          SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
+
+          BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
+          FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
+          BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
+          FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
+          SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
+          SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
+          SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
+
+          GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
+          if [ -n "$GRAFANA_URL" ]; then
+            SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
+          fi
+          # shellcheck disable=SC2086
+          python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
+
+      - name: Generate charts
+        if: success()
+        env:
+          BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
+          FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
+        run: |
+          CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
+          FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
+          BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
+          FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
+          BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
+          CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
+          CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
+          CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
+          CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
+          # shellcheck disable=SC2086
+          uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
+
+      - name: Upload results
+        if: "!cancelled()"
+        uses: actions/upload-artifact@v7
+        with:
+          name: bench-scheduled-results
+          path: ${{ env.BENCH_WORK_DIR }}
+
+      - name: Push charts
+        id: push-charts
+        if: success()
+        run: |
+          RUN_ID=${{ github.run_id }}
+          CHART_DIR="nightly/${RUN_ID}"
+          CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
+
+          TMP_DIR=$(mktemp -d)
+          if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
+            true
+          else
+            git init "${TMP_DIR}"
+            git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
+          fi
+
+          mkdir -p "${TMP_DIR}/${CHART_DIR}"
+          cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
+          git -C "${TMP_DIR}" add "${CHART_DIR}"
+          git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
+            commit -m "nightly bench charts for run ${RUN_ID}"
+          git -C "${TMP_DIR}" push origin HEAD:main
+          echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          rm -rf "${TMP_DIR}"
+
+      - name: Write job summary
+        if: success()
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const fs = require('fs');
+            const { verdict, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
+
+            let summary;
+            try {
+              summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+            } catch (e) {
+              await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
+              return;
+            }
+
+            const repo = `${context.repo.owner}/${context.repo.repo}`;
+            const commitUrl = `https://github.com/${repo}/commit`;
+            const { emoji, label } = verdict(summary.changes);
+            const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
+            const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
+            const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+
+            let md = `# ${emoji} Nightly Regression: ${label}\n\n`;
+            md += `**Baseline:** ${baselineLink}\n`;
+            md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
+            md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
+
+            const rows = metricRows(summary);
+            md += `| Metric | Baseline | Feature | Change |\n`;
+            md += `|--------|----------|---------|--------|\n`;
+            for (const r of rows) {
+              md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
+            }
+            md += '\n';
+
+            const wtRows = waitTimeRows(summary);
+            if (wtRows.length > 0) {
+              md += `### Wait Time Breakdown\n\n`;
+              md += `| Metric | Baseline | Feature |\n`;
+              md += `|--------|----------|--------|\n`;
+              for (const r of wtRows) {
+                md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
+              }
+              md += '\n';
+            }
+
+            // Charts
+            const chartSha = '${{ steps.push-charts.outputs.sha }}';
+            if (chartSha) {
+              const runId = '${{ github.run_id }}';
+              const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`;
+              const charts = [
+                { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
+                { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
+                { file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
+              ];
+              md += `### Charts\n\n`;
+              for (const chart of charts) {
+                md += `<details><summary>${chart.label}</summary>\n\n`;
+                md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
+                md += `</details>\n\n`;
+              }
+            }
+
+            const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}';
+            if (grafanaUrl) {
+              md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
+            }
+
+            try {
+              const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
+              if (errors.trim()) md += '\n' + errors + '\n';
+            } catch {}
+
+            await core.summary.addRaw(md).write();
+
+      - name: Send Slack notification (success)
+        if: success() && env.BENCH_NO_SLACK != 'true'
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const fs = require('fs');
+            const { verdict, fmtChange, fmtMs, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
+
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) {
+              core.info('Slack credentials not set, skipping notification');
+              return;
+            }
+
+            let summary;
+            try {
+              summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+            } catch (e) {
+              core.warning('Could not read summary.json for Slack notification');
+              return;
+            }
+
+            // Only notify on significant changes (regression OR improvement)
+            const changes = summary.changes || {};
+            const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
+            if (!hasSignificant) {
+              core.info('No significant changes detected, skipping nightly Slack notification');
+              return;
+            }
+
+            const SLACK_VERDICT = {
+              '⚠️': ':warning:',
+              '❌': ':x:',
+              '✅': ':white_check_mark:',
+              '⚪': ':white_circle:',
+            };
+
+            const repo = `${context.repo.owner}/${context.repo.repo}`;
+            const { emoji, label } = verdict(changes);
+            const headerEmoji = SLACK_VERDICT[emoji] || emoji;
+            const commitUrl = `https://github.com/${repo}/commit`;
+            const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
+            const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
+            const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+
+            function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
+
+            const sectionText = [
+              '*Nightly Regression*',
+              '',
+              `*Baseline:* ${baselineLink}`,
+              `*Feature:* ${featureLink}`,
+              blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '),
+            ].join('\n');
+
+            const rows = metricRows(summary);
+            const tableRows = [
+              [cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
+              ...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
+            ];
+
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: `${headerEmoji} Nightly: ${label}`, emoji: true },
+              },
+              {
+                type: 'section',
+                text: { type: 'mrkdwn', text: sectionText },
+              },
+              {
+                type: 'table',
+                column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }, { align: 'right' }],
+                rows: tableRows,
+              },
+              {
+                type: 'actions',
+                elements: [
+                  {
+                    type: 'button',
+                    text: { type: 'plain_text', text: 'CI :github:', emoji: true },
+                    url: jobUrl,
+                    action_id: 'ci_button',
+                  },
+                  {
+                    type: 'button',
+                    text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
+                    url: diffUrl,
+                    action_id: 'diff_button',
+                  },
+                ],
+              },
+            ];
+
+            const text = `Nightly Regression: ${summary.baseline.name} vs ${summary.feature.name}`;
+            const resp = await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: {
+                'Authorization': `Bearer ${token}`,
+                'Content-Type': 'application/json',
+              },
+              body: JSON.stringify({ channel, blocks, text, unfurl_links: false }),
+            });
+            const data = await resp.json();
+            if (!data.ok) {
+              core.warning(`Slack API error: ${JSON.stringify(data)}`);
+              return;
+            }
+
+            // Post wait time breakdown as threaded reply
+            const wtRows = waitTimeRows(summary);
+            if (data.ts && wtRows.length > 0) {
+              const waitTableRows = [
+                [cell('Wait Time'), cell('Baseline'), cell('Feature')],
+                ...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
+              ];
+              await fetch('https://slack.com/api/chat.postMessage', {
+                method: 'POST',
+                headers: {
+                  'Authorization': `Bearer ${token}`,
+                  'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                  channel,
+                  thread_ts: data.ts,
+                  blocks: [{
+                    type: 'table',
+                    column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }],
+                    rows: waitTableRows,
+                  }],
+                  text: 'Wait time breakdown',
+                  unfurl_links: false,
+                }),
+              });
+            }
+
+      - name: Send Slack notification (failure)
+        if: failure()
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) return;
+
+            const steps_status = [
+              ['building binaries', '${{ steps.build.outcome }}'],
+              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
+              ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
+              ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
+              ['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
+              ['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
+            ];
+            const failed = steps_status.find(([, o]) => o === 'failure');
+            const failedStep = failed ? failed[0] : 'unknown step';
+
+            const repo = `${context.repo.owner}/${context.repo.repo}`;
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: ':rotating_light: Nightly Bench Failed', emoji: true },
+              },
+              {
+                type: 'section',
+                text: { type: 'mrkdwn', text: `*Nightly regression* failed while *${failedStep}*` },
+              },
+              {
+                type: 'actions',
+                elements: [{
+                  type: 'button',
+                  text: { type: 'plain_text', text: 'View Logs :github:', emoji: true },
+                  url: jobUrl,
+                  action_id: 'ci_button',
+                }],
+              },
+            ];
+
+            await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: {
+                'Authorization': `Bearer ${token}`,
+                'Content-Type': 'application/json',
+              },
+              body: JSON.stringify({
+                channel,
+                blocks,
+                text: `Nightly bench failed while ${failedStep}`,
+                unfurl_links: false,
+              }),
+            });
+
+      - name: Restore system settings
+        if: always()
+        run: |
+          sudo systemctl start irqbalance cron atd 2>/dev/null || true
+
+  # ---------------------------------------------------------------------------
+  # Job 3: Save state on success
+  # ---------------------------------------------------------------------------
+  save-state:
+    needs: [resolve-refs, bench-scheduled]
+    if: success()
+    name: save-state
+    runs-on: ubuntu-latest
+    steps:
+      - name: Write state file
+        run: |
+          mkdir -p .nightly-state
+          echo "${{ needs.resolve-refs.outputs.feature-ref }}" > .nightly-state/last-feature-ref
+
+      - name: Save nightly state
+        uses: actions/cache/save@v4
+        with:
+          path: .nightly-state
+          key: bench-scheduled-state-${{ needs.resolve-refs.outputs.feature-ref }}

.github/workflows/docker-test.yml

@@ -6,7 +6,7 @@ on:
       hive_target:
         required: true
         type: string
-        description: "Docker bake target to build (e.g. hive-stable, hive-edge)"
+        description: "Docker bake target to build (e.g. hive)"
       artifact_name:
         required: false
         type: string
@@ -76,7 +76,7 @@ jobs:
             *.dockerfile=Dockerfile
 
       - name: Upload reth image
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: ${{ inputs.artifact_name }}
-          path: ./artifacts
+          path: ./artifacts

.github/workflows/docker.yml

@@ -28,12 +28,30 @@ on:
         required: false
         type: boolean
         default: false
+      pgo:
+        description: "Enable PGO profiling"
+        required: false
+        type: boolean
+        default: false
+      pgo_blocks:
+        description: "Number of blocks to execute for PGO profiling"
+        required: false
+        type: string
+        default: "20"
 
 jobs:
+  collect-pgo-profile:
+    if: github.repository == 'paradigmxyz/reth' && github.event_name == 'workflow_dispatch' && inputs.pgo
+    uses: ./.github/workflows/pgo-profile.yml
+    with:
+      pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
+    secrets: inherit
+
   build:
-    if: github.repository == 'paradigmxyz/reth'
+    if: github.repository == 'paradigmxyz/reth' && !failure() && !cancelled()
     name: Build Docker images
     runs-on: ubuntu-24.04
+    needs: collect-pgo-profile
     permissions:
       packages: write
       contents: read
@@ -45,7 +63,7 @@ jobs:
         uses: depot/setup-action@v1
 
       - name: Log in to GHCR
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}
@@ -58,6 +76,30 @@ jobs:
           echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
           echo "dirty=false" >> "$GITHUB_OUTPUT"
 
+      - name: Download pre-collected PGO profile
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.pgo }}
+        uses: actions/download-artifact@v7
+        with:
+          name: pgo-profdata
+          path: dist
+
+      - name: Configure PGO build args
+        id: pgo
+        run: |
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ "${{ inputs.pgo }}" == "true" ]]; then
+            if [ ! -f dist/merged.profdata ]; then
+              echo "::error::Expected dist/merged.profdata from collect-pgo-profile job"
+              exit 1
+            fi
+            echo "use_pgo_bolt=true" >> "$GITHUB_OUTPUT"
+            echo "pgo_profdata=dist/merged.profdata" >> "$GITHUB_OUTPUT"
+            echo "Using pre-collected PGO profile from collect-pgo-profile job"
+          else
+            echo "use_pgo_bolt=false" >> "$GITHUB_OUTPUT"
+            echo "pgo_profdata=" >> "$GITHUB_OUTPUT"
+            echo "PGO disabled"
+          fi
+
       - name: Determine build parameters
         id: params
         run: |
@@ -107,6 +149,9 @@ jobs:
           push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
           set: |
             ${{ steps.params.outputs.ethereum_set }}
+            *.args.USE_PGO_BOLT=${{ steps.pgo.outputs.use_pgo_bolt }}
+            *.args.PGO_PROFDATA=${{ steps.pgo.outputs.pgo_profdata }}
+            *.args.STRIP_SYMBOLS=false
 
       - name: Verify image architectures
         env:

.github/workflows/e2e.yml

@@ -63,6 +63,6 @@ jobs:
         run: |
           cargo nextest run \
             --no-fail-fast \
-            --locked --features "edge" \
+            --locked \
             -p reth-e2e-test-utils \
             -E 'binary(rocksdb)'

.github/workflows/hive.yml

@@ -5,7 +5,10 @@ name: hive
 on:
   workflow_dispatch:
   schedule:
-    - cron: "0 0 * * *"
+    - cron: "0 */6 * * *"
+  pull_request:
+    branches:
+      - "**"
 
 env:
   CARGO_TERM_COLOR: always
@@ -15,30 +18,26 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-reth-stable:
-    uses: ./.github/workflows/docker-test.yml
+  prepare-reth-stable:
+    uses: ./.github/workflows/prepare-reth.yml
     with:
-      hive_target: hive-stable
+      image_tag: ghcr.io/paradigmxyz/reth:latest
+      binary_name: reth
+      cargo_features: "asm-keccak"
       artifact_name: "reth-stable"
-    secrets: inherit
-
-  build-reth-edge:
-    uses: ./.github/workflows/docker-test.yml
-    with:
-      hive_target: hive-edge
-      artifact_name: "reth-edge"
-    secrets: inherit
 
   prepare-hive:
     if: github.repository == 'paradigmxyz/reth'
     timeout-minutes: 45
-    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
+    runs-on:
+      group: Reth
     steps:
       - uses: actions/checkout@v6
       - name: Checkout hive tests
         uses: actions/checkout@v6
         with:
-          repository: ethereum/hive
+          repository: Soubhik-10/hive # rm later and use ethereum/hive
+          ref: master
           path: hivetests
 
       - name: Get hive commit hash
@@ -75,7 +74,7 @@ jobs:
           chmod +x hive
 
       - name: Upload hive assets
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: hive_assets
           path: ./hive_assets
@@ -84,7 +83,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        storage: [stable, edge]
+        storage: [stable]
         # ethereum/rpc to be deprecated:
         # https://github.com/ethereum/hive/pull/1117
         scenario:
@@ -125,26 +124,28 @@ jobs:
           - sim: ethereum/rpc-compat
             include:
               - eth_blockNumber
-              - eth_call
-              - eth_chainId
-              - eth_createAccessList
-              - eth_estimateGas
-              - eth_feeHistory
-              - eth_getBalance
-              - eth_getBlockBy
-              - eth_getBlockTransactionCountBy
-              - eth_getCode
-              - eth_getProof
-              - eth_getStorage
-              - eth_getTransactionBy
-              - eth_getTransactionCount
-              - eth_getTransactionReceipt
-              - eth_sendRawTransaction
-              - eth_syncing
-              # debug_ rpc methods
-              - debug_
+              # - eth_call
+              # - eth_chainId
+              # - eth_createAccessList
+              # - eth_estimateGas
+              # - eth_feeHistory
+              # - eth_getBalance
+              # - eth_getBlockBy
+              # - eth_getBlockTransactionCountBy
+              # - eth_getCode
+              # - eth_getProof
+              # - eth_getStorage
+              # - eth_getTransactionBy
+              # - eth_getTransactionCount
+              # - eth_getTransactionReceipt
+              # - eth_sendRawTransaction
+              # - eth_syncing
+              # # debug_ rpc methods
+              # - debug_
 
           # consume-engine
+          - sim: ethereum/eels/consume-engine
+            limit: .*tests/amsterdam.*
           - sim: ethereum/eels/consume-engine
             limit: .*tests/osaka.*
           - sim: ethereum/eels/consume-engine
@@ -165,6 +166,8 @@ jobs:
             limit: .*tests/paris.*
 
           # consume-rlp
+          - sim: ethereum/eels/consume-rlp
+            limit: .*tests/amsterdam.*
           - sim: ethereum/eels/consume-rlp
             limit: .*tests/osaka.*
           - sim: ethereum/eels/consume-rlp
@@ -184,11 +187,11 @@ jobs:
           - sim: ethereum/eels/consume-rlp
             limit: .*tests/paris.*
     needs:
-      - build-reth-stable
-      - build-reth-edge
+      - prepare-reth-stable
       - prepare-hive
     name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
-    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
+    # Use larger runners for eels tests to avoid OOM runner crashes
+    runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
     permissions:
       issues: write
     steps:
@@ -197,13 +200,13 @@ jobs:
           fetch-depth: 0
 
       - name: Download hive assets
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: hive_assets
           path: /tmp
 
       - name: Download reth image
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: reth-${{ matrix.storage }}
           path: /tmp
@@ -219,7 +222,7 @@ jobs:
       - name: Checkout hive tests
         uses: actions/checkout@v6
         with:
-          repository: ethereum/hive
+          repository: Soubhik-10/hive
           ref: master
           path: hivetests
 

.github/workflows/integration.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   test:
-    name: test / ${{ matrix.network }} / ${{ matrix.storage }}
+    name: test / ${{ matrix.network }}
     if: github.event_name != 'schedule'
     runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
     env:
@@ -30,7 +30,6 @@ jobs:
     strategy:
       matrix:
         network: ["ethereum"]
-        storage: ["stable", "edge"]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v6
@@ -47,7 +46,7 @@ jobs:
         run: |
           cargo nextest run \
             --no-fail-fast \
-            --locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
+            --locked --features "asm-keccak ${{ matrix.network }}" \
             --workspace --exclude ef-tests \
             -E "kind(test) and not binary(e2e_testsuite)"
 

.github/workflows/kurtosis.yml

@@ -40,7 +40,7 @@ jobs:
           fetch-depth: 0
 
       - name: Download reth image
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: artifacts
           path: /tmp

.github/workflows/pgo-profile.yml

@@ -0,0 +1,107 @@
+name: pgo-profile
+
+on:
+  workflow_call:
+    inputs:
+      pgo_blocks:
+        description: "Number of blocks to execute for PGO profiling"
+        required: false
+        type: string
+        default: "20"
+  workflow_dispatch:
+    inputs:
+      pgo_blocks:
+        description: "Number of blocks to execute for PGO profiling"
+        required: false
+        type: string
+        default: "20"
+
+jobs:
+  collect:
+    name: collect PGO profiles
+    runs-on: [self-hosted, Linux, X64]
+    timeout-minutes: 180
+    env:
+      SCHELK_MOUNT: /reth-bench
+      BENCH_RPC_URL: https://ethereum.reth.rs/rpc
+      RUSTC_WRAPPER: "sccache"
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
+      - uses: rui314/setup-mold@v1
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          target: x86_64-unknown-linux-gnu
+      - uses: mozilla-actions/sccache-action@v0.0.9
+        continue-on-error: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends \
+            dmsetup lsb-release wget linux-tools-"$(uname -r)" || \
+            sudo apt-get install -y --no-install-recommends linux-tools-generic
+
+      - name: Download snapshot if needed
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCH_REPO: ${{ github.repository }}
+        run: |
+          if ! .github/scripts/bench-reth-snapshot.sh --check; then
+            echo "Snapshot outdated or missing, downloading..."
+            .github/scripts/bench-reth-snapshot.sh
+          fi
+
+      - name: Mount snapshot
+        run: |
+          sudo pkill -9 reth || true
+          sleep 1
+          if mountpoint -q "$SCHELK_MOUNT"; then
+            sudo umount -l "$SCHELK_MOUNT" || true
+            sudo schelk recover -y || true
+          fi
+          sudo schelk mount -y
+          sync
+          sudo sh -c 'echo 3 > /proc/sys/vm/drop_caches'
+
+      - name: Collect PGO profile
+        run: |
+          DATADIR="$SCHELK_MOUNT/datadir" \
+          RPC_URL="$BENCH_RPC_URL" \
+          PGO_BLOCKS="${{ inputs.pgo_blocks || '20' }}" \
+          BOLT_BLOCKS="${{ inputs.pgo_blocks || '20' }}" \
+          COLLECT_PGO_ONLY=true \
+          SKIP_BOLT=true \
+          PROFILE=maxperf-symbols \
+          FEATURES="jemalloc,asm-keccak,min-debug-logs" \
+          TARGET=x86_64-unknown-linux-gnu \
+          EXTRA_RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" \
+            .github/scripts/build_pgo_bolt.sh
+
+      - name: Show PGO profile stats
+        run: |
+          LLVM_PROFDATA=$(find "$(rustc --print sysroot)" -name llvm-profdata -type f | head -1)
+          if [ -z "$LLVM_PROFDATA" ]; then
+            echo "::error::llvm-profdata not found in rust toolchain"
+            exit 1
+          fi
+          "$LLVM_PROFDATA" show --detailed-summary --topn=20 target/pgo-profiles/merged.profdata
+
+      - name: Upload PGO profile
+        uses: actions/upload-artifact@v7
+        with:
+          name: pgo-profdata
+          path: target/pgo-profiles/merged.profdata
+          retention-days: 1
+
+      - name: Recover snapshot
+        if: always()
+        run: |
+          if mountpoint -q "$SCHELK_MOUNT"; then
+            sudo umount -l "$SCHELK_MOUNT" || true
+            sudo schelk recover -y || true
+          fi

.github/workflows/pr-audit.yml

@@ -0,0 +1,30 @@
+name: Pull request audit
+
+on:
+  pull_request:
+    types: [labeled]
+
+jobs:
+  publish:
+    runs-on: ubuntu-latest
+    if: github.event.label.name == 'cyclops'
+    steps:
+      - name: Publish event
+        run: |
+          set -euo pipefail
+
+          echo "${{ secrets.EVENTS_KEY }}" > ${{ runner.temp }}/key
+          echo "${{ secrets.EVENTS_CERT }}" > ${{ runner.temp }}/cert
+
+          curl -sf -o /dev/null -X POST ${{ secrets.EVENTS_ARGS }} \
+            -H "Content-Type: application/json" \
+            --key ${{ runner.temp }}/key \
+            --cert ${{ runner.temp }}/cert \
+            -d '{
+              "repository": "${{ github.repository }}",
+              "event": "pr_audit",
+              "data": {
+                "pr_number": ${{ github.event.pull_request.number }},
+                "sha": "${{ github.event.pull_request.head.sha }}"
+              }
+            }'

.github/workflows/prepare-reth.yml

@@ -0,0 +1,61 @@
+name: Prepare Reth Image
+
+on:
+  workflow_call:
+    inputs:
+      image_tag:
+        required: true
+        type: string
+        description: "Docker image tag to use"
+      binary_name:
+        required: false
+        type: string
+        default: "reth"
+        description: "Binary name to build (reth or op-reth)"
+      cargo_features:
+        required: false
+        type: string
+        default: "asm-keccak"
+        description: "Cargo features to enable"
+      cargo_package:
+        required: false
+        type: string
+        description: "Optional cargo package path"
+      artifact_name:
+        required: false
+        type: string
+        default: "artifacts"
+        description: "Name for the uploaded artifact"
+
+jobs:
+  prepare-reth:
+    if: github.repository == 'paradigmxyz/reth'
+    timeout-minutes: 45
+    runs-on: depot-ubuntu-latest-4
+    steps:
+      - uses: actions/checkout@v6
+      - run: mkdir artifacts
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build and export reth image
+        uses: docker/build-push-action@v6
+        with:
+          context: .
+          file: .github/scripts/hive/Dockerfile
+          tags: ${{ inputs.image_tag }}
+          outputs: type=docker,dest=./artifacts/reth_image.tar
+          build-args: |
+            CARGO_BIN=${{ inputs.binary_name }}
+            MANIFEST_PATH=${{ inputs.cargo_package }}
+            FEATURES=${{ inputs.cargo_features }}
+          cache-from: type=gha
+          cache-to: type=gha,mode=max
+
+      - name: Upload reth image
+        id: upload
+        uses: actions/upload-artifact@v6
+        with:
+          name: ${{ inputs.artifact_name }}
+          path: ./artifacts

.github/workflows/release-reproducible.yml

@@ -52,7 +52,7 @@ jobs:
         uses: docker/setup-buildx-action@v3
 
       - name: Log in to GitHub Container Registry
-        uses: docker/login-action@v3
+        uses: docker/login-action@v4
         with:
           registry: ghcr.io
           username: ${{ github.actor }}

.github/workflows/release.yml

@@ -13,6 +13,14 @@ on:
         description: "Enable dry run mode (builds artifacts but skips uploads and release creation)"
         type: boolean
         default: false
+      pgo:
+        description: "Enable PGO profiling"
+        type: boolean
+        default: false
+      pgo_blocks:
+        description: "Number of blocks to execute for PGO profiling on self-hosted runner"
+        type: string
+        default: "20"
 
 env:
   REPO_NAME: ${{ github.repository_owner }}/reth
@@ -37,7 +45,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Extract version
-        run: echo "VERSION=${GITHUB_REF_NAME}" >> $GITHUB_OUTPUT
+        run: echo "VERSION=${GITHUB_REF_NAME//\//-}" >> $GITHUB_OUTPUT
         id: extract_version
     outputs:
       VERSION: ${{ steps.extract_version.outputs.VERSION }}
@@ -69,26 +77,22 @@ jobs:
       fail-fast: true
       matrix:
         configs:
-          - target: x86_64-unknown-linux-gnu
-            os: ubuntu-24.04
-            profile: maxperf
-            allow_fail: false
           - target: aarch64-unknown-linux-gnu
-            os: ubuntu-24.04
+            os: ubuntu-24.04-arm
             profile: maxperf
             allow_fail: false
+            rustflags: ""
+            native: true
           - target: x86_64-apple-darwin
             os: macos-14
             profile: maxperf
             allow_fail: false
+            rustflags: "-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"
           - target: aarch64-apple-darwin
             os: macos-14
             profile: maxperf
             allow_fail: false
-          - target: riscv64gc-unknown-linux-gnu
-            os: ubuntu-24.04
-            profile: maxperf
-            allow_fail: true
+            rustflags: ""
         build:
           - command: build
             binary: reth
@@ -100,6 +104,7 @@ jobs:
           target: ${{ matrix.configs.target }}
       - uses: mozilla-actions/sccache-action@v0.0.9
       - name: Install cross main
+        if: ${{ !matrix.configs.native }}
         id: cross_main
         run: |
           cargo install cross --locked --git https://github.com/cross-rs/cross
@@ -114,7 +119,12 @@ jobs:
           echo "MACOSX_DEPLOYMENT_TARGET=$(xcrun -sdk macosx --show-sdk-platform-version)" >> $GITHUB_ENV
 
       - name: Build Reth
-        run: make PROFILE=${{ matrix.configs.profile }} ${{ matrix.build.command }}-${{ matrix.configs.target }}
+        run: |
+          if [ "${{ matrix.configs.native }}" = "true" ]; then
+            make PROFILE=${{ matrix.configs.profile }} EXTRA_RUSTFLAGS="${{ matrix.configs.rustflags }}" ${{ matrix.build.command }}-native-${{ matrix.configs.target }}
+          else
+            make PROFILE=${{ matrix.configs.profile }} EXTRA_RUSTFLAGS="${{ matrix.configs.rustflags }}" ${{ matrix.build.command }}-${{ matrix.configs.target }}
+          fi
       - name: Move binary
         run: |
           mkdir artifacts
@@ -135,23 +145,105 @@ jobs:
 
       - name: Upload artifact
         if: ${{ github.event.inputs.dry_run != 'true' }}
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
           path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz
 
       - name: Upload signature
         if: ${{ github.event.inputs.dry_run != 'true' }}
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
           path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
 
+  collect-pgo-profile:
+    if: github.event_name == 'workflow_dispatch' && inputs.pgo
+    uses: ./.github/workflows/pgo-profile.yml
+    with:
+      pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
+    secrets: inherit
+
+  build-pgo:
+    if: github.event_name == 'workflow_dispatch' && inputs.pgo
+    name: build release (x86_64-linux PGO+BOLT)
+    runs-on: [self-hosted, Linux, X64]
+    needs: [extract-version, collect-pgo-profile]
+    timeout-minutes: 120
+    env:
+      RUSTC_WRAPPER: "sccache"
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
+      - uses: rui314/setup-mold@v1
+      - uses: dtolnay/rust-toolchain@stable
+        with:
+          target: x86_64-unknown-linux-gnu
+      - uses: mozilla-actions/sccache-action@v0.0.9
+        continue-on-error: true
+      - uses: Swatinem/rust-cache@v2
+        with:
+          cache-on-failure: true
+
+      - name: Download pre-collected PGO profile
+        uses: actions/download-artifact@v7
+        with:
+          name: pgo-profdata
+          path: dist
+
+      - name: Verify PGO profile artifact
+        run: |
+          test -f dist/merged.profdata
+          ls -lh dist/merged.profdata
+
+      - name: Build Reth with PGO+BOLT
+        run: |
+          SKIP_BOLT=true \
+          PGO_PROFDATA="$PWD/dist/merged.profdata" \
+          PROFILE=maxperf-symbols \
+          FEATURES="jemalloc,asm-keccak,min-debug-logs" \
+          TARGET=x86_64-unknown-linux-gnu \
+          EXTRA_RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq" \
+            .github/scripts/build_pgo_bolt.sh
+
+      - name: Move binary
+        run: |
+          mkdir artifacts
+          mv target/maxperf-symbols/reth ./artifacts
+
+      - name: Configure GPG and create artifacts
+        env:
+          GPG_SIGNING_KEY: ${{ secrets.GPG_SIGNING_KEY }}
+          GPG_PASSPHRASE: ${{ secrets.GPG_PASSPHRASE }}
+        run: |
+          export GPG_TTY=$(tty)
+          echo -n "$GPG_SIGNING_KEY" | base64 --decode | gpg --batch --import
+          cd artifacts
+          tar -czf reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz reth*
+          echo "$GPG_PASSPHRASE" | gpg --passphrase-fd 0 --pinentry-mode loopback --batch -ab reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
+          mv *tar.gz* ..
+        shell: bash
+
+      - name: Upload artifact
+        if: ${{ github.event.inputs.dry_run != 'true' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
+          path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
+
+      - name: Upload signature
+        if: ${{ github.event.inputs.dry_run != 'true' }}
+        uses: actions/upload-artifact@v7
+        with:
+          name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
+          path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
+
   draft-release:
     name: draft release
     runs-on: ubuntu-latest
-    needs: [build, extract-version]
-    if: ${{ github.event.inputs.dry_run != 'true' }}
+    needs: [build, build-pgo, extract-version]
+    if: ${{ !failure() && !cancelled() && github.event.inputs.dry_run != 'true' }}
     env:
       VERSION: ${{ needs.extract-version.outputs.VERSION }}
     permissions:
@@ -164,7 +256,7 @@ jobs:
         with:
           fetch-depth: 0
       - name: Download artifacts
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
       - name: Generate full changelog
         id: changelog
         run: |

.github/workflows/reproducible-build.yml

@@ -43,7 +43,7 @@ jobs:
           echo "Binaries SHA256 on ${{ matrix.machine }}: $(cat checksum.sha256)"
 
       - name: Upload the hash
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: checksum-${{ matrix.machine }}
           path: |
@@ -56,12 +56,12 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Download artifacts from machine-1
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: checksum-machine-1
           path: machine-1/
       - name: Download artifacts from machine-2
-        uses: actions/download-artifact@v7
+        uses: actions/download-artifact@v8
         with:
           name: checksum-machine-2
           path: machine-2/

.github/workflows/stage.yml

@@ -51,15 +51,12 @@ jobs:
       - name: Run execution stage
         run: |
           reth stage run execution --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
-      - name: Run account-hashing stage
-        run: |
-          reth stage run account-hashing --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
-      - name: Run storage hashing stage
-        run: |
-          reth stage run storage-hashing --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
-      - name: Run hashing stage
-        run: |
-          reth stage run hashing --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints
+      # NOTE: account-hashing, storage-hashing, and hashing stages are omitted.
+      # With storage v2 (now default), these stages are no-ops because the
+      # execution stage writes directly to HashedAccounts/HashedStorages.
+      # Running them here is harmful: `stage run` unwinds before executing,
+      # and the unwind reverts the hashed state that execution wrote, but
+      # the no-op execute never restores it — causing merkle to fail.
       - name: Run merkle stage
         run: |
           reth stage run merkle --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints

.github/workflows/unit.yml

@@ -19,15 +19,13 @@ concurrency:
 
 jobs:
   test:
-    name: test / ${{ matrix.type }} / ${{ matrix.storage }}
+    name: test / ${{ matrix.type }}
     runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
     env:
       RUST_BACKTRACE: 1
-      EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
     strategy:
       matrix:
         type: [ethereum]
-        storage: [stable, edge]
         include:
           - type: ethereum
             features: asm-keccak ethereum
@@ -50,14 +48,14 @@ jobs:
         run: |
           cargo nextest run \
             --no-fail-fast \
-            --features "${{ matrix.features }} $EDGE_FEATURES" --locked \
+            --features "${{ matrix.features }}" --locked \
             ${{ matrix.exclude_args }} --workspace \
             --exclude ef-tests --no-tests=warn \
             -E "!kind(test) and not binary(e2e_testsuite)"
 
   state:
     name: Ethereum state tests
-    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
     env:
       RUST_LOG: info,sync=error
       RUST_BACKTRACE: 1

CLAUDE.md

@@ -172,10 +172,97 @@ Before submitting changes, ensure:
 2. **Clippy**: No warnings
 3. **Tests Pass**: All unit and integration tests
 4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
-5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
+5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below)
+6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
+
+### CLI Reference Docs (`book` CI Job)
+
+The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless.
+
+When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running:
+
+```bash
+make update-book-cli
+```
+
+This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes.
+
+The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`.
 
 ### Opening PRs against <https://github.com/paradigmxyz/reth>
 
+#### Titles
+
+Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope:
+
+```
+<type>(<scope>): <short description>
+```
+
+**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore`
+
+**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net`
+
+Examples:
+- `fix(rpc): correct gas estimation for ERC-20 transfers`
+- `perf: batch trie updates to reduce cursor overhead`
+- `feat(engine): add new_payload_interval metric`
+
+#### Descriptions
+
+Keep it short. Say what changed and why — nothing more.
+
+**Do:**
+- Write 1–3 sentences summarizing the change
+- Explain _why_ if the diff doesn't make it obvious
+- Link related issues or EIPs
+- Include benchmark numbers for perf changes
+
+**Don't:**
+- List every file changed — that's what the diff is for
+- Repeat the title in the body
+- Add "Files changed" or "Changes" sections
+- Write walls of text that go stale when the diff is updated
+- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage"
+
+**Template:**
+
+```
+Closes #<issue>
+
+<what changed, 1-3 sentences>
+
+<why, if not obvious from the diff>
+```
+
+**Good example:**
+
+```
+Closes #16800
+
+Adds fallback for external IP resolution so node startup doesn't fail
+when STUN is unreachable. Falls back to the configured default.
+```
+
+**Bad example:**
+
+```
+## Summary
+This PR introduces comprehensive improvements to the IP resolution system.
+
+## Changes
+- Modified `crates/net/discv4/src/lib.rs` to add fallback
+- Modified `crates/net/discv4/src/config.rs` to add default IP
+- Added tests in `crates/net/discv4/src/tests/ip.rs`
+
+## Files Changed
+- crates/net/discv4/src/lib.rs
+- crates/net/discv4/src/config.rs
+- crates/net/discv4/src/tests/ip.rs
+```
+
+#### Labels and CI
+
 Label PRs appropriately, first check the available labels and then apply the relevant ones:
 * when changes are RPC related, add A-rpc label
 * when changes are docs related, add C-docs label
@@ -455,5 +542,8 @@ cargo build --release
 cargo check --workspace --all-features
 
 # Check documentation
-cargo docs --document-private-items 
+cargo docs --document-private-items
+
+# Regenerate CLI reference docs (after CLI changes)
+make update-book-cli
 ```

Cargo.toml

@@ -1,5 +1,5 @@
 [workspace.package]
-version = "1.11.0"
+version = "1.11.3"
 edition = "2024"
 rust-version = "1.93"
 license = "MIT OR Apache-2.0"
@@ -10,7 +10,6 @@ exclude = [".github/"]
 [workspace]
 members = [
     "bin/reth-bench/",
-    "bin/reth-bench-compare/",
     "bin/reth/",
     "crates/storage/rpc-provider/",
     "crates/chain-state/",
@@ -169,6 +168,7 @@ rust.rust_2018_idioms = { level = "deny", priority = -1 }
 rust.unreachable_pub = "warn"
 rust.unused_must_use = "deny"
 rust.rust_2024_incompatible_pat = "warn"
+rust.unexpected_cfgs = { level = "warn", check-cfg = ['cfg(tokio_unstable)'] }
 rustdoc.all = "warn"
 # rust.unnameable-types = "warn"
 
@@ -322,7 +322,6 @@ reth = { path = "bin/reth" }
 reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" }
 reth-basic-payload-builder = { path = "crates/payload/basic" }
 reth-bench = { path = "bin/reth-bench" }
-reth-bench-compare = { path = "bin/reth-bench-compare" }
 reth-chain-state = { path = "crates/chain-state" }
 reth-chainspec = { path = "crates/chainspec", default-features = false }
 reth-cli = { path = "crates/cli/cli" }
@@ -397,7 +396,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
 reth-payload-primitives = { path = "crates/payload/primitives" }
 reth-payload-validator = { path = "crates/payload/validator" }
 reth-payload-util = { path = "crates/payload/util" }
-reth-primitives = { path = "crates/primitives", default-features = false }
+reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
 reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
 reth-provider = { path = "crates/storage/provider" }
 reth-prune = { path = "crates/prune/prune" }
@@ -435,25 +434,24 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
 reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-features = false }
 
 # revm
-revm = { version = "34.0.0", default-features = false }
-revm-bytecode = { version = "8.0.0", default-features = false }
-revm-database = { version = "10.0.0", default-features = false }
-revm-state = { version = "9.0.0", default-features = false }
-revm-primitives = { version = "22.0.0", default-features = false }
-revm-interpreter = { version = "32.0.0", default-features = false }
-revm-database-interface = { version = "9.0.0", default-features = false }
-op-revm = { version = "15.0.0", default-features = false }
-revm-inspectors = "0.34.2"
+revm = { version = "36.0.0", default-features = false }
+revm-bytecode = { version = "9.0.0", default-features = false }
+revm-database = { version = "12.0.0", default-features = false }
+revm-state = { version = "10.0.0", default-features = false }
+revm-primitives = { version = "22.1.0", default-features = false }
+revm-interpreter = { version = "34.0.0", default-features = false }
+revm-database-interface = { version = "10.0.0", default-features = false }
+revm-inspectors = "0.36.0"
 
 # eth
-alloy-dyn-abi = "1.5.6"
-alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
-alloy-sol-types = { version = "1.5.6", default-features = false }
+alloy-dyn-abi = "1.5.0"
+alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
+alloy-sol-types = { version = "1.5.0", default-features = false }
 
 alloy-chains = { version = "0.2.5", default-features = false }
 alloy-eip2124 = { version = "0.2.0", default-features = false }
-alloy-eip7928 = { version = "0.3.0", default-features = false }
-alloy-evm = { version = "0.27.2", default-features = false }
+alloy-eip7928 = { version = "0.3.3", default-features = false, features = ["rlp"] }
+alloy-evm = { version = "0.29.2", default-features = false }
 alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
 alloy-trie = { version = "0.9.4", default-features = false }
 
@@ -488,7 +486,6 @@ alloy-transport-ipc = { version = "1.7.3", default-features = false }
 alloy-transport-ws = { version = "1.7.3", default-features = false }
 
 # op
-alloy-op-evm = { version = "0.27.2", default-features = false }
 alloy-op-hardforks = "0.4.4"
 op-alloy-rpc-types = { version = "0.23.1", default-features = false }
 op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -507,6 +504,7 @@ bincode = "1.3"
 bitflags = "2.4"
 boyer-moore-magiclen = "0.2.16"
 bytes = { version = "1.11.1", default-features = false }
+blake3 = "1.8"
 brotli = "8"
 cfg-if = "1.0"
 clap = "4"
@@ -522,6 +520,7 @@ humantime = "2.1"
 humantime-serde = "1.1"
 itertools = { version = "0.14", default-features = false }
 linked_hash_set = "0.1"
+libc = "0.2"
 lz4 = "1.28.1"
 modular-bitfield = "0.13.1"
 notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
@@ -540,11 +539,13 @@ serde_with = { version = "3", default-features = false, features = ["macros"] }
 sha2 = { version = "0.10", default-features = false }
 shellexpand = "3.0.0"
 shlex = "1.3"
+slotmap = "1"
 smallvec = "1"
 strum = { version = "0.27", default-features = false }
 strum_macros = "0.27"
 syn = "2.0"
 thiserror = { version = "2.0.0", default-features = false }
+thread-priority = "3.0.0"
 tar = "0.4.44"
 tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
 tracing-appender = "0.2"
@@ -664,6 +665,7 @@ cipher = "0.4.3"
 comfy-table = "7.0"
 concat-kdf = "0.1.0"
 crossbeam-channel = "0.5.13"
+crossbeam-queue = "0.3"
 crossbeam-utils = "0.8"
 crossterm = "0.29.0"
 csv = "1.3.0"
@@ -740,10 +742,8 @@ ipnet = "2.11"
 # alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
 
 # op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
-# op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
 # op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
 # op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
-# op-alloy-rpc-jsonrpsee = { git = "https://github.com/alloy-rs/op-alloy", rev = "a79d6fc" }
 #
 # revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "1207e33" }
 #
@@ -753,12 +753,71 @@ ipnet = "2.11"
 # jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
 # jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
 
-# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
-# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
+# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "9bc2dba" }
 
 # revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
 
 # alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
 # alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
 
-nybbles = { git = "https://github.com/alloy-rs/nybbles", rev = "8df38fb" }
+# =============================================================================
+# BAL devnet patches (EIP-7778, EIP-7928 block access lists)
+# =============================================================================
+
+# revm staging patches
+revm = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-database = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-state = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-context = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-handler = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+op-revm = { git = "https://github.com/bluealloy/revm", rev = "fa5a6914398c9b178422efc1edd1d2ab33dad923" }
+# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", branch = "staging" }
+
+# alloy-evm bal-devnet2 patches
+alloy-evm = { git = "https://github.com/alloy-rs/evm", branch = "bal-devnet2" }
+
+# alloy bal-devnet2 patches
+alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-consensus-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-node-bindings = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-any = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+alloy-tx-macros = { git = "https://github.com/alloy-rs/alloy", branch = "bal-devnet2" }
+
+# op-alloy bal-devnet2 patches
+op-alloy-consensus = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
+op-alloy-network = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
+op-alloy-rpc-types = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }
+op-alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/op-alloy", branch = "bal-devnet2" }

Dockerfile.depot

@@ -1,8 +1,10 @@
 # syntax=docker/dockerfile:1
 
 # Dockerfile for reth, optimized for Depot builds
+# Supports PGO+BOLT optimization for maximum performance
 # Usage:
 #   reth: --build-arg BINARY=reth
+#   PGO+BOLT: --build-arg USE_PGO_BOLT=true (Linux x86_64/aarch64 only)
 
 FROM rust:1.93 AS builder
 WORKDIR /app
@@ -43,6 +45,18 @@ ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
 ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
 ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
 
+# Enable PGO+BOLT optimization (Linux only)
+ARG USE_PGO_BOLT=false
+ENV USE_PGO_BOLT=$USE_PGO_BOLT
+
+# Optional path to a pre-collected merged.profdata file in build context.
+ARG PGO_PROFDATA=""
+ENV PGO_PROFDATA=$PGO_PROFDATA
+
+# Whether to strip debug symbols from PGO-built binaries.
+ARG STRIP_SYMBOLS=true
+ENV STRIP_SYMBOLS=$STRIP_SYMBOLS
+
 # Build application
 # Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
 ARG TARGETPLATFORM
@@ -53,12 +67,21 @@ RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
     --mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
     export RUSTC_WRAPPER=sccache SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev SCCACHE_DIR=/sccache && \
     sccache --start-server && \
-    if [ -n "$RUSTFLAGS" ]; then \
-        export RUSTFLAGS="$RUSTFLAGS"; \
-    elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
-        export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
+    if [ "$USE_PGO_BOLT" = "true" ] && [ "$TARGETPLATFORM" = "linux/amd64" ] && [ -n "$PGO_PROFDATA" ] && [ -f "$PGO_PROFDATA" ]; then \
+        apt-get update && apt-get install -y -qq lsb-release wget sudo && \
+        BINARY="$BINARY" PROFILE="$BUILD_PROFILE" FEATURES="$FEATURES" SKIP_BOLT=true STRIP_SYMBOLS="$STRIP_SYMBOLS" PGO_PROFDATA="$PGO_PROFDATA" \
+            ./.github/scripts/build_pgo_bolt.sh; \
+    else \
+        if [ "$USE_PGO_BOLT" = "true" ]; then \
+            echo "PGO requested but pre-collected profile missing at '${PGO_PROFDATA:-<unset>}' - falling back to non-PGO build"; \
+        fi; \
+        if [ -n "$RUSTFLAGS" ]; then \
+            export RUSTFLAGS="$RUSTFLAGS"; \
+        elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
+            export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
+        fi && \
+        cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml; \
     fi && \
-    cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml && \
     sccache --show-stats
 
 # Copy binary to a known location (ARG not resolved in COPY)

Makefile

@@ -17,6 +17,9 @@ FEATURES ?=
 # Cargo profile for builds. Default is for local builds, CI uses an override.
 PROFILE ?= release
 
+# Extra RUSTFLAGS to append to build targets (e.g., "-C target-cpu=x86-64-v3")
+EXTRA_RUSTFLAGS ?=
+
 # Extra flags for Cargo
 CARGO_INSTALL_EXTRA_FLAGS ?=
 
@@ -26,7 +29,7 @@ EF_TESTS_URL := https://github.com/ethereum/tests/archive/refs/tags/$(EF_TESTS_T
 EF_TESTS_DIR := ./testing/ef-tests/ethereum-tests
 
 # The release tag of https://github.com/ethereum/execution-spec-tests to use for EEST tests
-EEST_TESTS_TAG := v4.5.0
+EEST_TESTS_TAG := bal@v5.0.0
 EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
 EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
 
@@ -74,7 +77,7 @@ build-debug: ## Build the reth binary into `target/debug` directory.
 	cargo build --bin reth --features "$(FEATURES)"
 # Builds the reth binary natively.
 build-native-%:
-	cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
+	$(if $(EXTRA_RUSTFLAGS),RUSTFLAGS="$(EXTRA_RUSTFLAGS)") cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
 
 # The following commands use `cross` to build a cross-compile.
 #
@@ -92,11 +95,12 @@ build-native-%:
 # on other systems. JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to use 64-KiB
 # pages. See: https://github.com/paradigmxyz/reth/issues/6742
 build-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
+build-native-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
 
 # Note: The additional rustc compiler flags are for intrinsics needed by MDBX.
 # See: https://github.com/cross-rs/cross/wiki/FAQ#undefined-reference-with-build-std
 build-%:
-	RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \
+	RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc $(EXTRA_RUSTFLAGS)" \
 		cross build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
 
 # Unfortunately we can't easily use cross to build for Darwin because of licensing issues.

bin/reth-bench-compare/Cargo.toml

@@ -1,102 +0,0 @@
-[package]
-name = "reth-bench-compare"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-license.workspace = true
-homepage.workspace = true
-repository.workspace = true
-description = "Automated reth benchmark comparison between git references"
-
-[lints]
-workspace = true
-
-[[bin]]
-name = "reth-bench-compare"
-path = "src/main.rs"
-
-[dependencies]
-# reth
-reth-cli-runner.workspace = true
-reth-cli-util.workspace = true
-reth-node-core.workspace = true
-reth-tracing.workspace = true
-reth-chainspec.workspace = true
-
-# alloy
-alloy-provider = { workspace = true, features = ["reqwest-rustls-tls"], default-features = false }
-alloy-rpc-client = { workspace = true, features = ["pubsub"] }
-alloy-rpc-types-eth.workspace = true
-alloy-transport-ws.workspace = true
-alloy-primitives.workspace = true
-
-# CLI and argument parsing
-clap = { workspace = true, features = ["derive", "env"] }
-eyre.workspace = true
-
-# Async runtime
-tokio = { workspace = true, features = ["full"] }
-tracing.workspace = true
-
-# Serialization
-serde = { workspace = true, features = ["derive"] }
-serde_json.workspace = true
-
-# Time handling
-chrono = { workspace = true, features = ["serde"] }
-
-# Path manipulation
-shellexpand.workspace = true
-
-# CSV handling
-csv.workspace = true
-
-# Process management
-ctrlc.workspace = true
-shlex.workspace = true
-
-[target.'cfg(unix)'.dependencies]
-nix = { version = "0.31", features = ["signal", "process"] }
-
-[features]
-default = ["jemalloc"]
-
-asm-keccak = [
-    "reth-node-core/asm-keccak",
-    "alloy-primitives/asm-keccak",
-]
-
-jemalloc = [
-    "reth-cli-util/jemalloc",
-    "reth-node-core/jemalloc",
-]
-jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
-tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
-tracy = [
-    "reth-node-core/tracy",
-    "reth-tracing/tracy",
-]
-
-min-error-logs = [
-    "tracing/release_max_level_error",
-    "reth-node-core/min-error-logs",
-]
-min-warn-logs = [
-    "tracing/release_max_level_warn",
-    "reth-node-core/min-warn-logs",
-]
-min-info-logs = [
-    "tracing/release_max_level_info",
-    "reth-node-core/min-info-logs",
-]
-min-debug-logs = [
-    "tracing/release_max_level_debug",
-    "reth-node-core/min-debug-logs",
-]
-min-trace-logs = [
-    "tracing/release_max_level_trace",
-    "reth-node-core/min-trace-logs",
-]
-
-# no-op feature flag for CI matrices
-ethereum = []

bin/reth-bench-compare/README.md

@@ -1,50 +0,0 @@
-# reth-bench-compare
-
-Compare reth performance between two git references.
-
-## Usage
-
-```bash
-reth-bench-compare \
-  --baseline-ref main \
-  --feature-ref my-feature \
-  --blocks 100 \
-  --wait-for-persistence
-```
-
-## Arguments
-
-| Argument | Description | Default | Required |
-|----------|-------------|---------|----------|
-| `--baseline-ref <REF>` | Git reference for baseline | - | Yes |
-| `--feature-ref <REF>` | Git reference to compare | - | Yes |
-| `--blocks <N>` | Number of blocks to benchmark | `100` | No |
-| `--chain <CHAIN>` | Chain to benchmark | `mainnet` | No |
-| `--datadir <PATH>` | Data directory path | OS-specific | No |
-| `--rpc-url <URL>` | RPC endpoint for block data | Chain default | No |
-| `--output-dir <PATH>` | Output directory | `./reth-bench-compare` | No |
-| `--wait-for-persistence` | Wait for block persistence | `false` | No |
-| `--persistence-threshold <N>` | Wait after every N+1 blocks | `2` | No |
-| `--wait-time <DURATION>` | Fixed delay (legacy) | - | No |
-| `--warmup-blocks <N>` | Cache warmup blocks | Same as `--blocks` | No |
-| `--draw` | Generate charts (needs Python/uv) | `false` | No |
-| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
-| `-vvvv` | Debug logging | Info | No |
-| `--features <FEATURES>` | Extra Rust features for both builds | - | No |
-| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
-| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
-| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |
-| `--baseline-rustflags <FLAGS>` | RUSTFLAGS for baseline only | Inherits `--rustflags` | No |
-| `--feature-rustflags <FLAGS>` | RUSTFLAGS for feature only | Inherits `--rustflags` | No |
-| `--baseline-args <ARGS>` | Extra args for baseline node | - | No |
-| `--feature-args <ARGS>` | Extra args for feature node | - | No |
-| `--metrics-port <PORT>` | Metrics endpoint port | `5005` | No |
-| `--sudo` | Run with elevated privileges | `false` | No |
-
-## Output
-
-Results in `./reth-bench-compare/results/<timestamp>/`:
-- `comparison_report.json` - Metrics comparison
-- `per_block_comparison.csv` - Per-block statistics
-- `baseline/` and `feature/` - Individual run results
-- `latency_comparison.png` - Chart (if `--draw` used)

bin/reth-bench-compare/src/benchmark.rs

@@ -1,307 +0,0 @@
-//! Benchmark execution using reth-bench.
-
-use crate::cli::Args;
-use eyre::{eyre, Result, WrapErr};
-use std::{
-    path::Path,
-    sync::{Arc, Mutex},
-};
-use tokio::{
-    fs::File as AsyncFile,
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
-    process::Command,
-};
-use tracing::{debug, error, info, warn};
-
-/// Manages benchmark execution using reth-bench
-pub(crate) struct BenchmarkRunner {
-    rpc_url: String,
-    jwt_secret: String,
-    wait_time: Option<String>,
-    wait_for_persistence: bool,
-    persistence_threshold: Option<u64>,
-    warmup_blocks: u64,
-}
-
-impl BenchmarkRunner {
-    /// Create a new `BenchmarkRunner` from CLI arguments
-    pub(crate) fn new(args: &Args) -> Self {
-        Self {
-            rpc_url: args.get_rpc_url(),
-            jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(),
-            wait_time: args.wait_time.clone(),
-            wait_for_persistence: args.wait_for_persistence,
-            persistence_threshold: args.persistence_threshold,
-            warmup_blocks: args.get_warmup_blocks(),
-        }
-    }
-
-    /// Clear filesystem caches (page cache, dentries, and inodes)
-    pub(crate) async fn clear_fs_caches() -> Result<()> {
-        info!("Clearing filesystem caches...");
-
-        // First sync to ensure all pending writes are flushed
-        let sync_output =
-            Command::new("sync").output().await.wrap_err("Failed to execute sync command")?;
-
-        if !sync_output.status.success() {
-            return Err(eyre!("sync command failed"));
-        }
-
-        // Drop caches - requires sudo/root permissions
-        // 3 = drop pagecache, dentries, and inodes
-        let drop_caches_cmd = Command::new("sudo")
-            .args(["-n", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"])
-            .output()
-            .await;
-
-        match drop_caches_cmd {
-            Ok(output) if output.status.success() => {
-                info!("Successfully cleared filesystem caches");
-                Ok(())
-            }
-            Ok(output) => {
-                let stderr = String::from_utf8_lossy(&output.stderr);
-                if stderr.contains("sudo: a password is required") {
-                    warn!("Unable to clear filesystem caches: sudo password required");
-                    warn!(
-                        "For optimal benchmarking, configure passwordless sudo for cache clearing:"
-                    );
-                    warn!("  echo '$USER ALL=(ALL) NOPASSWD: /bin/sh -c echo\\\\ [0-9]\\\\ \\\\>\\\\ /proc/sys/vm/drop_caches' | sudo tee /etc/sudoers.d/drop_caches");
-                    Ok(())
-                } else {
-                    Err(eyre!("Failed to clear filesystem caches: {}", stderr))
-                }
-            }
-            Err(e) => {
-                warn!("Unable to clear filesystem caches: {}", e);
-                Ok(())
-            }
-        }
-    }
-
-    /// Run a warmup benchmark for cache warming
-    pub(crate) async fn run_warmup(&self, from_block: u64) -> Result<()> {
-        let to_block = from_block + self.warmup_blocks;
-        info!(
-            "Running warmup benchmark from block {} to {} ({} blocks)",
-            from_block, to_block, self.warmup_blocks
-        );
-
-        // Build the reth-bench command for warmup (no output flag)
-        let mut cmd = Command::new("reth-bench");
-        cmd.args([
-            "new-payload-fcu",
-            "--rpc-url",
-            &self.rpc_url,
-            "--jwt-secret",
-            &self.jwt_secret,
-            "--from",
-            &from_block.to_string(),
-            "--to",
-            &to_block.to_string(),
-            "--wait-time=0ms", // Warmup should avoid persistence waits.
-        ]);
-
-        cmd.env("RUST_LOG_STYLE", "never")
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true);
-
-        // Set process group for consistent signal handling
-        #[cfg(unix)]
-        {
-            cmd.process_group(0);
-        }
-
-        debug!("Executing warmup reth-bench command: {:?}", cmd);
-
-        // Execute the warmup benchmark
-        let mut child = cmd.spawn().wrap_err("Failed to start warmup reth-bench process")?;
-
-        // Stream output at debug level
-        if let Some(stdout) = child.stdout.take() {
-            tokio::spawn(async move {
-                let reader = BufReader::new(stdout);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[WARMUP] {}", line);
-                }
-            });
-        }
-
-        if let Some(stderr) = child.stderr.take() {
-            tokio::spawn(async move {
-                let reader = BufReader::new(stderr);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[WARMUP] {}", line);
-                }
-            });
-        }
-
-        let status = child.wait().await.wrap_err("Failed to wait for warmup reth-bench")?;
-
-        if !status.success() {
-            return Err(eyre!("Warmup reth-bench failed with exit code: {:?}", status.code()));
-        }
-
-        info!("Warmup completed successfully");
-        Ok(())
-    }
-
-    /// Run a benchmark for the specified block range
-    pub(crate) async fn run_benchmark(
-        &self,
-        from_block: u64,
-        to_block: u64,
-        output_dir: &Path,
-    ) -> Result<()> {
-        info!(
-            "Running benchmark from block {} to {} (output: {:?})",
-            from_block, to_block, output_dir
-        );
-
-        // Ensure output directory exists
-        std::fs::create_dir_all(output_dir)
-            .wrap_err_with(|| format!("Failed to create output directory: {output_dir:?}"))?;
-
-        // Create log file path for reth-bench output
-        let log_file_path = output_dir.join("reth_bench.log");
-        info!("reth-bench logs will be saved to: {:?}", log_file_path);
-
-        // Build the reth-bench command
-        let mut cmd = Command::new("reth-bench");
-        cmd.args([
-            "new-payload-fcu",
-            "--rpc-url",
-            &self.rpc_url,
-            "--jwt-secret",
-            &self.jwt_secret,
-            "--from",
-            &from_block.to_string(),
-            "--to",
-            &to_block.to_string(),
-            "--output",
-            &output_dir.to_string_lossy(),
-        ]);
-
-        // Configure wait mode: both can be used together
-        // When both are set: wait at least wait_time, and also wait for persistence if needed
-        if let Some(ref wait_time) = self.wait_time {
-            cmd.args(["--wait-time", wait_time]);
-        }
-        if self.wait_for_persistence {
-            cmd.arg("--wait-for-persistence");
-
-            // Add persistence threshold if specified
-            if let Some(threshold) = self.persistence_threshold {
-                cmd.args(["--persistence-threshold", &threshold.to_string()]);
-            }
-        }
-
-        cmd.env("RUST_LOG_STYLE", "never")
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true);
-
-        // Set process group for consistent signal handling
-        #[cfg(unix)]
-        {
-            cmd.process_group(0);
-        }
-
-        // Debug log the command
-        debug!("Executing reth-bench command: {:?}", cmd);
-
-        // Execute the benchmark
-        let mut child = cmd.spawn().wrap_err("Failed to start reth-bench process")?;
-
-        // Capture stdout and stderr for error reporting
-        let stdout_lines = Arc::new(Mutex::new(Vec::new()));
-        let stderr_lines = Arc::new(Mutex::new(Vec::new()));
-
-        // Stream stdout with prefix at debug level, capture for error reporting, and write to log
-        // file
-        if let Some(stdout) = child.stdout.take() {
-            let stdout_lines_clone = stdout_lines.clone();
-            let log_file = AsyncFile::create(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = BufReader::new(stdout);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-BENCH] {}", line);
-                    if let Ok(mut captured) = stdout_lines_clone.lock() {
-                        captured.push(line.clone());
-                    }
-                    // Write to log file (reth-bench output already has timestamps if needed)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        // Stream stderr with prefix at debug level, capture for error reporting, and write to log
-        // file
-        if let Some(stderr) = child.stderr.take() {
-            let stderr_lines_clone = stderr_lines.clone();
-            let log_file = AsyncFile::options()
-                .create(true)
-                .append(true)
-                .open(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = BufReader::new(stderr);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-BENCH] {}", line);
-                    if let Ok(mut captured) = stderr_lines_clone.lock() {
-                        captured.push(line.clone());
-                    }
-                    // Write to log file (reth-bench output already has timestamps if needed)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        let status = child.wait().await.wrap_err("Failed to wait for reth-bench")?;
-
-        if !status.success() {
-            // Print all captured output when command fails
-            error!("reth-bench failed with exit code: {:?}", status.code());
-
-            if let Ok(stdout) = stdout_lines.lock() &&
-                !stdout.is_empty()
-            {
-                error!("reth-bench stdout:");
-                for line in stdout.iter() {
-                    error!("  {}", line);
-                }
-            }
-
-            if let Ok(stderr) = stderr_lines.lock() &&
-                !stderr.is_empty()
-            {
-                error!("reth-bench stderr:");
-                for line in stderr.iter() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!("reth-bench failed with exit code: {:?}", status.code()));
-        }
-
-        info!("Benchmark completed");
-        Ok(())
-    }
-}

bin/reth-bench-compare/src/comparison.rs

@@ -1,723 +0,0 @@
-//! Results comparison and report generation.
-
-use crate::cli::Args;
-use chrono::{DateTime, Utc};
-use csv::Reader;
-use eyre::{eyre, Result, WrapErr};
-use serde::{Deserialize, Serialize};
-use std::{
-    cmp::Ordering,
-    collections::HashMap,
-    fs,
-    path::{Path, PathBuf},
-};
-use tracing::{info, warn};
-
-/// Manages comparison between baseline and feature reference results
-pub(crate) struct ComparisonGenerator {
-    output_dir: PathBuf,
-    timestamp: String,
-    baseline_ref_name: String,
-    feature_ref_name: String,
-    baseline_results: Option<BenchmarkResults>,
-    feature_results: Option<BenchmarkResults>,
-    baseline_command: Option<String>,
-    feature_command: Option<String>,
-}
-
-/// Represents the results from a single benchmark run
-#[derive(Debug, Clone)]
-pub(crate) struct BenchmarkResults {
-    pub ref_name: String,
-    pub combined_latency_data: Vec<CombinedLatencyRow>,
-    pub summary: BenchmarkSummary,
-    pub start_timestamp: Option<DateTime<Utc>>,
-    pub end_timestamp: Option<DateTime<Utc>>,
-}
-
-/// Combined latency CSV row structure
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub(crate) struct CombinedLatencyRow {
-    pub block_number: u64,
-    #[serde(default)]
-    pub transaction_count: Option<u64>,
-    pub gas_used: u64,
-    pub new_payload_latency: u128,
-}
-
-/// Total gas CSV row structure
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub(crate) struct TotalGasRow {
-    pub block_number: u64,
-    #[serde(default)]
-    pub transaction_count: Option<u64>,
-    pub gas_used: u64,
-    pub time: u128,
-}
-
-/// Summary statistics for a benchmark run.
-///
-/// Latencies are derived from per-block `engine_newPayload` timings (converted from µs to ms):
-/// - `mean_new_payload_latency_ms`: arithmetic mean latency across blocks.
-/// - `median_new_payload_latency_ms`: p50 latency across blocks.
-/// - `p90_new_payload_latency_ms` / `p99_new_payload_latency_ms`: tail latencies across blocks.
-#[derive(Debug, Clone, Serialize)]
-pub(crate) struct BenchmarkSummary {
-    pub total_blocks: u64,
-    pub total_gas_used: u64,
-    pub total_duration_ms: u128,
-    pub mean_new_payload_latency_ms: f64,
-    pub median_new_payload_latency_ms: f64,
-    pub p90_new_payload_latency_ms: f64,
-    pub p99_new_payload_latency_ms: f64,
-    pub gas_per_second: f64,
-    pub blocks_per_second: f64,
-    pub min_block_number: u64,
-    pub max_block_number: u64,
-}
-
-/// Comparison report between two benchmark runs
-#[derive(Debug, Serialize)]
-pub(crate) struct ComparisonReport {
-    pub timestamp: String,
-    pub baseline: RefInfo,
-    pub feature: RefInfo,
-    pub comparison_summary: ComparisonSummary,
-    pub per_block_comparisons: Vec<BlockComparison>,
-}
-
-/// Information about a reference in the comparison
-#[derive(Debug, Serialize)]
-pub(crate) struct RefInfo {
-    pub ref_name: String,
-    pub summary: BenchmarkSummary,
-    pub start_timestamp: Option<DateTime<Utc>>,
-    pub end_timestamp: Option<DateTime<Utc>>,
-    pub reth_command: Option<String>,
-}
-
-/// Summary of the comparison between references.
-///
-/// Percent deltas are `(feature - baseline) / baseline * 100`:
-/// - `new_payload_latency_mean_change_percent`: percent changes of the per-block means.
-/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective
-///   per-block percentiles.
-/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the
-///   mean and median of per-block percent deltas (feature vs baseline), capturing block-level
-///   drift.
-/// - `per_block_latency_change_std_dev_percent`: standard deviation of per-block percent changes,
-///   measuring consistency of performance changes across blocks.
-/// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time
-///   across the run.
-///
-/// Positive means slower/higher; negative means faster/lower.
-#[derive(Debug, Serialize)]
-pub(crate) struct ComparisonSummary {
-    pub per_block_latency_change_mean_percent: f64,
-    pub per_block_latency_change_median_percent: f64,
-    pub per_block_latency_change_std_dev_percent: f64,
-    pub new_payload_total_latency_change_percent: f64,
-    pub new_payload_latency_mean_change_percent: f64,
-    pub new_payload_latency_p50_change_percent: f64,
-    pub new_payload_latency_p90_change_percent: f64,
-    pub new_payload_latency_p99_change_percent: f64,
-    pub gas_per_second_change_percent: f64,
-    pub blocks_per_second_change_percent: f64,
-}
-
-/// Per-block comparison data
-#[derive(Debug, Serialize)]
-pub(crate) struct BlockComparison {
-    pub block_number: u64,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub transaction_count: Option<u64>,
-    pub gas_used: u64,
-    pub baseline_new_payload_latency: u128,
-    pub feature_new_payload_latency: u128,
-    pub new_payload_latency_change_percent: f64,
-}
-
-impl ComparisonGenerator {
-    /// Create a new comparison generator
-    pub(crate) fn new(args: &Args) -> Self {
-        let now: DateTime<Utc> = Utc::now();
-        let timestamp = now.format("%Y%m%d_%H%M%S").to_string();
-
-        Self {
-            output_dir: args.output_dir_path(),
-            timestamp,
-            baseline_ref_name: args.baseline_ref.clone(),
-            feature_ref_name: args.feature_ref.clone(),
-            baseline_results: None,
-            feature_results: None,
-            baseline_command: None,
-            feature_command: None,
-        }
-    }
-
-    /// Get the output directory for a specific reference
-    pub(crate) fn get_ref_output_dir(&self, ref_type: &str) -> PathBuf {
-        self.output_dir.join("results").join(&self.timestamp).join(ref_type)
-    }
-
-    /// Get the main output directory for this comparison run
-    pub(crate) fn get_output_dir(&self) -> PathBuf {
-        self.output_dir.join("results").join(&self.timestamp)
-    }
-
-    /// Add benchmark results for a reference
-    pub(crate) fn add_ref_results(&mut self, ref_type: &str, output_path: &Path) -> Result<()> {
-        let ref_name = match ref_type {
-            "baseline" => &self.baseline_ref_name,
-            "feature" => &self.feature_ref_name,
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        };
-
-        let results = self.load_benchmark_results(ref_name, output_path)?;
-
-        match ref_type {
-            "baseline" => self.baseline_results = Some(results),
-            "feature" => self.feature_results = Some(results),
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        }
-
-        info!("Loaded benchmark results for {} reference", ref_type);
-
-        Ok(())
-    }
-
-    /// Set the benchmark run timestamps for a reference
-    pub(crate) fn set_ref_timestamps(
-        &mut self,
-        ref_type: &str,
-        start: DateTime<Utc>,
-        end: DateTime<Utc>,
-    ) -> Result<()> {
-        match ref_type {
-            "baseline" => {
-                if let Some(ref mut results) = self.baseline_results {
-                    results.start_timestamp = Some(start);
-                    results.end_timestamp = Some(end);
-                } else {
-                    return Err(eyre!("Baseline results not loaded yet"));
-                }
-            }
-            "feature" => {
-                if let Some(ref mut results) = self.feature_results {
-                    results.start_timestamp = Some(start);
-                    results.end_timestamp = Some(end);
-                } else {
-                    return Err(eyre!("Feature results not loaded yet"));
-                }
-            }
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        }
-
-        Ok(())
-    }
-
-    /// Set the reth command for a reference
-    pub(crate) fn set_ref_command(&mut self, ref_type: &str, command: String) -> Result<()> {
-        match ref_type {
-            "baseline" => {
-                self.baseline_command = Some(command);
-            }
-            "feature" => {
-                self.feature_command = Some(command);
-            }
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        }
-
-        Ok(())
-    }
-
-    /// Generate the final comparison report
-    pub(crate) async fn generate_comparison_report(&self) -> Result<()> {
-        info!("Generating comparison report...");
-
-        let baseline =
-            self.baseline_results.as_ref().ok_or_else(|| eyre!("Baseline results not loaded"))?;
-
-        let feature =
-            self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?;
-
-        let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?;
-        let comparison_summary = self.calculate_comparison_summary(
-            &baseline.summary,
-            &feature.summary,
-            &per_block_comparisons,
-        )?;
-
-        let report = ComparisonReport {
-            timestamp: self.timestamp.clone(),
-            baseline: RefInfo {
-                ref_name: baseline.ref_name.clone(),
-                summary: baseline.summary.clone(),
-                start_timestamp: baseline.start_timestamp,
-                end_timestamp: baseline.end_timestamp,
-                reth_command: self.baseline_command.clone(),
-            },
-            feature: RefInfo {
-                ref_name: feature.ref_name.clone(),
-                summary: feature.summary.clone(),
-                start_timestamp: feature.start_timestamp,
-                end_timestamp: feature.end_timestamp,
-                reth_command: self.feature_command.clone(),
-            },
-            comparison_summary,
-            per_block_comparisons,
-        };
-
-        // Write reports
-        self.write_comparison_reports(&report).await?;
-
-        // Print summary to console
-        self.print_comparison_summary(&report);
-
-        Ok(())
-    }
-
-    /// Load benchmark results from CSV files
-    fn load_benchmark_results(
-        &self,
-        ref_name: &str,
-        output_path: &Path,
-    ) -> Result<BenchmarkResults> {
-        let combined_latency_path = output_path.join("combined_latency.csv");
-        let total_gas_path = output_path.join("total_gas.csv");
-
-        let combined_latency_data = self.load_combined_latency_csv(&combined_latency_path)?;
-        let total_gas_data = self.load_total_gas_csv(&total_gas_path)?;
-
-        let summary = self.calculate_summary(&combined_latency_data, &total_gas_data)?;
-
-        Ok(BenchmarkResults {
-            ref_name: ref_name.to_string(),
-            combined_latency_data,
-            summary,
-            start_timestamp: None,
-            end_timestamp: None,
-        })
-    }
-
-    /// Load combined latency CSV data
-    fn load_combined_latency_csv(&self, path: &Path) -> Result<Vec<CombinedLatencyRow>> {
-        let mut reader = Reader::from_path(path)
-            .wrap_err_with(|| format!("Failed to open combined latency CSV: {path:?}"))?;
-
-        let mut rows = Vec::new();
-        for result in reader.deserialize() {
-            let row: CombinedLatencyRow = result
-                .wrap_err_with(|| format!("Failed to parse combined latency row in {path:?}"))?;
-            rows.push(row);
-        }
-
-        if rows.is_empty() {
-            return Err(eyre!("No data found in combined latency CSV: {:?}", path));
-        }
-
-        Ok(rows)
-    }
-
-    /// Load total gas CSV data
-    fn load_total_gas_csv(&self, path: &Path) -> Result<Vec<TotalGasRow>> {
-        let mut reader = Reader::from_path(path)
-            .wrap_err_with(|| format!("Failed to open total gas CSV: {path:?}"))?;
-
-        let mut rows = Vec::new();
-        for result in reader.deserialize() {
-            let row: TotalGasRow =
-                result.wrap_err_with(|| format!("Failed to parse total gas row in {path:?}"))?;
-            rows.push(row);
-        }
-
-        if rows.is_empty() {
-            return Err(eyre!("No data found in total gas CSV: {:?}", path));
-        }
-
-        Ok(rows)
-    }
-
-    /// Calculate summary statistics for a benchmark run.
-    ///
-    /// Computes latency statistics from per-block `new_payload_latency` values in `combined_data`
-    /// (converting from µs to ms), and throughput metrics using the total run duration from
-    /// `total_gas_data`. Percentiles (p50/p90/p99) use linear interpolation on sorted latencies.
-    fn calculate_summary(
-        &self,
-        combined_data: &[CombinedLatencyRow],
-        total_gas_data: &[TotalGasRow],
-    ) -> Result<BenchmarkSummary> {
-        if combined_data.is_empty() || total_gas_data.is_empty() {
-            return Err(eyre!("Cannot calculate summary for empty data"));
-        }
-
-        let total_blocks = combined_data.len() as u64;
-        let total_gas_used: u64 = combined_data.iter().map(|r| r.gas_used).sum();
-
-        let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds
-
-        let latencies_ms: Vec<f64> =
-            combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).collect();
-        let mean_new_payload_latency_ms: f64 =
-            latencies_ms.iter().sum::<f64>() / total_blocks as f64;
-
-        let mut sorted_latencies_ms = latencies_ms;
-        sorted_latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
-        let median_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.5);
-        let p90_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.9);
-        let p99_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.99);
-
-        let total_duration_seconds = total_duration_ms as f64 / 1000.0;
-        let gas_per_second = if total_duration_seconds > f64::EPSILON {
-            total_gas_used as f64 / total_duration_seconds
-        } else {
-            0.0
-        };
-
-        let blocks_per_second = if total_duration_seconds > f64::EPSILON {
-            total_blocks as f64 / total_duration_seconds
-        } else {
-            0.0
-        };
-
-        let min_block_number = combined_data.first().unwrap().block_number;
-        let max_block_number = combined_data.last().unwrap().block_number;
-
-        Ok(BenchmarkSummary {
-            total_blocks,
-            total_gas_used,
-            total_duration_ms,
-            mean_new_payload_latency_ms,
-            median_new_payload_latency_ms,
-            p90_new_payload_latency_ms,
-            p99_new_payload_latency_ms,
-            gas_per_second,
-            blocks_per_second,
-            min_block_number,
-            max_block_number,
-        })
-    }
-
-    /// Calculate comparison summary between baseline and feature
-    fn calculate_comparison_summary(
-        &self,
-        baseline: &BenchmarkSummary,
-        feature: &BenchmarkSummary,
-        per_block_comparisons: &[BlockComparison],
-    ) -> Result<ComparisonSummary> {
-        let calc_percent_change = |baseline: f64, feature: f64| -> f64 {
-            if baseline.abs() > f64::EPSILON {
-                ((feature - baseline) / baseline) * 100.0
-            } else {
-                0.0
-            }
-        };
-
-        // Calculate per-block statistics. "Per-block" means: for each block, compute the percent
-        // change (feature - baseline) / baseline * 100, then calculate statistics across those
-        // per-block percent changes. This captures how consistently the feature performs relative
-        // to baseline across all blocks.
-        let per_block_percent_changes: Vec<f64> =
-            per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect();
-        let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() {
-            0.0
-        } else {
-            per_block_percent_changes.iter().sum::<f64>() / per_block_percent_changes.len() as f64
-        };
-        let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() {
-            0.0
-        } else {
-            let mut sorted = per_block_percent_changes.clone();
-            sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
-            percentile(&sorted, 0.5)
-        };
-        let per_block_latency_change_std_dev_percent =
-            calculate_std_dev(&per_block_percent_changes, per_block_latency_change_mean_percent);
-
-        let baseline_total_latency_ms =
-            baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64;
-        let feature_total_latency_ms =
-            feature.mean_new_payload_latency_ms * feature.total_blocks as f64;
-        let new_payload_total_latency_change_percent =
-            calc_percent_change(baseline_total_latency_ms, feature_total_latency_ms);
-
-        Ok(ComparisonSummary {
-            per_block_latency_change_mean_percent,
-            per_block_latency_change_median_percent,
-            per_block_latency_change_std_dev_percent,
-            new_payload_total_latency_change_percent,
-            new_payload_latency_mean_change_percent: calc_percent_change(
-                baseline.mean_new_payload_latency_ms,
-                feature.mean_new_payload_latency_ms,
-            ),
-            new_payload_latency_p50_change_percent: calc_percent_change(
-                baseline.median_new_payload_latency_ms,
-                feature.median_new_payload_latency_ms,
-            ),
-            new_payload_latency_p90_change_percent: calc_percent_change(
-                baseline.p90_new_payload_latency_ms,
-                feature.p90_new_payload_latency_ms,
-            ),
-            new_payload_latency_p99_change_percent: calc_percent_change(
-                baseline.p99_new_payload_latency_ms,
-                feature.p99_new_payload_latency_ms,
-            ),
-            gas_per_second_change_percent: calc_percent_change(
-                baseline.gas_per_second,
-                feature.gas_per_second,
-            ),
-            blocks_per_second_change_percent: calc_percent_change(
-                baseline.blocks_per_second,
-                feature.blocks_per_second,
-            ),
-        })
-    }
-
-    /// Calculate per-block comparisons
-    fn calculate_per_block_comparisons(
-        &self,
-        baseline: &BenchmarkResults,
-        feature: &BenchmarkResults,
-    ) -> Result<Vec<BlockComparison>> {
-        let mut baseline_map: HashMap<u64, &CombinedLatencyRow> = HashMap::new();
-        for row in &baseline.combined_latency_data {
-            baseline_map.insert(row.block_number, row);
-        }
-
-        let mut comparisons = Vec::new();
-        for feature_row in &feature.combined_latency_data {
-            if let Some(baseline_row) = baseline_map.get(&feature_row.block_number) {
-                let calc_percent_change = |baseline: u128, feature: u128| -> f64 {
-                    if baseline > 0 {
-                        ((feature as f64 - baseline as f64) / baseline as f64) * 100.0
-                    } else {
-                        0.0
-                    }
-                };
-
-                let comparison = BlockComparison {
-                    block_number: feature_row.block_number,
-                    transaction_count: feature_row.transaction_count,
-                    gas_used: feature_row.gas_used,
-                    baseline_new_payload_latency: baseline_row.new_payload_latency,
-                    feature_new_payload_latency: feature_row.new_payload_latency,
-                    new_payload_latency_change_percent: calc_percent_change(
-                        baseline_row.new_payload_latency,
-                        feature_row.new_payload_latency,
-                    ),
-                };
-                comparisons.push(comparison);
-            } else {
-                warn!("Block {} not found in baseline data", feature_row.block_number);
-            }
-        }
-
-        Ok(comparisons)
-    }
-
-    /// Write comparison reports to files
-    async fn write_comparison_reports(&self, report: &ComparisonReport) -> Result<()> {
-        let report_dir = self.output_dir.join("results").join(&self.timestamp);
-        fs::create_dir_all(&report_dir)
-            .wrap_err_with(|| format!("Failed to create report directory: {report_dir:?}"))?;
-
-        // Write JSON report
-        let json_path = report_dir.join("comparison_report.json");
-        let json_content = serde_json::to_string_pretty(report)
-            .wrap_err("Failed to serialize comparison report to JSON")?;
-        fs::write(&json_path, json_content)
-            .wrap_err_with(|| format!("Failed to write JSON report: {json_path:?}"))?;
-
-        // Write CSV report for per-block comparisons
-        let csv_path = report_dir.join("per_block_comparison.csv");
-        let mut writer = csv::Writer::from_path(&csv_path)
-            .wrap_err_with(|| format!("Failed to create CSV writer: {csv_path:?}"))?;
-
-        for comparison in &report.per_block_comparisons {
-            writer.serialize(comparison).wrap_err("Failed to write comparison row to CSV")?;
-        }
-        writer.flush().wrap_err("Failed to flush CSV writer")?;
-
-        info!("Comparison reports written to: {:?}", report_dir);
-        Ok(())
-    }
-
-    /// Print comparison summary to console
-    fn print_comparison_summary(&self, report: &ComparisonReport) {
-        // Parse and format timestamp nicely
-        let formatted_timestamp = if let Ok(dt) = chrono::DateTime::parse_from_str(
-            &format!("{} +0000", report.timestamp.replace('_', " ")),
-            "%Y%m%d %H%M%S %z",
-        ) {
-            dt.format("%Y-%m-%d %H:%M:%S UTC").to_string()
-        } else {
-            // Fallback to original if parsing fails
-            report.timestamp.clone()
-        };
-
-        println!("\n=== BENCHMARK COMPARISON SUMMARY ===");
-        println!("Timestamp: {formatted_timestamp}");
-        println!("Baseline: {}", report.baseline.ref_name);
-        println!("Feature:  {}", report.feature.ref_name);
-        println!();
-
-        let summary = &report.comparison_summary;
-
-        println!("Performance Changes:");
-        println!(
-            "  NewPayload Latency per-block mean change:   {:+.2}%",
-            summary.per_block_latency_change_mean_percent
-        );
-        println!(
-            "  NewPayload Latency per-block median change: {:+.2}%",
-            summary.per_block_latency_change_median_percent
-        );
-        println!(
-            "  NewPayload Latency per-block std dev:       {:.2}%",
-            summary.per_block_latency_change_std_dev_percent
-        );
-        println!(
-            "  Total newPayload time change:                {:+.2}%",
-            summary.new_payload_total_latency_change_percent
-        );
-        println!(
-            "  NewPayload Latency mean:           {:+.2}%",
-            summary.new_payload_latency_mean_change_percent
-        );
-        println!(
-            "  NewPayload Latency p50:           {:+.2}%",
-            summary.new_payload_latency_p50_change_percent
-        );
-        println!(
-            "  NewPayload Latency p90:           {:+.2}%",
-            summary.new_payload_latency_p90_change_percent
-        );
-        println!(
-            "  NewPayload Latency p99:           {:+.2}%",
-            summary.new_payload_latency_p99_change_percent
-        );
-        println!(
-            "  Gas/Second:                      {:+.2}%",
-            summary.gas_per_second_change_percent
-        );
-        println!(
-            "  Blocks/Second:                   {:+.2}%",
-            summary.blocks_per_second_change_percent
-        );
-        println!();
-
-        println!("Baseline Summary:");
-        let baseline = &report.baseline.summary;
-        println!(
-            "  Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
-            baseline.total_blocks,
-            baseline.min_block_number,
-            baseline.max_block_number,
-            baseline.total_gas_used,
-            baseline.total_duration_ms as f64 / 1000.0
-        );
-        println!("  NewPayload latency (ms):");
-        println!(
-            "    mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
-            baseline.mean_new_payload_latency_ms,
-            baseline.median_new_payload_latency_ms,
-            baseline.p90_new_payload_latency_ms,
-            baseline.p99_new_payload_latency_ms
-        );
-        if let (Some(start), Some(end)) =
-            (&report.baseline.start_timestamp, &report.baseline.end_timestamp)
-        {
-            println!(
-                "  Started: {}, Ended: {}",
-                start.format("%Y-%m-%d %H:%M:%S UTC"),
-                end.format("%Y-%m-%d %H:%M:%S UTC")
-            );
-        }
-        if let Some(ref cmd) = report.baseline.reth_command {
-            println!("  Command: {}", cmd);
-        }
-        println!();
-
-        println!("Feature Summary:");
-        let feature = &report.feature.summary;
-        println!(
-            "  Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
-            feature.total_blocks,
-            feature.min_block_number,
-            feature.max_block_number,
-            feature.total_gas_used,
-            feature.total_duration_ms as f64 / 1000.0
-        );
-        println!("  NewPayload latency (ms):");
-        println!(
-            "    mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
-            feature.mean_new_payload_latency_ms,
-            feature.median_new_payload_latency_ms,
-            feature.p90_new_payload_latency_ms,
-            feature.p99_new_payload_latency_ms
-        );
-        if let (Some(start), Some(end)) =
-            (&report.feature.start_timestamp, &report.feature.end_timestamp)
-        {
-            println!(
-                "  Started: {}, Ended: {}",
-                start.format("%Y-%m-%d %H:%M:%S UTC"),
-                end.format("%Y-%m-%d %H:%M:%S UTC")
-            );
-        }
-        if let Some(ref cmd) = report.feature.reth_command {
-            println!("  Command: {}", cmd);
-        }
-        println!();
-    }
-}
-
-/// Calculate standard deviation from a set of values and their mean.
-///
-/// Computes the population standard deviation using the formula:
-/// `sqrt(sum((x - mean)²) / n)`
-///
-/// Returns 0.0 for empty input.
-fn calculate_std_dev(values: &[f64], mean: f64) -> f64 {
-    if values.is_empty() {
-        return 0.0;
-    }
-
-    let variance = values
-        .iter()
-        .map(|x| {
-            let diff = x - mean;
-            diff * diff
-        })
-        .sum::<f64>() /
-        values.len() as f64;
-
-    variance.sqrt()
-}
-
-/// Calculate percentile using linear interpolation on a sorted slice.
-///
-/// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls
-/// between two indices, linearly interpolates between those values. For example, with 100 values,
-/// p90 computes rank = 0.9 × 99 = 89.1, then returns `values[89] × 0.9 + values[90] × 0.1`.
-///
-/// Returns 0.0 for empty input.
-fn percentile(sorted_values: &[f64], percentile: f64) -> f64 {
-    if sorted_values.is_empty() {
-        return 0.0;
-    }
-
-    let clamped = percentile.clamp(0.0, 1.0);
-    let max_index = sorted_values.len() - 1;
-    let rank = clamped * max_index as f64;
-    let lower = rank.floor() as usize;
-    let upper = rank.ceil() as usize;
-
-    if lower == upper {
-        sorted_values[lower]
-    } else {
-        let weight = rank - lower as f64;
-        sorted_values[lower].mul_add(1.0 - weight, sorted_values[upper] * weight)
-    }
-}

bin/reth-bench-compare/src/compilation.rs

@@ -1,305 +0,0 @@
-//! Compilation operations for reth and reth-bench.
-
-use crate::git::GitManager;
-use eyre::{eyre, Result, WrapErr};
-use std::{fs, path::PathBuf, process::Command};
-use tracing::{debug, error, info, warn};
-
-/// Manages compilation operations for reth components
-#[derive(Debug)]
-pub(crate) struct CompilationManager {
-    repo_root: String,
-    output_dir: PathBuf,
-    git_manager: GitManager,
-}
-
-impl CompilationManager {
-    /// Create a new `CompilationManager`
-    pub(crate) const fn new(
-        repo_root: String,
-        output_dir: PathBuf,
-        git_manager: GitManager,
-    ) -> Result<Self> {
-        Ok(Self { repo_root, output_dir, git_manager })
-    }
-
-    /// Get the path to the cached binary using explicit commit hash
-    pub(crate) fn get_cached_binary_path_for_commit(&self, commit: &str) -> PathBuf {
-        let identifier = &commit[..8]; // Use first 8 chars of commit
-        self.output_dir.join("bin").join(format!("reth_{identifier}"))
-    }
-
-    /// Compile reth using cargo build and cache the binary
-    pub(crate) fn compile_reth(&self, commit: &str, features: &str, rustflags: &str) -> Result<()> {
-        // Validate that current git commit matches the expected commit
-        let current_commit = self.git_manager.get_current_commit()?;
-        if current_commit != commit {
-            return Err(eyre!(
-                "Git commit mismatch! Expected: {}, but currently at: {}",
-                &commit[..8],
-                &current_commit[..8]
-            ));
-        }
-
-        let cached_path = self.get_cached_binary_path_for_commit(commit);
-
-        // Check if cached binary already exists (since path contains commit hash, it's valid)
-        if cached_path.exists() {
-            info!("Using cached binary (commit: {})", &commit[..8]);
-            return Ok(());
-        }
-
-        info!("No cached binary found, compiling (commit: {})...", &commit[..8]);
-
-        let binary_name = "reth";
-
-        info!(
-            "Compiling {} with profiling configuration (commit: {})...",
-            binary_name,
-            &commit[..8]
-        );
-
-        let mut cmd = Command::new("cargo");
-        cmd.arg("build").arg("--profile").arg("profiling");
-
-        cmd.arg("--features").arg(features);
-        info!("Using features: {features}");
-
-        cmd.current_dir(&self.repo_root);
-
-        // Set RUSTFLAGS
-        cmd.env("RUSTFLAGS", rustflags);
-        info!("Using RUSTFLAGS: {rustflags}");
-
-        info!("Compiling {binary_name} with {cmd:?}");
-
-        let output = cmd.output().wrap_err("Failed to execute cargo build command")?;
-
-        // Print stdout and stderr with prefixes at debug level
-        let stdout = String::from_utf8_lossy(&output.stdout);
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        for line in stdout.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO] {}", line);
-            }
-        }
-
-        for line in stderr.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO] {}", line);
-            }
-        }
-
-        if !output.status.success() {
-            // Print all output when compilation fails
-            error!("Cargo build failed with exit code: {:?}", output.status.code());
-
-            if !stdout.trim().is_empty() {
-                error!("Cargo stdout:");
-                for line in stdout.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            if !stderr.trim().is_empty() {
-                error!("Cargo stderr:");
-                for line in stderr.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!("Compilation failed with exit code: {:?}", output.status.code()));
-        }
-
-        info!("{} compilation completed", binary_name);
-
-        // Copy the compiled binary to cache
-        let source_path =
-            PathBuf::from(&self.repo_root).join(format!("target/profiling/{}", binary_name));
-        if !source_path.exists() {
-            return Err(eyre!("Compiled binary not found at {:?}", source_path));
-        }
-
-        // Create bin directory if it doesn't exist
-        let bin_dir = self.output_dir.join("bin");
-        fs::create_dir_all(&bin_dir).wrap_err("Failed to create bin directory")?;
-
-        // Copy binary to cache
-        fs::copy(&source_path, &cached_path).wrap_err("Failed to copy binary to cache")?;
-
-        // Make the cached binary executable
-        #[cfg(unix)]
-        {
-            use std::os::unix::fs::PermissionsExt;
-            let mut perms = fs::metadata(&cached_path)?.permissions();
-            perms.set_mode(0o755);
-            fs::set_permissions(&cached_path, perms)?;
-        }
-
-        info!("Cached compiled binary at: {:?}", cached_path);
-        Ok(())
-    }
-
-    /// Check if reth-bench is available in PATH
-    pub(crate) fn is_reth_bench_available(&self) -> bool {
-        match Command::new("which").arg("reth-bench").output() {
-            Ok(output) => {
-                if output.status.success() {
-                    let path = String::from_utf8_lossy(&output.stdout);
-                    info!("Found reth-bench: {}", path.trim());
-                    true
-                } else {
-                    false
-                }
-            }
-            Err(_) => false,
-        }
-    }
-
-    /// Check if samply is available in PATH
-    pub(crate) fn is_samply_available(&self) -> bool {
-        match Command::new("which").arg("samply").output() {
-            Ok(output) => {
-                if output.status.success() {
-                    let path = String::from_utf8_lossy(&output.stdout);
-                    info!("Found samply: {}", path.trim());
-                    true
-                } else {
-                    false
-                }
-            }
-            Err(_) => false,
-        }
-    }
-
-    /// Install samply using cargo
-    pub(crate) fn install_samply(&self) -> Result<()> {
-        info!("Installing samply via cargo...");
-
-        let mut cmd = Command::new("cargo");
-        cmd.args(["install", "--locked", "samply"]);
-
-        info!("Installing samply with {cmd:?}");
-
-        let output = cmd.output().wrap_err("Failed to execute cargo install samply command")?;
-
-        // Print stdout and stderr with prefixes at debug level
-        let stdout = String::from_utf8_lossy(&output.stdout);
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        for line in stdout.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO-SAMPLY] {}", line);
-            }
-        }
-
-        for line in stderr.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO-SAMPLY] {}", line);
-            }
-        }
-
-        if !output.status.success() {
-            // Print all output when installation fails
-            error!("Cargo install samply failed with exit code: {:?}", output.status.code());
-
-            if !stdout.trim().is_empty() {
-                error!("Cargo stdout:");
-                for line in stdout.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            if !stderr.trim().is_empty() {
-                error!("Cargo stderr:");
-                for line in stderr.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!(
-                "samply installation failed with exit code: {:?}",
-                output.status.code()
-            ));
-        }
-
-        info!("Samply installation completed");
-        Ok(())
-    }
-
-    /// Ensure samply is available, installing if necessary
-    pub(crate) fn ensure_samply_available(&self) -> Result<()> {
-        if self.is_samply_available() {
-            Ok(())
-        } else {
-            warn!("samply not found in PATH, installing...");
-            self.install_samply()
-        }
-    }
-
-    /// Ensure reth-bench is available, compiling if necessary
-    pub(crate) fn ensure_reth_bench_available(&self) -> Result<()> {
-        if self.is_reth_bench_available() {
-            Ok(())
-        } else {
-            warn!("reth-bench not found in PATH, compiling and installing...");
-            self.compile_reth_bench()
-        }
-    }
-
-    /// Compile and install reth-bench using `make install-reth-bench`
-    pub(crate) fn compile_reth_bench(&self) -> Result<()> {
-        info!("Compiling and installing reth-bench...");
-
-        let mut cmd = Command::new("make");
-        cmd.arg("install-reth-bench").current_dir(&self.repo_root);
-
-        info!("Compiling reth-bench with {cmd:?}");
-
-        let output = cmd.output().wrap_err("Failed to execute make install-reth-bench command")?;
-
-        // Print stdout and stderr with prefixes at debug level
-        let stdout = String::from_utf8_lossy(&output.stdout);
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        for line in stdout.lines() {
-            if !line.trim().is_empty() {
-                debug!("[MAKE-BENCH] {}", line);
-            }
-        }
-
-        for line in stderr.lines() {
-            if !line.trim().is_empty() {
-                debug!("[MAKE-BENCH] {}", line);
-            }
-        }
-
-        if !output.status.success() {
-            // Print all output when compilation fails
-            error!("Make install-reth-bench failed with exit code: {:?}", output.status.code());
-
-            if !stdout.trim().is_empty() {
-                error!("Make stdout:");
-                for line in stdout.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            if !stderr.trim().is_empty() {
-                error!("Make stderr:");
-                for line in stderr.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!(
-                "reth-bench compilation failed with exit code: {:?}",
-                output.status.code()
-            ));
-        }
-
-        info!("Reth-bench compilation completed");
-        Ok(())
-    }
-}

bin/reth-bench-compare/src/git.rs

@@ -1,328 +0,0 @@
-//! Git operations for branch management.
-
-use eyre::{eyre, Result, WrapErr};
-use std::process::Command;
-use tracing::{info, warn};
-
-/// Manages git operations for branch switching
-#[derive(Debug, Clone)]
-pub(crate) struct GitManager {
-    repo_root: String,
-}
-
-impl GitManager {
-    /// Create a new `GitManager`, detecting the repository root
-    pub(crate) fn new() -> Result<Self> {
-        let output = Command::new("git")
-            .args(["rev-parse", "--show-toplevel"])
-            .output()
-            .wrap_err("Failed to execute git command - is git installed?")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Not in a git repository or git command failed"));
-        }
-
-        let repo_root = String::from_utf8(output.stdout)
-            .wrap_err("Git output is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        let manager = Self { repo_root };
-        info!(
-            "Detected git repository at: {}, current reference: {}",
-            manager.repo_root(),
-            manager.get_current_ref()?
-        );
-
-        Ok(manager)
-    }
-
-    /// Get the current git branch name
-    pub(crate) fn get_current_branch(&self) -> Result<String> {
-        let output = Command::new("git")
-            .args(["branch", "--show-current"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current branch")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Failed to determine current branch"));
-        }
-
-        let branch = String::from_utf8(output.stdout)
-            .wrap_err("Branch name is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        if branch.is_empty() {
-            return Err(eyre!("Not on a named branch (detached HEAD?)"));
-        }
-
-        Ok(branch)
-    }
-
-    /// Get the current git reference (branch name, tag, or commit hash)
-    pub(crate) fn get_current_ref(&self) -> Result<String> {
-        // First try to get branch name
-        if let Ok(branch) = self.get_current_branch() {
-            return Ok(branch);
-        }
-
-        // If not on a branch, check if we're on a tag
-        let tag_output = Command::new("git")
-            .args(["describe", "--exact-match", "--tags", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to check for tag")?;
-
-        if tag_output.status.success() {
-            let tag = String::from_utf8(tag_output.stdout)
-                .wrap_err("Tag name is not valid UTF-8")?
-                .trim()
-                .to_string();
-            return Ok(tag);
-        }
-
-        // If not on a branch or tag, return the commit hash
-        let commit_output = Command::new("git")
-            .args(["rev-parse", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current commit")?;
-
-        if !commit_output.status.success() {
-            return Err(eyre!("Failed to get current commit hash"));
-        }
-
-        let commit_hash = String::from_utf8(commit_output.stdout)
-            .wrap_err("Commit hash is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        Ok(commit_hash)
-    }
-
-    /// Check if the git working directory has uncommitted changes to tracked files
-    pub(crate) fn validate_clean_state(&self) -> Result<()> {
-        let output = Command::new("git")
-            .args(["status", "--porcelain"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to check git status")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Git status command failed"));
-        }
-
-        let status_output =
-            String::from_utf8(output.stdout).wrap_err("Git status output is not valid UTF-8")?;
-
-        // Check for uncommitted changes to tracked files
-        // Status codes: M = modified, A = added, D = deleted, R = renamed, C = copied, U = updated
-        // ?? = untracked files (we want to ignore these)
-        let has_uncommitted_changes = status_output.lines().any(|line| {
-            if line.len() >= 2 {
-                let status = &line[0..2];
-                // Ignore untracked files (??) and ignored files (!!)
-                !matches!(status, "??" | "!!")
-            } else {
-                false
-            }
-        });
-
-        if has_uncommitted_changes {
-            warn!("Git working directory has uncommitted changes to tracked files:");
-            for line in status_output.lines() {
-                if line.len() >= 2 && !matches!(&line[0..2], "??" | "!!") {
-                    warn!("  {}", line);
-                }
-            }
-            return Err(eyre!(
-                "Git working directory has uncommitted changes to tracked files. Please commit or stash changes before running benchmark comparison."
-            ));
-        }
-
-        // Check if there are untracked files and log them as info
-        let untracked_files: Vec<&str> =
-            status_output.lines().filter(|line| line.starts_with("??")).collect();
-
-        if !untracked_files.is_empty() {
-            info!(
-                "Git working directory has {} untracked files (this is OK)",
-                untracked_files.len()
-            );
-        }
-
-        info!("Git working directory is clean (no uncommitted changes to tracked files)");
-        Ok(())
-    }
-
-    /// Fetch all refs from remote to ensure we have latest branches and tags
-    pub(crate) fn fetch_all(&self) -> Result<()> {
-        let output = Command::new("git")
-            .args(["fetch", "--all", "--tags", "--quiet", "--force"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to fetch latest refs")?;
-
-        if output.status.success() {
-            info!("Fetched latest refs");
-        } else {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            // Only warn if there's actual error content, not just fetch progress
-            if !stderr.trim().is_empty() && !stderr.contains("-> origin/") {
-                warn!("Git fetch encountered issues (continuing anyway): {}", stderr);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Validate that the specified git references exist (branches, tags, or commits)
-    pub(crate) fn validate_refs(&self, refs: &[&str]) -> Result<()> {
-        for &git_ref in refs {
-            // Try to resolve the ref similar to `git checkout` by peeling to a commit.
-            // First try the ref as-is with ^{commit}, then fall back to origin/{ref}^{commit}.
-            let as_is = format!("{git_ref}^{{commit}}");
-            let ref_check = Command::new("git")
-                .args(["rev-parse", "--verify", &as_is])
-                .current_dir(&self.repo_root)
-                .output();
-
-            let found = if let Ok(output) = ref_check &&
-                output.status.success()
-            {
-                info!("Validated reference exists: {}", git_ref);
-                true
-            } else {
-                // Try remote-only branches via origin/{ref}
-                let origin_ref = format!("origin/{git_ref}^{{commit}}");
-                let origin_check = Command::new("git")
-                    .args(["rev-parse", "--verify", &origin_ref])
-                    .current_dir(&self.repo_root)
-                    .output();
-
-                if let Ok(output) = origin_check &&
-                    output.status.success()
-                {
-                    info!("Validated remote reference exists: origin/{}", git_ref);
-                    true
-                } else {
-                    false
-                }
-            };
-
-            if !found {
-                return Err(eyre!(
-                    "Git reference '{}' does not exist as branch, tag, or commit (tried '{}' and 'origin/{}^{{commit}}')",
-                    git_ref,
-                    format!("{git_ref}^{{commit}}"),
-                    git_ref,
-                ));
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Switch to the specified git reference (branch, tag, or commit)
-    pub(crate) fn switch_ref(&self, git_ref: &str) -> Result<()> {
-        // First checkout the reference
-        let output = Command::new("git")
-            .args(["checkout", git_ref])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err_with(|| format!("Failed to switch to reference '{git_ref}'"))?;
-
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            return Err(eyre!("Failed to switch to reference '{}': {}", git_ref, stderr));
-        }
-
-        // Check if this is a branch that tracks a remote and pull latest changes
-        let is_branch = Command::new("git")
-            .args(["show-ref", "--verify", "--quiet", &format!("refs/heads/{git_ref}")])
-            .current_dir(&self.repo_root)
-            .status()
-            .map(|s| s.success())
-            .unwrap_or(false);
-
-        if is_branch {
-            // Check if the branch tracks a remote
-            let tracking_output = Command::new("git")
-                .args([
-                    "rev-parse",
-                    "--abbrev-ref",
-                    "--symbolic-full-name",
-                    &format!("{git_ref}@{{upstream}}"),
-                ])
-                .current_dir(&self.repo_root)
-                .output();
-
-            if let Ok(output) = tracking_output &&
-                output.status.success()
-            {
-                let upstream = String::from_utf8_lossy(&output.stdout).trim().to_string();
-                if !upstream.is_empty() && upstream != format!("{git_ref}@{{upstream}}") {
-                    // Branch tracks a remote, pull latest changes
-                    info!("Pulling latest changes for branch: {}", git_ref);
-
-                    let pull_output = Command::new("git")
-                        .args(["pull", "--ff-only"])
-                        .current_dir(&self.repo_root)
-                        .output()
-                        .wrap_err_with(|| {
-                            format!("Failed to pull latest changes for branch '{git_ref}'")
-                        })?;
-
-                    if pull_output.status.success() {
-                        info!("Successfully pulled latest changes for branch: {}", git_ref);
-                    } else {
-                        let stderr = String::from_utf8_lossy(&pull_output.stderr);
-                        warn!("Failed to pull latest changes for branch '{}': {}", git_ref, stderr);
-                        // Continue anyway, we'll use whatever version we have
-                    }
-                }
-            }
-        }
-
-        // Verify the checkout succeeded by checking the current commit
-        let current_commit_output = Command::new("git")
-            .args(["rev-parse", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current commit")?;
-
-        if !current_commit_output.status.success() {
-            return Err(eyre!("Failed to verify git checkout"));
-        }
-
-        info!("Switched to reference: {}", git_ref);
-        Ok(())
-    }
-
-    /// Get the current commit hash
-    pub(crate) fn get_current_commit(&self) -> Result<String> {
-        let output = Command::new("git")
-            .args(["rev-parse", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current commit")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Failed to get current commit hash"));
-        }
-
-        let commit_hash = String::from_utf8(output.stdout)
-            .wrap_err("Commit hash is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        Ok(commit_hash)
-    }
-
-    /// Get the repository root path
-    pub(crate) fn repo_root(&self) -> &str {
-        &self.repo_root
-    }
-}

bin/reth-bench-compare/src/main.rs

@@ -1,47 +0,0 @@
-//! # reth-bench-compare
-//!
-//! Automated tool for comparing reth performance between two git branches.
-//! This tool automates the complete workflow of compiling, running, and benchmarking
-//! reth on different branches to provide meaningful performance comparisons.
-
-#![doc(
-    html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
-    html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
-    issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
-)]
-#![cfg_attr(not(test), warn(unused_crate_dependencies))]
-
-#[global_allocator]
-static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
-
-use alloy_primitives as _;
-
-mod benchmark;
-mod cli;
-mod comparison;
-mod compilation;
-mod git;
-mod node;
-
-use clap::Parser;
-use cli::{run_comparison, Args};
-use eyre::Result;
-use reth_cli_runner::CliRunner;
-
-fn main() -> Result<()> {
-    // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
-    if std::env::var_os("RUST_BACKTRACE").is_none() {
-        unsafe {
-            std::env::set_var("RUST_BACKTRACE", "1");
-        }
-    }
-
-    let args = Args::parse();
-
-    // Initialize tracing
-    let _guard = args.init_tracing()?;
-
-    // Run until either exit or sigint or sigterm
-    let runner = CliRunner::try_default_runtime()?;
-    runner.run_command_until_exit(|ctx| run_comparison(args, ctx))
-}

bin/reth-bench-compare/src/node.rs

@@ -1,695 +0,0 @@
-//! Node management for starting, stopping, and controlling reth instances.
-
-use crate::cli::Args;
-use alloy_provider::{Provider, ProviderBuilder};
-use alloy_rpc_client::RpcClient;
-use alloy_rpc_types_eth::SyncStatus;
-use alloy_transport_ws::WsConnect;
-use eyre::{eyre, OptionExt, Result, WrapErr};
-#[cfg(unix)]
-use nix::sys::signal::{killpg, Signal};
-#[cfg(unix)]
-use nix::unistd::Pid;
-use reth_chainspec::Chain;
-use std::{fs, path::PathBuf, time::Duration};
-use tokio::{
-    fs::File as AsyncFile,
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader as AsyncBufReader},
-    process::Command,
-    time::{sleep, timeout},
-};
-use tracing::{debug, info, warn};
-
-/// Default websocket RPC port used by reth
-const DEFAULT_WS_RPC_PORT: u16 = 8546;
-
-/// Manages reth node lifecycle and operations
-pub(crate) struct NodeManager {
-    datadir: Option<String>,
-    metrics_port: u16,
-    chain: Chain,
-    use_sudo: bool,
-    binary_path: Option<std::path::PathBuf>,
-    enable_profiling: bool,
-    output_dir: PathBuf,
-    additional_reth_args: Vec<String>,
-    comparison_dir: Option<PathBuf>,
-    tracing_endpoint: Option<String>,
-    otlp_max_queue_size: usize,
-}
-
-impl NodeManager {
-    /// Create a new `NodeManager` with configuration from CLI args
-    pub(crate) fn new(args: &Args) -> Self {
-        Self {
-            datadir: Some(args.datadir_path().to_string_lossy().to_string()),
-            metrics_port: args.metrics_port,
-            chain: args.chain,
-            use_sudo: args.sudo,
-            binary_path: None,
-            enable_profiling: args.profile,
-            output_dir: args.output_dir_path(),
-            // Filter out empty strings to prevent invalid arguments being passed to reth node
-            additional_reth_args: args
-                .reth_args
-                .iter()
-                .filter(|s| !s.is_empty())
-                .cloned()
-                .collect(),
-            comparison_dir: None,
-            tracing_endpoint: args.traces.otlp.as_ref().map(|u| u.to_string()),
-            otlp_max_queue_size: args.otlp_max_queue_size,
-        }
-    }
-
-    /// Set the comparison directory path for logging
-    pub(crate) fn set_comparison_dir(&mut self, dir: PathBuf) {
-        self.comparison_dir = Some(dir);
-    }
-
-    /// Get the log file path for a given reference type
-    fn get_log_file_path(&self, ref_type: &str) -> Result<PathBuf> {
-        let comparison_dir = self
-            .comparison_dir
-            .as_ref()
-            .ok_or_eyre("Comparison directory not set. Call set_comparison_dir first.")?;
-
-        // The comparison directory already contains the full path to results/<timestamp>
-        let log_dir = comparison_dir.join(ref_type);
-
-        // Create the directory if it doesn't exist
-        fs::create_dir_all(&log_dir)
-            .wrap_err(format!("Failed to create log directory: {:?}", log_dir))?;
-
-        let log_file = log_dir.join("reth_node.log");
-        Ok(log_file)
-    }
-
-    /// Get the perf event max sample rate from the system, capped at 10000
-    fn get_perf_sample_rate(&self) -> Option<String> {
-        let perf_rate_file = "/proc/sys/kernel/perf_event_max_sample_rate";
-        if let Ok(content) = fs::read_to_string(perf_rate_file) {
-            let rate_str = content.trim();
-            if !rate_str.is_empty() {
-                if let Ok(system_rate) = rate_str.parse::<u32>() {
-                    let capped_rate = std::cmp::min(system_rate, 10000);
-                    info!(
-                        "Detected perf_event_max_sample_rate: {}, using: {}",
-                        system_rate, capped_rate
-                    );
-                    return Some(capped_rate.to_string());
-                }
-                warn!("Failed to parse perf_event_max_sample_rate: {}", rate_str);
-            }
-        }
-        None
-    }
-
-    /// Get the absolute path to samply using 'which' command
-    async fn get_samply_path(&self) -> Result<String> {
-        let output = Command::new("which")
-            .arg("samply")
-            .output()
-            .await
-            .wrap_err("Failed to execute 'which samply' command")?;
-
-        if !output.status.success() {
-            return Err(eyre!("samply not found in PATH"));
-        }
-
-        let samply_path = String::from_utf8(output.stdout)
-            .wrap_err("samply path is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        if samply_path.is_empty() {
-            return Err(eyre!("which samply returned empty path"));
-        }
-
-        Ok(samply_path)
-    }
-
-    /// Build reth arguments as a vector of strings
-    fn build_reth_args(
-        &self,
-        binary_path_str: &str,
-        additional_args: &[String],
-        ref_type: &str,
-    ) -> (Vec<String>, String) {
-        let mut reth_args = vec![binary_path_str.to_string(), "node".to_string()];
-
-        // Add chain argument (skip for mainnet as it's the default)
-        let chain_str = self.chain.to_string();
-        if chain_str != "mainnet" {
-            reth_args.extend_from_slice(&["--chain".to_string(), chain_str.clone()]);
-        }
-
-        // Add datadir if specified
-        if let Some(ref datadir) = self.datadir {
-            reth_args.extend_from_slice(&["--datadir".to_string(), datadir.clone()]);
-        }
-
-        // Add reth-specific arguments
-        let metrics_arg = format!("0.0.0.0:{}", self.metrics_port);
-        reth_args.extend_from_slice(&[
-            "--engine.accept-execution-requests-hash".to_string(),
-            "--metrics".to_string(),
-            metrics_arg,
-            "--http".to_string(),
-            "--http.api".to_string(),
-            "eth,reth".to_string(),
-            "--ws".to_string(),
-            "--ws.api".to_string(),
-            "eth,reth".to_string(),
-            "--disable-discovery".to_string(),
-            "--trusted-only".to_string(),
-            "--disable-tx-gossip".to_string(),
-        ]);
-
-        // Add tracing arguments if OTLP endpoint is configured
-        if let Some(ref endpoint) = self.tracing_endpoint {
-            info!("Enabling OTLP tracing export to: {} (service: reth-{})", endpoint, ref_type);
-            // Endpoint requires equals per clap settings in reth
-            reth_args.push(format!("--tracing-otlp={}", endpoint));
-        }
-
-        // Add any additional arguments passed via command line (common to both baseline and
-        // feature)
-        reth_args.extend_from_slice(&self.additional_reth_args);
-
-        // Add reference-specific additional arguments
-        reth_args.extend_from_slice(additional_args);
-
-        (reth_args, chain_str)
-    }
-
-    /// Create a command for profiling mode
-    async fn create_profiling_command(
-        &self,
-        ref_type: &str,
-        reth_args: &[String],
-    ) -> Result<Command> {
-        // Create profiles directory if it doesn't exist
-        let profile_dir = self.output_dir.join("profiles");
-        fs::create_dir_all(&profile_dir).wrap_err("Failed to create profiles directory")?;
-
-        let profile_path = profile_dir.join(format!("{}.json.gz", ref_type));
-        info!("Starting reth node with samply profiling...");
-        info!("Profile output: {:?}", profile_path);
-
-        // Get absolute path to samply
-        let samply_path = self.get_samply_path().await?;
-
-        let mut cmd = if self.use_sudo {
-            let mut sudo_cmd = Command::new("sudo");
-            sudo_cmd.arg(&samply_path);
-            sudo_cmd
-        } else {
-            Command::new(&samply_path)
-        };
-
-        // Add samply arguments
-        cmd.args(["record", "--save-only", "-o", &profile_path.to_string_lossy()]);
-
-        // Add rate argument if available
-        if let Some(rate) = self.get_perf_sample_rate() {
-            cmd.args(["--rate", &rate]);
-        }
-
-        // Add separator and complete reth command
-        cmd.arg("--");
-        cmd.args(reth_args);
-
-        // Enable tracing-samply
-        if supports_samply_flags(&reth_args[0]) {
-            cmd.arg("--log.samply");
-        }
-
-        // Set environment variable to disable log styling
-        cmd.env("RUST_LOG_STYLE", "never");
-
-        Ok(cmd)
-    }
-
-    /// Create a command for direct reth execution
-    fn create_direct_command(&self, reth_args: &[String]) -> Command {
-        let binary_path = &reth_args[0];
-
-        let mut cmd = if self.use_sudo {
-            info!("Starting reth node with sudo...");
-            let mut sudo_cmd = Command::new("sudo");
-            sudo_cmd.args(reth_args);
-            sudo_cmd
-        } else {
-            info!("Starting reth node...");
-            let mut reth_cmd = Command::new(binary_path);
-            reth_cmd.args(&reth_args[1..]); // Skip the binary path since it's the command
-            reth_cmd
-        };
-
-        // Set environment variable to disable log styling
-        cmd.env("RUST_LOG_STYLE", "never");
-
-        cmd
-    }
-
-    /// Start a reth node using the specified binary path and return the process handle
-    /// along with the formatted reth command string for reporting.
-    pub(crate) async fn start_node(
-        &mut self,
-        binary_path: &std::path::Path,
-        _git_ref: &str,
-        ref_type: &str,
-        additional_args: &[String],
-    ) -> Result<(tokio::process::Child, String)> {
-        // Store the binary path for later use (e.g., in unwind_to_block)
-        self.binary_path = Some(binary_path.to_path_buf());
-
-        let binary_path_str = binary_path.to_string_lossy();
-        let (reth_args, _) = self.build_reth_args(&binary_path_str, additional_args, ref_type);
-
-        // Format the reth command string for reporting
-        let reth_command = shlex::try_join(reth_args.iter().map(|s| s.as_str()))
-            .wrap_err("Failed to format reth command string")?;
-
-        // Log additional arguments if any
-        if !self.additional_reth_args.is_empty() {
-            info!("Using common additional reth arguments: {:?}", self.additional_reth_args);
-        }
-        if !additional_args.is_empty() {
-            info!("Using reference-specific additional reth arguments: {:?}", additional_args);
-        }
-
-        let mut cmd = if self.enable_profiling {
-            self.create_profiling_command(ref_type, &reth_args).await?
-        } else {
-            self.create_direct_command(&reth_args)
-        };
-
-        // Set process group for better signal handling
-        #[cfg(unix)]
-        {
-            cmd.process_group(0);
-        }
-
-        // Set high queue size to prevent trace dropping during benchmarks
-        if self.tracing_endpoint.is_some() {
-            cmd.env("OTEL_BSP_MAX_QUEUE_SIZE", self.otlp_max_queue_size.to_string()); // Traces
-            cmd.env("OTEL_BLRP_MAX_QUEUE_SIZE", "10000"); // Logs
-
-            // Set service name to differentiate baseline vs feature runs in Jaeger
-            cmd.env("OTEL_SERVICE_NAME", format!("reth-{}", ref_type));
-        }
-
-        debug!("Executing reth command: {cmd:?}");
-
-        let mut child = cmd
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true) // Kill on drop so that on Ctrl-C for parent process we stop all child processes
-            .spawn()
-            .wrap_err("Failed to start reth node")?;
-
-        info!(
-            "Reth node started with PID: {:?} (binary: {})",
-            child.id().ok_or_eyre("Reth node is not running")?,
-            binary_path_str
-        );
-
-        // Prepare log file path
-        let log_file_path = self.get_log_file_path(ref_type)?;
-        info!("Reth node logs will be saved to: {:?}", log_file_path);
-
-        // Stream stdout and stderr with prefixes at debug level and to log file
-        if let Some(stdout) = child.stdout.take() {
-            let log_file = AsyncFile::create(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stdout);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH] {}", line);
-                    // Write to log file (reth already includes timestamps)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        if let Some(stderr) = child.stderr.take() {
-            let log_file = AsyncFile::options()
-                .create(true)
-                .append(true)
-                .open(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stderr);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH] {}", line);
-                    // Write to log file (reth already includes timestamps)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        // Give the node a moment to start up
-        sleep(Duration::from_secs(5)).await;
-
-        Ok((child, reth_command))
-    }
-
-    /// Wait for the node to be ready and return its current tip.
-    ///
-    /// Fails early if the node process exits before becoming ready.
-    pub(crate) async fn wait_for_node_ready_and_get_tip(
-        &self,
-        child: &mut tokio::process::Child,
-    ) -> Result<u64> {
-        info!("Waiting for node to be ready and synced...");
-
-        let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync
-        let check_interval = Duration::from_secs(2);
-        let rpc_url = "http://localhost:8545";
-
-        // Create Alloy provider
-        let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
-        let provider = ProviderBuilder::new().connect_http(url);
-
-        let start_time = tokio::time::Instant::now();
-        let mut iteration = 0;
-
-        timeout(max_wait, async {
-            loop {
-                iteration += 1;
-                debug!(
-                    "Readiness check iteration {} (elapsed: {:?})",
-                    iteration,
-                    start_time.elapsed()
-                );
-
-                // Check if the node process has exited.
-                if let Some(status) = child.try_wait()? {
-                    return Err(eyre!("Node process exited unexpectedly with {status}"));
-                }
-
-                // First check if RPC is up and node is not syncing
-                match provider.syncing().await {
-                    Ok(sync_result) => {
-                        match sync_result {
-                            SyncStatus::Info(sync_info) => {
-                                debug!("Node is still syncing {sync_info:?}, waiting...");
-                            }
-                            _ => {
-                                debug!("HTTP RPC is up and node is not syncing, checking block number...");
-                                // Node is not syncing, now get the tip
-                                match provider.get_block_number().await {
-                                    Ok(tip) => {
-                                        debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
-
-                                        // Verify WebSocket RPC is ready (public endpoint, no JWT required)
-                                        let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
-                                        debug!("Attempting WebSocket connection to {} (public endpoint)", ws_url);
-                                        let ws_connect = WsConnect::new(&ws_url);
-
-                                        match RpcClient::connect_pubsub(ws_connect).await
-                                        {
-                                            Ok(_) => {
-                                                info!(
-                                                    "Node is ready (HTTP and WebSocket) at block: {} (took {:?}, {} iterations)",
-                                                    tip, start_time.elapsed(), iteration
-                                                );
-                                                return Ok(tip);
-                                            }
-                                            Err(e) => {
-                                                debug!(
-                                                    "HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
-                                                    iteration, e
-                                                );
-                                                debug!("WebSocket error details: {}", e);
-                                            }
-                                        }
-                                    }
-                                    Err(e) => {
-                                        debug!("Failed to get block number (iteration {}): {:?}", iteration, e);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    Err(e) => {
-                        debug!("Node RPC not ready yet or failed to check sync status (iteration {}): {:?}", iteration, e);
-                    }
-                }
-
-                debug!("Sleeping for {:?} before next check", check_interval);
-                sleep(check_interval).await;
-            }
-        })
-        .await
-        .wrap_err("Timed out waiting for node to be ready and synced")?
-    }
-
-    /// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
-    ///
-    /// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
-    /// the node is still syncing.
-    pub(crate) async fn wait_for_rpc_and_get_tip(
-        &self,
-        child: &mut tokio::process::Child,
-    ) -> Result<u64> {
-        info!("Waiting for node RPC to be ready (skipping sync wait)...");
-
-        let max_wait = Duration::from_secs(60);
-        let check_interval = Duration::from_secs(2);
-        let rpc_url = "http://localhost:8545";
-
-        let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
-        let provider = ProviderBuilder::new().connect_http(url);
-
-        let start_time = tokio::time::Instant::now();
-        let mut iteration = 0;
-
-        timeout(max_wait, async {
-            loop {
-                iteration += 1;
-                debug!(
-                    "RPC readiness check iteration {} (elapsed: {:?})",
-                    iteration,
-                    start_time.elapsed()
-                );
-
-                if let Some(status) = child.try_wait()? {
-                    return Err(eyre!("Node process exited unexpectedly with {status}"));
-                }
-
-                match provider.get_block_number().await {
-                    Ok(tip) => {
-                        debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
-
-                        let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
-                        let ws_connect = WsConnect::new(&ws_url);
-
-                        match RpcClient::connect_pubsub(ws_connect).await {
-                            Ok(_) => {
-                                info!(
-                                    "Node RPC is ready at block: {} (took {:?}, {} iterations)",
-                                    tip,
-                                    start_time.elapsed(),
-                                    iteration
-                                );
-                                return Ok(tip);
-                            }
-                            Err(e) => {
-                                debug!(
-                                    "HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
-                                    iteration, e
-                                );
-                            }
-                        }
-                    }
-                    Err(e) => {
-                        debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
-                    }
-                }
-
-                sleep(check_interval).await;
-            }
-        })
-        .await
-        .wrap_err("Timed out waiting for node RPC to be ready")?
-    }
-
-    /// Stop the reth node gracefully
-    pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
-        let pid = child.id().ok_or_eyre("Child process ID should be available")?;
-
-        // Check if the process has already exited
-        match child.try_wait() {
-            Ok(Some(status)) => {
-                info!("Reth node (PID: {}) has already exited with status: {:?}", pid, status);
-                return Ok(());
-            }
-            Ok(None) => {
-                // Process is still running, proceed to stop it
-                info!("Stopping process gracefully with SIGINT (PID: {})...", pid);
-            }
-            Err(e) => {
-                return Err(eyre!("Failed to check process status: {}", e));
-            }
-        }
-
-        #[cfg(unix)]
-        {
-            // Send SIGINT to process group to mimic Ctrl-C behavior
-            let nix_pgid = Pid::from_raw(pid as i32);
-
-            match killpg(nix_pgid, Signal::SIGINT) {
-                Ok(()) => {}
-                Err(nix::errno::Errno::ESRCH) => {
-                    info!("Process group {} has already exited", pid);
-                }
-                Err(e) => {
-                    return Err(eyre!("Failed to send SIGINT to process group {}: {}", pid, e));
-                }
-            }
-        }
-
-        #[cfg(not(unix))]
-        {
-            // On non-Unix systems, fall back to using external kill command
-            let output = Command::new("taskkill")
-                .args(["/PID", &pid.to_string(), "/F"])
-                .output()
-                .await
-                .wrap_err("Failed to execute taskkill command")?;
-
-            if !output.status.success() {
-                let stderr = String::from_utf8_lossy(&output.stderr);
-                // Check if the error is because the process doesn't exist
-                if stderr.contains("not found") || stderr.contains("not exist") {
-                    info!("Process {} has already exited", pid);
-                } else {
-                    return Err(eyre!("Failed to kill process {}: {}", pid, stderr));
-                }
-            }
-        }
-
-        // Wait for the process to exit
-        match child.wait().await {
-            Ok(status) => {
-                info!("Reth node (PID: {}) exited with status: {:?}", pid, status);
-            }
-            Err(e) => {
-                // If we get an error here, it might be because the process already exited
-                debug!("Error waiting for process exit (may have already exited): {}", e);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Unwind the node to a specific block
-    pub(crate) async fn unwind_to_block(&self, block_number: u64) -> Result<()> {
-        if self.use_sudo {
-            info!("Unwinding node to block: {} (with sudo)", block_number);
-        } else {
-            info!("Unwinding node to block: {}", block_number);
-        }
-
-        // Use the binary path from the last start_node call, or fallback to default
-        let binary_path = self
-            .binary_path
-            .as_ref()
-            .map(|p| p.to_string_lossy().to_string())
-            .unwrap_or_else(|| "./target/profiling/reth".to_string());
-
-        let mut cmd = if self.use_sudo {
-            let mut sudo_cmd = Command::new("sudo");
-            sudo_cmd.args([&binary_path, "stage", "unwind"]);
-            sudo_cmd
-        } else {
-            let mut reth_cmd = Command::new(&binary_path);
-            reth_cmd.args(["stage", "unwind"]);
-            reth_cmd
-        };
-
-        // Add chain argument (skip for mainnet as it's the default)
-        let chain_str = self.chain.to_string();
-        if chain_str != "mainnet" {
-            cmd.args(["--chain", &chain_str]);
-        }
-
-        // Add datadir if specified
-        if let Some(ref datadir) = self.datadir {
-            cmd.args(["--datadir", datadir]);
-        }
-
-        cmd.args(["to-block", &block_number.to_string()]);
-
-        // Set environment variable to disable log styling
-        cmd.env("RUST_LOG_STYLE", "never");
-
-        // Debug log the command
-        debug!("Executing reth unwind command: {:?}", cmd);
-
-        let mut child = cmd
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .spawn()
-            .wrap_err("Failed to start unwind command")?;
-
-        // Stream stdout and stderr with prefixes in real-time
-        if let Some(stdout) = child.stdout.take() {
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stdout);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-UNWIND] {}", line);
-                }
-            });
-        }
-
-        if let Some(stderr) = child.stderr.take() {
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stderr);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-UNWIND] {}", line);
-                }
-            });
-        }
-
-        // Wait for the command to complete
-        let status = child.wait().await.wrap_err("Failed to wait for unwind command")?;
-
-        if !status.success() {
-            return Err(eyre!("Unwind command failed with exit code: {:?}", status.code()));
-        }
-
-        info!("Unwound to block: {}", block_number);
-        Ok(())
-    }
-}
-
-fn supports_samply_flags(bin: &str) -> bool {
-    let mut cmd = std::process::Command::new(bin);
-    // NOTE: The flag to check must come before --help.
-    // We pass --help as a shortcut to not execute any command.
-    cmd.args(["--log.samply", "--help"]);
-    debug!(?cmd, "Checking samply flags support");
-    let Ok(output) = cmd.output() else {
-        return false;
-    };
-    debug!(?output, "Samply flags support check");
-    output.status.success()
-}

bin/reth-bench/Cargo.toml

@@ -17,7 +17,6 @@ workspace = true
 reth-cli-runner.workspace = true
 reth-cli-util.workspace = true
 reth-engine-primitives.workspace = true
-reth-ethereum-primitives.workspace = true
 reth-fs-util.workspace = true
 reth-node-api.workspace = true
 reth-node-core.workspace = true
@@ -25,13 +24,11 @@ reth-primitives-traits.workspace = true
 reth-rpc-api.workspace = true
 
 reth-tracing.workspace = true
-reth-chainspec.workspace = true
 
 # alloy
 alloy-eips.workspace = true
 alloy-json-rpc.workspace = true
-alloy-consensus.workspace = true
-alloy-network.workspace = true
+
 alloy-primitives = { workspace = true, features = ["rand"] }
 alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
 alloy-pubsub.workspace = true

bin/reth-bench/README.md

@@ -35,6 +35,10 @@ The `new-payload-fcu` command supports two optional waiting modes that can be us
 - `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms` or `--wait-time 400` for 400ms)
 - `--wait-for-persistence`: Waits for blocks to be persisted using the `reth_subscribePersistedBlock` subscription
 
+Both `new-payload-fcu` and `new-payload-only` support `--rpc-block-fetch-retries <RETRIES>`
+to control how many times block fetches are retried after an RPC failure. The default is `10`.
+Use `--rpc-block-fetch-retries forever` to keep retrying indefinitely.
+
 When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
 
 By default, the WebSocket URL for persistence subscriptions is derived from `--engine-rpc-url` (converting to ws:// on port 8546). Use `--ws-rpc-url` to override this.

bin/reth-bench/src/bench/context.rs

@@ -7,7 +7,7 @@ use alloy_primitives::address;
 use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
 use alloy_rpc_client::ClientBuilder;
 use alloy_rpc_types_engine::JwtSecret;
-use alloy_transport::layers::RetryBackoffLayer;
+use alloy_transport::layers::{RateLimitRetryPolicy, RetryBackoffLayer};
 use reqwest::Url;
 use reth_node_core::args::BenchmarkArgs;
 use tracing::info;
@@ -31,6 +31,12 @@ pub(crate) struct BenchContext {
     pub(crate) is_optimism: bool,
     /// Whether to use `reth_newPayload` endpoint instead of `engine_newPayload*`.
     pub(crate) use_reth_namespace: bool,
+    /// Whether to fetch and replay RLP-encoded blocks.
+    pub(crate) rlp_blocks: bool,
+    /// Whether to skip waiting for persistence (pass `wait_for_persistence: false`).
+    pub(crate) no_wait_for_persistence: bool,
+    /// Whether to skip waiting for caches (pass `wait_for_caches: false`).
+    pub(crate) no_wait_for_caches: bool,
 }
 
 impl BenchContext {
@@ -51,9 +57,16 @@ impl BenchContext {
             }
         }
 
-        // set up alloy client for blocks
+        // set up alloy client for blocks, retrying on 429/503 (default) and 502
+        let retry_policy =
+            RateLimitRetryPolicy::default().or(|err: &alloy_transport::TransportError| -> bool {
+                err.as_transport_err()
+                    .and_then(|t| t.as_http_error())
+                    .is_some_and(|e| e.status == 502)
+            });
+        let max_retries = bench_args.rpc_block_fetch_retries.as_max_retries();
         let client = ClientBuilder::default()
-            .layer(RetryBackoffLayer::new(10, 800, u64::MAX))
+            .layer(RetryBackoffLayer::new_with_policy(max_retries, 800, u64::MAX, retry_policy))
             .http(rpc_url.parse()?);
         let block_provider = RootProvider::<AnyNetwork>::new(client);
 
@@ -86,9 +99,11 @@ impl BenchContext {
 
         // Computes the block range for the benchmark.
         //
-        // - If `--advance` is provided, fetches the latest block and sets:
+        // - If `--advance` is provided, fetches the latest block from the engine and sets:
         //     - `from = head + 1`
         //     - `to = head + advance`
+        // - If only `--to` is provided, fetches the latest block from the engine and sets:
+        //     - `from = head`
         // - Otherwise, uses the values from `--from` and `--to`.
         let (from, to) = if let Some(advance) = bench_args.advance {
             if advance == 0 {
@@ -101,6 +116,14 @@ impl BenchContext {
                 .ok_or_else(|| eyre::eyre!("Failed to fetch latest block for --advance"))?;
             let head_number = head_block.header.number;
             (Some(head_number), Some(head_number + advance))
+        } else if bench_args.from.is_none() && bench_args.to.is_some() {
+            let head_block = auth_provider
+                .get_block_by_number(BlockNumberOrTag::Latest)
+                .await?
+                .ok_or_else(|| eyre::eyre!("Failed to fetch latest block from engine"))?;
+            let head_number = head_block.header.number;
+            info!(target: "reth-bench", "No --from provided, derived from engine head: {}", head_number);
+            (Some(head_number), bench_args.to)
         } else {
             (bench_args.from, bench_args.to)
         };
@@ -112,7 +135,7 @@ impl BenchContext {
             .full()
             .await?
             .ok_or_else(|| eyre::eyre!("Failed to fetch latest block from RPC"))?;
-        let mut benchmark_mode = BenchMode::new(from, to, latest_block.into_inner().number())?;
+        let mut benchmark_mode = BenchMode::new(from, to, latest_block.into_inner().number());
 
         let first_block = match benchmark_mode {
             BenchMode::Continuous(start) => {
@@ -142,7 +165,10 @@ impl BenchContext {
         };
 
         let next_block = first_block.header.number + 1;
-        let use_reth_namespace = bench_args.reth_new_payload;
+        let rlp_blocks = bench_args.rlp_blocks;
+        let use_reth_namespace = bench_args.reth_new_payload || rlp_blocks;
+        let no_wait_for_persistence = bench_args.no_wait_for_persistence;
+        let no_wait_for_caches = bench_args.no_wait_for_caches;
         Ok(Self {
             auth_provider,
             block_provider,
@@ -150,6 +176,9 @@ impl BenchContext {
             next_block,
             is_optimism,
             use_reth_namespace,
+            rlp_blocks,
+            no_wait_for_persistence,
+            no_wait_for_caches,
         })
     }
 }

bin/reth-bench/src/bench/gas_limit_ramp.rs

@@ -1,244 +0,0 @@
-//! Benchmarks empty block processing by ramping the block gas limit.
-
-use crate::{
-    authenticated_transport::AuthenticatedTransportConnect,
-    bench::{
-        helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
-        output::GasRampPayloadFile,
-    },
-    valid_payload::{call_forkchoice_updated, call_new_payload_with_reth, payload_to_new_payload},
-};
-use alloy_eips::BlockNumberOrTag;
-use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
-use alloy_rpc_client::ClientBuilder;
-use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
-
-use clap::Parser;
-use reqwest::Url;
-use reth_chainspec::ChainSpec;
-use reth_cli_runner::CliContext;
-use reth_ethereum_primitives::TransactionSigned;
-use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
-use std::{path::PathBuf, time::Instant};
-use tracing::info;
-
-/// `reth benchmark gas-limit-ramp` command.
-#[derive(Debug, Parser)]
-pub struct Command {
-    /// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
-    #[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
-    blocks: Option<u64>,
-
-    /// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
-    /// reaches or exceeds this value. Mutually exclusive with --blocks.
-    /// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
-    /// billion).
-    #[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
-    target_gas_limit: Option<u64>,
-
-    /// The Engine API RPC URL.
-    #[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
-    engine_rpc_url: String,
-
-    /// Path to the JWT secret for Engine API authentication.
-    #[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
-    jwt_secret: PathBuf,
-
-    /// Output directory for benchmark results and generated payloads.
-    #[arg(long, value_name = "OUTPUT")]
-    output: PathBuf,
-
-    /// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
-    ///
-    /// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
-    /// directly, waits for persistence and cache updates to complete before processing,
-    /// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
-    #[arg(long, default_value = "false", verbatim_doc_comment)]
-    reth_new_payload: bool,
-}
-
-/// Mode for determining when to stop ramping.
-#[derive(Debug, Clone, Copy)]
-enum RampMode {
-    /// Ramp for a fixed number of blocks.
-    Blocks(u64),
-    /// Ramp until reaching or exceeding target gas limit.
-    TargetGasLimit(u64),
-}
-
-impl Command {
-    /// Execute `benchmark gas-limit-ramp` command.
-    pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
-        let mode = match (self.blocks, self.target_gas_limit) {
-            (Some(blocks), None) => {
-                if blocks == 0 {
-                    return Err(eyre::eyre!("--blocks must be greater than 0"));
-                }
-                RampMode::Blocks(blocks)
-            }
-            (None, Some(target)) => {
-                if target == 0 {
-                    return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
-                }
-                RampMode::TargetGasLimit(target)
-            }
-            _ => {
-                return Err(eyre::eyre!(
-                    "Exactly one of --blocks or --target-gas-limit must be specified"
-                ));
-            }
-        };
-
-        // Ensure output directory exists
-        if self.output.is_file() {
-            return Err(eyre::eyre!("Output path must be a directory"));
-        }
-        if !self.output.exists() {
-            std::fs::create_dir_all(&self.output)?;
-            info!(target: "reth-bench", "Created output directory: {:?}", self.output);
-        }
-
-        // Set up authenticated provider (used for both Engine API and eth_ methods)
-        let jwt = std::fs::read_to_string(&self.jwt_secret)?;
-        let jwt = JwtSecret::from_hex(jwt)?;
-        let auth_url = Url::parse(&self.engine_rpc_url)?;
-
-        info!(target: "reth-bench", "Connecting to Engine RPC at {}", auth_url);
-        let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
-        let client = ClientBuilder::default().connect_with(auth_transport).await?;
-        let provider = RootProvider::<AnyNetwork>::new(client);
-
-        // Get chain spec - required for fork detection
-        let chain_id = provider.get_chain_id().await?;
-        let chain_spec = ChainSpec::from_chain_id(chain_id)
-            .ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
-
-        // Fetch the current head block as parent
-        let parent_block = provider
-            .get_block_by_number(BlockNumberOrTag::Latest)
-            .full()
-            .await?
-            .ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
-
-        let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
-
-        let canonical_parent = parent_header.number;
-        let start_block = canonical_parent + 1;
-
-        match mode {
-            RampMode::Blocks(blocks) => {
-                info!(
-                    target: "reth-bench",
-                    canonical_parent,
-                    start_block,
-                    end_block = start_block + blocks - 1,
-                    "Starting gas limit ramp benchmark (block count mode)"
-                );
-            }
-            RampMode::TargetGasLimit(target) => {
-                info!(
-                    target: "reth-bench",
-                    canonical_parent,
-                    start_block,
-                    current_gas_limit = parent_header.gas_limit,
-                    target_gas_limit = target,
-                    "Starting gas limit ramp benchmark (target gas limit mode)"
-                );
-            }
-        }
-        if self.reth_new_payload {
-            info!("Using reth_newPayload endpoint");
-        }
-
-        let mut blocks_processed = 0u64;
-        let total_benchmark_duration = Instant::now();
-
-        while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
-            let timestamp = parent_header.timestamp.saturating_add(1);
-
-            let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
-            let new_payload_version = request.new_payload_version;
-
-            let (payload, sidecar) = build_payload(&provider, request).await?;
-
-            let mut block =
-                payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
-
-            let max_increase = max_gas_limit_increase(parent_header.gas_limit);
-            let gas_limit =
-                parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
-
-            block.header.gas_limit = gas_limit;
-
-            let block_hash = block.header.hash_slow();
-            // Regenerate the payload from the modified block, but keep the original sidecar
-            // which contains the actual execution requests data (not just the hash)
-            let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
-            let (version, params, execution_data) = payload_to_new_payload(
-                payload,
-                sidecar,
-                false,
-                block.header.withdrawals_root,
-                Some(new_payload_version),
-            )?;
-
-            // Save payload to file with version info for replay
-            let payload_path =
-                self.output.join(format!("payload_block_{}.json", block.header.number));
-            let file = GasRampPayloadFile {
-                version: version as u8,
-                block_hash,
-                params: params.clone(),
-                execution_data: Some(execution_data.clone()),
-            };
-            let payload_json = serde_json::to_string_pretty(&file)?;
-            std::fs::write(&payload_path, &payload_json)?;
-            info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
-
-            let reth_data = self.reth_new_payload.then_some(execution_data);
-            let _ = call_new_payload_with_reth(&provider, version, params, reth_data).await?;
-
-            let forkchoice_state = ForkchoiceState {
-                head_block_hash: block_hash,
-                safe_block_hash: block_hash,
-                finalized_block_hash: block_hash,
-            };
-            call_forkchoice_updated(&provider, version, forkchoice_state, None).await?;
-
-            parent_header = block.header;
-            parent_hash = block_hash;
-            blocks_processed += 1;
-
-            let progress = match mode {
-                RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
-                RampMode::TargetGasLimit(target) => {
-                    let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
-                    format!("{pct:.1}%")
-                }
-            };
-            info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
-        }
-
-        let final_gas_limit = parent_header.gas_limit;
-        info!(
-            target: "reth-bench",
-            total_duration=?total_benchmark_duration.elapsed(),
-            blocks_processed,
-            final_gas_limit,
-            "Benchmark complete"
-        );
-
-        Ok(())
-    }
-}
-
-const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
-    (parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
-}
-
-const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
-    match mode {
-        RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
-        RampMode::TargetGasLimit(target) => current_gas_limit >= target,
-    }
-}

bin/reth-bench/src/bench/generate_big_block.rs

@@ -773,6 +773,7 @@ impl Command {
                 suggested_fee_recipient: alloy_primitives::Address::ZERO,
                 withdrawals: Some(vec![]),
                 parent_beacon_block_root: Some(B256::ZERO),
+                slot_number: None,
             },
             transactions: transactions.to_vec(),
             extra_data: None,

bin/reth-bench/src/bench/helpers.rs

@@ -1,6 +1,5 @@
 //! Common helpers for reth-bench commands.
 
-use crate::valid_payload::call_forkchoice_updated;
 use eyre::Result;
 use std::{
     io::{BufReader, Read},
@@ -70,180 +69,6 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
     }
 }
 
-use alloy_consensus::Header;
-use alloy_eips::eip4844::kzg_to_versioned_hash;
-use alloy_primitives::{Address, B256};
-use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
-use alloy_rpc_types_engine::{
-    CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
-    PayloadAttributes, PayloadId,
-};
-use eyre::OptionExt;
-use reth_chainspec::{ChainSpec, EthereumHardforks};
-use reth_node_api::EngineApiMessageVersion;
-use tracing::debug;
-
-/// Prepared payload request data for triggering block building.
-pub(crate) struct PayloadRequest {
-    /// The payload attributes for the new block.
-    pub(crate) attributes: PayloadAttributes,
-    /// The forkchoice state pointing to the parent block.
-    pub(crate) forkchoice_state: ForkchoiceState,
-    /// The engine API version for FCU calls.
-    pub(crate) fcu_version: EngineApiMessageVersion,
-    /// The getPayload version to use (1-5).
-    pub(crate) get_payload_version: u8,
-    /// The newPayload version to use.
-    pub(crate) new_payload_version: EngineApiMessageVersion,
-}
-
-/// Prepare payload attributes and forkchoice state for a new block.
-pub(crate) fn prepare_payload_request(
-    chain_spec: &ChainSpec,
-    timestamp: u64,
-    parent_hash: B256,
-) -> PayloadRequest {
-    let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
-    let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
-    let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
-    let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
-
-    // FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
-    let fcu_version = if cancun_active {
-        EngineApiMessageVersion::V3
-    } else if shanghai_active {
-        EngineApiMessageVersion::V2
-    } else {
-        EngineApiMessageVersion::V1
-    };
-
-    // getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
-    // newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
-    let (get_payload_version, new_payload_version) = if osaka_active {
-        (5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
-    } else if prague_active {
-        (4, EngineApiMessageVersion::V4)
-    } else if cancun_active {
-        (3, EngineApiMessageVersion::V3)
-    } else if shanghai_active {
-        (2, EngineApiMessageVersion::V2)
-    } else {
-        (1, EngineApiMessageVersion::V1)
-    };
-
-    PayloadRequest {
-        attributes: PayloadAttributes {
-            timestamp,
-            prev_randao: B256::ZERO,
-            suggested_fee_recipient: Address::ZERO,
-            withdrawals: shanghai_active.then(Vec::new),
-            parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
-        },
-        forkchoice_state: ForkchoiceState {
-            head_block_hash: parent_hash,
-            safe_block_hash: parent_hash,
-            finalized_block_hash: parent_hash,
-        },
-        fcu_version,
-        get_payload_version,
-        new_payload_version,
-    }
-}
-
-/// Trigger payload building via FCU and retrieve the built payload.
-///
-/// This sends a forkchoiceUpdated with payload attributes to start building,
-/// then calls getPayload to retrieve the result.
-pub(crate) async fn build_payload(
-    provider: &RootProvider<AnyNetwork>,
-    request: PayloadRequest,
-) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
-    let fcu_result = call_forkchoice_updated(
-        provider,
-        request.fcu_version,
-        request.forkchoice_state,
-        Some(request.attributes.clone()),
-    )
-    .await?;
-
-    let payload_id =
-        fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
-
-    get_payload_with_sidecar(
-        provider,
-        request.get_payload_version,
-        payload_id,
-        request.attributes.parent_beacon_block_root,
-    )
-    .await
-}
-
-/// Convert an RPC block to a consensus header and block hash.
-pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
-    let block_hash = block.header.hash;
-    let header = block.header.inner.clone().into_header_with_defaults();
-    (header, block_hash)
-}
-
-/// Compute versioned hashes from KZG commitments.
-fn versioned_hashes_from_commitments(
-    commitments: &[alloy_primitives::FixedBytes<48>],
-) -> Vec<B256> {
-    commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
-}
-
-/// Fetch an execution payload using the appropriate engine API version.
-pub(crate) async fn get_payload_with_sidecar(
-    provider: &RootProvider<AnyNetwork>,
-    version: u8,
-    payload_id: PayloadId,
-    parent_beacon_block_root: Option<B256>,
-) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
-    debug!(target: "reth-bench", get_payload_version = ?version, ?payload_id, "Sending getPayload");
-
-    match version {
-        1 => {
-            let payload = provider.get_payload_v1(payload_id).await?;
-            Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
-        }
-        2 => {
-            let envelope = provider.get_payload_v2(payload_id).await?;
-            let payload = match envelope.execution_payload {
-                alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
-                alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
-            };
-            Ok((payload, ExecutionPayloadSidecar::none()))
-        }
-        3 => {
-            let envelope = provider.get_payload_v3(payload_id).await?;
-            let versioned_hashes =
-                versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
-            let cancun_fields = CancunPayloadFields {
-                parent_beacon_block_root: parent_beacon_block_root
-                    .ok_or_eyre("parent_beacon_block_root required for V3")?,
-                versioned_hashes,
-            };
-            Ok((
-                ExecutionPayload::V3(envelope.execution_payload),
-                ExecutionPayloadSidecar::v3(cancun_fields),
-            ))
-        }
-        4 => {
-            let envelope = provider.get_payload_v4(payload_id).await?;
-            Ok(envelope.into_payload_and_sidecar(
-                parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V4")?,
-            ))
-        }
-        5 => {
-            let envelope = provider.get_payload_v5(payload_id).await?;
-            Ok(envelope.into_payload_and_sidecar(
-                parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V5")?,
-            ))
-        }
-        _ => panic!("This tool does not support getPayload versions past v5"),
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

bin/reth-bench/src/bench/mod.rs

@@ -6,7 +6,6 @@ use reth_node_core::args::LogArgs;
 use reth_tracing::FileWorkerGuard;
 
 mod context;
-mod gas_limit_ramp;
 mod generate_big_block;
 pub(crate) mod helpers;
 pub use generate_big_block::{
@@ -16,7 +15,6 @@ pub(crate) mod metrics_scraper;
 mod new_payload_fcu;
 mod new_payload_only;
 mod output;
-mod persistence_waiter;
 mod replay_payloads;
 mod send_invalid_payload;
 mod send_payload;
@@ -37,9 +35,6 @@ pub enum Subcommands {
     /// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
     NewPayloadFcu(new_payload_fcu::Command),
 
-    /// Benchmark which builds empty blocks with a ramped gas limit.
-    GasLimitRamp(gas_limit_ramp::Command),
-
     /// Benchmark which only calls subsequent `newPayload` calls.
     NewPayloadOnly(new_payload_only::Command),
 
@@ -99,7 +94,6 @@ impl BenchmarkCommand {
 
         match self.command {
             Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
-            Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
             Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
             Subcommands::SendPayload(command) => command.execute(ctx).await,
             Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,

bin/reth-bench/src/bench/new_payload_fcu.rs

@@ -1,13 +1,5 @@
 //! Runs the `reth bench` command, calling first newPayload for each block, then calling
 //! forkchoiceUpdated.
-//!
-//! Supports configurable waiting behavior:
-//! - **`--wait-time`**: Fixed sleep interval between blocks.
-//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
-//!   `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
-//!   threshold. This ensures the benchmark doesn't outpace persistence.
-//!
-//! Both options can be used together or independently.
 
 use crate::{
     bench::{
@@ -17,16 +9,16 @@ use crate::{
         output::{
             write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
         },
-        persistence_waiter::{
-            derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
-        },
     },
-    valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload_with_reth},
+    valid_payload::{
+        block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
+    },
 };
-use alloy_provider::Provider;
+use alloy_provider::{ext::DebugApi, Provider};
 use alloy_rpc_types_engine::ForkchoiceState;
 use clap::Parser;
 use eyre::{Context, OptionExt};
+use futures::{stream, StreamExt, TryStreamExt};
 use reth_cli_runner::CliContext;
 use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
 use reth_node_core::args::BenchmarkArgs;
@@ -47,16 +39,6 @@ pub struct Command {
     #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
     wait_time: Option<Duration>,
 
-    /// Wait for blocks to be persisted before sending the next batch.
-    ///
-    /// When enabled, waits for every Nth block to be persisted using the
-    /// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
-    /// doesn't outpace persistence.
-    ///
-    /// The subscription uses the regular RPC websocket endpoint (no JWT required).
-    #[arg(long, default_value = "false", verbatim_doc_comment)]
-    wait_for_persistence: bool,
-
     /// Engine persistence threshold used for deciding when to wait for persistence.
     ///
     /// The benchmark waits after every `(threshold + 1)` blocks. By default this
@@ -104,54 +86,17 @@ impl Command {
         if let Some(duration) = self.wait_time {
             info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
         }
-        if self.wait_for_persistence {
-            info!(
-                target: "reth-bench",
-                "Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
-                self.persistence_threshold + 1,
-                self.persistence_threshold
-            );
-        }
-
-        // Set up waiter based on configured options
-        // When both are set: wait at least wait_time, and also wait for persistence if needed
-        let mut waiter = match (self.wait_time, self.wait_for_persistence) {
-            (Some(duration), true) => {
-                let ws_url = derive_ws_rpc_url(
-                    self.benchmark.ws_rpc_url.as_deref(),
-                    &self.benchmark.engine_rpc_url,
-                )?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_duration_and_subscription(
-                    duration,
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
-            (None, true) => {
-                let ws_url = derive_ws_rpc_url(
-                    self.benchmark.ws_rpc_url.as_deref(),
-                    &self.benchmark.engine_rpc_url,
-                )?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_subscription(
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (None, false) => None,
-        };
 
         let BenchContext {
             benchmark_mode,
             block_provider,
             auth_provider,
-            mut next_block,
+            next_block,
             is_optimism,
             use_reth_namespace,
+            rlp_blocks,
+            no_wait_for_persistence,
+            no_wait_for_caches,
         } = BenchContext::new(&self.benchmark, self.rpc_url).await?;
 
         let total_blocks = benchmark_mode.total_blocks();
@@ -159,69 +104,85 @@ impl Command {
         let mut metrics_scraper = MetricsScraper::maybe_new(self.benchmark.metrics_url.clone());
 
         if use_reth_namespace {
-            info!("Using reth_newPayload endpoint");
+            info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
         }
 
         let buffer_size = self.rpc_block_buffer_size;
 
-        // Use a oneshot channel to propagate errors from the spawned task
-        let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
-        let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
+        let mut blocks = Box::pin(
+            stream::iter((next_block..)
+                .take_while(|next_block| {
+                    benchmark_mode.contains(*next_block)
+                }))
+                .map(|next_block| {
+                    let block_provider = block_provider.clone();
+                    async move {
+                        let block_res = block_provider
+                            .get_block_by_number(next_block.into())
+                            .full()
+                            .await
+                            .wrap_err_with(|| {
+                                format!("Failed to fetch block by number {next_block}")
+                            });
+                        let block =
+                            match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
+                                Ok(block) => block,
+                                Err(e) => {
+                                    tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
+                                    return Err(e)
+                                }
+                            };
 
-        tokio::task::spawn(async move {
-            while benchmark_mode.contains(next_block) {
-                let block_res = block_provider
-                    .get_block_by_number(next_block.into())
-                    .full()
-                    .await
-                    .wrap_err_with(|| format!("Failed to fetch block by number {next_block}"));
-                let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
-                    Ok(block) => block,
-                    Err(e) => {
-                        tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
-                        let _ = error_sender.send(e);
-                        break;
+                        let rlp = if rlp_blocks {
+                            let rlp = match block_provider
+                                .debug_get_raw_block(next_block.into())
+                                .await
+                            {
+                                Ok(rlp) => rlp,
+                                Err(e) => {
+                                    tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
+                                    return Err(e.into())
+                                }
+                            };
+                            Some(rlp)
+                        } else {
+                            None
+                        };
+
+                        let head_block_hash = block.header.hash;
+                        let safe_block_hash = block_provider
+                            .get_block_by_number(block.header.number.saturating_sub(32).into());
+
+                        let finalized_block_hash = block_provider
+                            .get_block_by_number(block.header.number.saturating_sub(64).into());
+
+                        let (safe, finalized) =
+                            tokio::join!(safe_block_hash, finalized_block_hash);
+
+                        let safe_block_hash = match safe {
+                            Ok(Some(block)) => block.header.hash,
+                            Ok(None) | Err(_) => head_block_hash,
+                        };
+
+                        let finalized_block_hash = match finalized {
+                            Ok(Some(block)) => block.header.hash,
+                            Ok(None) | Err(_) => head_block_hash,
+                        };
+
+                        Ok((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
                     }
-                };
-
-                let head_block_hash = block.header.hash;
-                let safe_block_hash = block_provider
-                    .get_block_by_number(block.header.number.saturating_sub(32).into());
-
-                let finalized_block_hash = block_provider
-                    .get_block_by_number(block.header.number.saturating_sub(64).into());
-
-                let (safe, finalized) = tokio::join!(safe_block_hash, finalized_block_hash,);
-
-                let safe_block_hash = match safe {
-                    Ok(Some(block)) => block.header.hash,
-                    Ok(None) | Err(_) => head_block_hash,
-                };
-
-                let finalized_block_hash = match finalized {
-                    Ok(Some(block)) => block.header.hash,
-                    Ok(None) | Err(_) => head_block_hash,
-                };
-
-                next_block += 1;
-                if let Err(e) = sender
-                    .send((block, head_block_hash, safe_block_hash, finalized_block_hash))
-                    .await
-                {
-                    tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
-                    break;
-                }
-            }
-        });
+                })
+                .buffered(buffer_size),
+        );
 
         let mut results = Vec::new();
         let mut blocks_processed = 0u64;
         let total_benchmark_duration = Instant::now();
         let mut total_wait_time = Duration::ZERO;
 
-        while let Some((block, head, safe, finalized)) = {
+        while let Some((block, head, safe, finalized, rlp)) = {
             let wait_start = Instant::now();
-            let result = receiver.recv().await;
+            let result = blocks.try_next().await?;
             total_wait_time += wait_start.elapsed();
             result
         } {
@@ -238,11 +199,17 @@ impl Command {
                 finalized_block_hash: finalized,
             };
 
-            let (version, params, execution_data) = block_to_new_payload(block, is_optimism)?;
+            let (version, params) = block_to_new_payload(
+                block,
+                is_optimism,
+                rlp,
+                use_reth_namespace,
+                no_wait_for_persistence,
+                no_wait_for_caches,
+            )?;
             let start = Instant::now();
-            let reth_data = use_reth_namespace.then_some(execution_data);
             let server_timings =
-                call_new_payload_with_reth(&auth_provider, version, params, reth_data).await?;
+                call_new_payload_with_reth(&auth_provider, version, params).await?;
 
             let np_latency =
                 server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
@@ -261,7 +228,7 @@ impl Command {
             };
 
             let fcu_start = Instant::now();
-            call_forkchoice_updated(&auth_provider, version, forkchoice_state, None).await?;
+            call_forkchoice_updated_with_reth(&auth_provider, version, forkchoice_state).await?;
             let fcu_latency = fcu_start.elapsed();
 
             let total_latency = if server_timings.is_some() {
@@ -297,8 +264,8 @@ impl Command {
                 warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
             }
 
-            if let Some(w) = &mut waiter {
-                w.on_block(block_number).await?;
+            if let Some(wait_time) = self.wait_time {
+                tokio::time::sleep(wait_time).await;
             }
 
             let gas_row =
@@ -306,15 +273,6 @@ impl Command {
             results.push((gas_row, combined_result));
         }
 
-        // Check if the spawned task encountered an error
-        if let Ok(error) = error_receiver.try_recv() {
-            return Err(error);
-        }
-
-        // Drop waiter - we don't need to wait for final blocks to persist
-        // since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
-        drop(waiter);
-
         let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
             results.into_iter().unzip();
 

bin/reth-bench/src/bench/new_payload_only.rs

@@ -11,7 +11,7 @@ use crate::{
     },
     valid_payload::{block_to_new_payload, call_new_payload_with_reth},
 };
-use alloy_provider::Provider;
+use alloy_provider::{ext::DebugApi, Provider};
 use clap::Parser;
 use csv::Writer;
 use eyre::{Context, OptionExt};
@@ -51,6 +51,9 @@ impl Command {
             mut next_block,
             is_optimism,
             use_reth_namespace,
+            rlp_blocks,
+            no_wait_for_persistence,
+            no_wait_for_caches,
         } = BenchContext::new(&self.benchmark, self.rpc_url).await?;
 
         let total_blocks = benchmark_mode.total_blocks();
@@ -83,8 +86,21 @@ impl Command {
                     }
                 };
 
+                let rlp = if rlp_blocks {
+                    let Ok(rlp) = block_provider.debug_get_raw_block(next_block.into()).await
+                    else {
+                        tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}");
+                        let _ = error_sender
+                            .send(eyre::eyre!("Failed to fetch raw block {next_block}"));
+                        break;
+                    };
+                    Some(rlp)
+                } else {
+                    None
+                };
+
                 next_block += 1;
-                if let Err(e) = sender.send(block).await {
+                if let Err(e) = sender.send((block, rlp)).await {
                     tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
                     break;
                 }
@@ -96,7 +112,7 @@ impl Command {
         let total_benchmark_duration = Instant::now();
         let mut total_wait_time = Duration::ZERO;
 
-        while let Some(block) = {
+        while let Some((block, rlp)) = {
             let wait_start = Instant::now();
             let result = receiver.recv().await;
             total_wait_time += wait_start.elapsed();
@@ -108,12 +124,18 @@ impl Command {
 
             debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
 
-            let (version, params, execution_data) = block_to_new_payload(block, is_optimism)?;
+            let (version, params) = block_to_new_payload(
+                block,
+                is_optimism,
+                rlp,
+                use_reth_namespace,
+                no_wait_for_persistence,
+                no_wait_for_caches,
+            )?;
 
             let start = Instant::now();
-            let reth_data = use_reth_namespace.then_some(execution_data);
             let server_timings =
-                call_new_payload_with_reth(&auth_provider, version, params, reth_data).await?;
+                call_new_payload_with_reth(&auth_provider, version, params).await?;
 
             let latency =
                 server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());

bin/reth-bench/src/bench/output.rs

@@ -1,11 +1,10 @@
 //! Contains various benchmark output formats, either for logging or for
 //! serialization to / from files.
 
-use alloy_primitives::B256;
 use csv::Writer;
 use eyre::OptionExt;
 use reth_primitives_traits::constants::GIGAGAS;
-use serde::{ser::SerializeStruct, Deserialize, Serialize};
+use serde::{ser::SerializeStruct, Serialize};
 use std::{fs, path::Path, time::Duration};
 use tracing::info;
 
@@ -18,20 +17,6 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
 /// This is the suffix for new payload output csv files.
 pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
 
-/// Serialized format for gas ramp payloads on disk.
-#[derive(Debug, Serialize, Deserialize)]
-pub(crate) struct GasRampPayloadFile {
-    /// Engine API version (1-5).
-    pub(crate) version: u8,
-    /// The block hash for FCU.
-    pub(crate) block_hash: B256,
-    /// The params to pass to newPayload.
-    pub(crate) params: serde_json::Value,
-    /// The execution data for `reth_newPayload`.
-    #[serde(skip_serializing_if = "Option::is_none", default)]
-    pub(crate) execution_data: Option<alloy_rpc_types_engine::ExecutionData>,
-}
-
 /// This represents the results of a single `newPayload` call in the benchmark, containing the gas
 /// used and the `newPayload` latency.
 #[derive(Debug)]
@@ -114,16 +99,27 @@ impl CombinedResult {
 
 impl std::fmt::Display for CombinedResult {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        let np = &self.new_payload_result;
         write!(
             f,
             "Block {} processed at {:.4} Ggas/s, used {} total gas. Combined: {:.4} Ggas/s. fcu: {:?}, newPayload: {:?}",
             self.block_number,
-            self.new_payload_result.gas_per_second() / GIGAGAS as f64,
-            self.new_payload_result.gas_used,
+            np.gas_per_second() / GIGAGAS as f64,
+            np.gas_used,
             self.combined_gas_per_second() / GIGAGAS as f64,
             self.fcu_latency,
-            self.new_payload_result.latency
-        )
+            np.latency,
+        )?;
+        if !np.execution_cache_wait.is_zero() {
+            write!(f, ", execution cache wait: {:?}", np.execution_cache_wait)?;
+        }
+        if !np.sparse_trie_wait.is_zero() {
+            write!(f, ", trie cache wait: {:?}", np.sparse_trie_wait)?;
+        }
+        if let Some(d) = np.persistence_wait {
+            write!(f, ", persistence wait: {d:?}")?;
+        }
+        Ok(())
     }
 }
 

bin/reth-bench/src/bench/persistence_waiter.rs

@@ -1,334 +0,0 @@
-//! Persistence waiting utilities for benchmarks.
-//!
-//! Provides waiting behavior to control benchmark pacing:
-//! - **Fixed duration waits**: Sleep for a fixed time between blocks
-//! - **Persistence-based waits**: Wait for blocks to be persisted using
-//!   `reth_subscribePersistedBlock` subscription
-//! - **Combined mode**: Wait at least the fixed duration, and also wait for persistence if the
-//!   block hasn't been persisted yet (whichever takes longer)
-
-use alloy_eips::BlockNumHash;
-use alloy_network::Ethereum;
-use alloy_provider::{Provider, RootProvider};
-use alloy_pubsub::SubscriptionStream;
-use alloy_rpc_client::RpcClient;
-use alloy_transport_ws::WsConnect;
-use eyre::Context;
-use futures::StreamExt;
-use std::time::Duration;
-use tracing::{debug, info};
-
-/// Default `WebSocket` RPC port for reth.
-const DEFAULT_WS_RPC_PORT: u16 = 8546;
-use url::Url;
-
-/// Returns the websocket RPC URL used for the persistence subscription.
-///
-/// Preference:
-/// - If `ws_rpc_url` is provided, use it directly.
-/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
-///
-/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
-/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
-/// Since we may only have the engine URL by default, we convert the scheme
-/// (http→ws, https→wss) and force the port to 8546.
-pub(crate) fn derive_ws_rpc_url(
-    ws_rpc_url: Option<&str>,
-    engine_rpc_url: &str,
-) -> eyre::Result<Url> {
-    if let Some(ws_url) = ws_rpc_url {
-        let parsed: Url = ws_url
-            .parse()
-            .wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
-        info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
-        Ok(parsed)
-    } else {
-        let derived = engine_url_to_ws_url(engine_rpc_url)?;
-        debug!(
-            target: "reth-bench",
-            engine_url = %engine_rpc_url,
-            %derived,
-            "Derived WebSocket RPC URL from engine RPC URL"
-        );
-        Ok(derived)
-    }
-}
-
-/// Converts an engine API URL to the default RPC websocket URL.
-///
-/// Transformations:
-/// - `http`  → `ws`
-/// - `https` → `wss`
-/// - `ws` / `wss` keep their scheme
-/// - Port is always set to `8546`, reth's default RPC websocket port.
-///
-/// This is used when we only know the engine API URL (typically `:8551`) but
-/// need to connect to the node's WS RPC endpoint for persistence events.
-fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
-    let url: Url = engine_url
-        .parse()
-        .wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
-
-    let mut ws_url = url.clone();
-
-    match ws_url.scheme() {
-        "http" => ws_url
-            .set_scheme("ws")
-            .map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
-        "https" => ws_url
-            .set_scheme("wss")
-            .map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
-        "ws" | "wss" => {}
-        scheme => {
-            return Err(eyre::eyre!(
-            "Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
-        ))
-        }
-    }
-
-    ws_url
-        .set_port(Some(DEFAULT_WS_RPC_PORT))
-        .map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
-
-    Ok(ws_url)
-}
-
-/// Waits until the persistence subscription reports that `target` has been persisted.
-///
-/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
-/// - the subscription stream ends unexpectedly, or
-/// - `timeout` elapses before `target` is observed.
-async fn wait_for_persistence(
-    stream: &mut SubscriptionStream<BlockNumHash>,
-    target: u64,
-    last_persisted: &mut u64,
-    timeout: Duration,
-) -> eyre::Result<()> {
-    tokio::time::timeout(timeout, async {
-        while *last_persisted < target {
-            match stream.next().await {
-                Some(persisted) => {
-                    *last_persisted = persisted.number;
-                    debug!(
-                        target: "reth-bench",
-                        persisted_block = ?last_persisted,
-                        "Received persistence notification"
-                    );
-                }
-                None => {
-                    return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
-                }
-            }
-        }
-        Ok(())
-    })
-    .await
-    .map_err(|_| {
-        eyre::eyre!(
-            "Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
-            target,
-            timeout,
-            last_persisted
-        )
-    })?
-}
-
-/// Wrapper that keeps both the subscription stream and the underlying provider alive.
-/// The provider must be kept alive for the subscription to continue receiving events.
-pub(crate) struct PersistenceSubscription {
-    _provider: RootProvider<Ethereum>,
-    stream: SubscriptionStream<BlockNumHash>,
-}
-
-impl PersistenceSubscription {
-    const fn new(
-        provider: RootProvider<Ethereum>,
-        stream: SubscriptionStream<BlockNumHash>,
-    ) -> Self {
-        Self { _provider: provider, stream }
-    }
-
-    const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
-        &mut self.stream
-    }
-}
-
-/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
-///
-/// The `keepalive_interval` is set to match `persistence_timeout` so that the `WebSocket`
-/// connection is not dropped during long MDBX commits that block the server from responding
-/// to pings.
-pub(crate) async fn setup_persistence_subscription(
-    ws_url: Url,
-    persistence_timeout: Duration,
-) -> eyre::Result<PersistenceSubscription> {
-    info!(target: "reth-bench", "Connecting to WebSocket at {} for persistence subscription", ws_url);
-
-    let ws_connect =
-        WsConnect::new(ws_url.to_string()).with_keepalive_interval(persistence_timeout);
-    let client = RpcClient::connect_pubsub(ws_connect)
-        .await
-        .wrap_err("Failed to connect to WebSocket RPC endpoint")?;
-    let provider: RootProvider<Ethereum> = RootProvider::new(client);
-
-    let subscription = provider
-        .subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
-        .await
-        .wrap_err("Failed to subscribe to persistence notifications")?;
-
-    info!(target: "reth-bench", "Subscribed to persistence notifications");
-
-    Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
-}
-
-/// Encapsulates the block waiting logic.
-///
-/// Provides a simple `on_block()` interface that handles both:
-/// - Fixed duration waits (when `wait_time` is set)
-/// - Persistence-based waits (when `subscription` is set)
-///
-/// For persistence mode, waits after every `(threshold + 1)` blocks.
-pub(crate) struct PersistenceWaiter {
-    wait_time: Option<Duration>,
-    subscription: Option<PersistenceSubscription>,
-    blocks_sent: u64,
-    last_persisted: u64,
-    threshold: u64,
-    timeout: Duration,
-}
-
-impl PersistenceWaiter {
-    pub(crate) const fn with_duration(wait_time: Duration) -> Self {
-        Self {
-            wait_time: Some(wait_time),
-            subscription: None,
-            blocks_sent: 0,
-            last_persisted: 0,
-            threshold: 0,
-            timeout: Duration::ZERO,
-        }
-    }
-
-    pub(crate) const fn with_subscription(
-        subscription: PersistenceSubscription,
-        threshold: u64,
-        timeout: Duration,
-    ) -> Self {
-        Self {
-            wait_time: None,
-            subscription: Some(subscription),
-            blocks_sent: 0,
-            last_persisted: 0,
-            threshold,
-            timeout,
-        }
-    }
-
-    /// Creates a waiter that combines both duration and persistence waiting.
-    ///
-    /// Waits at least `wait_time` between blocks, and also waits for persistence
-    /// if the block hasn't been persisted yet (whichever takes longer).
-    pub(crate) const fn with_duration_and_subscription(
-        wait_time: Duration,
-        subscription: PersistenceSubscription,
-        threshold: u64,
-        timeout: Duration,
-    ) -> Self {
-        Self {
-            wait_time: Some(wait_time),
-            subscription: Some(subscription),
-            blocks_sent: 0,
-            last_persisted: 0,
-            threshold,
-            timeout,
-        }
-    }
-
-    /// Called once per block. Waits based on the configured mode.
-    ///
-    /// When both `wait_time` and `subscription` are set (combined mode):
-    /// - Always waits at least `wait_time`
-    /// - Additionally waits for persistence if we're at a persistence checkpoint
-    #[allow(clippy::manual_is_multiple_of)]
-    pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
-        // Always wait for the fixed duration if configured
-        if let Some(wait_time) = self.wait_time {
-            tokio::time::sleep(wait_time).await;
-        }
-
-        // Check persistence if subscription is configured
-        let Some(ref mut subscription) = self.subscription else {
-            return Ok(());
-        };
-
-        self.blocks_sent += 1;
-
-        if self.blocks_sent % (self.threshold + 1) == 0 {
-            debug!(
-                target: "reth-bench",
-                target_block = ?block_number,
-                last_persisted = self.last_persisted,
-                blocks_sent = self.blocks_sent,
-                "Waiting for persistence"
-            );
-
-            wait_for_persistence(
-                subscription.stream_mut(),
-                block_number,
-                &mut self.last_persisted,
-                self.timeout,
-            )
-            .await?;
-
-            debug!(
-                target: "reth-bench",
-                persisted = self.last_persisted,
-                "Persistence caught up"
-            );
-        }
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::time::Instant;
-
-    #[test]
-    fn test_engine_url_to_ws_url() {
-        // http -> ws, always uses port 8546
-        let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
-        assert_eq!(result.as_str(), "ws://localhost:8546/");
-
-        // https -> wss
-        let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
-        assert_eq!(result.as_str(), "wss://localhost:8546/");
-
-        // Custom engine port still maps to 8546
-        let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
-        assert_eq!(result.port(), Some(8546));
-
-        // Already ws passthrough
-        let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
-        assert_eq!(result.scheme(), "ws");
-
-        // Invalid inputs
-        assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
-        assert!(engine_url_to_ws_url("not a valid url").is_err());
-    }
-
-    #[tokio::test]
-    async fn test_waiter_with_duration() {
-        let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
-
-        let start = Instant::now();
-        waiter.on_block(1).await.unwrap();
-        waiter.on_block(2).await.unwrap();
-        waiter.on_block(3).await.unwrap();
-
-        // Should have waited ~3ms total
-        assert!(start.elapsed() >= Duration::from_millis(3));
-    }
-}

bin/reth-bench/src/bench/replay_payloads.rs

@@ -1,15 +1,4 @@
 //! Command for replaying pre-generated payloads from disk.
-//!
-//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
-//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
-//!
-//! Supports configurable waiting behavior:
-//! - **`--wait-time`**: Fixed sleep interval between blocks.
-//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
-//!   `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
-//!   threshold. This ensures the benchmark doesn't outpace persistence.
-//!
-//! Both options can be used together or independently.
 
 use crate::{
     authenticated_transport::AuthenticatedTransportConnect,
@@ -17,17 +6,13 @@ use crate::{
         helpers::parse_duration,
         metrics_scraper::MetricsScraper,
         output::{
-            write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
-            TotalGasOutput, TotalGasRow,
-        },
-        persistence_waiter::{
-            derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
+            write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
         },
     },
-    valid_payload::{call_forkchoice_updated, call_new_payload_with_reth},
+    valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
 };
 use alloy_primitives::B256;
-use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
+use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
 use alloy_rpc_client::ClientBuilder;
 use alloy_rpc_types_engine::{
     CancunPayloadFields, ExecutionData, ExecutionPayloadEnvelopeV4, ExecutionPayloadSidecar,
@@ -36,13 +21,13 @@ use alloy_rpc_types_engine::{
 use clap::Parser;
 use eyre::Context;
 use reth_cli_runner::CliContext;
-use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
 use reth_node_api::EngineApiMessageVersion;
+use reth_rpc_api::RethNewPayloadInput;
 use std::{
     path::PathBuf,
     time::{Duration, Instant},
 };
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
 use url::Url;
 
 /// `reth bench replay-payloads` command
@@ -72,9 +57,9 @@ pub struct Command {
     #[arg(long, value_name = "SKIP", default_value = "0")]
     skip: usize,
 
-    /// Optional directory containing gas ramp payloads to replay first.
-    /// These are replayed before the main payloads to warm up the gas limit.
-    #[arg(long, value_name = "GAS_RAMP_DIR")]
+    /// Deprecated: gas ramp is no longer needed. Use `--testing.skip-gas-limit-ramp-check`
+    /// and `--testing.gas-limit` on the reth node instead. This flag is accepted but ignored.
+    #[arg(long, value_name = "GAS_RAMP_DIR", hide = true)]
     gas_ramp_dir: Option<PathBuf>,
 
     /// Optional output directory for benchmark results (CSV files).
@@ -88,47 +73,6 @@ pub struct Command {
     #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
     wait_time: Option<Duration>,
 
-    /// Wait for blocks to be persisted before sending the next batch.
-    ///
-    /// When enabled, waits for every Nth block to be persisted using the
-    /// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
-    /// doesn't outpace persistence.
-    ///
-    /// The subscription uses the regular RPC websocket endpoint (no JWT required).
-    #[arg(long, default_value = "false", verbatim_doc_comment)]
-    wait_for_persistence: bool,
-
-    /// Engine persistence threshold used for deciding when to wait for persistence.
-    ///
-    /// The benchmark waits after every `(threshold + 1)` blocks. By default this
-    /// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
-    /// at blocks 3, 6, 9, etc.
-    #[arg(
-        long = "persistence-threshold",
-        value_name = "PERSISTENCE_THRESHOLD",
-        default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
-        verbatim_doc_comment
-    )]
-    persistence_threshold: u64,
-
-    /// Timeout for waiting on persistence at each checkpoint.
-    ///
-    /// Must be long enough to account for the persistence thread being blocked
-    /// by pruning after the previous save.
-    #[arg(
-        long = "persistence-timeout",
-        value_name = "PERSISTENCE_TIMEOUT",
-        value_parser = parse_duration,
-        default_value = "120s",
-        verbatim_doc_comment
-    )]
-    persistence_timeout: Duration,
-
-    /// Optional `WebSocket` RPC URL for persistence subscription.
-    /// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
-    #[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
-    ws_rpc_url: Option<String>,
-
     /// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
     ///
     /// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
@@ -137,6 +81,20 @@ pub struct Command {
     #[arg(long, default_value = "false", verbatim_doc_comment)]
     reth_new_payload: bool,
 
+    /// Skip waiting for in-flight persistence before processing.
+    ///
+    /// Only works with `--reth-new-payload`. When set, passes `wait_for_persistence: false`
+    /// to the `reth_newPayload` endpoint.
+    #[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
+    no_wait_for_persistence: bool,
+
+    /// Skip waiting for execution cache and sparse trie locks before processing.
+    ///
+    /// Only works with `--reth-new-payload`. When set, passes `wait_for_caches: false`
+    /// to the `reth_newPayload` endpoint.
+    #[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
+    no_wait_for_caches: bool,
+
     /// Optional Prometheus metrics endpoint to scrape after each block.
     ///
     /// When provided, reth-bench will fetch metrics from this URL after each
@@ -156,16 +114,6 @@ struct LoadedPayload {
     block_hash: B256,
 }
 
-/// A gas ramp payload loaded from disk.
-struct GasRampPayload {
-    /// Block number from filename.
-    block_number: u64,
-    /// Engine API version for newPayload.
-    version: EngineApiMessageVersion,
-    /// The file contents.
-    file: GasRampPayloadFile,
-}
-
 impl Command {
     /// Execute the `replay-payloads` command.
     pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
@@ -175,44 +123,10 @@ impl Command {
         if let Some(duration) = self.wait_time {
             info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
         }
-        if self.wait_for_persistence {
-            info!(
-                target: "reth-bench",
-                "Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
-                self.persistence_threshold + 1,
-                self.persistence_threshold
-            );
-        }
         if self.reth_new_payload {
-            info!("Using reth_newPayload endpoint");
+            info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
         }
 
-        // Set up waiter based on configured options
-        // When both are set: wait at least wait_time, and also wait for persistence if needed
-        let mut waiter = match (self.wait_time, self.wait_for_persistence) {
-            (Some(duration), true) => {
-                let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_duration_and_subscription(
-                    duration,
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
-            (None, true) => {
-                let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_subscription(
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (None, false) => None,
-        };
-
         let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
 
         // Set up authenticated engine provider
@@ -242,18 +156,16 @@ impl Command {
             "Using initial parent block"
         );
 
-        // Load all payloads upfront to avoid I/O delays between phases
-        let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
-            let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
-            if payloads.is_empty() {
-                return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
-            }
-            info!(target: "reth-bench", count = payloads.len(), "Loaded gas ramp payloads from disk");
-            payloads
-        } else {
-            Vec::new()
-        };
+        // Warn if deprecated --gas-ramp-dir is passed
+        if self.gas_ramp_dir.is_some() {
+            warn!(
+                target: "reth-bench",
+                "--gas-ramp-dir is deprecated and ignored. Use --testing.skip-gas-limit-ramp-check \
+                 and --testing.gas-limit on the reth node instead."
+            );
+        }
 
+        // Load all payloads upfront to avoid I/O delays between phases
         let payloads = self.load_payloads()?;
         if payloads.is_empty() {
             return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
@@ -262,47 +174,6 @@ impl Command {
 
         let mut parent_hash = initial_parent_hash;
 
-        // Replay gas ramp payloads first
-        for (i, payload) in gas_ramp_payloads.iter().enumerate() {
-            info!(
-                target: "reth-bench",
-                gas_ramp_payload = i + 1,
-                total = gas_ramp_payloads.len(),
-                block_number = payload.block_number,
-                block_hash = %payload.file.block_hash,
-                "Executing gas ramp payload (newPayload + FCU)"
-            );
-
-            let reth_data =
-                if self.reth_new_payload { payload.file.execution_data.clone() } else { None };
-            let _ = call_new_payload_with_reth(
-                &auth_provider,
-                payload.version,
-                payload.file.params.clone(),
-                reth_data,
-            )
-            .await?;
-
-            let fcu_state = ForkchoiceState {
-                head_block_hash: payload.file.block_hash,
-                safe_block_hash: parent_hash,
-                finalized_block_hash: parent_hash,
-            };
-            call_forkchoice_updated(&auth_provider, payload.version, fcu_state, None).await?;
-
-            info!(target: "reth-bench", gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
-
-            if let Some(w) = &mut waiter {
-                w.on_block(payload.block_number).await?;
-            }
-
-            parent_hash = payload.file.block_hash;
-        }
-
-        if !gas_ramp_payloads.is_empty() {
-            info!(target: "reth-bench", count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
-        }
-
         let mut results = Vec::new();
         let total_benchmark_duration = Instant::now();
 
@@ -336,31 +207,41 @@ impl Command {
                 "Sending newPayload"
             );
 
-            let params = serde_json::to_value((
-                execution_payload.clone(),
-                Vec::<B256>::new(),
-                B256::ZERO,
-                envelope.execution_requests.to_vec(),
-            ))?;
+            let (version, params) = if self.reth_new_payload {
+                let reth_data = ExecutionData {
+                    payload: execution_payload.clone().into(),
+                    sidecar: ExecutionPayloadSidecar::v4(
+                        CancunPayloadFields {
+                            versioned_hashes: Vec::new(),
+                            parent_beacon_block_root: B256::ZERO,
+                        },
+                        PraguePayloadFields {
+                            requests: envelope.execution_requests.clone().into(),
+                        },
+                    ),
+                };
+                (
+                    None,
+                    serde_json::to_value((
+                        RethNewPayloadInput::ExecutionData(reth_data),
+                        self.no_wait_for_persistence.then_some(false),
+                        self.no_wait_for_caches.then_some(false),
+                    ))?,
+                )
+            } else {
+                (
+                    Some(EngineApiMessageVersion::V4),
+                    serde_json::to_value((
+                        execution_payload.clone(),
+                        Vec::<B256>::new(),
+                        B256::ZERO,
+                        envelope.execution_requests.to_vec(),
+                    ))?,
+                )
+            };
 
-            let reth_data = self.reth_new_payload.then(|| ExecutionData {
-                payload: execution_payload.clone().into(),
-                sidecar: ExecutionPayloadSidecar::v4(
-                    CancunPayloadFields {
-                        versioned_hashes: Vec::new(),
-                        parent_beacon_block_root: B256::ZERO,
-                    },
-                    PraguePayloadFields { requests: envelope.execution_requests.clone().into() },
-                ),
-            });
-
-            let server_timings = call_new_payload_with_reth(
-                &auth_provider,
-                EngineApiMessageVersion::V4,
-                params,
-                reth_data,
-            )
-            .await?;
+            let server_timings =
+                call_new_payload_with_reth(&auth_provider, version, params).await?;
 
             let np_latency =
                 server_timings.as_ref().map(|t| t.latency).unwrap_or_else(|| start.elapsed());
@@ -384,10 +265,8 @@ impl Command {
                 finalized_block_hash: parent_hash,
             };
 
-            debug!(target: "reth-bench", method = "engine_forkchoiceUpdatedV3", ?fcu_state, "Sending forkchoiceUpdated");
-
             let fcu_start = Instant::now();
-            let fcu_result = auth_provider.fork_choice_updated_v3(fcu_state, None).await?;
+            call_forkchoice_updated_with_reth(&auth_provider, version, fcu_state).await?;
             let fcu_latency = fcu_start.elapsed();
 
             let total_latency =
@@ -412,22 +291,13 @@ impl Command {
                 tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
             }
 
-            if let Some(w) = &mut waiter {
-                w.on_block(block_number).await?;
-            }
-
             let gas_row =
                 TotalGasRow { block_number, transaction_count, gas_used, time: current_duration };
             results.push((gas_row, combined_result));
 
-            debug!(target: "reth-bench", ?fcu_result, "Payload executed successfully");
             parent_hash = block_hash;
         }
 
-        // Drop waiter - we don't need to wait for final blocks to persist
-        // since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
-        drop(waiter);
-
         let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
             results.into_iter().unzip();
 
@@ -514,60 +384,4 @@ impl Command {
 
         Ok(payloads)
     }
-
-    /// Load and parse gas ramp payload files from a directory.
-    fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
-        let mut payloads = Vec::new();
-
-        let entries: Vec<_> = std::fs::read_dir(dir)
-            .wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
-            .filter_map(|e| e.ok())
-            .filter(|e| {
-                e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
-                    e.file_name().to_string_lossy().starts_with("payload_block_")
-            })
-            .collect();
-
-        // Parse filenames to get block numbers and sort
-        let mut indexed_paths: Vec<(u64, PathBuf)> = entries
-            .into_iter()
-            .filter_map(|e| {
-                let name = e.file_name();
-                let name_str = name.to_string_lossy();
-                // Extract block number from "payload_block_NNN.json"
-                let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
-                let block_number: u64 = block_str.parse().ok()?;
-                Some((block_number, e.path()))
-            })
-            .collect();
-
-        indexed_paths.sort_by_key(|(num, _)| *num);
-
-        for (block_number, path) in indexed_paths {
-            let content = std::fs::read_to_string(&path)
-                .wrap_err_with(|| format!("Failed to read {:?}", path))?;
-            let file: GasRampPayloadFile = serde_json::from_str(&content)
-                .wrap_err_with(|| format!("Failed to parse {:?}", path))?;
-
-            let version = match file.version {
-                1 => EngineApiMessageVersion::V1,
-                2 => EngineApiMessageVersion::V2,
-                3 => EngineApiMessageVersion::V3,
-                4 => EngineApiMessageVersion::V4,
-                5 => EngineApiMessageVersion::V5,
-                v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
-            };
-
-            info!(
-                block_number,
-                block_hash = %file.block_hash,
-                path = %path.display(),
-                "Loaded gas ramp payload"
-            );
-
-            payloads.push(GasRampPayload { block_number, version, file });
-        }
-
-        Ok(payloads)
-    }
 }

bin/reth-bench/src/bench/send_invalid_payload/mod.rs

@@ -240,6 +240,7 @@ impl Command {
             ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
             ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
             ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
+            ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
         };
 
         let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
@@ -254,6 +255,9 @@ impl Command {
                         ExecutionPayload::V1(p) => p.block_hash,
                         ExecutionPayload::V2(p) => p.payload_inner.block_hash,
                         ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash,
+                        ExecutionPayload::V4(p) => {
+                            p.payload_inner.payload_inner.payload_inner.block_hash
+                        }
                     }
                 }
             };
@@ -262,6 +266,9 @@ impl Command {
                 ExecutionPayload::V1(p) => p.block_hash = new_hash,
                 ExecutionPayload::V2(p) => p.payload_inner.block_hash = new_hash,
                 ExecutionPayload::V3(p) => p.payload_inner.payload_inner.block_hash = new_hash,
+                ExecutionPayload::V4(p) => {
+                    p.payload_inner.payload_inner.payload_inner.block_hash = new_hash
+                }
             }
         }
 

bin/reth-bench/src/bench_mode.rs

@@ -34,17 +34,16 @@ impl BenchMode {
     }
 
     /// Create a [`BenchMode`] from optional `from` and `to` fields.
-    pub fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Result<Self, eyre::Error> {
+    ///
+    /// If only `--to` is provided, `from` is derived as `latest_block + 1`.
+    pub const fn new(from: Option<u64>, to: Option<u64>, latest_block: u64) -> Self {
         // If neither `--from` nor `--to` are provided, we will run the benchmark continuously,
         // starting at the latest block.
         match (from, to) {
-            (Some(from), Some(to)) => Ok(Self::Range(from..=to)),
-            (None, None) => Ok(Self::Continuous(latest_block)),
-            (Some(start), None) => Ok(Self::Continuous(start)),
-            _ => {
-                // both or neither are allowed, everything else is ambiguous
-                Err(eyre::eyre!("`from` and `to` must be provided together, or not at all."))
-            }
+            (Some(from), Some(to)) => Self::Range(from..=to),
+            (None, None) => Self::Continuous(latest_block),
+            (Some(start), None) => Self::Continuous(start),
+            (None, Some(to)) => Self::Range(latest_block + 1..=to),
         }
     }
 }

bin/reth-bench/src/valid_payload.rs

@@ -3,7 +3,7 @@
 //! before sending additional calls.
 
 use alloy_eips::eip7685::Requests;
-use alloy_primitives::B256;
+use alloy_primitives::{Bytes, B256};
 use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
 use alloy_rpc_types_engine::{
     ExecutionData, ExecutionPayload, ExecutionPayloadInputV2, ExecutionPayloadSidecar,
@@ -12,6 +12,7 @@ use alloy_rpc_types_engine::{
 use alloy_transport::TransportResult;
 use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
 use reth_node_api::EngineApiMessageVersion;
+use reth_rpc_api::RethNewPayloadInput;
 use serde::Deserialize;
 use std::time::Duration;
 use tracing::{debug, error};
@@ -166,10 +167,27 @@ where
 /// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
 ///
 /// Returns `(version, versioned_params, execution_data)`.
+///
+/// When `no_wait_for_persistence` or `no_wait_for_caches` is `true` and using `reth_newPayload`,
+/// passes the corresponding `wait_for_*: false` to skip that wait.
 pub(crate) fn block_to_new_payload(
     block: AnyRpcBlock,
     is_optimism: bool,
-) -> eyre::Result<(EngineApiMessageVersion, serde_json::Value, ExecutionData)> {
+    rlp: Option<Bytes>,
+    reth_new_payload: bool,
+    no_wait_for_persistence: bool,
+    no_wait_for_caches: bool,
+) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
+    if let Some(rlp) = rlp {
+        return Ok((
+            None,
+            serde_json::to_value((
+                RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),
+                no_wait_for_persistence.then_some(false),
+                no_wait_for_caches.then_some(false),
+            ))?,
+        ));
+    }
     let block = block
         .into_inner()
         .map_header(|header| header.map(|h| h.into_header_with_defaults()))
@@ -181,7 +199,21 @@ pub(crate) fn block_to_new_payload(
 
     // Convert to execution payload
     let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
-    payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)
+    let (version, params, execution_data) =
+        payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)?;
+
+    if reth_new_payload {
+        Ok((
+            None,
+            serde_json::to_value((
+                RethNewPayloadInput::ExecutionData(execution_data),
+                no_wait_for_persistence.then_some(false),
+                no_wait_for_caches.then_some(false),
+            ))?,
+        ))
+    } else {
+        Ok((Some(version), params))
+    }
 }
 
 /// Converts an execution payload and sidecar into versioned engine API params and an
@@ -198,6 +230,20 @@ pub(crate) fn payload_to_new_payload(
     let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
 
     let (version, params) = match payload {
+        ExecutionPayload::V4(payload) => {
+            let cancun = sidecar.cancun().unwrap();
+            let prague = sidecar.prague().unwrap();
+            let requests = prague.requests.requests_hash();
+            (
+                EngineApiMessageVersion::V5,
+                serde_json::to_value((
+                    payload,
+                    cancun.versioned_hashes.clone(),
+                    cancun.parent_beacon_block_root,
+                    requests,
+                ))?,
+            )
+        }
         ExecutionPayload::V3(payload) => {
             let cancun = sidecar.cancun().unwrap();
 
@@ -219,8 +265,8 @@ pub(crate) fn payload_to_new_payload(
                         ))?,
                     )
                 } else {
-                    // Extract actual Requests from RequestsOrHash
-                    let requests = prague.requests.requests_hash();
+                    // Preserve the original RequestsOrHash payload for engine_newPayloadV4.
+                    let requests = prague.requests.clone();
                     (
                         version,
                         serde_json::to_value((
@@ -266,17 +312,15 @@ pub(crate) fn payload_to_new_payload(
 #[allow(dead_code)]
 pub(crate) async fn call_new_payload<N: Network, P: Provider<N>>(
     provider: P,
-    version: EngineApiMessageVersion,
+    version: Option<EngineApiMessageVersion>,
     params: serde_json::Value,
-) -> TransportResult<Option<NewPayloadTimingBreakdown>> {
-    call_new_payload_with_reth(provider, version, params, None).await
+) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
+    call_new_payload_with_reth(provider, version, params).await
 }
 
 /// Response from `reth_newPayload` endpoint, which includes server-measured latency.
 #[derive(Debug, Deserialize)]
 struct RethPayloadStatus {
-    #[serde(flatten)]
-    status: PayloadStatus,
     latency_us: u64,
     #[serde(default)]
     persistence_wait_us: Option<u64>,
@@ -300,72 +344,50 @@ pub(crate) struct NewPayloadTimingBreakdown {
 }
 
 /// Calls either `engine_newPayload*` or `reth_newPayload` depending on whether
-/// `reth_execution_data` is provided.
+/// `version` is provided.
 ///
-/// When `reth_execution_data` is `Some`, uses the `reth_newPayload` endpoint which takes
-/// `ExecutionData` directly and waits for persistence and cache updates to complete.
+/// When `version` is `None`, uses `reth_newPayload` endpoint with provided params.
 ///
 /// Returns the server-reported timing breakdown when using the reth namespace, or `None` for
 /// the standard engine namespace.
 pub(crate) async fn call_new_payload_with_reth<N: Network, P: Provider<N>>(
     provider: P,
-    version: EngineApiMessageVersion,
+    version: Option<EngineApiMessageVersion>,
     params: serde_json::Value,
-    reth_execution_data: Option<ExecutionData>,
-) -> TransportResult<Option<NewPayloadTimingBreakdown>> {
-    if let Some(execution_data) = reth_execution_data {
-        let method = "reth_newPayload";
-        let reth_params = serde_json::to_value((execution_data.clone(),))
-            .expect("ExecutionData serialization cannot fail");
+) -> eyre::Result<Option<NewPayloadTimingBreakdown>> {
+    let method = version.map(|v| v.method_name()).unwrap_or("reth_newPayload");
 
-        debug!(target: "reth-bench", method, "Sending newPayload");
+    debug!(target: "reth-bench", method, "Sending newPayload");
 
-        let mut resp: RethPayloadStatus = provider.client().request(method, &reth_params).await?;
+    let resp = loop {
+        let resp: serde_json::Value = provider.client().request(method, &params).await?;
+        let status = PayloadStatus::deserialize(&resp)?;
 
-        while !resp.status.is_valid() {
-            if resp.status.is_invalid() {
-                error!(target: "reth-bench", status=?resp.status, "Invalid {method}");
-                return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
-                    std::io::Error::other(format!("Invalid {method}: {:?}", resp.status)),
-                )))
-            }
-            if resp.status.is_syncing() {
-                return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
-                    "invalid range: no canonical state found for parent of requested block",
-                ))
-            }
-            resp = provider.client().request(method, &reth_params).await?;
+        if status.is_valid() {
+            break resp;
         }
-
-        Ok(Some(NewPayloadTimingBreakdown {
-            latency: Duration::from_micros(resp.latency_us),
-            persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
-            execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
-            sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
-        }))
-    } else {
-        let method = version.method_name();
-
-        debug!(target: "reth-bench", method, "Sending newPayload");
-
-        let mut status: PayloadStatus = provider.client().request(method, &params).await?;
-
-        while !status.is_valid() {
-            if status.is_invalid() {
-                error!(target: "reth-bench", ?status, ?params, "Invalid {method}",);
-                return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
-                    std::io::Error::other(format!("Invalid {method}: {status:?}")),
-                )))
-            }
-            if status.is_syncing() {
-                return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
-                    "invalid range: no canonical state found for parent of requested block",
-                ))
-            }
-            status = provider.client().request(method, &params).await?;
+        if status.is_invalid() {
+            return Err(eyre::eyre!("Invalid {method}: {status:?}"));
         }
-        Ok(None)
+        if status.is_syncing() {
+            return Err(eyre::eyre!(
+                "invalid range: no canonical state found for parent of requested block"
+            ));
+        }
+    };
+
+    if version.is_some() {
+        return Ok(None);
     }
+
+    let resp: RethPayloadStatus = serde_json::from_value(resp)?;
+
+    Ok(Some(NewPayloadTimingBreakdown {
+        latency: Duration::from_micros(resp.latency_us),
+        persistence_wait: resp.persistence_wait_us.map(Duration::from_micros),
+        execution_cache_wait: Duration::from_micros(resp.execution_cache_wait_us),
+        sparse_trie_wait: Duration::from_micros(resp.sparse_trie_wait_us),
+    }))
 }
 
 /// Calls the correct `engine_forkchoiceUpdated` method depending on the given
@@ -379,9 +401,12 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
     forkchoice_state: ForkchoiceState,
     payload_attributes: Option<PayloadAttributes>,
 ) -> TransportResult<ForkchoiceUpdated> {
-    // FCU V3 is used for both Cancun and Prague (there is no FCU V4)
+    // FCU V3 is used for Cancun, Prague, and Amsterdam (there is no FCU V4-V6)
     match message_version {
-        EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
+        EngineApiMessageVersion::V3 |
+        EngineApiMessageVersion::V4 |
+        EngineApiMessageVersion::V5 |
+        EngineApiMessageVersion::V6 => {
             provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
         }
         EngineApiMessageVersion::V2 => {
@@ -392,3 +417,47 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
         }
     }
 }
+
+/// Calls either `reth_forkchoiceUpdated` or the standard `engine_forkchoiceUpdated*` depending
+/// on `use_reth`.
+///
+/// When `use_reth` is true, uses the `reth_forkchoiceUpdated` endpoint which sends a regular FCU
+/// with no payload attributes.
+pub(crate) async fn call_forkchoice_updated_with_reth<
+    N: Network,
+    P: Provider<N> + EngineApiValidWaitExt<N>,
+>(
+    provider: P,
+    message_version: Option<EngineApiMessageVersion>,
+    forkchoice_state: ForkchoiceState,
+) -> TransportResult<ForkchoiceUpdated> {
+    if let Some(message_version) = message_version {
+        call_forkchoice_updated(provider, message_version, forkchoice_state, None).await
+    } else {
+        let method = "reth_forkchoiceUpdated";
+        let reth_params = serde_json::to_value((forkchoice_state,))
+            .expect("ForkchoiceState serialization cannot fail");
+
+        debug!(target: "reth-bench", method, "Sending forkchoiceUpdated");
+
+        loop {
+            let resp: ForkchoiceUpdated = provider.client().request(method, &reth_params).await?;
+
+            if resp.is_valid() {
+                break Ok(resp)
+            }
+
+            if resp.is_invalid() {
+                error!(target: "reth-bench", ?resp, "Invalid {method}");
+                return Err(alloy_json_rpc::RpcError::LocalUsageError(Box::new(
+                    std::io::Error::other(format!("Invalid {method}: {resp:?}")),
+                )))
+            }
+            if resp.is_syncing() {
+                return Err(alloy_json_rpc::RpcError::UnsupportedFeature(
+                    "invalid range: no canonical state found for parent of requested block",
+                ))
+            }
+        }
+    }
+}

bin/reth/Cargo.toml

@@ -30,7 +30,8 @@ workspace = true
 # reth
 reth-ethereum-cli.workspace = true
 reth-chainspec.workspace = true
-reth-primitives.workspace = true
+reth-primitives-traits.workspace = true
+reth-ethereum-primitives.workspace = true
 reth-db = { workspace = true, features = ["mdbx"] }
 reth-provider.workspace = true
 reth-revm.workspace = true
@@ -69,7 +70,7 @@ aquamarine.workspace = true
 clap = { workspace = true, features = ["derive", "env"] }
 
 [dev-dependencies]
-alloy-node-bindings = "1.6.3"
+alloy-node-bindings = "1.5.2"
 alloy-provider = { workspace = true, features = ["reqwest"] }
 alloy-rpc-types-eth.workspace = true
 backon.workspace = true
@@ -88,7 +89,6 @@ default = [
     "keccak-cache-global",
     "asm-keccak",
     "min-debug-logs",
-    "rocksdb",
 ]
 
 otlp = [
@@ -110,7 +110,6 @@ dev = ["reth-ethereum-cli/dev"]
 
 asm-keccak = [
     "reth-node-core/asm-keccak",
-    "reth-primitives/asm-keccak",
     "reth-ethereum-cli/asm-keccak",
     "reth-node-ethereum/asm-keccak",
     "alloy-primitives/asm-keccak",
@@ -125,6 +124,7 @@ jemalloc = [
     "reth-node-core/jemalloc",
     "reth-node-metrics/jemalloc",
     "reth-ethereum-cli/jemalloc",
+    "reth-provider/jemalloc",
 ]
 jemalloc-prof = [
     "reth-cli-util/jemalloc",
@@ -191,8 +191,6 @@ min-trace-logs = [
 ]
 
 trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"]
-rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
-edge = ["rocksdb"]
 
 [[bin]]
 name = "reth"

bin/reth/src/lib.rs

@@ -124,9 +124,11 @@ pub mod providers {
     pub use reth_provider::*;
 }
 
-/// Re-exported from `reth_primitives`.
+/// Re-exported primitives.
+#[allow(ambiguous_glob_reexports)]
 pub mod primitives {
-    pub use reth_primitives::*;
+    pub use reth_ethereum_primitives::*;
+    pub use reth_primitives_traits::*;
 }
 
 /// Re-exported from `reth_ethereum_consensus`.