Merge branch 'main' into dan/static-file-split

Co-authored-by: Amp <amp@ampcode.com>

.changelog/dry-lakes-sleep.md

@@ -0,0 +1,6 @@
+---
+reth-trie-common: minor
+reth-trie: minor
+---
+
+Added `contains_range` method to `PrefixSet` for checking if any key falls within a half-open range. Added prefix set support to `ProofCalculator` via `with_prefix_set`, enabling stale cached hash invalidation and branch collapse detection when keys are inserted or removed; propagated storage prefix sets through `SyncAccountValueEncoder`.

.changelog/dull-clams-pack.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Refactored arena trie internals by adding a `BranchChildIdx::sibling()` helper, deduplicating `Index`/`NodeArena` type aliases, and replacing `is_empty()` with a `drop_root()` method. Fixed a bug where `cursor.pop()` was called before checking if the leaf was the root node, which could cause incorrect dirty-state propagation.

.changelog/dull-seals-laugh.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: minor
+---
+
+Fixed a bug in `ArenaParallelSparseTrie` where subtrie updates that would completely empty a subtrie were incorrectly dispatched to parallel workers instead of being processed inline, preventing correct branch collapse detection when blinded siblings are present. Refactored the `SparseTrie` test suite to accept a `fn() -> T` factory instead of requiring `T: Default`, enabling a new `arena_parallel_sparse_trie_always_parallel` test variant that exercises all tests with parallelism thresholds set to 1. Added `test_branch_collapse_multi_empty_subtries_blinded_remaining` to cover the case where removing multiple revealed leaves empties their subtries and leaves a single blinded sibling requiring a proof.

.changelog/fine-horses-whisper.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Fixed another branch collapse edge case where `check_subtrie_collapse_needs_proof` incorrectly compared removal count against total update count (including `Touched` entries), causing it to skip proof requests for blinded siblings and panic when the subtrie emptied. Added a regression test covering the removals + `Touched` + blinded sibling scenario.

.changelog/fix-download-api-url.md

@@ -0,0 +1,5 @@
+---
+reth-cli-commands: patch
+---
+
+Added `snapshot_api_url` field to `DownloadDefaults` so downstream projects can override the snapshot discovery API endpoint. Previously, `discover_manifest_url`, `fetch_snapshot_api_entries`, and `print_snapshot_listing` used a hardcoded `snapshots.reth.rs` URL that bypassed the `DownloadDefaults` override mechanism.

.changelog/kind-eagles-hum.md

@@ -0,0 +1,6 @@
+---
+reth-primitives-traits: major
+reth-downloaders: patch
+---
+
+Removed the local `size` module from `reth-primitives-traits` and replaced it with `alloy_consensus::InMemorySize`. Simplified `SignedTransaction` to a blanket impl covering all types satisfying the required bounds, removing `is_system_tx`, `auto_impl` attributes, and explicit impls for `EthereumTxEnvelope` and OP types. Updated import paths in `reth-downloaders` accordingly.

.changelog/kind-frogs-meow.md

@@ -0,0 +1,13 @@
+---
+reth-primitives-traits: minor
+reth-engine-local: patch
+reth-evm: patch
+reth-node-builder: patch
+reth-payload-primitives: patch
+reth-rpc-convert: patch
+reth-rpc-eth-api: patch
+reth-db-api: patch
+reth-db: patch
+---
+
+Removed the unused `Extended` type and `op` feature (including `op-alloy-consensus` dependency) from `reth-primitives-traits`. Updated all dependent crates to remove the now-unnecessary `reth-primitives-traits/op` feature flag propagation.

.changelog/nice-whales-buzz.md

@@ -0,0 +1,5 @@
+---
+reth-engine-tree: patch
+---
+
+Added idle-time pre-computation of account trie upper hashes in the sparse trie payload processor when no pending proof results are available.

.changelog/odd-birds-meow.md

@@ -0,0 +1,6 @@
+---
+reth-engine-local: patch
+reth-node-builder: patch
+---
+
+Removed the `op` feature flag and `OpPayloadAttributes` `PayloadAttributesBuilder` implementation from `reth-engine-local`, along with the `op-alloy-rpc-types-engine` dependency. Updated `reth-node-builder` to no longer enable the removed `op` feature on `reth-engine-local`.

.changelog/rich-pigs-shake.md

@@ -0,0 +1,6 @@
+---
+reth-payload-primitives: patch
+reth-engine-local: patch
+---
+
+Removed the `op` feature and `op-alloy-rpc-types-engine` dependency from `reth-payload-primitives`, along with the `ExecutionPayload` impl for `OpExecutionData`. Updated `reth-engine-local` to drop the corresponding feature flag dependency.

.changelog/safe-waves-cook.md

@@ -0,0 +1,5 @@
+---
+reth-engine-tree: patch
+---
+
+Downgraded per-transaction prewarm span from `debug_span!` to `trace_span!` to reduce noise in debug-level logging.

.changelog/tidy-ducks-bake.md

@@ -0,0 +1,5 @@
+---
+reth-trie-sparse: patch
+---
+
+Fixed a panic in `ParallelSparseTrie::reveal_nodes` when a boundary node's upper parent is absent or non-branch (e.g. when an upper extension crosses the boundary). The code now skips gracefully instead of unwrapping. Added a regression test covering this case.

.changelog/warm-slugs-crawl.md

@@ -0,0 +1,5 @@
+---
+reth-transaction-pool: minor
+---
+
+Added `TransactionValidationTaskExecutor::spawn` as a dedicated constructor that encapsulates spawning validation tasks on a runtime, and refactored `EthTransactionValidatorBuilder::build_with_tasks` to use it.

.changelog/warm-trees-grow.md

@@ -0,0 +1,7 @@
+---
+reth-engine-primitives: patch
+reth-engine-tree: patch
+reth-node-core: patch
+---
+
+Removed `--engine.enable-arena-sparse-trie` CLI flag and made the arena-based sparse trie the default implementation. The hash-map-based `ParallelSparseTrie` variant is no longer selectable.

.changelog/zesty-bees-drink.md

@@ -0,0 +1,8 @@
+---
+reth-engine-primitives: minor
+reth-engine-tree: major
+reth-node-core: minor
+reth-cli-commands: minor
+---
+
+Added persistence backpressure to the engine tree: when the canonical-minus-persisted block gap exceeds a configurable threshold (`--engine.persistence-backpressure-threshold`, default 16), the engine loop stalls on the persistence receiver instead of processing new incoming messages. Added CLI argument, cross-field validation, metrics (`backpressure_active`, `backpressure_stall_duration`), and tests.

.config/zepter.yaml

@@ -12,7 +12,7 @@ workflows:
         # Check that `A` activates the features of `B`.
         "propagate-feature",
         # These are the features to check:
-        "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global",
+        "--features=std,op,dev,asm-keccak,jemalloc,jemalloc-prof,tracy-allocator,tracy,serde-bincode-compat,serde,test-utils,arbitrary,bench,alloy-compat,min-error-logs,min-warn-logs,min-info-logs,min-debug-logs,min-trace-logs,otlp,otlp-logs,js-tracer,portable,keccak-cache-global,trie-debug",
         # Do not try to add a new section to `[features]` of `A` only because `B` exposes that feature. There are edge-cases where this is still needed, but we can add them manually.
         "--left-side-feature-missing=ignore",
         # Ignore the case that `A` it outside of the workspace. Otherwise it will report errors in external dependencies that we have no influence on.

.github/actionlint.yaml

@@ -5,3 +5,4 @@ self-hosted-runner:
     - depot-ubuntu-latest-4
     - depot-ubuntu-latest-8
     - depot-ubuntu-latest-16
+    - available

.github/scripts/bench-job-summary.js

@@ -0,0 +1,106 @@
+// Generates a rich GitHub Actions job summary for reth-bench results.
+//
+// Reads from environment:
+//   BENCH_WORK_DIR       – Directory containing summary.json
+//   BENCH_PR             – PR number (may be empty)
+//   BENCH_ACTOR          – GitHub user who triggered the bench
+//   BENCH_CORES          – CPU core limit (0 = all)
+//   BENCH_WARMUP_BLOCKS  – Number of warmup blocks
+//   BENCH_SAMPLY         – 'true' if samply profiling was enabled
+//   BENCH_ABBA           – 'true' if ABBA interleaved order was used
+//
+// Usage from actions/github-script:
+//   const jobSummary = require('./.github/scripts/bench-job-summary.js');
+//   await jobSummary({ core, context, chartSha, grafanaUrl, runId });
+
+const fs = require('fs');
+const { verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
+
+module.exports = async function ({ core, context, chartSha, grafanaUrl, runId }) {
+  let summary;
+  try {
+    summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+  } catch (e) {
+    await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
+    return;
+  }
+
+  const repo = `${context.repo.owner}/${context.repo.repo}`;
+  const prNumber = process.env.BENCH_PR;
+  const actor = process.env.BENCH_ACTOR;
+  const commitUrl = `https://github.com/${repo}/commit`;
+
+  const { emoji, label } = verdict(summary.changes);
+  const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
+  const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
+  const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+
+  // Header & metadata
+  const metaParts = [];
+  if (prNumber) metaParts.push(`**[PR #${prNumber}](https://github.com/${repo}/pull/${prNumber})**`);
+  metaParts.push(`triggered by @${actor}`);
+
+  let md = `# ${emoji} ${label}\n\n`;
+  md += metaParts.join(' · ') + '\n\n';
+  md += `**Baseline:** ${baselineLink}\n`;
+  md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
+  md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
+
+  // Main comparison table
+  const rows = metricRows(summary);
+  md += `| Metric | Baseline | Feature | Change |\n`;
+  md += `|--------|----------|---------|--------|\n`;
+  for (const r of rows) {
+    md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
+  }
+  md += '\n';
+
+  // Wait time breakdown
+  const wtRows = waitTimeRows(summary);
+  if (wtRows.length > 0) {
+    md += `### Wait Time Breakdown\n\n`;
+    md += `| Metric | Baseline | Feature |\n`;
+    md += `|--------|----------|--------|\n`;
+    for (const r of wtRows) {
+      md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
+    }
+    md += '\n';
+  }
+
+  // Charts
+  if (chartSha) {
+    const prNum = prNumber || '0';
+    const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/pr/${prNum}/${runId}`;
+    const charts = [
+      { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
+      { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
+      { file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
+    ];
+    md += `### Charts\n\n`;
+    for (const chart of charts) {
+      md += `<details><summary>${chart.label}</summary>\n\n`;
+      md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
+      md += `</details>\n\n`;
+    }
+  }
+
+  // Samply profiles
+  const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
+  const samplyLinks = Object.entries(samplyUrls).map(([run, url]) => `- **${run}**: [Firefox Profiler](${url})`);
+  if (samplyLinks.length > 0) {
+    md += `### Samply Profiles\n\n${samplyLinks.join('\n')}\n\n`;
+  }
+
+  // Grafana
+  if (grafanaUrl) {
+    md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
+  }
+
+  // Node errors
+  try {
+    const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
+    if (errors.trim()) md += '\n' + errors + '\n';
+  } catch {}
+
+  await core.summary.addRaw(md).write();
+};

.github/scripts/bench-reth-build.sh

@@ -11,10 +11,11 @@
 #              optional branch-sha is the PR head commit for cache key
 #
 # Outputs:
-#   baseline: <source-dir>/target/profiling/reth
-#   feature:  <source-dir>/target/profiling/reth, reth-bench installed to cargo bin
+#   baseline: <source-dir>/target/profiling/reth (or reth-bb if BENCH_BIG_BLOCKS=true)
+#   feature:  <source-dir>/target/profiling/reth (or reth-bb), reth-bench installed to cargo bin
 #
 # Required: mc (MinIO client) with a configured alias
+# Optional env: BENCH_BIG_BLOCKS (true/false) — build reth-bb instead of reth
 set -euo pipefail
 
 MC="mc"
@@ -22,6 +23,16 @@ MODE="$1"
 SOURCE_DIR="$2"
 COMMIT="$3"
 
+BIG_BLOCKS="${BENCH_BIG_BLOCKS:-false}"
+# The node binary to build: reth-bb for big blocks, reth otherwise
+if [ "$BIG_BLOCKS" = "true" ]; then
+  NODE_BIN="reth-bb"
+  NODE_PKG="-p reth-bb"
+else
+  NODE_BIN="reth"
+  NODE_PKG="--bin reth"
+fi
+
 # Tracy support: when BENCH_TRACY is "on" or "full", add Tracy cargo features
 # and frame pointers for accurate stack traces.
 EXTRA_FEATURES=""
@@ -62,18 +73,18 @@ case "$MODE" in
     mkdir -p "${SOURCE_DIR}/target/profiling"
 
     CACHE_VALID=false
-    if $MC stat "${BUCKET}/reth" &>/dev/null; then
-      echo "Cache hit for baseline (${COMMIT}), downloading binary..."
-      $MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
-      chmod +x "${SOURCE_DIR}/target/profiling/reth"
-      if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
+    if $MC stat "${BUCKET}/${NODE_BIN}" &>/dev/null; then
+      echo "Cache hit for baseline (${COMMIT}), downloading ${NODE_BIN}..."
+      $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}"
+      chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}"
+      if verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
         CACHE_VALID=true
       else
         echo "Cached baseline binary is stale, rebuilding..."
       fi
     fi
     if [ "$CACHE_VALID" = false ]; then
-      echo "Building baseline (${COMMIT}) from source..."
+      echo "Building baseline ${NODE_BIN} (${COMMIT}) from source..."
       cd "${SOURCE_DIR}"
       FEATURES_ARG=""
       WORKSPACE_ARG=""
@@ -84,8 +95,8 @@ case "$MODE" in
       fi
       # shellcheck disable=SC2086
       RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
-        cargo build --profile profiling --bin reth $WORKSPACE_ARG $FEATURES_ARG
-      $MC cp target/profiling/reth "${BUCKET}/reth"
+        cargo build --profile profiling $NODE_PKG $WORKSPACE_ARG $FEATURES_ARG
+      $MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
     fi
     ;;
 
@@ -94,32 +105,34 @@ case "$MODE" in
     BUCKET="minio/reth-binaries/${BRANCH_SHA}${BUILD_SUFFIX}"
 
     CACHE_VALID=false
-    if $MC stat "${BUCKET}/reth" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
+    if $MC stat "${BUCKET}/${NODE_BIN}" &>/dev/null && $MC stat "${BUCKET}/reth-bench" &>/dev/null; then
       echo "Cache hit for ${BRANCH_SHA}, downloading binaries..."
       mkdir -p "${SOURCE_DIR}/target/profiling"
-      $MC cp "${BUCKET}/reth" "${SOURCE_DIR}/target/profiling/reth"
+      $MC cp "${BUCKET}/${NODE_BIN}" "${SOURCE_DIR}/target/profiling/${NODE_BIN}"
       $MC cp "${BUCKET}/reth-bench" /home/ubuntu/.cargo/bin/reth-bench
-      chmod +x "${SOURCE_DIR}/target/profiling/reth" /home/ubuntu/.cargo/bin/reth-bench
-      if verify_binary "${SOURCE_DIR}/target/profiling/reth" "${COMMIT}"; then
+      chmod +x "${SOURCE_DIR}/target/profiling/${NODE_BIN}" /home/ubuntu/.cargo/bin/reth-bench
+      if verify_binary "${SOURCE_DIR}/target/profiling/${NODE_BIN}" "${COMMIT}"; then
         CACHE_VALID=true
       else
         echo "Cached feature binary is stale, rebuilding..."
       fi
     fi
     if [ "$CACHE_VALID" = false ]; then
-      echo "Building feature (${COMMIT}) from source..."
+      echo "Building feature ${NODE_BIN} (${COMMIT}) from source..."
       cd "${SOURCE_DIR}"
       rustup show active-toolchain || rustup default stable
       if [ -n "$EXTRA_FEATURES" ]; then
         # Can't use `make profiling` when adding features; build explicitly
         # --workspace is needed for cross-package feature syntax (tracy-client/ondemand)
         RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
-          cargo build --profile profiling --workspace --bin reth --features "${EXTRA_FEATURES}"
+          cargo build --profile profiling --workspace $NODE_PKG --features "${EXTRA_FEATURES}"
       else
-        make profiling
+        # shellcheck disable=SC2086
+        RUSTFLAGS="-C target-cpu=native${EXTRA_RUSTFLAGS}" \
+          cargo build --profile profiling $NODE_PKG
       fi
       make install-reth-bench
-      $MC cp target/profiling/reth "${BUCKET}/reth"
+      $MC cp "target/profiling/${NODE_BIN}" "${BUCKET}/${NODE_BIN}"
       $MC cp "$(which reth-bench)" "${BUCKET}/reth-bench"
     fi
     ;;

.github/scripts/bench-reth-run.sh

@@ -11,12 +11,18 @@
 #               BENCH_WAIT_TIME (duration like 500ms, default empty)
 #               BENCH_BASELINE_ARGS (extra reth node args for baseline runs)
 #               BENCH_FEATURE_ARGS (extra reth node args for feature runs)
+#               BENCH_OTLP_TRACES_ENDPOINT (OTLP HTTP endpoint for traces, e.g. https://host/insert/opentelemetry/v1/traces)
+#               BENCH_OTLP_LOGS_ENDPOINT (OTLP HTTP endpoint for logs, e.g. https://host/insert/opentelemetry/v1/logs)
 set -euo pipefail
 
 LABEL="$1"
 BINARY="$2"
 OUTPUT_DIR="$3"
-DATADIR="$SCHELK_MOUNT/datadir"
+DATADIR_NAME="datadir"
+if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
+  DATADIR_NAME="datadir-big-blocks"
+fi
+DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
 mkdir -p "$OUTPUT_DIR"
 LOG="${OUTPUT_DIR}/node.log"
 
@@ -118,9 +124,12 @@ RETH_ARGS=(
   --no-persist-peers
 )
 
-# Big blocks mode requires the testing API and skip-invalid-transactions
-if [ "$BIG_BLOCKS" = "true" ]; then
-  RETH_ARGS+=(--http.api eth,net,web3,reth,testing --testing.skip-invalid-transactions)
+# Gate flag on binary support (older baselines may not have it).
+# Uses --help which exits immediately via clap without node init.
+SYNC_STATE_IDLE=false
+if "$BINARY" node --help 2>/dev/null | grep -qF -- '--debug.startup-sync-state-idle'; then
+  RETH_ARGS+=(--debug.startup-sync-state-idle)
+  SYNC_STATE_IDLE=true
 fi
 
 # Append per-label extra node args (baseline or feature)
@@ -139,6 +148,14 @@ if [ -n "${BENCH_METRICS_ADDR:-}" ]; then
   RETH_ARGS+=(--metrics "$BENCH_METRICS_ADDR")
 fi
 
+# OTLP traces and logs export
+if [ -n "${BENCH_OTLP_TRACES_ENDPOINT:-}" ]; then
+  RETH_ARGS+=(--tracing-otlp="${BENCH_OTLP_TRACES_ENDPOINT}" --tracing-otlp.service-name=reth-bench)
+fi
+if [ -n "${BENCH_OTLP_LOGS_ENDPOINT:-}" ]; then
+  RETH_ARGS+=(--logs-otlp="${BENCH_OTLP_LOGS_ENDPOINT}" --logs-otlp.filter=debug)
+fi
+
 # Tracy profiling: add --log.tracy flags and set environment
 if [ "${BENCH_TRACY:-off}" != "off" ]; then
   RETH_ARGS+=(--log.tracy --log.tracy.filter "${BENCH_TRACY_FILTER:-debug}")
@@ -149,16 +166,29 @@ if [ "${BENCH_TRACY:-off}" != "off" ]; then
   fi
 fi
 
+SUDO_ENV=()
+if [ -n "${OTEL_RESOURCE_ATTRIBUTES:-}" ]; then
+  SUDO_ENV+=("OTEL_RESOURCE_ATTRIBUTES=${OTEL_RESOURCE_ATTRIBUTES}")
+  SUDO_ENV+=("OTEL_BSP_MAX_QUEUE_SIZE=65536" "OTEL_BLRP_MAX_QUEUE_SIZE=65536")
+fi
+
+# Limit reth memory to 95% of available RAM to prevent OOM kills
+TOTAL_MEM_KB=$(awk '/^MemTotal:/ {print $2}' /proc/meminfo)
+MEM_LIMIT=$(( TOTAL_MEM_KB * 95 / 100 * 1024 ))
+echo "Memory limit: $(( MEM_LIMIT / 1024 / 1024 ))MB (95% of $(( TOTAL_MEM_KB / 1024 ))MB)"
+
 if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
   RETH_ARGS+=(--log.samply)
   SAMPLY="$(which samply)"
-  sudo taskset -c "$RETH_CPUS" nice -n -20 \
+  sudo systemd-run --scope -p MemoryMax="$MEM_LIMIT" -p AllowedCPUs="$RETH_CPUS" \
+    env "${SUDO_ENV[@]}" nice -n -20 \
     "$SAMPLY" record --save-only --presymbolicate --rate 10000 \
     --output "$OUTPUT_DIR/samply-profile.json.gz" \
     -- "$BINARY" "${RETH_ARGS[@]}" \
     > "$LOG" 2>&1 &
 else
-  sudo taskset -c "$RETH_CPUS" nice -n -20 "$BINARY" "${RETH_ARGS[@]}" \
+  sudo systemd-run --scope -p MemoryMax="$MEM_LIMIT" -p AllowedCPUs="$RETH_CPUS" \
+    env "${SUDO_ENV[@]}" nice -n -20 "$BINARY" "${RETH_ARGS[@]}" \
     > "$LOG" 2>&1 &
 fi
 
@@ -171,7 +201,7 @@ for i in $(seq 1 60); do
     -H 'Content-Type: application/json' \
     -d '{"jsonrpc":"2.0","method":"eth_blockNumber","params":[],"id":1}' \
     > /dev/null 2>&1; then
-    echo "reth (${LABEL}) is ready after ${i}s"
+    echo "reth (${LABEL}) RPC is up after ${i}s"
     break
   fi
   if [ "$i" -eq 60 ]; then
@@ -182,6 +212,29 @@ for i in $(seq 1 60); do
   sleep 1
 done
 
+# Wait for the pipeline to finish (eth_syncing returns false) so the
+# engine is in live mode and can accept newPayload calls.
+# Only possible when --debug.startup-sync-state-idle is supported.
+if [ "$SYNC_STATE_IDLE" = "true" ]; then
+  for i in $(seq 1 300); do
+    SYNC_RESULT=$(curl -sf http://127.0.0.1:8545 -X POST \
+      -H 'Content-Type: application/json' \
+      -d '{"jsonrpc":"2.0","method":"eth_syncing","params":[],"id":1}' 2>/dev/null || true)
+    if [ -n "$SYNC_RESULT" ] && jq -e '.result == false' <<< "$SYNC_RESULT" > /dev/null 2>&1; then
+      echo "reth (${LABEL}) pipeline finished after ${i}s, engine is live"
+      break
+    fi
+    if [ "$i" -eq 300 ]; then
+      echo "::error::reth (${LABEL}) pipeline did not finish within 300s"
+      cat "$LOG"
+      exit 1
+    fi
+    sleep 1
+  done
+else
+  echo "reth (${LABEL}) binary does not support --debug.startup-sync-state-idle, skipping sync wait"
+fi
+
 # Run reth-bench with high priority but as the current user so output
 # files are not root-owned (avoids EACCES on next checkout).
 BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
@@ -189,19 +242,15 @@ BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
 # Build optional flags
 EXTRA_BENCH_ARGS=()
 if [ "${BENCH_RETH_NEW_PAYLOAD:-true}" != "false" ]; then
-  EXTRA_BENCH_ARGS+=(--reth-new-payload)
+  EXTRA_BENCH_ARGS+=(--reth-new-payload --wait-for-persistence)
 fi
 if [ -n "${BENCH_WAIT_TIME:-}" ]; then
   EXTRA_BENCH_ARGS+=(--wait-time "$BENCH_WAIT_TIME")
 fi
 
 if [ "$BIG_BLOCKS" = "true" ]; then
-  # Big blocks mode: replay pre-generated payloads with gas ramp
-  BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
-  # Count gas ramp blocks for reporting
-  GAS_RAMP_COUNT=$(find "$BIG_BLOCKS_DIR/gas-ramp-dir" -name '*.json' | wc -l)
-  echo "$GAS_RAMP_COUNT" > "$OUTPUT_DIR/gas_ramp_blocks.txt"
-  echo "Gas ramp blocks: $GAS_RAMP_COUNT"
+  # Big blocks mode: replay pre-generated payloads
+  BIG_BLOCKS_DIR="${BENCH_BIG_BLOCKS_DIR:-${BENCH_WORK_DIR}/big-blocks}"
 
   # Start tracy-capture so profile only covers the benchmark
   if [ "${BENCH_TRACY:-off}" != "off" ]; then
@@ -211,10 +260,18 @@ if [ "$BIG_BLOCKS" = "true" ]; then
     sleep 0.5  # give tracy-capture time to connect
   fi
 
+  BB_BENCH_ARGS=(--reth-new-payload --wait-for-persistence)
+  if [ -n "${BENCH_WAIT_TIME:-}" ]; then
+    BB_BENCH_ARGS+=(--wait-time "$BENCH_WAIT_TIME")
+  fi
+  # Limit number of payloads if blocks count is specified
+  if [ "${BENCH_BLOCKS:-0}" -gt 0 ] 2>/dev/null; then
+    BB_BENCH_ARGS+=(--count "$BENCH_BLOCKS")
+  fi
+
   echo "Running big blocks benchmark (replay-payloads)..."
   $BENCH_NICE "$RETH_BENCH" replay-payloads \
-    "${EXTRA_BENCH_ARGS[@]}" \
-    --gas-ramp-dir "$BIG_BLOCKS_DIR/gas-ramp-dir" \
+    "${BB_BENCH_ARGS[@]}" \
     --payload-dir "$BIG_BLOCKS_DIR/payloads" \
     --engine-rpc-url http://127.0.0.1:8551 \
     --jwt-secret "$DATADIR/jwt.hex" \

.github/scripts/bench-reth-snapshot.sh

@@ -1,15 +1,17 @@
 #!/usr/bin/env bash
 #
-# Downloads the latest nightly snapshot into the schelk volume with
-# progress reporting to the GitHub PR comment.
+# Downloads the latest snapshot into the schelk volume using
+# `reth download` with progress reporting to the GitHub PR comment.
 #
-# Skips the download if the local ETag marker matches the remote one.
+# Skips the download if the manifest content hasn't changed since
+# the last successful download (checked via SHA-256 of the manifest).
 #
 # Usage: bench-reth-snapshot.sh [--check]
-#   --check   Only check if a download is needed; exits 0 if up-to-date, 1 if not.
+#   --check   Only check if a download is needed; exits 0 if up-to-date, 10 if not.
 #
 # Required env:
 #   SCHELK_MOUNT       – schelk mount point (e.g. /reth-bench)
+#   BENCH_RETH_BINARY  – path to the reth binary
 #   GITHUB_TOKEN       – token for GitHub API calls (only for download)
 #   BENCH_COMMENT_ID   – PR comment ID to update (optional)
 #   BENCH_REPO         – owner/repo (e.g. paradigmxyz/reth)
@@ -18,52 +20,74 @@
 #   BENCH_CONFIG       – config summary line
 set -euo pipefail
 
-BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous.tar.zst"
-DATADIR="$SCHELK_MOUNT/datadir"
-ETAG_FILE="$HOME/.reth-bench-snapshot-etag"
-
-# Get remote metadata via JSON for reliable parsing
-MC_STAT=$(mc stat --json "$BUCKET" 2>/dev/null || true)
-REMOTE_ETAG=$(echo "$MC_STAT" | jq -r '.etag // empty')
-if [ -z "$REMOTE_ETAG" ]; then
-  echo "::warning::Failed to get ETag from mc stat, will re-download"
-  REMOTE_ETAG="unknown-$(date +%s)"
+MC="mc"
+BUCKET="minio/reth-snapshots"
+# Allow overriding the snapshot name (e.g. for big-blocks mode where the
+# big-blocks manifest specifies which base snapshot to use).
+SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable}"
+MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
+DATADIR_NAME="datadir"
+HASH_MODE_SUFFIX=""
+if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
+  DATADIR_NAME="datadir-big-blocks"
+  HASH_MODE_SUFFIX="-big-blocks"
 fi
+DATADIR="$SCHELK_MOUNT/$DATADIR_NAME"
+HASH_FILE="$HOME/.reth-bench-snapshot-hash${HASH_MODE_SUFFIX}"
 
-LOCAL_ETAG=""
-[ -f "$ETAG_FILE" ] && LOCAL_ETAG=$(cat "$ETAG_FILE")
+# Fetch manifest and compute content hash for reliable freshness check
+MANIFEST_CONTENT=$($MC cat "${BUCKET}/${MANIFEST_PATH}" 2>/dev/null) || {
+  echo "::error::Failed to fetch snapshot manifest from ${BUCKET}/${MANIFEST_PATH}"
+  exit 2
+}
+REMOTE_HASH=$(echo "$MANIFEST_CONTENT" | sha256sum | awk '{print $1}')
 
-if [ "$REMOTE_ETAG" = "$LOCAL_ETAG" ]; then
-  echo "Snapshot is up-to-date (ETag: ${REMOTE_ETAG})"
-  if [ "${1:-}" = "--check" ]; then
-    exit 0
-  fi
+LOCAL_HASH=""
+[ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
+
+if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
+  echo "Snapshot is up-to-date (manifest hash: ${REMOTE_HASH:0:16}…)"
   exit 0
 fi
 
-echo "Snapshot needs update (local: ${LOCAL_ETAG:-<none>}, remote: ${REMOTE_ETAG})"
+echo "Snapshot needs update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
 if [ "${1:-}" = "--check" ]; then
+  exit 10
+fi
+
+RETH="${BENCH_RETH_BINARY:?BENCH_RETH_BINARY must be set}"
+if [ ! -x "$RETH" ]; then
+  echo "::error::reth binary not found or not executable at $RETH"
   exit 1
 fi
 
-# Get compressed size for progress tracking
-TOTAL_BYTES=$(echo "$MC_STAT" | jq -r '.size // empty')
-if [ -z "$TOTAL_BYTES" ] || [ "$TOTAL_BYTES" = "0" ]; then
-  echo "::error::Failed to get snapshot size from mc stat"
+# Resolve the MinIO HTTP endpoint from the mc alias so reth can
+# fetch archives over HTTP (the manifest's embedded base_url points
+# to the cluster-internal address which is unreachable from runners).
+MINIO_ENDPOINT=$($MC alias list minio --json 2>/dev/null | jq -r '.URL // empty') || true
+if [ -z "$MINIO_ENDPOINT" ]; then
+  echo "::error::Failed to resolve MinIO endpoint from mc alias 'minio'"
   exit 1
 fi
-echo "Snapshot size: $TOTAL_BYTES bytes ($(numfmt --to=iec "$TOTAL_BYTES"))"
+BASE_URL="${MINIO_ENDPOINT}/reth-snapshots/${SNAPSHOT_NAME}"
+
+# Rewrite manifest's base_url with the runner-reachable endpoint
+MANIFEST_TMP=$(mktemp --suffix=.json)
+trap 'rm -f -- "$MANIFEST_TMP"' EXIT
+echo "$MANIFEST_CONTENT" \
+  | jq --arg base "$BASE_URL" '.base_url = $base' > "$MANIFEST_TMP"
 
 # Prepare mount
 mountpoint -q "$SCHELK_MOUNT" && sudo schelk recover -y || true
 sudo schelk mount -y
 sudo rm -rf "$DATADIR"
 sudo mkdir -p "$DATADIR"
+# reth download runs as current user (not root), needs write access
+sudo chown -R "$(id -u):$(id -g)" "$DATADIR"
 
 update_comment() {
-  local pct="$1"
+  local status="$1"
   [ -z "${BENCH_COMMENT_ID:-}" ] && return 0
-  local status="Building binaries & downloading snapshot… ${pct}%"
   local body
   body="$(printf 'cc @%s\n\n🚀 Benchmark started! [View job](%s)\n\n⏳ **Status:** %s\n\n%s' \
     "$BENCH_ACTOR" "$BENCH_JOB_URL" "$status" "$BENCH_CONFIG")"
@@ -75,53 +99,31 @@ update_comment() {
     > /dev/null 2>&1 || true
 }
 
-# Track compressed bytes flowing through the pipe
-DL_BYTES_FILE=$(mktemp)
-echo 0 > "$DL_BYTES_FILE"
+update_comment "Downloading snapshot…"
 
-# Start progress reporter in background
-(
-  while true; do
-    sleep 10
-    CURRENT=$(cat "$DL_BYTES_FILE" 2>/dev/null || echo 0)
-    if [ "$TOTAL_BYTES" -gt 0 ]; then
-      PCT=$(( CURRENT * 100 / TOTAL_BYTES ))
-      [ "$PCT" -gt 100 ] && PCT=100
-      echo "Snapshot download: $(numfmt --to=iec "$CURRENT") / $(numfmt --to=iec "$TOTAL_BYTES") (${PCT}%)"
-      update_comment "$PCT"
-    fi
-  done
-) &
-PROGRESS_PID=$!
-trap 'kill $PROGRESS_PID 2>/dev/null || true; rm -f "$DL_BYTES_FILE"' EXIT
+# Download using reth download (manifest-path with rewritten base_url)
+"$RETH" download \
+  --manifest-path "$MANIFEST_TMP" \
+  -y \
+  --minimal \
+  --datadir "$DATADIR"
 
-# Download and extract; python byte counter tracks compressed bytes received
-mc cat "$BUCKET" | python3 -c "
-import sys
-count = 0
-while True:
-    data = sys.stdin.buffer.read(1048576)
-    if not data:
-        break
-    count += len(data)
-    sys.stdout.buffer.write(data)
-    with open('$DL_BYTES_FILE', 'w') as f:
-        f.write(str(count))
-" | pzstd -d -p 6 | sudo tar -xf - -C "$DATADIR"
-
-# Stop progress reporter
-kill $PROGRESS_PID 2>/dev/null || true
-wait $PROGRESS_PID 2>/dev/null || true
-
-update_comment "100"
+update_comment "Downloading snapshot… done"
 echo "Snapshot download complete"
 
+# Sanity check: verify expected directories exist
+if [ ! -d "$DATADIR/db" ] || [ ! -d "$DATADIR/static_files" ]; then
+  echo "::error::Snapshot download did not produce expected directory layout (missing db/ or static_files/)"
+  ls -la "$DATADIR" || true
+  exit 1
+fi
+
 # Promote the new snapshot to become the schelk baseline (virgin volume).
 # This copies changed blocks from scratch → virgin so that future
 # `schelk recover` calls restore to this new state.
 sync
 sudo schelk promote -y
 
-# Save ETag marker
-echo "$REMOTE_ETAG" > "$ETAG_FILE"
-echo "Snapshot promoted to schelk baseline (ETag: ${REMOTE_ETAG})"
+# Save manifest hash
+echo "$REMOTE_HASH" > "$HASH_FILE"
+echo "Snapshot promoted to schelk baseline (manifest hash: ${REMOTE_HASH:0:16}…)"

.github/scripts/bench-reth-summary.py

@@ -472,7 +472,6 @@ def main():
     parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
     parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
     parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode")
-    parser.add_argument("--gas-ramp-blocks", type=int, default=0, help="Number of gas ramp blocks (big blocks mode)")
     parser.add_argument("--grafana-url", default=None, help="Grafana dashboard URL for this benchmark run")
     args = parser.parse_args()
 
@@ -554,7 +553,6 @@ def main():
     summary = {
         "blocks": paired_stats["blocks"],
         "big_blocks": args.big_blocks,
-        "gas_ramp_blocks": args.gas_ramp_blocks,
         "baseline": {
             "name": baseline_name,
             "ref": baseline_ref,

.github/scripts/bench-scheduled-refs.sh

@@ -0,0 +1,231 @@
+#!/usr/bin/env bash
+#
+# Resolves baseline and feature refs for scheduled benchmark runs.
+#
+# Supports two modes:
+#   nightly — Queries the latest successful scheduled docker.yml run via
+#             GitHub API to find the nightly Docker image commit. Compares
+#             with the last successful feature ref to detect staleness.
+#   hourly  — Compares origin/main HEAD against the last successfully
+#             benchmarked commit (falls back to HEAD~1 on first run).
+#             Checks for in-progress sibling runs to avoid overlap.
+#
+# Usage: bench-scheduled-refs.sh <force> <mode>
+#   force — "true" to run even if no new commit (bypass skip logic)
+#   mode  — "nightly" or "hourly"
+#
+# Outputs (via GITHUB_OUTPUT):
+#   baseline-ref    — commit SHA for baseline
+#   feature-ref     — commit SHA for feature
+#   should-skip     — "true" if no new commit since last run or sibling in progress
+#   is-stale        — "true" if latest nightly build is >24h old (nightly only)
+#   stale-age-hours — age of the nightly build in hours (nightly only)
+#   nightly-created — ISO timestamp of the nightly build (nightly only)
+#
+# Reads:
+#   .nightly-state/last-feature-ref  (nightly, from GH Actions cache)
+#   .hourly-state/last-feature-ref   (hourly, from GH Actions cache)
+#
+# Requires: gh (GitHub CLI), jq, date, git (hourly mode)
+set -euo pipefail
+
+FORCE="${1:-false}"
+MODE="${2:-nightly}"
+REPO="${GITHUB_REPOSITORY:-paradigmxyz/reth}"
+
+echo "Mode: $MODE, Force: $FORCE"
+
+# ==========================================================================
+# Hourly mode: compare origin/main HEAD vs HEAD~1
+# ==========================================================================
+if [ "$MODE" = "hourly" ]; then
+
+  # --- Step 1: Resolve feature ref from git ---
+  echo "::group::Resolving hourly refs from git"
+  git fetch origin main --quiet
+  FEATURE_REF=$(git rev-parse origin/main)
+  echo "Feature (HEAD): $FEATURE_REF"
+  echo "::endgroup::"
+
+  # --- Step 2: Check for in-progress sibling runs ---
+  echo "::group::Checking for in-progress sibling runs"
+  CURRENT_RUN_ID="${GITHUB_RUN_ID:-0}"
+  IN_PROGRESS=$(gh run list \
+    -R "$REPO" \
+    --workflow=bench-scheduled.yml \
+    --status=in_progress \
+    --json databaseId \
+    --jq "[.[] | select(.databaseId != $CURRENT_RUN_ID)] | length")
+
+  SHOULD_SKIP="false"
+  if [ "$IN_PROGRESS" -gt 0 ]; then
+    echo "::warning::Previous bench run still in progress ($IN_PROGRESS sibling run(s) found). Skipping."
+    SHOULD_SKIP="true"
+    # Output a flag so the workflow can send a Slack alert
+    echo "long-running=true" >> "$GITHUB_OUTPUT"
+  else
+    echo "No in-progress sibling runs"
+    echo "long-running=false" >> "$GITHUB_OUTPUT"
+  fi
+  echo "::endgroup::"
+
+  # --- Step 3: Read last successful feature ref from cache ---
+  echo "::group::Reading cached state"
+  LAST_FEATURE_REF=""
+  STATE_FILE=".hourly-state/last-feature-ref"
+  if [ -f "$STATE_FILE" ]; then
+    LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
+    echo "Previous feature ref: $LAST_FEATURE_REF"
+  else
+    echo "No cached state found (first run)"
+  fi
+  echo "::endgroup::"
+
+  # --- Step 4: Determine baseline and skip logic ---
+  echo "::group::Resolving baseline and skip logic"
+  if [ "$SHOULD_SKIP" = "true" ]; then
+    BASELINE_REF=$(git rev-parse origin/main~1)
+    echo "Already marked skip (sibling in progress)"
+  elif [ -z "$LAST_FEATURE_REF" ]; then
+    # First run: no previous state, fall back to HEAD~1
+    BASELINE_REF=$(git rev-parse origin/main~1)
+    echo "First run — using HEAD~1 as baseline"
+  elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
+    BASELINE_REF="$LAST_FEATURE_REF"
+    if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
+      echo "No new commits on main, but force=true — running anyway"
+    else
+      SHOULD_SKIP="true"
+      echo "No new commits on main since last run — will skip"
+    fi
+  else
+    # Normal case: use last benchmarked commit as baseline
+    BASELINE_REF="$LAST_FEATURE_REF"
+    echo "New commit(s) on main detected — comparing against last benchmarked commit"
+  fi
+
+  echo "Baseline: $BASELINE_REF"
+  echo "Feature:  $FEATURE_REF"
+  echo "Skip:     $SHOULD_SKIP"
+  echo "::endgroup::"
+
+  # --- Step 5: Write outputs ---
+  {
+    echo "baseline-ref=$BASELINE_REF"
+    echo "feature-ref=$FEATURE_REF"
+    echo "should-skip=$SHOULD_SKIP"
+    echo "is-stale=false"
+    echo "stale-age-hours=0"
+    echo "nightly-created="
+  } >> "$GITHUB_OUTPUT"
+  exit 0
+fi
+
+# ==========================================================================
+# Nightly mode: query latest Docker nightly build (original logic)
+# ==========================================================================
+
+# --- Step 1: Query latest successful scheduled docker.yml run ---
+echo "::group::Querying latest nightly docker build"
+
+RUNS_JSON=$(gh run list \
+  -R "$REPO" \
+  --workflow=docker.yml \
+  --event=schedule \
+  --status=completed \
+  --limit 5 \
+  --json headSha,createdAt,conclusion)
+
+# Find the most recent successful run
+LATEST=$(echo "$RUNS_JSON" | jq -r '[.[] | select(.conclusion == "success")] | first // empty')
+
+if [ -z "$LATEST" ]; then
+  echo "::error::No successful scheduled docker.yml run found in the last 5 runs"
+  echo "Runs found: $RUNS_JSON"
+  exit 1
+fi
+
+FEATURE_REF=$(echo "$LATEST" | jq -r '.headSha')
+CREATED_AT=$(echo "$LATEST" | jq -r '.createdAt')
+echo "Latest nightly commit: $FEATURE_REF"
+echo "Built at: $CREATED_AT"
+echo "::endgroup::"
+
+# --- Step 2: Staleness check ---
+echo "::group::Checking staleness"
+NOW_EPOCH=$(date +%s)
+# Handle both GNU date (-d) and BSD date (-j -f) for cross-platform compat
+CREATED_EPOCH=$(date -d "$CREATED_AT" +%s 2>/dev/null || \
+  date -j -f "%Y-%m-%dT%H:%M:%SZ" "$CREATED_AT" +%s 2>/dev/null || \
+  date -j -f "%Y-%m-%dT%T%z" "$CREATED_AT" +%s 2>/dev/null || \
+  { echo "::error::Cannot parse date: $CREATED_AT"; exit 1; })
+
+AGE_SECONDS=$(( NOW_EPOCH - CREATED_EPOCH ))
+AGE_HOURS=$(( AGE_SECONDS / 3600 ))
+IS_STALE="false"
+
+if [ "$AGE_HOURS" -gt 24 ]; then
+  IS_STALE="true"
+  echo "::warning::STALE NIGHTLY: Build is ${AGE_HOURS}h old (>24h threshold)"
+  echo "This indicates the nightly docker build failed — no new image was produced"
+else
+  echo "Nightly build age: ${AGE_HOURS}h (within 24h threshold)"
+fi
+echo "::endgroup::"
+
+# --- Step 3: Read last successful feature ref from cache ---
+echo "::group::Reading cached state"
+LAST_FEATURE_REF=""
+STATE_FILE=".nightly-state/last-feature-ref"
+if [ -f "$STATE_FILE" ]; then
+  LAST_FEATURE_REF=$(tr -d '[:space:]' < "$STATE_FILE")
+  echo "Previous feature ref: $LAST_FEATURE_REF"
+else
+  echo "No cached state found (first run)"
+fi
+echo "::endgroup::"
+
+# --- Step 4: Determine baseline and skip logic ---
+echo "::group::Resolving refs"
+SHOULD_SKIP="false"
+BASELINE_REF="$FEATURE_REF"  # default for first run
+
+if [ "$IS_STALE" = "true" ]; then
+  # Stale = error path, don't skip (will alert and fail downstream)
+  SHOULD_SKIP="false"
+  BASELINE_REF="${LAST_FEATURE_REF:-$FEATURE_REF}"
+  echo "Stale nightly detected — will alert and fail"
+elif [ -z "$LAST_FEATURE_REF" ]; then
+  # First run: baseline = feature (self-comparison to establish baseline)
+  BASELINE_REF="$FEATURE_REF"
+  echo "First run — will benchmark nightly against itself to establish baseline"
+elif [ "$LAST_FEATURE_REF" = "$FEATURE_REF" ]; then
+  # No new nightly since last successful run
+  if [ "$FORCE" = "true" ] || [ "$FORCE" = "--force" ]; then
+    echo "No new nightly, but force=true — running anyway"
+    BASELINE_REF="$LAST_FEATURE_REF"
+  else
+    SHOULD_SKIP="true"
+    echo "No new nightly since last run — will skip"
+  fi
+else
+  # Normal case: new nightly available
+  BASELINE_REF="$LAST_FEATURE_REF"
+  echo "New nightly detected"
+fi
+
+echo "Baseline: $BASELINE_REF"
+echo "Feature:  $FEATURE_REF"
+echo "Skip:     $SHOULD_SKIP"
+echo "Stale:    $IS_STALE"
+echo "::endgroup::"
+
+# --- Step 5: Write outputs ---
+{
+  echo "baseline-ref=$BASELINE_REF"
+  echo "feature-ref=$FEATURE_REF"
+  echo "should-skip=$SHOULD_SKIP"
+  echo "is-stale=$IS_STALE"
+  echo "stale-age-hours=$AGE_HOURS"
+  echo "nightly-created=$CREATED_AT"
+} >> "$GITHUB_OUTPUT"

.github/scripts/bench-slack-notify.js

@@ -18,6 +18,7 @@
 
 const fs = require('fs');
 const path = require('path');
+const { fmtChange, fmtMs, verdict, loadSamplyUrls, blocksLabel, metricRows, waitTimeRows } = require('./bench-utils');
 
 const SLACK_API = 'https://slack.com/api/chat.postMessage';
 
@@ -61,41 +62,17 @@ function cell(text) {
   return { type: 'raw_text', text: s || ' ' };
 }
 
+// Slack shortcodes for verdict (Block Kit header doesn't support unicode emoji)
+const SLACK_VERDICT = {
+  '⚠️': ':warning:',
+  '❌': ':x:',
+  '✅': ':white_check_mark:',
+  '⚪': ':white_circle:',
+};
+
 function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, repo, samplyUrls }) {
-  const b = summary.baseline.stats;
-  const f = summary.feature.stats;
-  const c = summary.changes;
-
-  const sigEmoji = { good: '\u2705', bad: '\u274c', neutral: '\u26aa' };
-
-  function fmtMs(v) { return v.toFixed(2) + 'ms'; }
-  function fmtMgas(v) { return v.toFixed(2); }
-  function fmtS(v) { return v.toFixed(2) + 's'; }
-  function fmtChange(ch) {
-    if (!ch.pct && !ch.ci_pct) return ' ';
-    const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
-    const ciStr = ch.ci_pct ? ` (\u00b1${ch.ci_pct.toFixed(2)}%)` : '';
-    return `${pctStr}${ciStr} ${sigEmoji[ch.sig]}`;
-  }
-
-  // Overall result for header
-  const vals = Object.values(c);
-  const hasBad = vals.some(v => v.sig === 'bad');
-  const hasGood = vals.some(v => v.sig === 'good');
-  let headerEmoji, headerResult;
-  if (hasBad && hasGood) {
-    headerEmoji = ':warning:';
-    headerResult = 'Mixed Results';
-  } else if (hasBad) {
-    headerEmoji = ':x:';
-    headerResult = 'Regression';
-  } else if (hasGood) {
-    headerEmoji = ':white_check_mark:';
-    headerResult = 'Improvement';
-  } else {
-    headerEmoji = ':white_circle:';
-    headerResult = 'No Difference';
-  }
+  const { emoji, label } = verdict(summary.changes);
+  const headerEmoji = SLACK_VERDICT[emoji] || emoji;
 
   const prUrl = prNumber ? `https://github.com/${repo}/pull/${prNumber}` : '';
   const commitUrl = `https://github.com/${repo}/commit`;
@@ -120,19 +97,7 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
   if (fl1) featureLine += ` | <${fl1}|Samply 1>`;
   if (fl2) featureLine += ` | <${fl2}|Samply 2>`;
 
-  const cores = process.env.BENCH_CORES || '0';
-  const countsParts = [];
-  if (summary.big_blocks) {
-    const gasRamp = summary.gas_ramp_blocks || 0;
-    if (gasRamp > 0) countsParts.push(`*Gas Ramp:* ${gasRamp}`);
-    countsParts.push(`*Big Blocks:* ${summary.blocks}`);
-  } else {
-    const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
-    if (warmup) countsParts.push(`*Warmup:* ${warmup}`);
-    countsParts.push(`*Blocks:* ${summary.blocks}`);
-  }
-  if (cores !== '0') countsParts.push(`*Cores:* ${cores}`);
-  const countsLine = countsParts.join(' | ');
+  const countsLine = blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | ');
 
   const baselineArgs = process.env.BENCH_BASELINE_ARGS || '';
   const featureArgs = process.env.BENCH_FEATURE_ARGS || '';
@@ -159,10 +124,17 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
     },
   ];
 
+  // Build table rows from shared metricRows
+  const rows = metricRows(summary);
+  const tableRows = [
+    [cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
+    ...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
+  ];
+
   const blocks = [
     {
       type: 'header',
-      text: { type: 'plain_text', text: `${headerEmoji} ${headerResult}`, emoji: true },
+      text: { type: 'plain_text', text: `${headerEmoji} ${label}`, emoji: true },
     },
     {
       type: 'section',
@@ -176,16 +148,7 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
         { align: 'right' },
         { align: 'right' },
       ],
-      rows: [
-        [cell('Metric'),  cell('Baseline'), cell('Feature'), cell('Change')],
-        [cell('Mean'),     cell(fmtMs(b.mean_ms)),      cell(fmtMs(f.mean_ms)),      cell(fmtChange(c.mean))],
-        [cell('StdDev'),   cell(fmtMs(b.stddev_ms)),    cell(fmtMs(f.stddev_ms)),    cell(' ')],
-        [cell('P50'),      cell(fmtMs(b.p50_ms)),       cell(fmtMs(f.p50_ms)),       cell(fmtChange(c.p50))],
-        [cell('P90'),      cell(fmtMs(b.p90_ms)),       cell(fmtMs(f.p90_ms)),       cell(fmtChange(c.p90))],
-        [cell('P99'),      cell(fmtMs(b.p99_ms)),       cell(fmtMs(f.p99_ms)),       cell(fmtChange(c.p99))],
-        [cell('Mgas/s'),   cell(fmtMgas(b.mean_mgas_s)), cell(fmtMgas(f.mean_mgas_s)), cell(fmtChange(c.mgas_s))],
-        [cell('Wall Clock'), cell(fmtS(b.wall_clock_s)), cell(fmtS(f.wall_clock_s)), cell(fmtChange(c.wall_clock))],
-      ],
+      rows: tableRows,
     },
     {
       type: 'actions',
@@ -195,16 +158,12 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
 
   // Wait times as a separate table block (sent as threaded reply due to Slack one-table limit)
   const threadBlocks = [];
-  const waitTimes = summary.wait_times || {};
-  const waitKeys = Object.keys(waitTimes);
-  if (waitKeys.length > 0) {
-    const waitRows = [
+  const wtRows = waitTimeRows(summary);
+  if (wtRows.length > 0) {
+    const waitTableRows = [
       [cell('Wait Time'), cell('Baseline'), cell('Feature')],
+      ...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
     ];
-    for (const key of waitKeys) {
-      const wt = waitTimes[key];
-      waitRows.push([cell(wt.title), cell(fmtMs(wt.baseline.mean_ms)), cell(fmtMs(wt.feature.mean_ms))]);
-    }
     threadBlocks.push({
       type: 'table',
       column_settings: [
@@ -212,7 +171,7 @@ function buildSuccessBlocks({ summary, prNumber, actor, actorSlackId, jobUrl, re
         { align: 'right' },
         { align: 'right' },
       ],
-      rows: waitRows,
+      rows: waitTableRows,
     });
   }
 
@@ -274,16 +233,7 @@ async function success({ core, context }) {
   const jobUrl = process.env.BENCH_JOB_URL ||
     `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
 
-  // Load samply profile URLs (files exist when samply profiling was enabled)
-  const samplyUrls = {};
-  for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
-    try {
-      const url = fs.readFileSync(
-        path.join(process.env.BENCH_WORK_DIR, run, 'samply-profile-url.txt'), 'utf8'
-      ).trim();
-      if (url) samplyUrls[run] = url;
-    } catch {}
-  }
+  const samplyUrls = loadSamplyUrls(process.env.BENCH_WORK_DIR);
 
   const slackUsers = loadSlackUsers(process.env.GITHUB_WORKSPACE || '.');
   const actorSlackId = slackUsers[actor];

.github/scripts/bench-slack-users.json

@@ -20,5 +20,6 @@
   "SuperFluffy": "U095BKHB2Q4",
   "kamsz": "U0A2563UBRD",
   "zerosnacks": "U09FARPMN74",
-  "samczsun": "U096R14E4H3"
+  "samczsun": "U096R14E4H3",
+  "laibe": "U09FARE0B9Q"
 }

.github/scripts/bench-utils.js

@@ -0,0 +1,95 @@
+// Shared utilities for reth-bench result rendering.
+//
+// Used by bench-job-summary.js and bench-slack-notify.js.
+
+const fs = require('fs');
+const path = require('path');
+
+const SIG_EMOJI = { good: '✅', bad: '❌', neutral: '⚪' };
+
+function fmtMs(v) { return v.toFixed(2) + 'ms'; }
+function fmtMgas(v) { return v.toFixed(2); }
+function fmtS(v) { return v.toFixed(2) + 's'; }
+
+function fmtChange(ch) {
+  if (!ch || (!ch.pct && !ch.ci_pct)) return '';
+  const pctStr = `${ch.pct >= 0 ? '+' : ''}${ch.pct.toFixed(2)}%`;
+  const ciStr = ch.ci_pct ? ` (±${ch.ci_pct.toFixed(2)}%)` : '';
+  return `${pctStr}${ciStr} ${SIG_EMOJI[ch.sig]}`;
+}
+
+function verdict(changes) {
+  const vals = Object.values(changes);
+  const hasBad = vals.some(v => v.sig === 'bad');
+  const hasGood = vals.some(v => v.sig === 'good');
+  if (hasBad && hasGood) return { emoji: '⚠️', label: 'Mixed Results' };
+  if (hasBad) return { emoji: '❌', label: 'Regression' };
+  if (hasGood) return { emoji: '✅', label: 'Improvement' };
+  return { emoji: '⚪', label: 'No Difference' };
+}
+
+function loadSamplyUrls(workDir) {
+  const urls = {};
+  for (const run of ['baseline-1', 'baseline-2', 'feature-1', 'feature-2']) {
+    try {
+      const url = fs.readFileSync(path.join(workDir, run, 'samply-profile-url.txt'), 'utf8').trim();
+      if (url) urls[run] = url;
+    } catch {}
+  }
+  return urls;
+}
+
+function blocksLabel(summary) {
+  const parts = [];
+  if (summary.big_blocks) {
+    parts.push({ key: 'Big Blocks', value: summary.blocks });
+  } else {
+    const warmup = summary.warmup_blocks || process.env.BENCH_WARMUP_BLOCKS || '';
+    if (warmup) parts.push({ key: 'Warmup', value: warmup });
+    parts.push({ key: 'Blocks', value: summary.blocks });
+  }
+  const cores = process.env.BENCH_CORES || '0';
+  if (cores !== '0') parts.push({ key: 'Cores', value: cores });
+  return parts;
+}
+
+// The 7 metric rows shared by all renderers.
+// Returns an array of { label, baseline, feature, change } objects.
+function metricRows(summary) {
+  const b = summary.baseline.stats;
+  const f = summary.feature.stats;
+  const c = summary.changes;
+  return [
+    { label: 'Mean',       baseline: fmtMs(b.mean_ms),       feature: fmtMs(f.mean_ms),       change: fmtChange(c.mean) },
+    { label: 'StdDev',     baseline: fmtMs(b.stddev_ms),     feature: fmtMs(f.stddev_ms),     change: '' },
+    { label: 'P50',        baseline: fmtMs(b.p50_ms),        feature: fmtMs(f.p50_ms),        change: fmtChange(c.p50) },
+    { label: 'P90',        baseline: fmtMs(b.p90_ms),        feature: fmtMs(f.p90_ms),        change: fmtChange(c.p90) },
+    { label: 'P99',        baseline: fmtMs(b.p99_ms),        feature: fmtMs(f.p99_ms),        change: fmtChange(c.p99) },
+    { label: 'Mgas/s',     baseline: fmtMgas(b.mean_mgas_s), feature: fmtMgas(f.mean_mgas_s), change: fmtChange(c.mgas_s) },
+    { label: 'Wall Clock', baseline: fmtS(b.wall_clock_s),   feature: fmtS(f.wall_clock_s),   change: fmtChange(c.wall_clock) },
+  ];
+}
+
+// Wait time rows: one row per metric showing mean values.
+function waitTimeRows(summary) {
+  const waitTimes = summary.wait_times || {};
+  const rows = [];
+  for (const key of Object.keys(waitTimes)) {
+    const wt = waitTimes[key];
+    rows.push({ title: wt.title, baseline: fmtMs(wt.baseline.mean_ms), feature: fmtMs(wt.feature.mean_ms) });
+  }
+  return rows;
+}
+
+module.exports = {
+  SIG_EMOJI,
+  fmtMs,
+  fmtMgas,
+  fmtS,
+  fmtChange,
+  verdict,
+  loadSamplyUrls,
+  blocksLabel,
+  metricRows,
+  waitTimeRows,
+};

.github/scripts/check_rv32imac.sh

@@ -2,9 +2,6 @@
 set -uo pipefail
 
 crates_to_check=(
-    reth-codecs-derive
-    reth-primitives
-    reth-primitives-traits
     reth-network-peers
     reth-trie-common
     reth-trie-sparse

.github/scripts/check_wasm.sh

@@ -22,6 +22,7 @@ exclude_crates=(
   reth-downloaders
   reth-e2e-test-utils
   reth-engine-service
+  reth-execution-cache
   reth-engine-tree
   reth-engine-util
   reth-eth-wire
@@ -55,6 +56,7 @@ exclude_crates=(
   reth-ress-provider
   # The following are not supposed to be working
   reth # all of the crates below
+  reth-bb # binary-only, uses tokio features unsupported on wasm
   reth-storage-rpc-provider
   reth-invalid-block-hooks # reth-provider
   reth-libmdbx # mdbx

.github/scripts/hive/Dockerfile

@@ -7,7 +7,7 @@ FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
 WORKDIR /app
 
 # Install system dependencies
-RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config
+RUN apt-get update && apt-get install -y libclang-dev pkg-config
 
 # 
 # We prepare the build plan

.github/scripts/hive/build_simulators.sh

@@ -11,8 +11,14 @@ go build .
 
 # Run each hive command in the background for each simulator and wait
 echo "Building images"
-# TODO: test code has been moved from https://github.com/ethereum/execution-spec-tests to https://github.com/ethereum/execution-specs  we need to pin eels branch with `--sim.buildarg branch=<release-branch-name>` once we have the fusaka release tagged on the new repo
-./hive -client reth --sim "ethereum/eels" --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz -sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/eels/consume-engine" \
+    --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
+    --sim.buildarg branch=forks/osaka \
+    --sim.timelimit 1s || true &
+./hive -client reth --sim "ethereum/eels/consume-rlp" \
+    --sim.buildarg fixtures=https://github.com/ethereum/execution-spec-tests/releases/download/v5.3.0/fixtures_develop.tar.gz \
+    --sim.buildarg branch=forks/osaka \
+    --sim.timelimit 1s || true &
 ./hive -client reth --sim "ethereum/engine" -sim.timelimit 1s || true &
 ./hive -client reth --sim "devp2p" -sim.timelimit 1s || true &
 ./hive -client reth --sim "ethereum/rpc-compat" -sim.timelimit 1s || true &

.github/scripts/hive/expected_failures.yaml

@@ -1,9 +1,6 @@
 # tracked by https://github.com/paradigmxyz/reth/issues/13879
 rpc-compat:
   - debug_getRawBlock/get-invalid-number (reth)
-  - debug_getRawHeader/get-invalid-number (reth)
-  - debug_getRawReceipts/get-invalid-number (reth)
-  - debug_getRawReceipts/get-block-n (reth)
   - debug_getRawTransaction/get-invalid-hash (reth)
 
   - eth_getStorageAt/get-storage-invalid-key-too-large (reth)

.github/scripts/hive/run_simulator.sh

@@ -13,7 +13,13 @@ if [[ "${sim}" == *"eels"* ]]; then
 fi
 
 run_hive() {
-    hive --sim "${sim}" --sim.limit "${limit}" --sim.parallelism "${parallelism}" --client reth 2>&1 | tee /tmp/log || true
+    hive \
+  --sim "${sim}" \
+  --sim.limit "${limit}" \
+  --sim.limit.exact=false \
+  --sim.parallelism "${parallelism}" \
+  --client reth \
+  2>&1 | tee /tmp/log || true
 }
 
 check_log() {

.github/workflows/bench-scheduled.yml

@@ -0,0 +1,966 @@
+# Scheduled regression benchmarks (nightly + hourly).
+#
+# Two modes:
+#   nightly — Compares the previous nightly Docker build against the current one.
+#             Runs daily after docker.yml produces a new nightly image.
+#   hourly  — Compares main HEAD against the last benchmarked commit to catch
+#             regressions quickly. Falls back to HEAD~1 on first run.
+#             Skips if no new commits or if a previous run is still in progress.
+#
+# State is persisted between runs via GitHub Actions cache: each successful
+# run saves the feature commit SHA so the next run knows what to compare against.
+
+on:
+  schedule:
+    # Nightly: compares previous vs current nightly Docker build
+    - cron: "30 5 * * *"
+    # Hourly: compares main HEAD vs last benchmarked commit, skips if no new commits
+    - cron: "0 * * * *"
+  workflow_dispatch:
+    inputs:
+      force:
+        description: "Force run even if no new commit (bypass skip logic)"
+        required: false
+        default: false
+        type: boolean
+      no_slack:
+        description: "Suppress Slack notifications"
+        required: false
+        default: true
+        type: boolean
+      mode:
+        description: "Benchmark mode"
+        required: false
+        default: "nightly"
+        type: choice
+        options:
+          - nightly
+          - hourly
+
+env:
+  CARGO_TERM_COLOR: always
+  RUSTC_WRAPPER: "sccache"
+
+name: bench-scheduled
+
+permissions:
+  contents: read
+  actions: read
+
+jobs:
+  # ---------------------------------------------------------------------------
+  # Job 1: Resolve refs, check staleness, manage state
+  # ---------------------------------------------------------------------------
+  resolve-refs:
+    name: resolve-refs
+    runs-on: ubuntu-latest
+    outputs:
+      mode: ${{ steps.mode.outputs.mode }}
+      baseline-ref: ${{ steps.refs.outputs.baseline-ref }}
+      feature-ref: ${{ steps.refs.outputs.feature-ref }}
+      should-skip: ${{ steps.refs.outputs.should-skip }}
+      is-stale: ${{ steps.refs.outputs.is-stale }}
+      stale-age-hours: ${{ steps.refs.outputs.stale-age-hours }}
+      nightly-created: ${{ steps.refs.outputs.nightly-created }}
+      long-running: ${{ steps.refs.outputs.long-running }}
+    steps:
+      - uses: actions/checkout@v6
+        with:
+          sparse-checkout: .github/scripts
+          sparse-checkout-cone-mode: true
+          fetch-depth: 2
+
+      - name: Detect mode
+        id: mode
+        run: |
+          # Maps cron schedules to modes (must match the schedule entries above)
+          if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then
+            MODE="${{ inputs.mode || 'nightly' }}"
+          elif [ "${{ github.event.schedule }}" = "30 5 * * *" ]; then
+            MODE="nightly"
+          else
+            MODE="hourly"
+          fi
+          echo "mode=$MODE" >> "$GITHUB_OUTPUT"
+          echo "Detected mode: $MODE"
+
+      - name: Restore state cache
+        id: state-cache
+        uses: actions/cache/restore@v4
+        with:
+          path: .${{ steps.mode.outputs.mode == 'hourly' && 'hourly' || 'nightly' }}-state
+          key: bench-${{ steps.mode.outputs.mode }}-state-dummy
+          restore-keys: |
+            bench-${{ steps.mode.outputs.mode }}-state-
+
+      - name: Resolve refs
+        id: refs
+        env:
+          GH_TOKEN: ${{ github.token }}
+          GITHUB_REPOSITORY: ${{ github.repository }}
+          GITHUB_RUN_ID: ${{ github.run_id }}
+        run: |
+          FORCE="${{ inputs.force || 'false' }}"
+          MODE="${{ steps.mode.outputs.mode }}"
+          .github/scripts/bench-scheduled-refs.sh "$FORCE" "$MODE"
+
+      - name: Alert on long-running hourly
+        if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true'
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) return;
+
+            const repo = '${{ github.repository }}';
+            const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: ':warning: Hourly Bench: previous run still in progress', emoji: true },
+              },
+              {
+                type: 'section',
+                text: {
+                  type: 'mrkdwn',
+                  text: 'A previous hourly benchmark run is still in progress. This invocation will be skipped.\nThis may indicate a long-running or stuck job.',
+                },
+              },
+              {
+                type: 'actions',
+                elements: [{
+                  type: 'button',
+                  text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
+                  url: runUrl,
+                  action_id: 'ci_button',
+                }],
+              },
+            ];
+            await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: { 'Authorization': `Bearer ${token}`, 'Content-Type': 'application/json' },
+              body: JSON.stringify({ channel, blocks, text: 'Hourly bench: previous run still in progress', unfurl_links: false }),
+            });
+
+      - name: Alert on stale nightly
+        if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true'
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) {
+              core.warning('Slack credentials not set, skipping stale nightly alert');
+              return;
+            }
+
+            const ageHours = '${{ steps.refs.outputs.stale-age-hours }}';
+            const created = '${{ steps.refs.outputs.nightly-created }}';
+            const featureRef = '${{ steps.refs.outputs.feature-ref }}';
+            const shortSha = featureRef.slice(0, 8);
+            const repo = '${{ github.repository }}';
+            const runUrl = `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: ':rotating_light: Nightly Regression: nightly build is stale', emoji: true },
+              },
+              {
+                type: 'section',
+                text: {
+                  type: 'mrkdwn',
+                  text: [
+                    '*Nightly regression did not run* — nightly build is stale',
+                    '',
+                    `The latest nightly image was built from a commit that is *${ageHours}h old* (threshold: 24h).`,
+                    `This means today's nightly docker build likely failed and no new image was produced.`,
+                    '',
+                    `Stale commit: \`${shortSha}\` (built at ${created})`,
+                    '',
+                    '*Action required:* Check the <https://github.com/' + repo + '/actions/workflows/docker.yml|docker.yml> workflow for failures.',
+                  ].join('\n'),
+                },
+              },
+              {
+                type: 'actions',
+                elements: [
+                  {
+                    type: 'button',
+                    text: { type: 'plain_text', text: 'View Run :github:', emoji: true },
+                    url: runUrl,
+                    action_id: 'ci_button',
+                  },
+                ],
+              },
+            ];
+
+            const resp = await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: {
+                'Authorization': `Bearer ${token}`,
+                'Content-Type': 'application/json',
+              },
+              body: JSON.stringify({
+                channel,
+                blocks,
+                text: 'Nightly regression: nightly build is stale',
+                unfurl_links: false,
+              }),
+            });
+            const data = await resp.json();
+            if (!data.ok) {
+              core.warning(`Slack API error: ${JSON.stringify(data)}`);
+            }
+
+      - name: Fail on stale nightly
+        if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true'
+        run: |
+          echo "::error::Nightly build is stale (>24h old). Aborting."
+          exit 1
+
+  # ---------------------------------------------------------------------------
+  # Job 2: Run the benchmark
+  # ---------------------------------------------------------------------------
+  bench-scheduled:
+    needs: resolve-refs
+    if: |
+      needs.resolve-refs.outputs.should-skip != 'true' &&
+      needs.resolve-refs.outputs.is-stale != 'true'
+    name: bench-scheduled
+    runs-on: [self-hosted, Linux, X64, available]
+    timeout-minutes: 120
+    env:
+      BENCH_RPC_URL: https://ethereum.reth.rs/rpc
+      SCHELK_MOUNT: /reth-bench
+      BENCH_WORK_DIR: ${{ github.workspace }}/bench-work
+      BENCH_PR: ""
+      BENCH_MODE: ${{ needs.resolve-refs.outputs.mode }}
+      BENCH_ACTOR: "${{ needs.resolve-refs.outputs.mode }}-regression"
+      BENCH_BLOCKS: "2000"
+      BENCH_WARMUP_BLOCKS: "500"
+      BENCH_SAMPLY: "false"
+      BENCH_CORES: "0"
+      BENCH_BIG_BLOCKS: "false"
+      BENCH_RETH_NEW_PAYLOAD: "true"
+      BENCH_WAIT_TIME: ""
+      BENCH_BASELINE_ARGS: ""
+      BENCH_FEATURE_ARGS: ""
+      BENCH_ABBA: "true"
+      BENCH_COMMENT_ID: ""
+      BENCH_NO_SLACK: ${{ github.event_name == 'workflow_dispatch' && inputs.no_slack == true && 'true' || 'false' }}
+      BENCH_METRICS_ADDR: "127.0.0.1:9100"
+      BENCH_OTLP_TRACES_ENDPOINT: ${{ secrets.BENCH_OTLP_TRACES_ENDPOINT }}
+      BENCH_OTLP_LOGS_ENDPOINT: ${{ secrets.BENCH_OTLP_LOGS_ENDPOINT }}
+      BASELINE_REF: ${{ needs.resolve-refs.outputs.baseline-ref }}
+      FEATURE_REF: ${{ needs.resolve-refs.outputs.feature-ref }}
+    steps:
+      - name: Clean up previous bench-work
+        run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
+
+      - uses: actions/checkout@v6
+        with:
+          submodules: true
+          fetch-depth: 0
+          ref: ${{ needs.resolve-refs.outputs.feature-ref }}
+
+      - name: Resolve job URL
+        id: job-url
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              run_id: context.runId,
+            });
+            const job = jobs.jobs.find(j => j.name === 'bench-scheduled');
+            const jobUrl = job ? job.html_url : `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
+            core.exportVariable('BENCH_JOB_URL', jobUrl);
+
+      - uses: dtolnay/rust-toolchain@stable
+      - uses: mozilla-actions/sccache-action@v0.0.9
+        continue-on-error: true
+
+      - name: Install dependencies
+        env:
+          DEREK_TOKEN: ${{ secrets.DEREK_TOKEN }}
+        run: |
+          mkdir -p "$HOME/.local/bin"
+
+          # apt packages
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends \
+            python3 make jq zstd curl dmsetup \
+            linux-tools-"$(uname -r)" || \
+            sudo apt-get install -y --no-install-recommends linux-tools-generic
+
+          # mc (MinIO client)
+          if ! command -v mc &>/dev/null; then
+            curl -sSfL https://dl.min.io/client/mc/release/linux-amd64/mc -o "$HOME/.local/bin/mc"
+            chmod +x "$HOME/.local/bin/mc"
+          fi
+
+          # uv (Python package manager)
+          if ! command -v uv &>/dev/null; then
+            curl -LsSf https://astral.sh/uv/install.sh | env UV_INSTALL_DIR="$HOME/.local/bin" sh
+          fi
+
+          # Configure git auth for private repos
+          git config --global url."https://x-access-token:${DEREK_TOKEN}@github.com/".insteadOf "https://github.com/"
+
+          # thin-provisioning-tools (era_invalidate, required by schelk)
+          if ! command -v era_invalidate &>/dev/null; then
+            git clone --depth 1 https://github.com/jthornber/thin-provisioning-tools /tmp/tpt
+            sudo make -C /tmp/tpt install
+            rm -rf /tmp/tpt
+          fi
+
+          # schelk (snapshot rollback tool, invoked via sudo)
+          if ! sudo sh -c 'command -v schelk' &>/dev/null; then
+            cargo install --git https://github.com/tempoxyz/schelk --locked
+            sudo install "$HOME/.cargo/bin/schelk" /usr/local/bin/
+          fi
+
+      - name: Check dependencies
+        run: |
+          export PATH="$HOME/.local/bin:$HOME/.cargo/bin:$PATH"
+          echo "$HOME/.local/bin" >> "$GITHUB_PATH"
+          echo "$HOME/.cargo/bin" >> "$GITHUB_PATH"
+          missing=()
+          for cmd in mc schelk cpupower taskset stdbuf python3 curl make uv pzstd jq; do
+            command -v "$cmd" &>/dev/null || missing+=("$cmd")
+          done
+          if [ ${#missing[@]} -gt 0 ]; then
+            echo "::error::Missing required tools: ${missing[*]}"
+            exit 1
+          fi
+          echo "All dependencies found"
+
+      - name: Resolve display names
+        id: refs
+        run: |
+          BASELINE_SHORT=$(echo "$BASELINE_REF" | cut -c1-8)
+          FEATURE_SHORT=$(echo "$FEATURE_REF" | cut -c1-8)
+          echo "baseline-name=${BENCH_MODE}-${BASELINE_SHORT}" >> "$GITHUB_OUTPUT"
+          echo "feature-name=${BENCH_MODE}-${FEATURE_SHORT}" >> "$GITHUB_OUTPUT"
+          echo "baseline-ref=$BASELINE_REF" >> "$GITHUB_OUTPUT"
+          echo "feature-ref=$FEATURE_REF" >> "$GITHUB_OUTPUT"
+
+      - name: Check if snapshot needs update
+        id: snapshot-check
+        run: |
+          set +e
+          .github/scripts/bench-reth-snapshot.sh --check
+          rc=$?
+          set -e
+          case "$rc" in
+            0)  echo "needed=false" >> "$GITHUB_OUTPUT" ;;
+            10) echo "needed=true"  >> "$GITHUB_OUTPUT" ;;
+            *)  echo "::error::Snapshot check failed (exit $rc)"
+                exit "$rc" ;;
+          esac
+
+      - name: Prepare source dirs
+        run: |
+          if [ -d ../reth-baseline ]; then
+            git -C ../reth-baseline fetch origin "$BASELINE_REF"
+          else
+            git clone . ../reth-baseline
+          fi
+          git -C ../reth-baseline checkout "$BASELINE_REF"
+
+          if [ -d ../reth-feature ]; then
+            git -C ../reth-feature fetch origin "$FEATURE_REF"
+          else
+            git clone . ../reth-feature
+          fi
+          git -C ../reth-feature checkout "$FEATURE_REF"
+
+      - name: Build binaries
+        id: build
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCH_REPO: ${{ github.repository }}
+        run: |
+          BASELINE_DIR="$(cd ../reth-baseline && pwd)"
+          FEATURE_DIR="$(cd ../reth-feature && pwd)"
+
+          .github/scripts/bench-reth-build.sh baseline "${BASELINE_DIR}" "$BASELINE_REF" &
+          PID_BASELINE=$!
+          .github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "$FEATURE_REF" &
+          PID_FEATURE=$!
+
+          FAIL=0
+          wait $PID_BASELINE || FAIL=1
+          wait $PID_FEATURE || FAIL=1
+          if [ $FAIL -ne 0 ]; then
+            echo "::error::One or more build tasks failed"
+            exit 1
+          fi
+
+      - name: Download snapshot
+        id: snapshot-download
+        if: steps.snapshot-check.outputs.needed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCH_REPO: ${{ github.repository }}
+          BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/reth
+        run: .github/scripts/bench-reth-snapshot.sh
+
+      # System tuning for reproducible benchmarks
+      - name: System setup
+        run: |
+          sudo cpupower frequency-set -g performance || true
+          # Disable turbo boost (Intel and AMD paths)
+          echo 1 | sudo tee /sys/devices/system/cpu/intel_pstate/no_turbo 2>/dev/null || true
+          echo 0 | sudo tee /sys/devices/system/cpu/cpufreq/boost 2>/dev/null || true
+          sudo swapoff -a || true
+          echo 0 | sudo tee /proc/sys/kernel/randomize_va_space || true
+          # Disable SMT (hyperthreading)
+          for cpu in /sys/devices/system/cpu/cpu*/topology/thread_siblings_list; do
+            first=$(cut -d, -f1 < "$cpu" | cut -d- -f1)
+            current=$(echo "$cpu" | grep -o 'cpu[0-9]*' | grep -o '[0-9]*')
+            if [ "$current" != "$first" ]; then
+              echo 0 | sudo tee "/sys/devices/system/cpu/cpu${current}/online" || true
+            fi
+          done
+          echo "Online CPUs: $(nproc)"
+          # Disable transparent huge pages
+          for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
+            [ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
+          done || true
+          # Prevent deep C-states
+          sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
+          # Move all IRQs to core 0
+          for irq in /proc/irq/*/smp_affinity_list; do
+            echo 0 | sudo tee "$irq" 2>/dev/null || true
+          done
+          # Stop noisy background services
+          sudo systemctl stop irqbalance cron atd unattended-upgrades snapd 2>/dev/null || true
+          echo "=== Benchmark environment ==="
+          uname -r
+          lscpu | grep -E 'Model name|CPU\(s\)|MHz|NUMA'
+          cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_governor
+          cat /sys/devices/system/cpu/cpu0/cpufreq/scaling_cur_freq
+          cat /sys/kernel/mm/transparent_hugepage/enabled 2>/dev/null || cat /sys/kernel/mm/transparent_hugepages/enabled 2>/dev/null || echo "THP: unknown"
+          free -h
+
+      - name: Pre-flight cleanup
+        run: |
+          sudo pkill -9 reth || true
+          sleep 1
+          if mountpoint -q "$SCHELK_MOUNT"; then
+            sudo umount -l "$SCHELK_MOUNT" || true
+            sudo schelk recover -y || true
+          fi
+          rm -rf "$BENCH_WORK_DIR"
+          mkdir -p "$BENCH_WORK_DIR"
+
+      - name: Start metrics proxy
+        run: |
+          BENCH_ID="${BENCH_MODE}-${{ github.run_id }}"
+          BENCH_REFERENCE_EPOCH=$(date +%s)
+          echo "BENCH_ID=${BENCH_ID}" >> "$GITHUB_ENV"
+          echo "BENCH_REFERENCE_EPOCH=${BENCH_REFERENCE_EPOCH}" >> "$GITHUB_ENV"
+
+          LABELS_FILE="/tmp/bench-metrics-labels.json"
+          echo '{}' > "$LABELS_FILE"
+          echo "BENCH_LABELS_FILE=${LABELS_FILE}" >> "$GITHUB_ENV"
+
+          python3 .github/scripts/bench-metrics-proxy.py \
+            --labels "$LABELS_FILE" \
+            --upstream "http://${BENCH_METRICS_ADDR}/" \
+            --subnet 10.10.0.0/24 \
+            --port 9090 &
+          PROXY_PID=$!
+          echo "BENCH_METRICS_PROXY_PID=${PROXY_PID}" >> "$GITHUB_ENV"
+          echo "Metrics proxy started (PID $PROXY_PID)"
+
+      # Interleaved run order (B-F-F-B) to reduce systematic bias
+      - name: "Run benchmark: baseline (1/2)"
+        id: run-baseline-1
+        run: |
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
+
+      - name: "Run benchmark: feature (1/2)"
+        id: run-feature-1
+        run: |
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
+
+      - name: "Run benchmark: feature (2/2)"
+        id: run-feature-2
+        run: |
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
+
+      - name: "Run benchmark: baseline (2/2)"
+        id: run-baseline-2
+        run: |
+          LAST_RUN_START=$(date +%s)
+          echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
+          cat > "$BENCH_LABELS_FILE" <<LABELS
+          {"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
+          LABELS
+          taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
+
+      - name: Stop metrics proxy & generate Grafana URL
+        id: metrics
+        if: "!cancelled()"
+        run: |
+          kill "$BENCH_METRICS_PROXY_PID" 2>/dev/null || true
+
+          LAST_RUN_DURATION=$(( $(date +%s) - BENCH_LAST_RUN_START ))
+          FROM_MS=$(( BENCH_REFERENCE_EPOCH * 1000 ))
+          TO_MS=$(( (BENCH_REFERENCE_EPOCH + LAST_RUN_DURATION) * 1000 ))
+          GRAFANA_URL="https://tempoxyz.grafana.net/d/reth-bench-ghr/reth-bench-ghr?orgId=1&from=${FROM_MS}&to=${TO_MS}&timezone=browser&var-datasource=ef57fux92e9z4e&var-job=reth-bench&var-benchmark_id=${BENCH_ID}&var-benchmark_run=\$__all"
+          echo "grafana-url=${GRAFANA_URL}" >> "$GITHUB_OUTPUT"
+          echo "Grafana URL: ${GRAFANA_URL}"
+
+      - name: Scan logs for errors
+        if: "!cancelled()"
+        run: |
+          ERRORS_FILE="$BENCH_WORK_DIR/errors.md"
+          found=false
+          for run_dir in baseline-1 feature-1 feature-2 baseline-2; do
+            LOG="$BENCH_WORK_DIR/$run_dir/node.log"
+            if [ ! -f "$LOG" ]; then continue; fi
+
+            panics=$(grep -c -E 'panicked at' "$LOG" || true)
+            errors=$(grep -c ' ERROR ' "$LOG" || true)
+
+            if [ "$panics" -gt 0 ] || [ "$errors" -gt 0 ]; then
+              if [ "$found" = false ]; then
+                printf '### ⚠️ Node Errors\n\n' >> "$ERRORS_FILE"
+                found=true
+              fi
+              printf '<details><summary><b>%s</b>: %d panic(s), %d error(s)</summary>\n\n' "$run_dir" "$panics" "$errors" >> "$ERRORS_FILE"
+              if [ "$panics" -gt 0 ]; then
+                printf '**Panics:**\n```\n' >> "$ERRORS_FILE"
+                grep -E 'panicked at' "$LOG" | head -10 >> "$ERRORS_FILE"
+                printf '```\n' >> "$ERRORS_FILE"
+              fi
+              if [ "$errors" -gt 0 ]; then
+                printf '**Errors (first 20):**\n```\n' >> "$ERRORS_FILE"
+                grep ' ERROR ' "$LOG" | head -20 >> "$ERRORS_FILE"
+                printf '```\n' >> "$ERRORS_FILE"
+              fi
+              printf '\n</details>\n\n' >> "$ERRORS_FILE"
+            fi
+          done
+
+      - name: Parse results
+        id: results
+        if: success()
+        env:
+          BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
+          FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
+        run: |
+          SUMMARY_ARGS="--output-summary $BENCH_WORK_DIR/summary.json"
+          SUMMARY_ARGS="$SUMMARY_ARGS --output-markdown $BENCH_WORK_DIR/comment.md"
+          SUMMARY_ARGS="$SUMMARY_ARGS --repo ${{ github.repository }}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --baseline-ref ${BASELINE_REF}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --baseline-name ${BASELINE_NAME}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --feature-name ${FEATURE_NAME}"
+          SUMMARY_ARGS="$SUMMARY_ARGS --feature-ref ${FEATURE_REF}"
+
+          BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
+          FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
+          BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
+          FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
+          SUMMARY_ARGS="$SUMMARY_ARGS --baseline-csv $BASELINE_CSVS"
+          SUMMARY_ARGS="$SUMMARY_ARGS --feature-csv $FEATURE_CSVS"
+          SUMMARY_ARGS="$SUMMARY_ARGS --gas-csv $BENCH_WORK_DIR/feature-1/total_gas.csv"
+
+          GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
+          if [ -n "$GRAFANA_URL" ]; then
+            SUMMARY_ARGS="$SUMMARY_ARGS --grafana-url $GRAFANA_URL"
+          fi
+          # shellcheck disable=SC2086
+          python3 .github/scripts/bench-reth-summary.py $SUMMARY_ARGS
+
+      - name: Generate charts
+        if: success() && env.BENCH_MODE != 'hourly'
+        env:
+          BASELINE_NAME: ${{ steps.refs.outputs.baseline-name }}
+          FEATURE_NAME: ${{ steps.refs.outputs.feature-name }}
+        run: |
+          CHART_ARGS="--output-dir $BENCH_WORK_DIR/charts"
+          FEATURE_CSVS="$BENCH_WORK_DIR/feature-1/combined_latency.csv"
+          BASELINE_CSVS="$BENCH_WORK_DIR/baseline-1/combined_latency.csv"
+          FEATURE_CSVS="$FEATURE_CSVS $BENCH_WORK_DIR/feature-2/combined_latency.csv"
+          BASELINE_CSVS="$BASELINE_CSVS $BENCH_WORK_DIR/baseline-2/combined_latency.csv"
+          CHART_ARGS="$CHART_ARGS --feature $FEATURE_CSVS"
+          CHART_ARGS="$CHART_ARGS --baseline $BASELINE_CSVS"
+          CHART_ARGS="$CHART_ARGS --baseline-name ${BASELINE_NAME}"
+          CHART_ARGS="$CHART_ARGS --feature-name ${FEATURE_NAME}"
+          # shellcheck disable=SC2086
+          uv run --with matplotlib python3 .github/scripts/bench-reth-charts.py $CHART_ARGS
+
+      - name: Upload results
+        if: "!cancelled()"
+        uses: actions/upload-artifact@v7
+        with:
+          name: bench-scheduled-results
+          path: ${{ env.BENCH_WORK_DIR }}
+
+      - name: Push charts
+        id: push-charts
+        if: success() && env.BENCH_MODE != 'hourly'
+        run: |
+          RUN_ID=${{ github.run_id }}
+          CHART_DIR="nightly/${RUN_ID}"
+          CHARTS_REPO="https://x-access-token:${{ secrets.DEREK_TOKEN }}@github.com/decofe/reth-bench-charts.git"
+
+          TMP_DIR=$(mktemp -d)
+          if git clone --depth 1 "${CHARTS_REPO}" "${TMP_DIR}" 2>/dev/null; then
+            true
+          else
+            git init "${TMP_DIR}"
+            git -C "${TMP_DIR}" remote add origin "${CHARTS_REPO}"
+          fi
+
+          mkdir -p "${TMP_DIR}/${CHART_DIR}"
+          cp "$BENCH_WORK_DIR"/charts/*.png "${TMP_DIR}/${CHART_DIR}/"
+          git -C "${TMP_DIR}" add "${CHART_DIR}"
+          git -C "${TMP_DIR}" -c user.name="github-actions" -c user.email="github-actions@github.com" \
+            commit -m "nightly bench charts for run ${RUN_ID}"
+          git -C "${TMP_DIR}" push origin HEAD:main
+          echo "sha=$(git -C "${TMP_DIR}" rev-parse HEAD)" >> "$GITHUB_OUTPUT"
+          rm -rf "${TMP_DIR}"
+
+      - name: Write job summary
+        if: success()
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const fs = require('fs');
+            const { verdict, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
+
+            let summary;
+            try {
+              summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+            } catch (e) {
+              await core.summary.addRaw('⚠️ Benchmark completed but failed to load summary.').write();
+              return;
+            }
+
+            const repo = `${context.repo.owner}/${context.repo.repo}`;
+            const commitUrl = `https://github.com/${repo}/commit`;
+            const { emoji, label } = verdict(summary.changes);
+            const baselineLink = `[\`${summary.baseline.name}\`](${commitUrl}/${summary.baseline.ref})`;
+            const featureLink = `[\`${summary.feature.name}\`](${commitUrl}/${summary.feature.ref})`;
+            const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+
+            const mode = process.env.BENCH_MODE || 'nightly';
+            const modeLabel = mode === 'hourly' ? 'Hourly Regression' : 'Nightly Regression';
+            let md = `# ${emoji} ${modeLabel}: ${label}\n\n`;
+            md += `**Baseline:** ${baselineLink}\n`;
+            md += `**Feature:** ${featureLink} ([diff](${diffUrl}))\n`;
+            md += blocksLabel(summary).map(p => `**${p.key}:** ${p.value}`).join(' · ') + '\n\n';
+
+            const rows = metricRows(summary);
+            md += `| Metric | Baseline | Feature | Change |\n`;
+            md += `|--------|----------|---------|--------|\n`;
+            for (const r of rows) {
+              md += `| ${r.label} | ${r.baseline} | ${r.feature} | ${r.change} |\n`;
+            }
+            md += '\n';
+
+            const wtRows = waitTimeRows(summary);
+            if (wtRows.length > 0) {
+              md += `### Wait Time Breakdown\n\n`;
+              md += `| Metric | Baseline | Feature |\n`;
+              md += `|--------|----------|--------|\n`;
+              for (const r of wtRows) {
+                md += `| ${r.title} | ${r.baseline} | ${r.feature} |\n`;
+              }
+              md += '\n';
+            }
+
+            // Charts
+            const chartSha = '${{ steps.push-charts.outputs.sha }}';
+            if (chartSha) {
+              const runId = '${{ github.run_id }}';
+              const baseUrl = `https://raw.githubusercontent.com/decofe/reth-bench-charts/${chartSha}/nightly/${runId}`;
+              const charts = [
+                { file: 'latency_throughput.png', label: 'Latency, Throughput & Diff' },
+                { file: 'wait_breakdown.png', label: 'Wait Time Breakdown' },
+                { file: 'gas_vs_latency.png', label: 'Gas vs Latency' },
+              ];
+              md += `### Charts\n\n`;
+              for (const chart of charts) {
+                md += `<details><summary>${chart.label}</summary>\n\n`;
+                md += `![${chart.label}](${baseUrl}/${chart.file})\n\n`;
+                md += `</details>\n\n`;
+              }
+            }
+
+            const grafanaUrl = '${{ steps.metrics.outputs.grafana-url }}';
+            if (grafanaUrl) {
+              md += `### Grafana Dashboard\n\n[View real-time metrics](${grafanaUrl})\n\n`;
+            }
+
+            try {
+              const errors = fs.readFileSync(process.env.BENCH_WORK_DIR + '/errors.md', 'utf8');
+              if (errors.trim()) md += '\n' + errors + '\n';
+            } catch {}
+
+            await core.summary.addRaw(md).write();
+
+      - name: Send Slack notification (success)
+        if: success() && env.BENCH_NO_SLACK != 'true'
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const fs = require('fs');
+            const { verdict, fmtChange, fmtMs, metricRows, waitTimeRows, blocksLabel } = require('./.github/scripts/bench-utils');
+
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) {
+              core.info('Slack credentials not set, skipping notification');
+              return;
+            }
+
+            let summary;
+            try {
+              summary = JSON.parse(fs.readFileSync(process.env.BENCH_WORK_DIR + '/summary.json', 'utf8'));
+            } catch (e) {
+              core.warning('Could not read summary.json for Slack notification');
+              return;
+            }
+
+            // Filter notifications based on mode
+            const changes = summary.changes || {};
+            const mode = process.env.BENCH_MODE || 'nightly';
+            const hasRegression = Object.values(changes).some(c => c.sig === 'bad');
+            const hasSignificant = Object.values(changes).some(c => c.sig === 'good' || c.sig === 'bad');
+
+            // Hourly mode: only notify on regressions
+            if (mode === 'hourly' && !hasRegression) {
+              core.info('Hourly mode: no regression detected, skipping Slack notification');
+              return;
+            }
+
+            // Nightly mode: notify on any significant change (regression or improvement)
+            if (!hasSignificant) {
+              core.info('No significant changes detected, skipping nightly Slack notification');
+              return;
+            }
+
+            const SLACK_VERDICT = {
+              '⚠️': ':warning:',
+              '❌': ':x:',
+              '✅': ':white_check_mark:',
+              '⚪': ':white_circle:',
+            };
+
+            const repo = `${context.repo.owner}/${context.repo.repo}`;
+            const { emoji, label } = verdict(changes);
+            const headerEmoji = SLACK_VERDICT[emoji] || emoji;
+            const commitUrl = `https://github.com/${repo}/commit`;
+            const baselineLink = `<${commitUrl}/${summary.baseline.ref}|${summary.baseline.name}>`;
+            const featureLink = `<${commitUrl}/${summary.feature.ref}|${summary.feature.name}>`;
+            const diffUrl = `https://github.com/${repo}/compare/${summary.baseline.ref}...${summary.feature.ref}`;
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+
+            function cell(text) { return { type: 'raw_text', text: String(text) || ' ' }; }
+
+            const modeLabel = mode === 'hourly' ? 'Hourly Regression' : 'Nightly Regression';
+            const sectionText = [
+              `*${modeLabel}*`,
+              '',
+              `*Baseline:* ${baselineLink}`,
+              `*Feature:* ${featureLink}`,
+              blocksLabel(summary).map(p => `*${p.key}:* ${p.value}`).join(' | '),
+            ].join('\n');
+
+            const rows = metricRows(summary);
+            const tableRows = [
+              [cell('Metric'), cell('Baseline'), cell('Feature'), cell('Change')],
+              ...rows.map(r => [cell(r.label), cell(r.baseline), cell(r.feature), cell(r.change || ' ')]),
+            ];
+
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: `${headerEmoji} ${modeLabel}: ${label}`, emoji: true },
+              },
+              {
+                type: 'section',
+                text: { type: 'mrkdwn', text: sectionText },
+              },
+              {
+                type: 'table',
+                column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }, { align: 'right' }],
+                rows: tableRows,
+              },
+              {
+                type: 'actions',
+                elements: [
+                  {
+                    type: 'button',
+                    text: { type: 'plain_text', text: 'CI :github:', emoji: true },
+                    url: jobUrl,
+                    action_id: 'ci_button',
+                  },
+                  {
+                    type: 'button',
+                    text: { type: 'plain_text', text: 'Diff :github:', emoji: true },
+                    url: diffUrl,
+                    action_id: 'diff_button',
+                  },
+                ],
+              },
+            ];
+
+            const text = `${modeLabel}: ${summary.baseline.name} vs ${summary.feature.name}`;
+            const resp = await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: {
+                'Authorization': `Bearer ${token}`,
+                'Content-Type': 'application/json',
+              },
+              body: JSON.stringify({ channel, blocks, text, unfurl_links: false }),
+            });
+            const data = await resp.json();
+            if (!data.ok) {
+              core.warning(`Slack API error: ${JSON.stringify(data)}`);
+              return;
+            }
+
+            // Post wait time breakdown as threaded reply
+            const wtRows = waitTimeRows(summary);
+            if (data.ts && wtRows.length > 0) {
+              const waitTableRows = [
+                [cell('Wait Time'), cell('Baseline'), cell('Feature')],
+                ...wtRows.map(r => [cell(r.title), cell(r.baseline), cell(r.feature)]),
+              ];
+              await fetch('https://slack.com/api/chat.postMessage', {
+                method: 'POST',
+                headers: {
+                  'Authorization': `Bearer ${token}`,
+                  'Content-Type': 'application/json',
+                },
+                body: JSON.stringify({
+                  channel,
+                  thread_ts: data.ts,
+                  blocks: [{
+                    type: 'table',
+                    column_settings: [{ align: 'left' }, { align: 'right' }, { align: 'right' }],
+                    rows: waitTableRows,
+                  }],
+                  text: 'Wait time breakdown',
+                  unfurl_links: false,
+                }),
+              });
+            }
+
+      - name: Send Slack notification (failure)
+        if: failure()
+        uses: actions/github-script@v8
+        env:
+          SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
+          SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
+        with:
+          script: |
+            const token = process.env.SLACK_BENCH_BOT_TOKEN;
+            const channel = process.env.SLACK_BENCH_CHANNEL;
+            if (!token || !channel) return;
+
+            const steps_status = [
+              ['building binaries', '${{ steps.build.outcome }}'],
+              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
+              ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
+              ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
+              ['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}'],
+              ['running baseline benchmark (2/2)', '${{ steps.run-baseline-2.outcome }}'],
+            ];
+            const failed = steps_status.find(([, o]) => o === 'failure');
+            const failedStep = failed ? failed[0] : 'unknown step';
+
+            const repo = `${context.repo.owner}/${context.repo.repo}`;
+            const jobUrl = process.env.BENCH_JOB_URL || `${context.serverUrl}/${repo}/actions/runs/${context.runId}`;
+
+            const mode = process.env.BENCH_MODE || 'nightly';
+            const modeLabel = mode === 'hourly' ? 'Hourly' : 'Nightly';
+
+            const blocks = [
+              {
+                type: 'header',
+                text: { type: 'plain_text', text: `:rotating_light: ${modeLabel} Bench Failed`, emoji: true },
+              },
+              {
+                type: 'section',
+                text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>` },
+              },
+              {
+                type: 'actions',
+                elements: [{
+                  type: 'button',
+                  text: { type: 'plain_text', text: 'View Logs :github:', emoji: true },
+                  url: jobUrl,
+                  action_id: 'ci_button',
+                }],
+              },
+            ];
+
+            await fetch('https://slack.com/api/chat.postMessage', {
+              method: 'POST',
+              headers: {
+                'Authorization': `Bearer ${token}`,
+                'Content-Type': 'application/json',
+              },
+              body: JSON.stringify({
+                channel,
+                blocks,
+                text: `${modeLabel} bench failed while ${failedStep}`,
+                unfurl_links: false,
+              }),
+            });
+
+      - name: Restore system settings
+        if: always()
+        run: |
+          sudo systemctl start irqbalance cron atd 2>/dev/null || true
+
+  # ---------------------------------------------------------------------------
+  # Job 3: Save state on success
+  # ---------------------------------------------------------------------------
+  save-state:
+    needs: [resolve-refs, bench-scheduled]
+    if: success()
+    name: save-state
+    runs-on: ubuntu-latest
+    steps:
+      - name: Write state file
+        run: |
+          MODE="${{ needs.resolve-refs.outputs.mode }}"
+          STATE_DIR=".${MODE}-state"
+          mkdir -p "$STATE_DIR"
+          echo "${{ needs.resolve-refs.outputs.feature-ref }}" > "$STATE_DIR/last-feature-ref"
+
+      - name: Save state
+        uses: actions/cache/save@v4
+        with:
+          path: .${{ needs.resolve-refs.outputs.mode }}-state
+          key: bench-${{ needs.resolve-refs.outputs.mode }}-state-${{ needs.resolve-refs.outputs.feature-ref }}

.github/workflows/bench.yml

@@ -12,10 +12,15 @@ on:
   workflow_dispatch:
     inputs:
       blocks:
-        description: "Number of blocks to benchmark (or 'big' for big blocks mode)"
+        description: "Number of blocks to benchmark"
         required: false
         default: "500"
         type: string
+      big_blocks:
+        description: "Use big blocks mode (pre-generated merged payloads with reth-bb)"
+        required: false
+        default: "false"
+        type: boolean
       warmup:
         description: "Number of warmup blocks"
         required: false
@@ -71,11 +76,14 @@ on:
         required: false
         default: "true"
         type: boolean
+      otlp:
+        description: "Export OTLP traces and logs"
+        required: false
+        default: "true"
+        type: boolean
 
 env:
   CARGO_TERM_COLOR: always
-  BASELINE: base
-  SEED: reth
   RUSTC_WRAPPER: "sccache"
   BENCH_RUNNERS: 2
 
@@ -110,6 +118,7 @@ jobs:
       baseline-args: ${{ steps.args.outputs.baseline-args }}
       feature-args: ${{ steps.args.outputs.feature-args }}
       abba: ${{ steps.args.outputs.abba }}
+      otlp: ${{ steps.args.outputs.otlp }}
       comment-id: ${{ steps.ack.outputs.comment-id }}
     steps:
       - name: Check org membership
@@ -148,9 +157,10 @@ jobs:
               samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
               var noSlack = '${{ github.event.inputs.no_slack }}' !== 'false' ? 'true' : 'false';
               cores = '${{ github.event.inputs.cores }}' || '0';
-              bigBlocks = blocks === 'big' ? 'true' : 'false';
+              bigBlocks = '${{ github.event.inputs.big_blocks }}' === 'true' ? 'true' : 'false';
               var rethNewPayload = '${{ github.event.inputs.reth_newPayload }}' !== 'false' ? 'true' : 'false';
               var abba = '${{ github.event.inputs.abba }}' !== 'false' ? 'true' : 'false';
+              var otlp = '${{ github.event.inputs.otlp }}' !== 'false' ? 'true' : 'false';
               var waitTime = '${{ github.event.inputs.wait_time }}' || '';
               var baselineNodeArgs = '${{ github.event.inputs.baseline_args }}' || '';
               var featureNodeArgs = '${{ github.event.inputs.feature_args }}' || '';
@@ -173,14 +183,13 @@ jobs:
               actor = context.payload.comment.user.login;
 
               const body = context.payload.comment.body.trim();
-              const intArgs = new Set(['warmup', 'cores']);
-              const intOrKeywordArgs = new Map([['blocks', new Set(['big'])]]);
+              const intArgs = new Set(['warmup', 'cores', 'blocks']);
               const refArgs = new Set(['baseline', 'feature']);
-              const boolArgs = new Set(['samply', 'no-slack']);
-              const boolDefaultTrue = new Set(['reth_newPayload', 'abba']);
+              const boolArgs = new Set(['samply', 'no-slack', 'big-blocks']);
+              const boolDefaultTrue = new Set(['reth_newPayload', 'abba', 'otlp']);
               const durationArgs = new Set(['wait-time']);
               const stringArgs = new Set(['baseline-args', 'feature-args']);
-              const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
+              const defaults = { blocks: '500', warmup: '100', baseline: '', feature: '', samply: 'false', 'no-slack': 'false', 'big-blocks': 'false', cores: '0', reth_newPayload: 'true', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
               const unknown = [];
               const invalid = [];
               const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -225,15 +234,6 @@ jobs:
                   } else {
                     defaults[key] = value;
                   }
-                } else if (intOrKeywordArgs.has(key)) {
-                  const keywords = intOrKeywordArgs.get(key);
-                  if (keywords.has(value)) {
-                    defaults[key] = value;
-                  } else if (/^\d+$/.test(value)) {
-                    defaults[key] = value;
-                  } else {
-                    invalid.push(`\`${key}=${value}\` (must be a positive integer or one of: ${[...keywords].join(', ')})`);
-                  }
                 } else if (refArgs.has(key)) {
                   if (!value) {
                     invalid.push(`\`${key}=\` (must be a git ref)`);
@@ -250,7 +250,7 @@ jobs:
               if (unknown.length) errors.push(`Unknown argument(s): \`${unknown.join('`, `')}\``);
               if (invalid.length) errors.push(`Invalid value(s): ${invalid.join(', ')}`);
               if (errors.length) {
-                const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N|big] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
+                const msg = `❌ **Invalid bench command**\n\n${errors.join('\n')}\n\n**Usage:** \`@decofe bench [blocks=N] [big-blocks] [warmup=N] [baseline=REF] [feature=REF] [samply] [no-slack] [cores=N] [reth_newPayload=true|false] [abba=true|false] [otlp=true|false] [wait-time=DURATION] [baseline-args="..."] [feature-args="..."]\``;
                 await github.rest.issues.createComment({
                   owner: context.repo.owner,
                   repo: context.repo.repo,
@@ -267,9 +267,10 @@ jobs:
               samply = defaults.samply;
               var noSlack = defaults['no-slack'];
               cores = defaults.cores;
-              bigBlocks = blocks === 'big' ? 'true' : 'false';
+              bigBlocks = defaults['big-blocks'];
               var rethNewPayload = defaults.reth_newPayload;
               var abba = defaults.abba;
+              var otlp = defaults.otlp;
               var waitTime = defaults['wait-time'];
               var baselineNodeArgs = defaults['baseline-args'];
               var featureNodeArgs = defaults['feature-args'];
@@ -308,6 +309,7 @@ jobs:
             core.setOutput('baseline-args', baselineNodeArgs);
             core.setOutput('feature-args', featureNodeArgs);
             core.setOutput('abba', abba);
+            core.setOutput('otlp', otlp);
 
       - name: Acknowledge request
         id: ack
@@ -375,6 +377,8 @@ jobs:
             const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
             const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
             const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
+            const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
+            const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
             const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
             const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
             const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
@@ -382,7 +386,7 @@ jobs:
             const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
             const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
             const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
-            const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
+            const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
 
             const { data: comment } = await github.rest.issues.createComment({
               owner: context.repo.owner,
@@ -417,6 +421,8 @@ jobs:
             const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
             const abbaEnabled = '${{ steps.args.outputs.abba }}' !== 'false';
             const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
+            const otlpEnabled = '${{ steps.args.outputs.otlp }}' !== 'false';
+            const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
             const waitTimeVal = '${{ steps.args.outputs.wait-time }}';
             const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
             const baselineArgsVal = '${{ steps.args.outputs.baseline-args }}';
@@ -424,7 +430,7 @@ jobs:
             const featureArgsVal = '${{ steps.args.outputs.feature-args }}';
             const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
             const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
-            const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
+            const config = `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`;
             const runUrl = `${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId}`;
 
             const numRunners = parseInt(process.env.BENCH_RUNNERS) || 1;
@@ -476,7 +482,7 @@ jobs:
   reth-bench:
     needs: reth-bench-ack
     name: reth-bench
-    runs-on: [self-hosted, Linux, X64]
+    runs-on: [self-hosted, Linux, X64, available]
     timeout-minutes: 120
     env:
       BENCH_RPC_URL: https://ethereum.reth.rs/rpc
@@ -494,9 +500,13 @@ jobs:
       BENCH_BASELINE_ARGS: ${{ needs.reth-bench-ack.outputs.baseline-args }}
       BENCH_FEATURE_ARGS: ${{ needs.reth-bench-ack.outputs.feature-args }}
       BENCH_ABBA: ${{ needs.reth-bench-ack.outputs.abba }}
+      BENCH_OTLP: ${{ needs.reth-bench-ack.outputs.otlp }}
       BENCH_COMMENT_ID: ${{ needs.reth-bench-ack.outputs.comment-id }}
       BENCH_NO_SLACK: ${{ needs.reth-bench-ack.outputs.no-slack }}
+      BENCH_NODE_BIN: ${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
       BENCH_METRICS_ADDR: "127.0.0.1:9100"
+      BENCH_OTLP_TRACES_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_TRACES_ENDPOINT || '' }}
+      BENCH_OTLP_LOGS_ENDPOINT: ${{ needs.reth-bench-ack.outputs.otlp != 'false' && secrets.BENCH_OTLP_LOGS_ENDPOINT || '' }}
     steps:
       - name: Clean up previous bench-work
         run: sudo rm -rf "$BENCH_WORK_DIR" 2>/dev/null || true
@@ -557,6 +567,8 @@ jobs:
             const rethNPNote = !rethNP ? ', reth_newPayload: `disabled`' : '';
             const abbaEnabled = (process.env.BENCH_ABBA || 'true') !== 'false';
             const abbaNote = !abbaEnabled ? ', abba: `disabled`' : '';
+            const otlpEnabled = (process.env.BENCH_OTLP || 'true') !== 'false';
+            const otlpNote = !otlpEnabled ? ', otlp: `disabled`' : '';
             const waitTimeVal = process.env.BENCH_WAIT_TIME || '';
             const waitTimeNote = waitTimeVal ? `, wait-time: \`${waitTimeVal}\`` : '';
             const baselineArgsVal = process.env.BENCH_BASELINE_ARGS || '';
@@ -564,7 +576,7 @@ jobs:
             const featureArgsVal = process.env.BENCH_FEATURE_ARGS || '';
             const featureArgsNote = featureArgsVal ? `, feature-args: \`${featureArgsVal}\`` : '';
             const blocksDesc = bigBlocks ? 'blocks: `big`' : `${blocks} blocks, ${warmup} warmup blocks`;
-            core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
+            core.exportVariable('BENCH_CONFIG', `**Config:** ${blocksDesc}, baseline: \`${baseline}\`, feature: \`${feature}\`${samplyNote}${noSlackNote}${coresNote}${rethNPNote}${abbaNote}${otlpNote}${waitTimeNote}${baselineArgsNote}${featureArgsNote}`);
 
             const { buildBody } = require('./.github/scripts/bench-update-status.js');
             await github.rest.issues.updateComment({
@@ -707,14 +719,52 @@ jobs:
             core.setOutput('feature-ref', featureRef);
             core.setOutput('feature-name', featureName);
 
+      - name: Check big-blocks freshness
+        if: env.BENCH_BIG_BLOCKS == 'true'
+        id: big-blocks-check
+        run: |
+          set -euo pipefail
+          MC="mc --config-dir /home/ubuntu/.mc"
+          MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
+          HASH_FILE="$HOME/.reth-bench-big-blocks-hash"
+          echo "Fetching big-blocks manifest from $MANIFEST..."
+          BB_MANIFEST=$($MC cat "$MANIFEST" 2>/dev/null) || {
+            echo "::error::Failed to fetch big-blocks manifest from $MANIFEST"
+            exit 1
+          }
+          BASE_SNAPSHOT=$(echo "$BB_MANIFEST" | jq -r '.base_snapshot // empty')
+          if [ -z "$BASE_SNAPSHOT" ]; then
+            echo "::error::Big-blocks manifest missing base_snapshot field"
+            exit 1
+          fi
+          echo "Big-blocks base snapshot: $BASE_SNAPSHOT"
+          echo "BENCH_SNAPSHOT_NAME=${BASE_SNAPSHOT}" >> "$GITHUB_ENV"
+
+          REMOTE_HASH=$(echo "$BB_MANIFEST" | sha256sum | awk '{print $1}')
+          LOCAL_HASH=""
+          [ -f "$HASH_FILE" ] && LOCAL_HASH=$(cat "$HASH_FILE")
+          if [ "$REMOTE_HASH" = "$LOCAL_HASH" ]; then
+            echo "Big blocks up-to-date (hash: ${REMOTE_HASH:0:16}…)"
+            echo "needed=false" >> "$GITHUB_OUTPUT"
+          else
+            echo "Big blocks need update (local: ${LOCAL_HASH:+${LOCAL_HASH:0:16}…}${LOCAL_HASH:-<none>}, remote: ${REMOTE_HASH:0:16}…)"
+            echo "needed=true" >> "$GITHUB_OUTPUT"
+            echo "remote-hash=${REMOTE_HASH}" >> "$GITHUB_OUTPUT"
+          fi
+
       - name: Check if snapshot needs update
         id: snapshot-check
         run: |
-          if .github/scripts/bench-reth-snapshot.sh --check; then
-            echo "needed=false" >> "$GITHUB_OUTPUT"
-          else
-            echo "needed=true" >> "$GITHUB_OUTPUT"
-          fi
+          set +e
+          .github/scripts/bench-reth-snapshot.sh --check
+          rc=$?
+          set -e
+          case "$rc" in
+            0)  echo "needed=false" >> "$GITHUB_OUTPUT" ;;
+            10) echo "needed=true"  >> "$GITHUB_OUTPUT" ;;
+            *)  echo "::error::Snapshot check failed (exit $rc)"
+                exit "$rc" ;;
+          esac
 
       - name: Update status (snapshot needed)
         if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
@@ -723,7 +773,7 @@ jobs:
           github-token: ${{ secrets.DEREK_PAT }}
           script: |
             const s = require('./.github/scripts/bench-update-status.js');
-            await s({github, context, status: 'Building binaries & downloading snapshot...'});
+            await s({github, context, status: 'Building binaries (snapshot update pending)...'});
 
       - name: Prepare source dirs
         run: |
@@ -743,12 +793,11 @@ jobs:
           fi
           git -C ../reth-feature checkout "$FEATURE_REF"
 
-      - name: Build binaries and download snapshot in parallel
+      - name: Build binaries
         id: build
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
           BENCH_REPO: ${{ github.repository }}
-          SNAPSHOT_NEEDED: ${{ steps.snapshot-check.outputs.needed }}
         run: |
           BASELINE_DIR="$(cd ../reth-baseline && pwd)"
           FEATURE_DIR="$(cd ../reth-feature && pwd)"
@@ -758,21 +807,23 @@ jobs:
           .github/scripts/bench-reth-build.sh feature "${FEATURE_DIR}" "${{ steps.refs.outputs.feature-ref }}" &
           PID_FEATURE=$!
 
-          PID_SNAPSHOT=
-          if [ "$SNAPSHOT_NEEDED" = "true" ]; then
-            .github/scripts/bench-reth-snapshot.sh &
-            PID_SNAPSHOT=$!
-          fi
-
           FAIL=0
           wait $PID_BASELINE || FAIL=1
           wait $PID_FEATURE || FAIL=1
-          [ -n "$PID_SNAPSHOT" ] && { wait $PID_SNAPSHOT || FAIL=1; }
           if [ $FAIL -ne 0 ]; then
-            echo "::error::One or more parallel tasks failed (builds / snapshot download)"
+            echo "::error::One or more build tasks failed"
             exit 1
           fi
 
+      - name: Download snapshot
+        id: snapshot-download
+        if: steps.snapshot-check.outputs.needed == 'true'
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          BENCH_REPO: ${{ github.repository }}
+          BENCH_RETH_BINARY: ${{ github.workspace }}/../reth-feature/target/profiling/${{ needs.reth-bench-ack.outputs.big-blocks == 'true' && 'reth-bb' || 'reth' }}
+        run: .github/scripts/bench-reth-snapshot.sh
+
       # System tuning for reproducible benchmarks
       - name: System setup
         run: |
@@ -828,20 +879,40 @@ jobs:
         if: env.BENCH_BIG_BLOCKS == 'true'
         run: |
           set -euo pipefail
+          BIG_BLOCKS_DIR="$HOME/.reth-bench-big-blocks"
+          echo "BENCH_BIG_BLOCKS_DIR=${BIG_BLOCKS_DIR}" >> "$GITHUB_ENV"
+          if [ "${{ steps.big-blocks-check.outputs.needed }}" = "false" ]; then
+            echo "Big blocks cached at $BIG_BLOCKS_DIR, skipping download"
+            echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
+            exit 0
+          fi
           MC="mc --config-dir /home/ubuntu/.mc"
-          BUCKET="minio/reth-snapshots/reth-1-minimal-nightly-previous-big-blocks.tar.zst"
-          BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
+          MANIFEST="minio/reth-snapshots/reth-1-minimal-stable-big-blocks.json"
           rm -rf "$BIG_BLOCKS_DIR"; mkdir -p "$BIG_BLOCKS_DIR"
-          echo "Downloading big blocks from $BUCKET..."
-          $MC cat "$BUCKET" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
+
+          # Download and parse manifest
+          echo "Downloading manifest from $MANIFEST..."
+          $MC cat "$MANIFEST" > "$BIG_BLOCKS_DIR/manifest.json"
+          UPLOAD_PATH=$(jq -r '.upload_path' "$BIG_BLOCKS_DIR/manifest.json")
+          COUNT=$(jq -r '.count' "$BIG_BLOCKS_DIR/manifest.json")
+          TARGET_GAS=$(jq -r '.target_gas' "$BIG_BLOCKS_DIR/manifest.json")
+          echo "Manifest: count=$COUNT, target_gas=$TARGET_GAS, archive=$UPLOAD_PATH"
+
+          # Download and extract archive
+          ARCHIVE="minio/$UPLOAD_PATH"
+          echo "Downloading big blocks from $ARCHIVE..."
+          $MC cat "$ARCHIVE" | pzstd -d -p 6 | tar -xf - -C "$BIG_BLOCKS_DIR"
           echo "Big blocks downloaded to $BIG_BLOCKS_DIR"
+
           # Verify expected directory structure
-          if [ ! -d "$BIG_BLOCKS_DIR/gas-ramp-dir" ] || [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
-            echo "::error::Big blocks archive missing expected gas-ramp-dir/ or payloads/ directories"
+          if [ ! -d "$BIG_BLOCKS_DIR/payloads" ]; then
+            echo "::error::Big blocks archive missing expected payloads/ directory"
             ls -laR "$BIG_BLOCKS_DIR"
             exit 1
           fi
           echo "Payload files: $(find "$BIG_BLOCKS_DIR/payloads" -name '*.json' | wc -l)"
+          # Save manifest hash for freshness check on next run
+          echo "${{ steps.big-blocks-check.outputs.remote-hash }}" > "$HOME/.reth-bench-big-blocks-hash"
 
       - name: Start metrics proxy
         run: |
@@ -878,43 +949,49 @@ jobs:
         id: run-baseline-1
         env:
           BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
+          OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=baseline-1,run_type=baseline,git_ref=${{ steps.refs.outputs.baseline-ref }}"
         run: |
           cat > "$BENCH_LABELS_FILE" <<LABELS
           {"benchmark_run":"baseline-1","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
           LABELS
-          taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-1"
+          taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-1"
 
       - name: "Run benchmark: feature (1/2)"
         id: run-feature-1
         env:
           FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
+          OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=feature-1,run_type=feature,git_ref=${{ steps.refs.outputs.feature-ref }}"
         run: |
           cat > "$BENCH_LABELS_FILE" <<LABELS
           {"benchmark_run":"feature-1","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
           LABELS
-          taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-1"
+          taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-1"
 
       - name: "Run benchmark: feature (2/2)"
+        if: env.BENCH_ABBA != 'false'
         id: run-feature-2
         env:
           FEATURE_REF: ${{ steps.refs.outputs.feature-ref }}
+          OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=feature-2,run_type=feature,git_ref=${{ steps.refs.outputs.feature-ref }}"
         run: |
           cat > "$BENCH_LABELS_FILE" <<LABELS
           {"benchmark_run":"feature-2","run_type":"feature","git_ref":"${FEATURE_REF}","bench_sha":"${FEATURE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"$(date +%s)","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
           LABELS
-          taskset -c 0 .github/scripts/bench-reth-run.sh feature ../reth-feature/target/profiling/reth "$BENCH_WORK_DIR/feature-2"
+          taskset -c 0 .github/scripts/bench-reth-run.sh feature "../reth-feature/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/feature-2"
 
       - name: "Run benchmark: baseline (2/2)"
+        if: env.BENCH_ABBA != 'false'
         id: run-baseline-2
         env:
           BASELINE_REF: ${{ steps.refs.outputs.baseline-ref }}
+          OTEL_RESOURCE_ATTRIBUTES: "benchmark_id=${{ env.BENCH_ID }},benchmark_run=baseline-2,run_type=baseline,git_ref=${{ steps.refs.outputs.baseline-ref }}"
         run: |
           LAST_RUN_START=$(date +%s)
           echo "BENCH_LAST_RUN_START=${LAST_RUN_START}" >> "$GITHUB_ENV"
           cat > "$BENCH_LABELS_FILE" <<LABELS
           {"benchmark_run":"baseline-2","run_type":"baseline","git_ref":"${BASELINE_REF}","bench_sha":"${BASELINE_REF}","benchmark_id":"${BENCH_ID}","run_start_epoch":"${LAST_RUN_START}","reference_epoch":"${BENCH_REFERENCE_EPOCH}"}
           LABELS
-          taskset -c 0 .github/scripts/bench-reth-run.sh baseline ../reth-baseline/target/profiling/reth "$BENCH_WORK_DIR/baseline-2"
+          taskset -c 0 .github/scripts/bench-reth-run.sh baseline "../reth-baseline/target/profiling/${BENCH_NODE_BIN}" "$BENCH_WORK_DIR/baseline-2"
 
       - name: Stop metrics proxy & generate Grafana URL
         id: metrics
@@ -1043,11 +1120,6 @@ jobs:
           fi
           if [ "${BENCH_BIG_BLOCKS:-false}" = "true" ]; then
             SUMMARY_ARGS="$SUMMARY_ARGS --big-blocks"
-            # Read gas ramp blocks count from first baseline run (same for all runs)
-            GAS_RAMP_FILE="$BENCH_WORK_DIR/baseline-1/gas_ramp_blocks.txt"
-            if [ -f "$GAS_RAMP_FILE" ]; then
-              SUMMARY_ARGS="$SUMMARY_ARGS --gas-ramp-blocks $(cat "$GAS_RAMP_FILE" | tr -d '[:space:]')"
-            fi
           fi
           GRAFANA_URL='${{ steps.metrics.outputs.grafana-url }}'
           if [ -n "$GRAFANA_URL" ]; then
@@ -1187,11 +1259,22 @@ jobs:
                 comment_id: parseInt(ackCommentId),
                 body,
               });
-            } else {
-              // No PR — write results to job summary
-              await core.summary.addRaw(body).write();
             }
 
+      - name: Write job summary
+        if: success()
+        uses: actions/github-script@v8
+        with:
+          script: |
+            const jobSummary = require('./.github/scripts/bench-job-summary.js');
+            await jobSummary({
+              core,
+              context,
+              chartSha: '${{ steps.push-charts.outputs.sha }}',
+              grafanaUrl: '${{ steps.metrics.outputs.grafana-url }}',
+              runId: '${{ github.run_id }}',
+            });
+
       - name: Send Slack notification (success)
         if: success() && env.BENCH_NO_SLACK != 'true'
         uses: actions/github-script@v8
@@ -1211,7 +1294,8 @@ jobs:
           script: |
             const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
             const steps_status = [
-              ['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
+              ['building binaries', '${{ steps.build.outcome }}'],
+              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
               ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
               ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
               ...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),
@@ -1246,7 +1330,8 @@ jobs:
           script: |
             const abba = (process.env.BENCH_ABBA || 'true') !== 'false';
             const steps_status = [
-              ['building binaries${{ steps.snapshot-check.outputs.needed == 'true' && ' & downloading snapshot' || '' }}', '${{ steps.build.outcome }}'],
+              ['building binaries', '${{ steps.build.outcome }}'],
+              ['downloading snapshot', '${{ steps.snapshot-download.outcome }}'],
               ['running baseline benchmark (1/2)', '${{ steps.run-baseline-1.outcome }}'],
               ['running feature benchmark (1/2)', '${{ steps.run-feature-1.outcome }}'],
               ...(abba ? [['running feature benchmark (2/2)', '${{ steps.run-feature-2.outcome }}']] : []),

.github/workflows/docker-test.yml

@@ -6,7 +6,7 @@ on:
       hive_target:
         required: true
         type: string
-        description: "Docker bake target to build (e.g. hive-stable, hive-edge)"
+        description: "Docker bake target to build (e.g. hive)"
       artifact_name:
         required: false
         type: string

.github/workflows/docker.yml

@@ -28,8 +28,8 @@ on:
         required: false
         type: boolean
         default: false
-      disable_pgo:
-        description: "Disable PGO profiling entirely"
+      pgo:
+        description: "Enable PGO profiling"
         required: false
         type: boolean
         default: false
@@ -41,10 +41,10 @@ on:
 
 jobs:
   collect-pgo-profile:
-    if: github.repository == 'paradigmxyz/reth' && !(github.event_name == 'workflow_dispatch' && inputs.disable_pgo)
+    if: github.repository == 'paradigmxyz/reth' && github.event_name == 'workflow_dispatch' && inputs.pgo
     uses: ./.github/workflows/pgo-profile.yml
     with:
-      pgo_blocks: ${{ github.event_name == 'workflow_dispatch' && inputs.pgo_blocks || '20' }}
+      pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
     secrets: inherit
 
   build:
@@ -77,7 +77,7 @@ jobs:
           echo "dirty=false" >> "$GITHUB_OUTPUT"
 
       - name: Download pre-collected PGO profile
-        if: ${{ !(github.event_name == 'workflow_dispatch' && inputs.disable_pgo) }}
+        if: ${{ github.event_name == 'workflow_dispatch' && inputs.pgo }}
         uses: actions/download-artifact@v7
         with:
           name: pgo-profdata
@@ -86,11 +86,7 @@ jobs:
       - name: Configure PGO build args
         id: pgo
         run: |
-          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ "${{ inputs.disable_pgo }}" == "true" ]]; then
-            echo "use_pgo_bolt=false" >> "$GITHUB_OUTPUT"
-            echo "pgo_profdata=" >> "$GITHUB_OUTPUT"
-            echo "PGO disabled via workflow input"
-          else
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]] && [[ "${{ inputs.pgo }}" == "true" ]]; then
             if [ ! -f dist/merged.profdata ]; then
               echo "::error::Expected dist/merged.profdata from collect-pgo-profile job"
               exit 1
@@ -98,6 +94,10 @@ jobs:
             echo "use_pgo_bolt=true" >> "$GITHUB_OUTPUT"
             echo "pgo_profdata=dist/merged.profdata" >> "$GITHUB_OUTPUT"
             echo "Using pre-collected PGO profile from collect-pgo-profile job"
+          else
+            echo "use_pgo_bolt=false" >> "$GITHUB_OUTPUT"
+            echo "pgo_profdata=" >> "$GITHUB_OUTPUT"
+            echo "PGO disabled"
           fi
 
       - name: Determine build parameters
@@ -183,6 +183,8 @@ jobs:
             *Run:* <https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}|View logs>
 
             *Action required:* Re-run the workflow or investigate the build failure.
+
+            <@U0AAA8F0JEM> investigate and re-run if flaky
           SLACK_FOOTER: "paradigmxyz/reth · docker.yml"
           MSG_MINIMAL: true
           SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

.github/workflows/e2e.yml

@@ -63,6 +63,6 @@ jobs:
         run: |
           cargo nextest run \
             --no-fail-fast \
-            --locked --features "edge" \
+            --locked \
             -p reth-e2e-test-utils \
             -E 'binary(rocksdb)'

.github/workflows/hive.yml

@@ -15,18 +15,11 @@ concurrency:
   cancel-in-progress: true
 
 jobs:
-  build-reth-stable:
+  build-reth:
     uses: ./.github/workflows/docker-test.yml
     with:
-      hive_target: hive-stable
-      artifact_name: "reth-stable"
-    secrets: inherit
-
-  build-reth-edge:
-    uses: ./.github/workflows/docker-test.yml
-    with:
-      hive_target: hive-edge
-      artifact_name: "reth-edge"
+      hive_target: hive
+      artifact_name: "reth"
     secrets: inherit
 
   prepare-hive:
@@ -84,7 +77,6 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        storage: [stable, edge]
         # ethereum/rpc to be deprecated:
         # https://github.com/ethereum/hive/pull/1117
         scenario:
@@ -184,10 +176,9 @@ jobs:
           - sim: ethereum/eels/consume-rlp
             limit: .*tests/paris.*
     needs:
-      - build-reth-stable
-      - build-reth-edge
+      - build-reth
       - prepare-hive
-    name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
+    name: ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
     # Use larger runners for eels tests to avoid OOM runner crashes
     runs-on: ${{ github.repository == 'paradigmxyz/reth' && (contains(matrix.scenario.sim, 'eels') && 'depot-ubuntu-latest-8' || 'depot-ubuntu-latest-4') || 'ubuntu-latest' }}
     permissions:
@@ -206,7 +197,7 @@ jobs:
       - name: Download reth image
         uses: actions/download-artifact@v8
         with:
-          name: reth-${{ matrix.storage }}
+          name: reth
           path: /tmp
 
       - name: Load Docker images

.github/workflows/integration.yml

@@ -22,7 +22,7 @@ concurrency:
 
 jobs:
   test:
-    name: test / ${{ matrix.network }} / ${{ matrix.storage }}
+    name: test / ${{ matrix.network }}
     if: github.event_name != 'schedule'
     runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
     env:
@@ -30,7 +30,6 @@ jobs:
     strategy:
       matrix:
         network: ["ethereum"]
-        storage: ["stable", "edge"]
     timeout-minutes: 60
     steps:
       - uses: actions/checkout@v6
@@ -47,7 +46,7 @@ jobs:
         run: |
           cargo nextest run \
             --no-fail-fast \
-            --locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
+            --locked --features "asm-keccak ${{ matrix.network }}" \
             --workspace --exclude ef-tests \
             -E "kind(test) and not binary(e2e_testsuite)"
 

.github/workflows/lint.yml

@@ -237,31 +237,6 @@ jobs:
       - name: Ensure no arbitrary or proptest dependency on default build
         run: cargo tree --package reth -e=features,no-dev | grep -Eq "arbitrary|proptest" && exit 1 || exit 0
 
-  # Checks that selected crates can compile with power set of features
-  features:
-    name: features
-    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
-    timeout-minutes: 30
-    steps:
-      - uses: actions/checkout@v6
-      - uses: rui314/setup-mold@v1
-      - uses: dtolnay/rust-toolchain@clippy
-      - uses: mozilla-actions/sccache-action@v0.0.9
-      - uses: Swatinem/rust-cache@v2
-        with:
-          cache-on-failure: true
-      - name: cargo install cargo-hack
-        uses: taiki-e/install-action@cargo-hack
-      - run: |
-          cargo hack check \
-            --package reth-codecs \
-            --package reth-primitives-traits \
-            --package reth-primitives \
-            --feature-powerset \
-            --depth 2
-        env:
-          RUSTFLAGS: -D warnings
-
   # Check crates correctly propagate features
   feature-propagation:
     runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest' || 'ubuntu-latest' }}
@@ -297,7 +272,6 @@ jobs:
       - typos
       - grafana
       - no-test-deps
-      - features
       - feature-propagation
       - deny
     timeout-minutes: 30

.github/workflows/release.yml

@@ -13,6 +13,10 @@ on:
         description: "Enable dry run mode (builds artifacts but skips uploads and release creation)"
         type: boolean
         default: false
+      pgo:
+        description: "Enable PGO profiling"
+        type: boolean
+        default: false
       pgo_blocks:
         description: "Number of blocks to execute for PGO profiling on self-hosted runner"
         type: string
@@ -154,12 +158,14 @@ jobs:
           path: ${{ matrix.build.binary }}-${{ needs.extract-version.outputs.VERSION }}-${{ matrix.configs.target }}.tar.gz.asc
 
   collect-pgo-profile:
+    if: github.event_name == 'workflow_dispatch' && inputs.pgo
     uses: ./.github/workflows/pgo-profile.yml
     with:
-      pgo_blocks: ${{ github.event_name == 'workflow_dispatch' && inputs.pgo_blocks || '20' }}
+      pgo_blocks: ${{ inputs.pgo_blocks || '20' }}
     secrets: inherit
 
   build-pgo:
+    if: github.event_name == 'workflow_dispatch' && inputs.pgo
     name: build release (x86_64-linux PGO+BOLT)
     runs-on: [self-hosted, Linux, X64]
     needs: [extract-version, collect-pgo-profile]
@@ -221,14 +227,14 @@ jobs:
 
       - name: Upload artifact
         if: ${{ github.event.inputs.dry_run != 'true' }}
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
           path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz
 
       - name: Upload signature
         if: ${{ github.event.inputs.dry_run != 'true' }}
-        uses: actions/upload-artifact@v6
+        uses: actions/upload-artifact@v7
         with:
           name: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
           path: reth-${{ needs.extract-version.outputs.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc
@@ -237,7 +243,7 @@ jobs:
     name: draft release
     runs-on: ubuntu-latest
     needs: [build, build-pgo, extract-version]
-    if: ${{ github.event.inputs.dry_run != 'true' }}
+    if: ${{ !failure() && !cancelled() && github.event.inputs.dry_run != 'true' }}
     env:
       VERSION: ${{ needs.extract-version.outputs.VERSION }}
     permissions:

.github/workflows/unit.yml

@@ -19,15 +19,13 @@ concurrency:
 
 jobs:
   test:
-    name: test / ${{ matrix.type }} / ${{ matrix.storage }}
+    name: test / ${{ matrix.type }}
     runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
     env:
       RUST_BACKTRACE: 1
-      EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
     strategy:
       matrix:
         type: [ethereum]
-        storage: [stable, edge]
         include:
           - type: ethereum
             features: asm-keccak ethereum
@@ -50,14 +48,14 @@ jobs:
         run: |
           cargo nextest run \
             --no-fail-fast \
-            --features "${{ matrix.features }} $EDGE_FEATURES" --locked \
+            --features "${{ matrix.features }}" --locked \
             ${{ matrix.exclude_args }} --workspace \
             --exclude ef-tests --no-tests=warn \
             -E "!kind(test) and not binary(e2e_testsuite)"
 
   state:
     name: Ethereum state tests
-    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-4' || 'ubuntu-latest' }}
+    runs-on: ${{ github.repository == 'paradigmxyz/reth' && 'depot-ubuntu-latest-8' || 'ubuntu-latest' }}
     env:
       RUST_LOG: info,sync=error
       RUST_BACKTRACE: 1

AGENTS.md

@@ -0,0 +1,549 @@
+# Reth Development Guide for AI Agents
+
+This guide provides comprehensive instructions for AI agents working on the Reth codebase. It covers the architecture, development workflows, and critical guidelines for effective contributions.
+
+## Project Overview
+
+Reth is a high-performance Ethereum execution client written in Rust, focusing on modularity, performance, and contributor-friendliness. The codebase is organized into well-defined crates with clear boundaries and responsibilities.
+
+## Architecture Overview
+
+### Core Components
+
+1. **Consensus (`crates/consensus/`)**: Validates blocks according to Ethereum consensus rules
+2. **Storage (`crates/storage/`)**: Hybrid database using MDBX + static files for optimal performance
+3. **Networking (`crates/net/`)**: P2P networking stack with discovery, sync, and transaction propagation
+4. **RPC (`crates/rpc/`)**: JSON-RPC server supporting all standard Ethereum APIs
+5. **Execution (`crates/evm/`, `crates/ethereum/`)**: Transaction execution and state transitions
+6. **Pipeline (`crates/stages/`)**: Staged sync architecture for blockchain synchronization
+7. **Trie (`crates/trie/`)**: Merkle Patricia Trie implementation with parallel state root computation
+8. **Node Builder (`crates/node/`)**: High-level node orchestration and configuration
+9. **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated)
+
+### Key Design Principles
+
+- **Modularity**: Each crate can be used as a standalone library
+- **Performance**: Extensive use of parallelism, memory-mapped I/O, and optimized data structures
+- **Extensibility**: Traits and generic types allow for different chain implementations
+- **Type Safety**: Strong typing throughout with minimal use of dynamic dispatch
+
+## Development Workflow
+
+### Code Style and Standards
+
+1. **Formatting**: Always use nightly rustfmt
+   ```bash
+   cargo +nightly fmt --all
+   ```
+
+2. **Linting**: Run clippy with all features
+   ```bash
+   cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features 
+   ```
+
+3. **Testing**: Use nextest for faster test execution
+   ```bash
+   cargo nextest run --workspace
+   ```
+
+### Common Contribution Types
+
+Based on actual recent PRs, here are typical contribution patterns:
+
+#### 1. Small Bug Fixes (1-10 lines)
+Real example: Fixing beacon block root handling ([#16767](https://github.com/paradigmxyz/reth/pull/16767))
+```rust
+// Changed a single line to fix logic error
+- parent_beacon_block_root: parent.parent_beacon_block_root(),
++ parent_beacon_block_root: parent.parent_beacon_block_root().map(|_| B256::ZERO),
+```
+
+#### 2. Integration with Upstream Changes
+Real example: Integrating revm updates ([#16752](https://github.com/paradigmxyz/reth/pull/16752))
+```rust
+// Update code to use new APIs from dependencies
+- if self.fork_tracker.is_shanghai_activated() {
+-     if let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) {
++ if let Some(init_code_size_limit) = self.fork_tracker.max_initcode_size() {
++     if let Err(err) = transaction.ensure_max_init_code_size(init_code_size_limit) {
+```
+
+#### 3. Adding Comprehensive Tests
+Real example: ETH69 protocol tests ([#16759](https://github.com/paradigmxyz/reth/pull/16759))
+```rust
+#[tokio::test(flavor = "multi_thread")]
+async fn test_eth69_peers_can_connect() {
+    // Create test network with specific protocol versions
+    let p0 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into()));
+    // Test connection and version negotiation
+}
+```
+
+#### 4. Making Components Generic
+Real example: Making EthEvmConfig generic over chainspec ([#16758](https://github.com/paradigmxyz/reth/pull/16758))
+```rust
+// Before: Hardcoded to ChainSpec
+- pub struct EthEvmConfig<EvmFactory = EthEvmFactory> {
+-     pub executor_factory: EthBlockExecutorFactory<RethReceiptBuilder, Arc<ChainSpec>, EvmFactory>,
+
+// After: Generic over any chain spec type
++ pub struct EthEvmConfig<C = ChainSpec, EvmFactory = EthEvmFactory>
++ where
++     C: EthereumHardforks,
++ {
++     pub executor_factory: EthBlockExecutorFactory<RethReceiptBuilder, Arc<C>, EvmFactory>,
+```
+
+#### 5. Resource Management Improvements
+Real example: ETL directory cleanup ([#16770](https://github.com/paradigmxyz/reth/pull/16770))
+```rust
+// Add cleanup logic on startup
++ if let Err(err) = fs::remove_dir_all(&etl_path) {
++     warn!(target: "reth::cli", ?etl_path, %err, "Failed to remove ETL path on launch");
++ }
+```
+
+#### 6. Feature Additions
+Real example: Sharded mempool support ([#16756](https://github.com/paradigmxyz/reth/pull/16756))
+```rust
+// Add new filtering policies for transaction announcements
+pub struct ShardedMempoolAnnouncementFilter<T> {
+    pub inner: T,
+    pub shard_bits: u8,
+    pub node_id: Option<B256>,
+}
+```
+
+### Testing Guidelines
+
+1. **Unit Tests**: Test individual functions and components
+2. **Integration Tests**: Test interactions between components
+3. **Benchmarks**: For performance-critical code
+4. **Fuzz Tests**: For parsing and serialization code
+5. **Property Tests**: For checking component correctness on a wide variety of inputs
+
+Example test structure:
+```rust
+#[cfg(test)]
+mod tests {
+    use super::*;
+    
+    #[test]
+    fn test_component_behavior() {
+        // Arrange
+        let component = Component::new();
+        
+        // Act
+        let result = component.operation();
+        
+        // Assert
+        assert_eq!(result, expected);
+    }
+}
+```
+
+### Performance Considerations
+
+1. **Avoid Allocations in Hot Paths**: Use references and borrowing
+2. **Parallel Processing**: Use rayon for CPU-bound parallel work
+3. **Async/Await**: Use tokio for I/O-bound operations
+4. **File Operations**: Use `reth_fs_util` instead of `std::fs` for better error handling
+
+### Common Pitfalls
+
+1. **Don't Block Async Tasks**: Use `spawn_blocking` for CPU-intensive work or work with lots of blocking I/O
+2. **Handle Errors Properly**: Use `?` operator and proper error types
+
+### What to Avoid
+
+Based on PR patterns, avoid:
+
+1. **Large, sweeping changes**: Keep PRs focused and reviewable
+2. **Mixing unrelated changes**: One logical change per PR
+3. **Ignoring CI failures**: All checks must pass
+4. **Incomplete implementations**: Finish features before submitting
+5. **Modifying libmdbx sources**: Never modify files in `crates/storage/libmdbx-rs/mdbx-sys/libmdbx/` - this is vendored third-party code
+
+### CI Requirements
+
+Before submitting changes, ensure:
+
+1. **Format Check**: `cargo +nightly fmt --all --check`
+2. **Clippy**: No warnings
+3. **Tests Pass**: All unit and integration tests
+4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
+5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below)
+6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
+
+### CLI Reference Docs (`book` CI Job)
+
+The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless.
+
+When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running:
+
+```bash
+make update-book-cli
+```
+
+This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes.
+
+The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`.
+
+### Opening PRs against <https://github.com/paradigmxyz/reth>
+
+#### Titles
+
+Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope:
+
+```
+<type>(<scope>): <short description>
+```
+
+**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore`
+
+**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net`
+
+Examples:
+- `fix(rpc): correct gas estimation for ERC-20 transfers`
+- `perf: batch trie updates to reduce cursor overhead`
+- `feat(engine): add new_payload_interval metric`
+
+#### Descriptions
+
+Keep it short. Say what changed and why — nothing more.
+
+**Do:**
+- Write 1–3 sentences summarizing the change
+- Explain _why_ if the diff doesn't make it obvious
+- Link related issues or EIPs
+- Include benchmark numbers for perf changes
+
+**Don't:**
+- List every file changed — that's what the diff is for
+- Repeat the title in the body
+- Add "Files changed" or "Changes" sections
+- Write walls of text that go stale when the diff is updated
+- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage"
+
+**Template:**
+
+```
+Closes #<issue>
+
+<what changed, 1-3 sentences>
+
+<why, if not obvious from the diff>
+```
+
+**Good example:**
+
+```
+Closes #16800
+
+Adds fallback for external IP resolution so node startup doesn't fail
+when STUN is unreachable. Falls back to the configured default.
+```
+
+**Bad example:**
+
+```
+## Summary
+This PR introduces comprehensive improvements to the IP resolution system.
+
+## Changes
+- Modified `crates/net/discv4/src/lib.rs` to add fallback
+- Modified `crates/net/discv4/src/config.rs` to add default IP
+- Added tests in `crates/net/discv4/src/tests/ip.rs`
+
+## Files Changed
+- crates/net/discv4/src/lib.rs
+- crates/net/discv4/src/config.rs
+- crates/net/discv4/src/tests/ip.rs
+```
+
+#### Labels and CI
+
+Label PRs appropriately, first check the available labels and then apply the relevant ones:
+* when changes are RPC related, add A-rpc label
+* when changes are docs related, add C-docs label
+* ... and so on, check the available labels for more options.
+* if being tasked to open a pr, ensure that all changes are properly formatted: `cargo +nightly fmt --all`
+
+If changes in reth include changes to dependencies, run commands `zepter` and `make lint-toml` before finalizing the pr. Assume `zepter` binary is installed.
+
+### Debugging Tips
+
+1. **Logging**: Use `tracing` crate with appropriate levels
+   ```rust
+   tracing::debug!(target: "reth::component", ?value, "description");
+   ```
+
+2. **Metrics**: Add metrics for monitoring
+   ```rust
+   metrics::counter!("reth_component_operations").increment(1);
+   ```
+
+3. **Test Isolation**: Use separate test databases/directories
+
+### Finding Where to Contribute
+
+1. **Check Issues**: Look for issues labeled `good-first-issue` or `help-wanted`
+2. **Review TODOs**: Search for `TODO` comments in the codebase
+3. **Improve Tests**: Areas with low test coverage are good targets
+4. **Documentation**: Improve code comments and documentation
+5. **Performance**: Profile and optimize hot paths (with benchmarks)
+
+### Common PR Patterns
+
+#### Small, Focused Changes
+Most PRs change only 1-5 files. Examples:
+- Single-line bug fixes
+- Adding a missing trait implementation
+- Updating error messages
+- Adding test cases for edge conditions
+
+#### Integration Work
+When dependencies update (especially revm), code needs updating:
+- Check for breaking API changes
+- Update to use new features (like EIP implementations)
+- Ensure compatibility with new versions
+
+#### Test Improvements
+Tests often need expansion for:
+- New protocol versions (ETH68, ETH69)
+- Edge cases in state transitions
+- Network behavior under specific conditions
+- Concurrent operations
+
+#### Making Code More Generic
+Common refactoring pattern:
+- Replace concrete types with generics
+- Add trait bounds for flexibility
+- Enable reuse across different chain types
+
+#### When to Comment
+
+Write comments that remain valuable after the PR is merged. Future readers won't have PR context - they only see the current code.
+
+##### ✅ DO: Add Value
+
+**Explain WHY and non-obvious behavior:**
+```rust
+// Process must handle allocations atomically to prevent race conditions
+// between dealloc on drop and concurrent limit checks
+unsafe impl GlobalAlloc for LimitedAllocator { ... }
+
+// Binary search requires sorted input. Panics on unsorted slices.
+fn find_index(items: &[Item], target: &Item) -> Option<usize>
+
+// Timeout set to 5s to match EVM block processing limits
+const TRACER_TIMEOUT: Duration = Duration::from_secs(5);
+```
+
+**Document constraints and assumptions:**
+```rust
+/// Returns heap size estimate.
+/// 
+/// Note: May undercount shared references (Rc/Arc). For precise
+/// accounting, combine with an allocator-based approach.
+fn deep_size_of(&self) -> usize
+```
+
+**Explain complex logic:**
+```rust
+// We reset limits at task start because tokio reuses threads in
+// spawn_blocking pool. Without reset, second task inherits first
+// task's allocation count and immediately hits limit.
+THREAD_ALLOCATED.with(|allocated| allocated.set(0));
+```
+
+##### ❌ DON'T: Describe Changes
+```rust
+// ❌ BAD - Describes the change, not the code
+// Changed from Vec to HashMap for O(1) lookups
+
+// ✅ GOOD - Explains the decision
+// HashMap provides O(1) symbol lookups during trace replay
+```
+```rust
+// ❌ BAD - PR-specific context
+// Fix for issue #234 where memory wasn't freed
+
+// ✅ GOOD - Documents the actual behavior
+// Explicitly drop allocations before limit check to ensure
+// accurate accounting
+```
+```rust
+// ❌ BAD - States the obvious
+// Increment counter
+counter += 1;
+
+// ✅ GOOD - Explains non-obvious purpose
+// Track allocations across all threads for global limit enforcement
+GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst);
+```
+
+✅ **Comment when:**
+- Non-obvious behavior or edge cases
+- Performance trade-offs
+- Safety requirements (unsafe blocks must always be documented)
+- Limitations or gotchas
+- Why simpler alternatives don't work
+
+❌ **Don't comment when:**
+- Code is self-explanatory
+- Just restating the code in English
+- Describing what changed in this PR
+
+##### The Test: "Will this make sense in 6 months?"
+
+Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful?
+
+
+#### Rust Style Guides
+
+##### Type Ordering in Files
+
+When defining structs, traits, and functions in a file, follow this ordering convention. The file's primary type (matching the file name) comes first, followed by supporting public types, then private types and helpers.
+
+```rust
+use ...;
+
+/// The primary type of this file (matches filename).
+pub struct PayloadProcessor { ... }
+
+impl PayloadProcessor { ... }
+
+// Followed by public auxiliary types that support the primary type
+
+/// Configuration for the processor.
+pub struct PayloadProcessorConfig { ... }
+
+/// Result type returned by processor operations.
+pub struct ProcessorResult { ... }
+
+// Followed by public traits related to the primary type
+
+pub trait ProcessorExt { ... }
+
+// Followed by private helper types
+
+struct InternalState { ... }
+
+// Followed by private helper functions
+
+fn validate_input() { ... }
+```
+
+❌ **Bad**: Adding new traits and auxiliary types **above** the file's primary type (see [#22133](https://github.com/paradigmxyz/reth/pull/22133)):
+
+```rust
+use ...;
+
+// ❌ BAD - new auxiliary struct added before the file's main type
+pub struct CacheWaitDurations { ... }
+
+// ❌ BAD - new trait added before the file's main type  
+pub trait WaitForCaches { ... }
+
+// The file's primary type is buried below unrelated additions
+pub struct PayloadProcessor { ... }
+```
+
+✅ **Good**: New types go **after** the primary type:
+
+```rust
+use ...;
+
+// ✅ The file's primary type stays at the top
+pub struct PayloadProcessor { ... }
+
+impl PayloadProcessor { ... }
+
+// ✅ Auxiliary types follow the primary type
+pub struct CacheWaitDurations { ... }
+
+pub trait WaitForCaches { ... }
+
+impl WaitForCaches for PayloadProcessor { ... }
+```
+
+### Example Contribution Workflow
+
+Let's say you want to fix a bug where external IP resolution fails on startup:
+
+1. **Create a branch**:
+   ```bash
+   git checkout -b fix-external-ip-resolution
+   ```
+
+2. **Find the relevant code**:
+   ```bash
+   # Search for IP resolution code
+   rg "external.*ip" --type rust
+   ```
+
+3. **Reason about the problem, when the problem is identified, make the fix**:
+   ```rust
+   // In crates/net/discv4/src/lib.rs
+   pub fn resolve_external_ip() -> Option<IpAddr> {
+       // Add fallback mechanism
+       nat::external_ip()
+           .or_else(|| nat::external_ip_from_stun())
+           .or_else(|| Some(DEFAULT_IP))
+   }
+   ```
+
+4. **Add a test**:
+   ```rust
+   #[test]
+   fn test_external_ip_fallback() {
+       // Test that resolution has proper fallbacks
+   }
+   ```
+
+5. **Run checks** (IMPORTANT!):
+   ```bash
+   cargo +nightly fmt --all
+   cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
+   cargo nextest run -p reth-discv4
+   ```
+
+6. **Commit with clear message**:
+   ```bash
+   git commit -m "fix: add fallback for external IP resolution
+
+   Previously, node startup could fail if external IP resolution
+   failed. This adds fallback mechanisms to ensure the node can
+   always start with a reasonable default."
+   ```
+
+## Quick Reference
+
+### Essential Commands
+
+```bash
+# Format code
+cargo +nightly fmt --all
+
+# Run lints
+cargo +nightly clippy --workspace --all-features
+
+# Run tests
+cargo nextest run --workspace
+
+# Run specific benchmark
+cargo bench --bench bench_name
+
+# Build optimized binary
+cargo build --release
+
+# Check compilation for all features
+cargo check --workspace --all-features
+
+# Check documentation
+cargo docs --document-private-items
+
+# Regenerate CLI reference docs (after CLI changes)
+make update-book-cli
+```

CLAUDE.md

@@ -1,549 +0,0 @@
-# Reth Development Guide for AI Agents
-
-This guide provides comprehensive instructions for AI agents working on the Reth codebase. It covers the architecture, development workflows, and critical guidelines for effective contributions.
-
-## Project Overview
-
-Reth is a high-performance Ethereum execution client written in Rust, focusing on modularity, performance, and contributor-friendliness. The codebase is organized into well-defined crates with clear boundaries and responsibilities.
-
-## Architecture Overview
-
-### Core Components
-
-1. **Consensus (`crates/consensus/`)**: Validates blocks according to Ethereum consensus rules
-2. **Storage (`crates/storage/`)**: Hybrid database using MDBX + static files for optimal performance
-3. **Networking (`crates/net/`)**: P2P networking stack with discovery, sync, and transaction propagation
-4. **RPC (`crates/rpc/`)**: JSON-RPC server supporting all standard Ethereum APIs
-5. **Execution (`crates/evm/`, `crates/ethereum/`)**: Transaction execution and state transitions
-6. **Pipeline (`crates/stages/`)**: Staged sync architecture for blockchain synchronization
-7. **Trie (`crates/trie/`)**: Merkle Patricia Trie implementation with parallel state root computation
-8. **Node Builder (`crates/node/`)**: High-level node orchestration and configuration
-9. **The Consensus Engine (`crates/engine/`)**: Handles processing blocks received from the consensus layer with the Engine API (newPayload, forkchoiceUpdated)
-
-### Key Design Principles
-
-- **Modularity**: Each crate can be used as a standalone library
-- **Performance**: Extensive use of parallelism, memory-mapped I/O, and optimized data structures
-- **Extensibility**: Traits and generic types allow for different chain implementations
-- **Type Safety**: Strong typing throughout with minimal use of dynamic dispatch
-
-## Development Workflow
-
-### Code Style and Standards
-
-1. **Formatting**: Always use nightly rustfmt
-   ```bash
-   cargo +nightly fmt --all
-   ```
-
-2. **Linting**: Run clippy with all features
-   ```bash
-   cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features 
-   ```
-
-3. **Testing**: Use nextest for faster test execution
-   ```bash
-   cargo nextest run --workspace
-   ```
-
-### Common Contribution Types
-
-Based on actual recent PRs, here are typical contribution patterns:
-
-#### 1. Small Bug Fixes (1-10 lines)
-Real example: Fixing beacon block root handling ([#16767](https://github.com/paradigmxyz/reth/pull/16767))
-```rust
-// Changed a single line to fix logic error
-- parent_beacon_block_root: parent.parent_beacon_block_root(),
-+ parent_beacon_block_root: parent.parent_beacon_block_root().map(|_| B256::ZERO),
-```
-
-#### 2. Integration with Upstream Changes
-Real example: Integrating revm updates ([#16752](https://github.com/paradigmxyz/reth/pull/16752))
-```rust
-// Update code to use new APIs from dependencies
-- if self.fork_tracker.is_shanghai_activated() {
--     if let Err(err) = transaction.ensure_max_init_code_size(MAX_INIT_CODE_BYTE_SIZE) {
-+ if let Some(init_code_size_limit) = self.fork_tracker.max_initcode_size() {
-+     if let Err(err) = transaction.ensure_max_init_code_size(init_code_size_limit) {
-```
-
-#### 3. Adding Comprehensive Tests
-Real example: ETH69 protocol tests ([#16759](https://github.com/paradigmxyz/reth/pull/16759))
-```rust
-#[tokio::test(flavor = "multi_thread")]
-async fn test_eth69_peers_can_connect() {
-    // Create test network with specific protocol versions
-    let p0 = PeerConfig::with_protocols(NoopProvider::default(), Some(EthVersion::Eth69.into()));
-    // Test connection and version negotiation
-}
-```
-
-#### 4. Making Components Generic
-Real example: Making EthEvmConfig generic over chainspec ([#16758](https://github.com/paradigmxyz/reth/pull/16758))
-```rust
-// Before: Hardcoded to ChainSpec
-- pub struct EthEvmConfig<EvmFactory = EthEvmFactory> {
--     pub executor_factory: EthBlockExecutorFactory<RethReceiptBuilder, Arc<ChainSpec>, EvmFactory>,
-
-// After: Generic over any chain spec type
-+ pub struct EthEvmConfig<C = ChainSpec, EvmFactory = EthEvmFactory>
-+ where
-+     C: EthereumHardforks,
-+ {
-+     pub executor_factory: EthBlockExecutorFactory<RethReceiptBuilder, Arc<C>, EvmFactory>,
-```
-
-#### 5. Resource Management Improvements
-Real example: ETL directory cleanup ([#16770](https://github.com/paradigmxyz/reth/pull/16770))
-```rust
-// Add cleanup logic on startup
-+ if let Err(err) = fs::remove_dir_all(&etl_path) {
-+     warn!(target: "reth::cli", ?etl_path, %err, "Failed to remove ETL path on launch");
-+ }
-```
-
-#### 6. Feature Additions
-Real example: Sharded mempool support ([#16756](https://github.com/paradigmxyz/reth/pull/16756))
-```rust
-// Add new filtering policies for transaction announcements
-pub struct ShardedMempoolAnnouncementFilter<T> {
-    pub inner: T,
-    pub shard_bits: u8,
-    pub node_id: Option<B256>,
-}
-```
-
-### Testing Guidelines
-
-1. **Unit Tests**: Test individual functions and components
-2. **Integration Tests**: Test interactions between components
-3. **Benchmarks**: For performance-critical code
-4. **Fuzz Tests**: For parsing and serialization code
-5. **Property Tests**: For checking component correctness on a wide variety of inputs
-
-Example test structure:
-```rust
-#[cfg(test)]
-mod tests {
-    use super::*;
-    
-    #[test]
-    fn test_component_behavior() {
-        // Arrange
-        let component = Component::new();
-        
-        // Act
-        let result = component.operation();
-        
-        // Assert
-        assert_eq!(result, expected);
-    }
-}
-```
-
-### Performance Considerations
-
-1. **Avoid Allocations in Hot Paths**: Use references and borrowing
-2. **Parallel Processing**: Use rayon for CPU-bound parallel work
-3. **Async/Await**: Use tokio for I/O-bound operations
-4. **File Operations**: Use `reth_fs_util` instead of `std::fs` for better error handling
-
-### Common Pitfalls
-
-1. **Don't Block Async Tasks**: Use `spawn_blocking` for CPU-intensive work or work with lots of blocking I/O
-2. **Handle Errors Properly**: Use `?` operator and proper error types
-
-### What to Avoid
-
-Based on PR patterns, avoid:
-
-1. **Large, sweeping changes**: Keep PRs focused and reviewable
-2. **Mixing unrelated changes**: One logical change per PR
-3. **Ignoring CI failures**: All checks must pass
-4. **Incomplete implementations**: Finish features before submitting
-5. **Modifying libmdbx sources**: Never modify files in `crates/storage/libmdbx-rs/mdbx-sys/libmdbx/` - this is vendored third-party code
-
-### CI Requirements
-
-Before submitting changes, ensure:
-
-1. **Format Check**: `cargo +nightly fmt --all --check`
-2. **Clippy**: No warnings
-3. **Tests Pass**: All unit and integration tests
-4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
-5. **CLI Docs** (if CLI changed): Run `make update-book-cli` (see below)
-6. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
-
-### CLI Reference Docs (`book` CI Job)
-
-The CLI reference pages under `docs/vocs/docs/pages/cli/` are **auto-generated** from the `reth` binary's `--help` output. **Do not edit these files manually** — any hand edits will be overwritten and CI will fail regardless.
-
-When you add, remove, or modify CLI commands, subcommands, or flags, regenerate the CLI docs by running:
-
-```bash
-make update-book-cli
-```
-
-This builds `reth` in debug mode and runs `docs/cli/update.sh` to regenerate all CLI pages. Commit the resulting changes.
-
-The `book` CI job (`.github/workflows/lint.yml`) enforces this by regenerating the docs and running `git diff --exit-code`. If the committed docs don't match the generated output, CI fails. Manually editing these pages is never productive — always use `make update-book-cli`.
-
-### Opening PRs against <https://github.com/paradigmxyz/reth>
-
-#### Titles
-
-Use [Conventional Commits](https://www.conventionalcommits.org/) with an optional scope:
-
-```
-<type>(<scope>): <short description>
-```
-
-**Types**: `feat`, `fix`, `perf`, `refactor`, `docs`, `test`, `chore`
-
-**Scope** (optional): crate or area, e.g. `evm`, `trie`, `rpc`, `engine`, `net`
-
-Examples:
-- `fix(rpc): correct gas estimation for ERC-20 transfers`
-- `perf: batch trie updates to reduce cursor overhead`
-- `feat(engine): add new_payload_interval metric`
-
-#### Descriptions
-
-Keep it short. Say what changed and why — nothing more.
-
-**Do:**
-- Write 1–3 sentences summarizing the change
-- Explain _why_ if the diff doesn't make it obvious
-- Link related issues or EIPs
-- Include benchmark numbers for perf changes
-
-**Don't:**
-- List every file changed — that's what the diff is for
-- Repeat the title in the body
-- Add "Files changed" or "Changes" sections
-- Write walls of text that go stale when the diff is updated
-- Use filler like "This PR introduces...", "comprehensive", "robust", "enhance", "leverage"
-
-**Template:**
-
-```
-Closes #<issue>
-
-<what changed, 1-3 sentences>
-
-<why, if not obvious from the diff>
-```
-
-**Good example:**
-
-```
-Closes #16800
-
-Adds fallback for external IP resolution so node startup doesn't fail
-when STUN is unreachable. Falls back to the configured default.
-```
-
-**Bad example:**
-
-```
-## Summary
-This PR introduces comprehensive improvements to the IP resolution system.
-
-## Changes
-- Modified `crates/net/discv4/src/lib.rs` to add fallback
-- Modified `crates/net/discv4/src/config.rs` to add default IP
-- Added tests in `crates/net/discv4/src/tests/ip.rs`
-
-## Files Changed
-- crates/net/discv4/src/lib.rs
-- crates/net/discv4/src/config.rs
-- crates/net/discv4/src/tests/ip.rs
-```
-
-#### Labels and CI
-
-Label PRs appropriately, first check the available labels and then apply the relevant ones:
-* when changes are RPC related, add A-rpc label
-* when changes are docs related, add C-docs label
-* ... and so on, check the available labels for more options.
-* if being tasked to open a pr, ensure that all changes are properly formatted: `cargo +nightly fmt --all`
-
-If changes in reth include changes to dependencies, run commands `zepter` and `make lint-toml` before finalizing the pr. Assume `zepter` binary is installed.
-
-### Debugging Tips
-
-1. **Logging**: Use `tracing` crate with appropriate levels
-   ```rust
-   tracing::debug!(target: "reth::component", ?value, "description");
-   ```
-
-2. **Metrics**: Add metrics for monitoring
-   ```rust
-   metrics::counter!("reth_component_operations").increment(1);
-   ```
-
-3. **Test Isolation**: Use separate test databases/directories
-
-### Finding Where to Contribute
-
-1. **Check Issues**: Look for issues labeled `good-first-issue` or `help-wanted`
-2. **Review TODOs**: Search for `TODO` comments in the codebase
-3. **Improve Tests**: Areas with low test coverage are good targets
-4. **Documentation**: Improve code comments and documentation
-5. **Performance**: Profile and optimize hot paths (with benchmarks)
-
-### Common PR Patterns
-
-#### Small, Focused Changes
-Most PRs change only 1-5 files. Examples:
-- Single-line bug fixes
-- Adding a missing trait implementation
-- Updating error messages
-- Adding test cases for edge conditions
-
-#### Integration Work
-When dependencies update (especially revm), code needs updating:
-- Check for breaking API changes
-- Update to use new features (like EIP implementations)
-- Ensure compatibility with new versions
-
-#### Test Improvements
-Tests often need expansion for:
-- New protocol versions (ETH68, ETH69)
-- Edge cases in state transitions
-- Network behavior under specific conditions
-- Concurrent operations
-
-#### Making Code More Generic
-Common refactoring pattern:
-- Replace concrete types with generics
-- Add trait bounds for flexibility
-- Enable reuse across different chain types
-
-#### When to Comment
-
-Write comments that remain valuable after the PR is merged. Future readers won't have PR context - they only see the current code.
-
-##### ✅ DO: Add Value
-
-**Explain WHY and non-obvious behavior:**
-```rust
-// Process must handle allocations atomically to prevent race conditions
-// between dealloc on drop and concurrent limit checks
-unsafe impl GlobalAlloc for LimitedAllocator { ... }
-
-// Binary search requires sorted input. Panics on unsorted slices.
-fn find_index(items: &[Item], target: &Item) -> Option<usize>
-
-// Timeout set to 5s to match EVM block processing limits
-const TRACER_TIMEOUT: Duration = Duration::from_secs(5);
-```
-
-**Document constraints and assumptions:**
-```rust
-/// Returns heap size estimate.
-/// 
-/// Note: May undercount shared references (Rc/Arc). For precise
-/// accounting, combine with an allocator-based approach.
-fn deep_size_of(&self) -> usize
-```
-
-**Explain complex logic:**
-```rust
-// We reset limits at task start because tokio reuses threads in
-// spawn_blocking pool. Without reset, second task inherits first
-// task's allocation count and immediately hits limit.
-THREAD_ALLOCATED.with(|allocated| allocated.set(0));
-```
-
-##### ❌ DON'T: Describe Changes
-```rust
-// ❌ BAD - Describes the change, not the code
-// Changed from Vec to HashMap for O(1) lookups
-
-// ✅ GOOD - Explains the decision
-// HashMap provides O(1) symbol lookups during trace replay
-```
-```rust
-// ❌ BAD - PR-specific context
-// Fix for issue #234 where memory wasn't freed
-
-// ✅ GOOD - Documents the actual behavior
-// Explicitly drop allocations before limit check to ensure
-// accurate accounting
-```
-```rust
-// ❌ BAD - States the obvious
-// Increment counter
-counter += 1;
-
-// ✅ GOOD - Explains non-obvious purpose
-// Track allocations across all threads for global limit enforcement
-GLOBAL_COUNTER.fetch_add(1, Ordering::SeqCst);
-```
-
-✅ **Comment when:**
-- Non-obvious behavior or edge cases
-- Performance trade-offs
-- Safety requirements (unsafe blocks must always be documented)
-- Limitations or gotchas
-- Why simpler alternatives don't work
-
-❌ **Don't comment when:**
-- Code is self-explanatory
-- Just restating the code in English
-- Describing what changed in this PR
-
-##### The Test: "Will this make sense in 6 months?"
-
-Before adding a comment, ask: Would someone reading just the current code (no PR, no history) find this helpful?
-
-
-#### Rust Style Guides
-
-##### Type Ordering in Files
-
-When defining structs, traits, and functions in a file, follow this ordering convention. The file's primary type (matching the file name) comes first, followed by supporting public types, then private types and helpers.
-
-```rust
-use ...;
-
-/// The primary type of this file (matches filename).
-pub struct PayloadProcessor { ... }
-
-impl PayloadProcessor { ... }
-
-// Followed by public auxiliary types that support the primary type
-
-/// Configuration for the processor.
-pub struct PayloadProcessorConfig { ... }
-
-/// Result type returned by processor operations.
-pub struct ProcessorResult { ... }
-
-// Followed by public traits related to the primary type
-
-pub trait ProcessorExt { ... }
-
-// Followed by private helper types
-
-struct InternalState { ... }
-
-// Followed by private helper functions
-
-fn validate_input() { ... }
-```
-
-❌ **Bad**: Adding new traits and auxiliary types **above** the file's primary type (see [#22133](https://github.com/paradigmxyz/reth/pull/22133)):
-
-```rust
-use ...;
-
-// ❌ BAD - new auxiliary struct added before the file's main type
-pub struct CacheWaitDurations { ... }
-
-// ❌ BAD - new trait added before the file's main type  
-pub trait WaitForCaches { ... }
-
-// The file's primary type is buried below unrelated additions
-pub struct PayloadProcessor { ... }
-```
-
-✅ **Good**: New types go **after** the primary type:
-
-```rust
-use ...;
-
-// ✅ The file's primary type stays at the top
-pub struct PayloadProcessor { ... }
-
-impl PayloadProcessor { ... }
-
-// ✅ Auxiliary types follow the primary type
-pub struct CacheWaitDurations { ... }
-
-pub trait WaitForCaches { ... }
-
-impl WaitForCaches for PayloadProcessor { ... }
-```
-
-### Example Contribution Workflow
-
-Let's say you want to fix a bug where external IP resolution fails on startup:
-
-1. **Create a branch**:
-   ```bash
-   git checkout -b fix-external-ip-resolution
-   ```
-
-2. **Find the relevant code**:
-   ```bash
-   # Search for IP resolution code
-   rg "external.*ip" --type rust
-   ```
-
-3. **Reason about the problem, when the problem is identified, make the fix**:
-   ```rust
-   // In crates/net/discv4/src/lib.rs
-   pub fn resolve_external_ip() -> Option<IpAddr> {
-       // Add fallback mechanism
-       nat::external_ip()
-           .or_else(|| nat::external_ip_from_stun())
-           .or_else(|| Some(DEFAULT_IP))
-   }
-   ```
-
-4. **Add a test**:
-   ```rust
-   #[test]
-   fn test_external_ip_fallback() {
-       // Test that resolution has proper fallbacks
-   }
-   ```
-
-5. **Run checks** (IMPORTANT!):
-   ```bash
-   cargo +nightly fmt --all
-   cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
-   cargo nextest run -p reth-discv4
-   ```
-
-6. **Commit with clear message**:
-   ```bash
-   git commit -m "fix: add fallback for external IP resolution
-
-   Previously, node startup could fail if external IP resolution
-   failed. This adds fallback mechanisms to ensure the node can
-   always start with a reasonable default."
-   ```
-
-## Quick Reference
-
-### Essential Commands
-
-```bash
-# Format code
-cargo +nightly fmt --all
-
-# Run lints
-cargo +nightly clippy --workspace --all-features
-
-# Run tests
-cargo nextest run --workspace
-
-# Run specific benchmark
-cargo bench --bench bench_name
-
-# Build optimized binary
-cargo build --release
-
-# Check compilation for all features
-cargo check --workspace --all-features
-
-# Check documentation
-cargo docs --document-private-items
-
-# Regenerate CLI reference docs (after CLI changes)
-make update-book-cli
-```

CLAUDE.md

@@ -0,0 +1 @@
+AGENTS.md

Cargo.toml

@@ -9,8 +9,8 @@ exclude = [".github/"]
 
 [workspace]
 members = [
+    "bin/reth-bb/",
     "bin/reth-bench/",
-    "bin/reth-bench-compare/",
     "bin/reth/",
     "crates/storage/rpc-provider/",
     "crates/chain-state/",
@@ -27,6 +27,7 @@ members = [
     "crates/engine/invalid-block-hooks/",
     "crates/engine/local",
     "crates/engine/primitives/",
+    "crates/engine/execution-cache/",
     "crates/engine/tree/",
     "crates/engine/util/",
     "crates/era",
@@ -77,8 +78,6 @@ members = [
     "crates/payload/primitives/",
     "crates/payload/validator/",
     "crates/payload/util/",
-    "crates/primitives-traits/",
-    "crates/primitives/",
     "crates/prune/db",
     "crates/prune/prune",
     "crates/prune/types",
@@ -100,8 +99,6 @@ members = [
     "crates/stages/types/",
     "crates/static-file/static-file",
     "crates/static-file/types/",
-    "crates/storage/codecs/",
-    "crates/storage/codecs/derive/",
     "crates/storage/db-api/",
     "crates/storage/db-common",
     "crates/storage/db-models/",
@@ -112,7 +109,6 @@ members = [
     "crates/storage/nippy-jar/",
     "crates/storage/provider/",
     "crates/storage/storage-api/",
-    "crates/storage/zstd-compressors/",
     "crates/tasks/",
     "crates/tokio-util/",
     "crates/tracing/",
@@ -324,15 +320,14 @@ reth = { path = "bin/reth" }
 reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" }
 reth-basic-payload-builder = { path = "crates/payload/basic" }
 reth-bench = { path = "bin/reth-bench" }
-reth-bench-compare = { path = "bin/reth-bench-compare" }
 reth-chain-state = { path = "crates/chain-state" }
 reth-chainspec = { path = "crates/chainspec", default-features = false }
 reth-cli = { path = "crates/cli/cli" }
 reth-cli-commands = { path = "crates/cli/commands" }
 reth-cli-runner = { path = "crates/cli/runner" }
 reth-cli-util = { path = "crates/cli/util" }
-reth-codecs = { path = "crates/storage/codecs" }
-reth-codecs-derive = { path = "crates/storage/codecs/derive" }
+reth-codecs = { version = "0.1.0", default-features = false }
+reth-codecs-derive = "0.1.0"
 reth-config = { path = "crates/config", default-features = false }
 reth-consensus = { path = "crates/consensus/consensus", default-features = false }
 reth-consensus-common = { path = "crates/consensus/common", default-features = false }
@@ -348,6 +343,7 @@ reth-downloaders = { path = "crates/net/downloaders" }
 reth-e2e-test-utils = { path = "crates/e2e-test-utils" }
 reth-ecies = { path = "crates/net/ecies" }
 reth-engine-local = { path = "crates/engine/local" }
+reth-execution-cache = { path = "crates/engine/execution-cache" }
 reth-engine-primitives = { path = "crates/engine/primitives", default-features = false }
 reth-engine-tree = { path = "crates/engine/tree" }
 reth-engine-util = { path = "crates/engine/util" }
@@ -399,8 +395,7 @@ reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
 reth-payload-primitives = { path = "crates/payload/primitives" }
 reth-payload-validator = { path = "crates/payload/validator" }
 reth-payload-util = { path = "crates/payload/util" }
-reth-primitives = { path = "crates/primitives", default-features = false, features = ["__internal"] }
-reth-primitives-traits = { path = "crates/primitives-traits", default-features = false }
+reth-primitives-traits = { version = "0.1.0", default-features = false }
 reth-provider = { path = "crates/storage/provider" }
 reth-prune = { path = "crates/prune/prune" }
 reth-prune-types = { path = "crates/prune/types", default-features = false }
@@ -416,6 +411,7 @@ reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = fal
 reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
 reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
 reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
+reth-rpc-traits = { version = "0.1.0", default-features = false }
 reth-stages = { path = "crates/stages/stages" }
 reth-stages-api = { path = "crates/stages/api" }
 reth-stages-types = { path = "crates/stages/types", default-features = false }
@@ -434,7 +430,7 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
 reth-trie-db = { path = "crates/trie/db" }
 reth-trie-parallel = { path = "crates/trie/parallel" }
 reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
-reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-features = false }
+reth-zstd-compressors = { version = "0.1.0", default-features = false }
 
 # revm
 revm = { version = "36.0.0", default-features = false }
@@ -444,7 +440,6 @@ revm-state = { version = "10.0.0", default-features = false }
 revm-primitives = { version = "22.1.0", default-features = false }
 revm-interpreter = { version = "34.0.0", default-features = false }
 revm-database-interface = { version = "10.0.0", default-features = false }
-op-revm = { version = "17.0.0", default-features = false }
 revm-inspectors = "0.36.0"
 
 # eth
@@ -452,42 +447,42 @@ alloy-dyn-abi = "1.5.6"
 alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
 alloy-sol-types = { version = "1.5.6", default-features = false }
 
-alloy-chains = { version = "0.2.5", default-features = false }
+alloy-chains = { version = "0.2.33", default-features = false }
 alloy-eip2124 = { version = "0.2.0", default-features = false }
 alloy-eip7928 = { version = "0.3.0", default-features = false }
-alloy-evm = { version = "0.29.2", default-features = false }
+alloy-evm = { version = "0.30.0", default-features = false }
 alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
 alloy-trie = { version = "0.9.4", default-features = false }
 
 alloy-hardforks = "0.4.5"
 
-alloy-consensus = { version = "1.7.3", default-features = false }
-alloy-contract = { version = "1.7.3", default-features = false }
-alloy-eips = { version = "1.7.3", default-features = false }
-alloy-genesis = { version = "1.7.3", default-features = false }
-alloy-json-rpc = { version = "1.7.3", default-features = false }
-alloy-network = { version = "1.7.3", default-features = false }
-alloy-network-primitives = { version = "1.7.3", default-features = false }
-alloy-provider = { version = "1.7.3", features = ["reqwest", "debug-api"], default-features = false }
-alloy-pubsub = { version = "1.7.3", default-features = false }
-alloy-rpc-client = { version = "1.7.3", default-features = false }
-alloy-rpc-types = { version = "1.7.3", features = ["eth"], default-features = false }
-alloy-rpc-types-admin = { version = "1.7.3", default-features = false }
-alloy-rpc-types-anvil = { version = "1.7.3", default-features = false }
-alloy-rpc-types-beacon = { version = "1.7.3", default-features = false }
-alloy-rpc-types-debug = { version = "1.7.3", default-features = false }
-alloy-rpc-types-engine = { version = "1.7.3", default-features = false }
-alloy-rpc-types-eth = { version = "1.7.3", default-features = false }
-alloy-rpc-types-mev = { version = "1.7.3", default-features = false }
-alloy-rpc-types-trace = { version = "1.7.3", default-features = false }
-alloy-rpc-types-txpool = { version = "1.7.3", default-features = false }
-alloy-serde = { version = "1.7.3", default-features = false }
-alloy-signer = { version = "1.7.3", default-features = false }
-alloy-signer-local = { version = "1.7.3", default-features = false }
-alloy-transport = { version = "1.7.3" }
-alloy-transport-http = { version = "1.7.3", features = ["reqwest-rustls-tls"], default-features = false }
-alloy-transport-ipc = { version = "1.7.3", default-features = false }
-alloy-transport-ws = { version = "1.7.3", default-features = false }
+alloy-consensus = { version = "1.8.2", default-features = false }
+alloy-contract = { version = "1.8.2", default-features = false }
+alloy-eips = { version = "1.8.2", default-features = false }
+alloy-genesis = { version = "1.8.2", default-features = false }
+alloy-json-rpc = { version = "1.8.2", default-features = false }
+alloy-network = { version = "1.8.2", default-features = false }
+alloy-network-primitives = { version = "1.8.2", default-features = false }
+alloy-provider = { version = "1.8.2", features = ["reqwest", "debug-api"], default-features = false }
+alloy-pubsub = { version = "1.8.2", default-features = false }
+alloy-rpc-client = { version = "1.8.2", default-features = false }
+alloy-rpc-types = { version = "1.8.2", features = ["eth"], default-features = false }
+alloy-rpc-types-admin = { version = "1.8.2", default-features = false }
+alloy-rpc-types-anvil = { version = "1.8.2", default-features = false }
+alloy-rpc-types-beacon = { version = "1.8.2", default-features = false }
+alloy-rpc-types-debug = { version = "1.8.2", default-features = false }
+alloy-rpc-types-engine = { version = "1.8.2", default-features = false }
+alloy-rpc-types-eth = { version = "1.8.2", default-features = false }
+alloy-rpc-types-mev = { version = "1.8.2", default-features = false }
+alloy-rpc-types-trace = { version = "1.8.2", default-features = false }
+alloy-rpc-types-txpool = { version = "1.8.2", default-features = false }
+alloy-serde = { version = "1.8.2", default-features = false }
+alloy-signer = { version = "1.8.2", default-features = false }
+alloy-signer-local = { version = "1.8.2", default-features = false }
+alloy-transport = { version = "1.8.2" }
+alloy-transport-http = { version = "1.8.2", features = ["reqwest-rustls-tls"], default-features = false }
+alloy-transport-ipc = { version = "1.8.2", default-features = false }
+alloy-transport-ws = { version = "1.8.2", default-features = false }
 
 # op
 op-alloy-rpc-types = { version = "0.24.0", default-features = false }
@@ -539,8 +534,7 @@ serde_json = { version = "1.0", default-features = false, features = ["alloc"] }
 serde_with = { version = "3", default-features = false, features = ["macros"] }
 sha2 = { version = "0.10", default-features = false }
 shlex = "1.3"
-# https://github.com/orlp/slotmap/pull/148
-slotmap = { git = "https://github.com/DaniPopes/slotmap.git", branch = "dani/shrink-methods" }
+slotmap = "1"
 smallvec = "1"
 strum = { version = "0.27", default-features = false }
 strum_macros = "0.27"
@@ -585,7 +579,7 @@ futures-util = { version = "0.3", default-features = false }
 hyper = "1.3"
 hyper-util = "0.1.5"
 pin-project = "1.0.12"
-reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
+reqwest = { version = "0.13", default-features = false, features = ["rustls", "stream"] }
 tracing-futures = "0.2"
 tower = "0.5"
 tower-http = "0.6"
@@ -651,6 +645,7 @@ ethereum_ssz_derive = "0.10.1"
 # allocators
 jemalloc_pprof = { version = "0.8", default-features = false }
 tikv-jemalloc-ctl = "0.6"
+tikv-jemalloc-sys = "0.6"
 tikv-jemallocator = "0.6"
 tracy-client = { version = "0.18.0", features = ["demangle"] }
 snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
@@ -685,7 +680,6 @@ memmap2 = "0.9.4"
 mev-share-sse = { version = "0.5.0", default-features = false }
 num-traits = "0.2.15"
 page_size = "0.6.0"
-parity-scale-codec = "3.2.1"
 plain_hasher = "0.2"
 pretty_assertions = "1.4"
 ratatui = { version = "0.30", default-features = false }
@@ -698,7 +692,7 @@ snap = "1.1.1"
 socket2 = { version = "0.6", default-features = false }
 sysinfo = { version = "0.38", default-features = false }
 tracing-journald = "0.3"
-tracing-logfmt = "=0.3.5"
+tracing-logfmt = "0.3.7"
 tracing-samply = "0.1"
 tracing-subscriber = { version = "0.3", default-features = false }
 tracing-tracy = "0.11"

Makefile

@@ -70,7 +70,7 @@ build-%-reproducible:
 	LC_ALL=C \
 	TZ=UTC \
 	JEMALLOC_OVERRIDE=/usr/lib/x86_64-linux-gnu/libjemalloc.a \
-	cargo build --bin reth --features "$(FEATURES) jemalloc-unprefixed" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu
+	cargo build --bin reth --features "$(FEATURES)" --profile "reproducible" --locked --target x86_64-unknown-linux-gnu
 
 .PHONY: build-debug
 build-debug: ## Build the reth binary into `target/debug` directory.

bin/reth-bb/Cargo.toml

@@ -0,0 +1,100 @@
+[package]
+name = "reth-bb"
+version.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+license.workspace = true
+homepage.workspace = true
+repository.workspace = true
+description = "Reth node configured for big block payload execution"
+
+[lints]
+workspace = true
+
+[dependencies]
+# reth
+reth-ethereum-cli.workspace = true
+reth-chainspec.workspace = true
+reth-ethereum-primitives.workspace = true
+reth-cli-util.workspace = true
+reth-node-core.workspace = true
+reth-node-ethereum.workspace = true
+reth-node-builder.workspace = true
+reth-node-api.workspace = true
+reth-ethereum-consensus.workspace = true
+reth-engine-primitives = { workspace = true, features = ["std"] }
+reth-engine-tree.workspace = true
+reth-primitives-traits.workspace = true
+reth-payload-primitives.workspace = true
+reth-provider.workspace = true
+reth-rpc-api.workspace = true
+reth-rpc-engine-api.workspace = true
+reth-evm.workspace = true
+reth-evm-ethereum.workspace = true
+reth-ethereum-forks.workspace = true
+reth-revm.workspace = true
+reth-consensus.workspace = true
+reth-chain-state.workspace = true
+reth-errors.workspace = true
+reth-storage-errors.workspace = true
+
+# alloy
+alloy-rpc-types = { workspace = true, features = ["engine"] }
+alloy-primitives.workspace = true
+alloy-rlp.workspace = true
+alloy-consensus.workspace = true
+alloy-eips.workspace = true
+alloy-evm.workspace = true
+
+# tracing
+tracing.workspace = true
+
+# misc
+clap = { workspace = true, features = ["derive", "env"] }
+jsonrpsee = { workspace = true, features = ["server", "macros"] }
+async-trait.workspace = true
+derive_more.workspace = true
+crossbeam-channel.workspace = true
+tokio = { workspace = true, features = ["sync"] }
+revm.workspace = true
+revm-primitives.workspace = true
+alloy-hardforks.workspace = true
+metrics.workspace = true
+
+# std
+eyre.workspace = true
+
+[features]
+default = [
+    "jemalloc",
+    "reth-cli-util/jemalloc",
+    "asm-keccak",
+    "min-debug-logs",
+]
+
+jemalloc = [
+    "reth-cli-util/jemalloc",
+    "reth-node-core/jemalloc",
+    "reth-ethereum-cli/jemalloc",
+    "reth-provider/jemalloc",
+]
+
+asm-keccak = [
+    "reth-node-core/asm-keccak",
+    "reth-ethereum-cli/asm-keccak",
+    "reth-node-ethereum/asm-keccak",
+    "alloy-primitives/asm-keccak",
+    "alloy-evm/asm-keccak",
+    "revm/asm-keccak",
+    "revm-primitives/asm-keccak",
+]
+
+min-debug-logs = [
+    "tracing/release_max_level_debug",
+    "reth-ethereum-cli/min-debug-logs",
+    "reth-node-core/min-debug-logs",
+]
+
+[[bin]]
+name = "reth-bb"
+path = "src/main.rs"

bin/reth-bb/README.md

@@ -0,0 +1,67 @@
+# reth-bb
+
+A modified reth node for benchmarking **big block** execution — payloads that merge transactions from multiple consecutive blocks into a single block to simulate high-gas workloads.
+
+> **Not for production use.** reth-bb disables some consensus-related validations to allow artificially large blocks. It is intended solely for performance benchmarking.
+
+## How it works
+
+reth-bb extends the standard Ethereum node with:
+
+1. **Multi-segment execution** — a custom `reth_newPayload` handler that accepts optional `BigBlockData` alongside the payload. When present, the block is executed in multiple segments, each with its own EVM environment (matching the original blocks that were merged).
+
+2. **Relaxed consensus** — the gas-limit bound-divisor check and blob gas validation are skipped, since merged blocks exceed single-block limits.
+
+## Quick start
+
+The full workflow has four steps: **build** binaries, **generate** big blocks,  **start** reth-bb, and **replay** the payloads.
+
+### Prerequisites
+
+- A synced reth datadir for the target chain (e.g. hoodi)
+- Rust toolchain
+
+### 1. Build
+
+```bash
+cargo build --profile profiling -p reth-bb -p reth-bench
+```
+
+### 2. Generate big blocks
+
+Fetch consecutive blocks from an RPC and merge them until a target gas is reached. Use `--from-block` set to the block number following the one the node is currently synced to (i.e. the next block the node would process):
+
+```bash
+reth-bench generate-big-block \
+    --rpc-url https://rpc.hoodi.ethpandaops.io \
+    --chain hoodi \
+    --from-block 910020 \
+    --target-gas 2G \
+    --num-big-blocks 5 \
+    --output-dir /tmp/payloads
+```
+
+This produces one JSON file per big block in the output directory.
+
+### 3. Start reth-bb
+
+```bash
+reth-bb node \
+    --datadir /data/reth/hoodi \
+    --chain hoodi \
+    --http --http.api debug,eth \
+    --authrpc.jwtsecret /tmp/jwt.hex \
+    -d
+```
+
+### 4. Replay payloads
+
+```bash
+reth-bench replay-payloads \
+    --engine-rpc-url http://localhost:8551 \
+    --jwt-secret /tmp/jwt.hex \
+    --payload-dir /tmp/payloads \
+    --reth-new-payload
+```
+
+The `--reth-new-payload` flag is required for big blocks — it uses the `reth_newPayload` endpoint which carries the multi-segment execution metadata.

bin/reth-bb/src/evm.rs

@@ -0,0 +1,570 @@
+//! Big-block executor.
+//!
+//! Provides [`BbBlockExecutor`] and [`BbBlockExecutorFactory`] which handle
+//! segment boundaries within big-block payloads.
+//!
+//! [`BbBlockExecutor`] wraps [`EthBlockExecutor`] and intercepts
+//! `execute_transaction` to apply segment-boundary changes.
+
+use crate::evm_config::BigBlockSegment;
+use alloy_eips::eip7685::Requests;
+use alloy_evm::{
+    block::{
+        BlockExecutionError, BlockExecutionResult, BlockExecutor, BlockExecutorFactory,
+        BlockExecutorFor, ExecutableTx, OnStateHook, StateChangeSource, StateDB,
+    },
+    eth::{EthBlockExecutionCtx, EthBlockExecutor, EthEvmContext, EthTxResult},
+    precompiles::PrecompilesMap,
+    Database, EthEvm, EthEvmFactory, Evm, FromRecoveredTx, FromTxWithEncoded,
+};
+use alloy_primitives::B256;
+use reth_ethereum_primitives::{Receipt, TransactionSigned};
+use reth_evm_ethereum::RethReceiptBuilder;
+use revm::{
+    context::{BlockEnv, TxEnv},
+    context_interface::result::{EVMError, HaltReason},
+    handler::PrecompileProvider,
+    interpreter::InterpreterResult,
+    primitives::hardfork::SpecId,
+    Inspector,
+};
+use std::sync::{Arc, Mutex};
+use tracing::{debug, trace};
+
+// ---------------------------------------------------------------------------
+// BbEvmPlan — runtime segment tracking state
+// ---------------------------------------------------------------------------
+
+/// Runtime state for segment boundary tracking.
+pub(crate) struct BbEvmPlan {
+    /// The segment boundaries and environments.
+    pub(crate) segments: Vec<BigBlockSegment>,
+    /// Index of the next segment to switch to (starts at 1).
+    pub(crate) next_segment: usize,
+    /// Number of user transactions executed so far.
+    pub(crate) tx_counter: usize,
+    /// Block hashes to seed for inter-segment BLOCKHASH resolution.
+    /// Includes both prior block hashes and inter-segment hashes.
+    pub(crate) block_hashes_to_seed: Vec<(u64, B256)>,
+}
+
+impl BbEvmPlan {
+    /// Creates a new `BbEvmPlan` from segments and hardfork flags.
+    pub(crate) fn new(segments: Vec<BigBlockSegment>) -> Self {
+        // Pre-compute all inter-segment block hashes.
+        let mut block_hashes_to_seed = Vec::new();
+        for seg in segments.iter().skip(1) {
+            let finished_block_number = seg.evm_env.block_env.number.saturating_to::<u64>() - 1;
+            let finished_block_hash = seg.ctx.parent_hash;
+            block_hashes_to_seed.push((finished_block_number, finished_block_hash));
+        }
+
+        Self { segments, next_segment: 1, tx_counter: 0, block_hashes_to_seed }
+    }
+
+    /// Returns the 256 block hashes relevant to a segment with the given block
+    /// number. BLOCKHASH can look back 256 blocks, so we select entries in
+    /// `[block_number - 256, block_number)`.
+    pub(crate) fn hashes_for_block(&self, block_number: u64) -> Vec<(u64, B256)> {
+        let min = block_number.saturating_sub(256);
+        self.block_hashes_to_seed
+            .iter()
+            .copied()
+            .filter(|(n, _)| *n >= min && *n < block_number)
+            .collect()
+    }
+}
+
+impl std::fmt::Debug for BbEvmPlan {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("BbEvmPlan")
+            .field("segments", &self.segments)
+            .field("next_segment", &self.next_segment)
+            .field("tx_counter", &self.tx_counter)
+            .field("block_hashes_to_seed", &self.block_hashes_to_seed)
+            .finish()
+    }
+}
+
+// ---------------------------------------------------------------------------
+// BbBlockExecutor — handles segment boundaries
+// ---------------------------------------------------------------------------
+
+/// Function pointer that seeds block hashes into the DB's block hash cache.
+///
+/// Injected from `ConfigureEvm::create_executor` where the concrete `State<DB>`
+/// type is known, allowing `BbBlockExecutor` to reseed the ring buffer at
+/// segment boundaries without requiring additional trait bounds on `DB`.
+pub(crate) type BlockHashSeeder<DB> = fn(&mut DB, &[(u64, B256)]);
+
+/// Block executor that wraps [`EthBlockExecutor`] and handles segment-boundary
+/// changes for big-block execution.
+///
+/// At segment boundaries, the inner executor is finished (applying its
+/// end-of-block logic: post-execution system calls, withdrawal balance
+/// increments) and a new one is constructed for the next segment (applying
+/// its start-of-block logic: EIP-2935/EIP-4788 system calls).
+///
+/// Gas counters reset at each boundary so that each segment's real gas limit
+/// is used (preserving correct GASLIMIT opcode behavior). Accumulated offsets
+/// are applied to receipts and totals in `finish()`.
+pub(crate) struct BbBlockExecutor<'a, DB, I, P, Spec>
+where
+    DB: Database,
+{
+    /// The inner executor. `None` transiently during `apply_segment_boundary`.
+    inner: Option<EthBlockExecutor<'a, EthEvm<DB, I, P>, Spec, RethReceiptBuilder>>,
+    plan: Option<BbEvmPlan>,
+    /// Requests accumulated from segments that have been finished at
+    /// boundaries. Merged into the final result in `finish()`.
+    accumulated_requests: Requests,
+    /// Cumulative gas used by all segments that have been finished at
+    /// boundaries. Added to receipts and the final gas total in `finish()`.
+    gas_used_offset: u64,
+    /// Cumulative blob gas used by all segments that have been finished at
+    /// boundaries.
+    blob_gas_used_offset: u64,
+    /// Shared state hook that survives inner executor finish/reconstruct
+    /// cycles at segment boundaries. Each inner executor receives a
+    /// forwarding hook that delegates to this shared instance.
+    shared_hook: Arc<Mutex<Option<Box<dyn OnStateHook>>>>,
+    /// Callback to reseed block hashes into the DB's cache at segment
+    /// boundaries. See [`BlockHashSeeder`].
+    block_hash_seeder: Option<BlockHashSeeder<DB>>,
+}
+
+impl<'a, DB, I, P, Spec> BbBlockExecutor<'a, DB, I, P, Spec>
+where
+    DB: StateDB,
+    I: Inspector<EthEvmContext<DB>>,
+    P: PrecompileProvider<EthEvmContext<DB>, Output = InterpreterResult>,
+    Spec: alloy_evm::eth::spec::EthExecutorSpec + Clone,
+    EthEvm<DB, I, P>: Evm<
+        DB = DB,
+        Tx = TxEnv,
+        HaltReason = HaltReason,
+        Error = EVMError<DB::Error>,
+        Spec = SpecId,
+        BlockEnv = BlockEnv,
+    >,
+    TxEnv: FromRecoveredTx<TransactionSigned> + FromTxWithEncoded<TransactionSigned>,
+{
+    pub(crate) fn new(
+        evm: EthEvm<DB, I, P>,
+        ctx: EthBlockExecutionCtx<'a>,
+        spec: Spec,
+        receipt_builder: RethReceiptBuilder,
+        plan: Option<BbEvmPlan>,
+        block_hash_seeder: Option<BlockHashSeeder<DB>>,
+    ) -> Self {
+        let inner = EthBlockExecutor::new(evm, ctx, spec, receipt_builder);
+        Self {
+            inner: Some(inner),
+            plan,
+            accumulated_requests: Requests::default(),
+            gas_used_offset: 0,
+            blob_gas_used_offset: 0,
+            shared_hook: Arc::new(Mutex::new(None)),
+            block_hash_seeder,
+        }
+    }
+
+    /// Creates a forwarding `OnStateHook` that delegates to the shared hook.
+    fn forwarding_hook(&self) -> Option<Box<dyn OnStateHook>> {
+        let shared = self.shared_hook.clone();
+        Some(Box::new(move |source: StateChangeSource, state: &revm::state::EvmState| {
+            if let Some(hook) = shared.lock().unwrap().as_mut() {
+                hook.on_state(source, state);
+            }
+        }))
+    }
+
+    const fn inner(&self) -> &EthBlockExecutor<'a, EthEvm<DB, I, P>, Spec, RethReceiptBuilder> {
+        self.inner.as_ref().expect("inner executor must exist")
+    }
+
+    const fn inner_mut(
+        &mut self,
+    ) -> &mut EthBlockExecutor<'a, EthEvm<DB, I, P>, Spec, RethReceiptBuilder> {
+        self.inner.as_mut().expect("inner executor must exist")
+    }
+
+    fn reseed_block_hashes_for(&mut self, block_number: u64) {
+        let Some(seeder) = self.block_hash_seeder else { return };
+        let hashes = match &self.plan {
+            Some(plan) => plan.hashes_for_block(block_number),
+            None => return,
+        };
+        seeder(self.inner_mut().evm_mut().db_mut(), &hashes);
+    }
+
+    fn maybe_apply_boundary(&mut self) -> Result<(), BlockExecutionError> {
+        loop {
+            let plan = match &self.plan {
+                Some(p) => p,
+                None => return Ok(()),
+            };
+
+            if plan.next_segment >= plan.segments.len() ||
+                plan.tx_counter != plan.segments[plan.next_segment].start_tx
+            {
+                return Ok(());
+            }
+
+            self.apply_segment_boundary()?;
+        }
+    }
+
+    fn apply_segment_boundary(&mut self) -> Result<(), BlockExecutionError> {
+        let plan = self.plan.as_mut().expect("plan must exist");
+        let seg_idx = plan.next_segment;
+        let prev_seg_idx = seg_idx - 1;
+
+        debug!(
+            target: "engine::bb::evm",
+            seg_idx,
+            tx_counter = plan.tx_counter,
+            "Applying segment boundary"
+        );
+
+        // Swap the inner executor's ctx to the finished segment's values so
+        // that finish() applies the correct withdrawals and post-execution
+        // system calls for that segment.
+        let prev_segment = &plan.segments[prev_seg_idx];
+        let prev_ctx = EthBlockExecutionCtx {
+            parent_hash: prev_segment.ctx.parent_hash,
+            parent_beacon_block_root: prev_segment.ctx.parent_beacon_block_root,
+            ommers: prev_segment.ctx.ommers,
+            withdrawals: prev_segment.ctx.withdrawals.clone(),
+            extra_data: prev_segment.ctx.extra_data.clone(),
+            tx_count_hint: prev_segment.ctx.tx_count_hint,
+        };
+
+        // Clone the next segment's data before we consume inner.
+        let new_segment = &plan.segments[seg_idx];
+        let new_block_env = new_segment.evm_env.block_env.clone();
+        let mut new_cfg_env = new_segment.evm_env.cfg_env.clone();
+        new_cfg_env.disable_base_fee = true;
+        let new_ctx = EthBlockExecutionCtx {
+            parent_hash: new_segment.ctx.parent_hash,
+            parent_beacon_block_root: new_segment.ctx.parent_beacon_block_root,
+            ommers: new_segment.ctx.ommers,
+            withdrawals: new_segment.ctx.withdrawals.clone(),
+            extra_data: new_segment.ctx.extra_data.clone(),
+            tx_count_hint: new_segment.ctx.tx_count_hint,
+        };
+
+        plan.next_segment += 1;
+
+        // Finish the inner executor for the completed segment. This applies
+        // post-execution system calls (EIP-7002/7251) and withdrawal balance
+        // increments via EthBlockExecutor::finish().
+        let mut inner = self.inner.take().expect("inner executor must exist");
+        inner.ctx = prev_ctx;
+        let spec = inner.spec.clone();
+        let receipt_builder = inner.receipt_builder;
+        let (mut evm, result) = inner.finish()?;
+
+        // Receipts already have globally-correct cumulative_gas_used (fixed
+        // up in commit_transaction). Update the offset with this segment's
+        // gas so that subsequent segments' receipts are adjusted correctly.
+        self.gas_used_offset += result.gas_used;
+        self.blob_gas_used_offset += result.blob_gas_used;
+        self.accumulated_requests.extend(result.requests);
+
+        let last_receipt_cumulative =
+            result.receipts.last().map(|r| r.cumulative_gas_used).unwrap_or(0);
+        let seg_block_number = prev_segment.evm_env.block_env.number.saturating_to::<u64>();
+        debug!(
+            target: "engine::bb::evm",
+            prev_seg_idx,
+            seg_block_number,
+            segment_gas_used = result.gas_used,
+            gas_used_offset = self.gas_used_offset,
+            last_receipt_cumulative,
+            receipt_count = result.receipts.len(),
+            "Finished segment"
+        );
+
+        // Swap EVM env to the next segment's values (using real gas_limit).
+        let ctx = evm.ctx_mut();
+        ctx.block = new_block_env;
+        ctx.cfg = new_cfg_env;
+
+        // Build a new inner executor for the next segment. gas_used starts
+        // at 0 so the per-transaction gas check uses this segment's real
+        // gas_limit correctly.
+        let mut new_inner = EthBlockExecutor::new(evm, new_ctx, spec, receipt_builder);
+
+        // Carry forward receipts from prior segments.
+        new_inner.receipts = result.receipts;
+
+        // Re-install the forwarding state hook so the parallel state root
+        // task continues to receive state changes.
+        if self.shared_hook.lock().unwrap().is_some() {
+            new_inner.set_state_hook(self.forwarding_hook());
+        }
+
+        self.inner = Some(new_inner);
+
+        // Reseed the block hash cache for the new segment's 256-block window
+        // before applying pre-execution changes (which may use BLOCKHASH).
+        let new_block_number = self.plan.as_ref().unwrap().segments[seg_idx]
+            .evm_env
+            .block_env
+            .number
+            .saturating_to::<u64>();
+        self.reseed_block_hashes_for(new_block_number);
+
+        // Apply pre-execution changes for the new segment (EIP-2935, EIP-4788).
+        self.inner_mut().apply_pre_execution_changes()?;
+
+        trace!(target: "engine::bb::evm", "Started segment {seg_idx}");
+
+        Ok(())
+    }
+}
+
+impl<'a, DB, I, P, Spec> BlockExecutor for BbBlockExecutor<'a, DB, I, P, Spec>
+where
+    DB: StateDB,
+    I: Inspector<EthEvmContext<DB>>,
+    P: PrecompileProvider<EthEvmContext<DB>, Output = InterpreterResult>,
+    Spec: alloy_evm::eth::spec::EthExecutorSpec + Clone,
+    EthEvm<DB, I, P>: Evm<
+        DB = DB,
+        Tx = TxEnv,
+        HaltReason = HaltReason,
+        Error = EVMError<DB::Error>,
+        Spec = SpecId,
+        BlockEnv = BlockEnv,
+    >,
+    TxEnv: FromRecoveredTx<TransactionSigned> + FromTxWithEncoded<TransactionSigned>,
+{
+    type Transaction = TransactionSigned;
+    type Receipt = Receipt;
+    type Evm = EthEvm<DB, I, P>;
+    type Result = EthTxResult<HaltReason, alloy_consensus::TxType>;
+
+    fn apply_pre_execution_changes(&mut self) -> Result<(), BlockExecutionError> {
+        // Swap the EVM's block_env and executor ctx to the first segment's
+        // values so that the initial EIP-2935/EIP-4788 system calls use the
+        // correct block number and parent hash. Without this, the outer big
+        // block header's block_number (which is synthetic) would be used,
+        // writing to wrong EIP-2935 slots and corrupting state.
+        if let Some(seg0) = self.plan.as_ref().map(|p| &p.segments[0]) {
+            let block_env = seg0.evm_env.block_env.clone();
+            let block_number = block_env.number.saturating_to::<u64>();
+            let mut cfg_env = seg0.evm_env.cfg_env.clone();
+            cfg_env.disable_base_fee = true;
+            let seg0_ctx = EthBlockExecutionCtx {
+                parent_hash: seg0.ctx.parent_hash,
+                parent_beacon_block_root: seg0.ctx.parent_beacon_block_root,
+                ommers: seg0.ctx.ommers,
+                withdrawals: seg0.ctx.withdrawals.clone(),
+                extra_data: seg0.ctx.extra_data.clone(),
+                tx_count_hint: seg0.ctx.tx_count_hint,
+            };
+
+            let inner = self.inner_mut();
+            let evm_ctx = inner.evm.ctx_mut();
+            evm_ctx.block = block_env;
+            evm_ctx.cfg = cfg_env;
+            inner.ctx = seg0_ctx;
+
+            self.reseed_block_hashes_for(block_number);
+        }
+
+        self.inner_mut().apply_pre_execution_changes()
+    }
+
+    fn execute_transaction_without_commit(
+        &mut self,
+        tx: impl ExecutableTx<Self>,
+    ) -> Result<Self::Result, BlockExecutionError> {
+        self.maybe_apply_boundary()?;
+        self.inner_mut().execute_transaction_without_commit(tx)
+    }
+
+    fn commit_transaction(&mut self, output: Self::Result) -> Result<u64, BlockExecutionError> {
+        let gas_used = self.inner_mut().commit_transaction(output)?;
+
+        // Fix up cumulative_gas_used on the just-committed receipt so that
+        // the receipt root task (which reads receipts incrementally) sees
+        // globally-correct values across all segments.
+        let offset = self.gas_used_offset;
+        if offset > 0 &&
+            let Some(receipt) = self.inner_mut().receipts.last_mut()
+        {
+            receipt.cumulative_gas_used += offset;
+        }
+
+        if let Some(plan) = &mut self.plan {
+            plan.tx_counter += 1;
+        }
+        Ok(gas_used)
+    }
+
+    fn finish(
+        mut self,
+    ) -> Result<(Self::Evm, BlockExecutionResult<Self::Receipt>), BlockExecutionError> {
+        // Swap the inner executor's ctx to the last segment's ctx so that
+        // EthBlockExecutor::finish() applies the correct withdrawal balance
+        // increments and post-execution system calls.
+        if let Some(last_seg) = self.plan.as_ref().map(|p| p.segments.last().unwrap()) {
+            let last_ctx = EthBlockExecutionCtx {
+                parent_hash: last_seg.ctx.parent_hash,
+                parent_beacon_block_root: last_seg.ctx.parent_beacon_block_root,
+                ommers: last_seg.ctx.ommers,
+                withdrawals: last_seg.ctx.withdrawals.clone(),
+                extra_data: last_seg.ctx.extra_data.clone(),
+                tx_count_hint: last_seg.ctx.tx_count_hint,
+            };
+            self.inner_mut().ctx = last_ctx;
+        }
+        let inner = self.inner.take().expect("inner executor must exist");
+        let (evm, mut result) = inner.finish()?;
+
+        // Receipts already have globally-correct cumulative_gas_used (fixed
+        // up in commit_transaction). Add the offset to the totals so they
+        // reflect gas across all segments.
+        let last_segment_gas = result.gas_used;
+        result.gas_used += self.gas_used_offset;
+        result.blob_gas_used += self.blob_gas_used_offset;
+
+        let last_receipt_cumulative =
+            result.receipts.last().map(|r| r.cumulative_gas_used).unwrap_or(0);
+        debug!(
+            target: "engine::bb::evm",
+            last_segment_gas,
+            gas_used_offset = self.gas_used_offset,
+            total_gas_used = result.gas_used,
+            last_receipt_cumulative,
+            receipt_count = result.receipts.len(),
+            "Finished final segment"
+        );
+
+        // Merge requests accumulated from earlier segment boundaries into
+        // the final result.
+        if !self.accumulated_requests.is_empty() {
+            let mut merged = self.accumulated_requests;
+            merged.extend(result.requests);
+            result.requests = merged;
+        }
+
+        Ok((evm, result))
+    }
+
+    fn set_state_hook(&mut self, hook: Option<Box<dyn OnStateHook>>) {
+        if self.plan.is_some() {
+            // Store the real hook in the shared slot and give the inner
+            // executor a lightweight forwarder. This way the hook survives
+            // inner executor finish/reconstruct cycles at segment boundaries.
+            *self.shared_hook.lock().unwrap() = hook;
+            let fwd = self.forwarding_hook();
+            self.inner_mut().set_state_hook(fwd);
+        } else {
+            self.inner_mut().set_state_hook(hook);
+        }
+    }
+
+    fn evm_mut(&mut self) -> &mut Self::Evm {
+        self.inner_mut().evm_mut()
+    }
+
+    fn evm(&self) -> &Self::Evm {
+        self.inner().evm()
+    }
+
+    fn receipts(&self) -> &[Self::Receipt] {
+        self.inner().receipts()
+    }
+}
+
+// ---------------------------------------------------------------------------
+// BbBlockExecutorFactory
+// ---------------------------------------------------------------------------
+
+/// Block executor factory that produces [`BbBlockExecutor`] for
+/// boundary-aware big-block execution.
+#[derive(Debug, Clone)]
+pub struct BbBlockExecutorFactory<Spec> {
+    receipt_builder: RethReceiptBuilder,
+    spec: Spec,
+    evm_factory: EthEvmFactory,
+    /// Staged plan consumed by the next [`BbBlockExecutor`].
+    pub(crate) staged_plan: Arc<Mutex<Option<BbEvmPlan>>>,
+}
+
+impl<Spec> BbBlockExecutorFactory<Spec> {
+    pub fn new(
+        receipt_builder: RethReceiptBuilder,
+        spec: Spec,
+        evm_factory: EthEvmFactory,
+    ) -> Self {
+        Self { receipt_builder, spec, evm_factory, staged_plan: Arc::new(Mutex::new(None)) }
+    }
+
+    pub const fn evm_factory(&self) -> &EthEvmFactory {
+        &self.evm_factory
+    }
+
+    pub const fn spec(&self) -> &Spec {
+        &self.spec
+    }
+
+    pub const fn receipt_builder(&self) -> &RethReceiptBuilder {
+        &self.receipt_builder
+    }
+
+    pub(crate) fn stage_plan(&self, plan: BbEvmPlan) {
+        *self.staged_plan.lock().unwrap() = Some(plan);
+    }
+
+    fn take_plan(&self) -> Option<BbEvmPlan> {
+        self.staged_plan.lock().unwrap().take()
+    }
+
+    pub(crate) fn create_executor_with_seeder<'a, DB, I>(
+        &'a self,
+        evm: EthEvm<DB, I, PrecompilesMap>,
+        ctx: EthBlockExecutionCtx<'a>,
+        block_hash_seeder: Option<BlockHashSeeder<DB>>,
+    ) -> BbBlockExecutor<'a, DB, I, PrecompilesMap, &'a Spec>
+    where
+        Spec: alloy_evm::eth::spec::EthExecutorSpec,
+        DB: StateDB + 'a,
+        I: Inspector<EthEvmContext<DB>> + 'a,
+    {
+        let plan = self.take_plan();
+        BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, block_hash_seeder)
+    }
+}
+
+impl<Spec> BlockExecutorFactory for BbBlockExecutorFactory<Spec>
+where
+    Spec: alloy_evm::eth::spec::EthExecutorSpec + 'static,
+    TxEnv: FromRecoveredTx<TransactionSigned> + FromTxWithEncoded<TransactionSigned>,
+{
+    type EvmFactory = EthEvmFactory;
+    type ExecutionCtx<'a> = EthBlockExecutionCtx<'a>;
+    type Transaction = TransactionSigned;
+    type Receipt = Receipt;
+
+    fn evm_factory(&self) -> &Self::EvmFactory {
+        &self.evm_factory
+    }
+
+    fn create_executor<'a, DB, I>(
+        &'a self,
+        evm: EthEvm<DB, I, PrecompilesMap>,
+        ctx: EthBlockExecutionCtx<'a>,
+    ) -> impl BlockExecutorFor<'a, Self, DB, I>
+    where
+        DB: StateDB + 'a,
+        I: Inspector<EthEvmContext<DB>> + 'a,
+    {
+        let plan = self.take_plan();
+        BbBlockExecutor::new(evm, ctx, &self.spec, self.receipt_builder, plan, None)
+    }
+}

bin/reth-bb/src/evm_config.rs

@@ -0,0 +1,291 @@
+//! Big-block EVM configuration.
+//!
+//! Wraps [`EthEvmConfig`] to create executors that handle multi-segment
+//! big-block execution internally. At transaction boundaries defined by
+//! [`BigBlockData`], the executor swaps the EVM environment (block env,
+//! cfg env) and applies pre/post execution changes for each segment.
+
+pub(crate) use reth_engine_primitives::BigBlockData;
+
+use crate::{
+    evm::{BbBlockExecutorFactory, BbEvmPlan},
+    BigBlockMap,
+};
+use alloy_consensus::Header;
+use alloy_evm::eth::EthBlockExecutionCtx;
+use alloy_primitives::B256;
+use alloy_rpc_types::engine::ExecutionData;
+use core::convert::Infallible;
+use reth_chainspec::{ChainSpec, EthChainSpec};
+use reth_ethereum_forks::Hardforks;
+use reth_ethereum_primitives::EthPrimitives;
+use reth_evm::{
+    ConfigureEngineEvm, ConfigureEvm, Database, EvmEnv, ExecutableTxIterator,
+    NextBlockEnvAttributes,
+};
+use reth_evm_ethereum::{EthBlockAssembler, EthEvmConfig, RethReceiptBuilder};
+use reth_primitives_traits::{SealedBlock, SealedHeader};
+use revm::primitives::hardfork::SpecId;
+use std::sync::Arc;
+use tracing::debug;
+
+use alloy_evm::{eth::spec::EthExecutorSpec, EthEvmFactory};
+use reth_evm::{EvmEnvFor, ExecutionCtxFor};
+
+// ---------------------------------------------------------------------------
+// Execution plan types
+// ---------------------------------------------------------------------------
+
+/// A single execution segment within a big block.
+#[derive(Debug, Clone)]
+pub(crate) struct BigBlockSegment {
+    /// Transaction index at which this segment starts.
+    pub start_tx: usize,
+    /// The EVM environment for this segment.
+    pub evm_env: EvmEnv,
+    /// The execution context for this segment.
+    pub ctx: EthBlockExecutionCtx<'static>,
+}
+
+// ---------------------------------------------------------------------------
+// BbEvmConfig
+// ---------------------------------------------------------------------------
+
+/// EVM configuration for big-block execution.
+///
+/// Wraps [`EthEvmConfig`] and a shared [`BigBlockMap`]. When a big-block
+/// payload is received, the plan is staged on the [`BbBlockExecutorFactory`]
+/// and consumed when the executor is created. Block hashes for inter-segment
+/// BLOCKHASH resolution are reseeded into `State::block_hashes` at each
+/// segment boundary via a [`BlockHashSeeder`](crate::evm::BlockHashSeeder)
+/// callback injected in [`ConfigureEvm::create_executor`].
+#[derive(Debug, Clone)]
+pub struct BbEvmConfig<C = ChainSpec> {
+    /// The inner Ethereum EVM configuration (used for env computation).
+    pub inner: EthEvmConfig<C>,
+    /// Shared map of pending big-block metadata.
+    pub pending: BigBlockMap,
+    /// Block executor factory for big-block execution.
+    executor_factory: BbBlockExecutorFactory<Arc<C>>,
+    /// Block assembler.
+    block_assembler: EthBlockAssembler<C>,
+}
+
+impl<C> BbEvmConfig<C> {
+    /// Creates a new big-block EVM configuration.
+    pub fn new(inner: EthEvmConfig<C>, pending: BigBlockMap) -> Self
+    where
+        C: Clone,
+    {
+        let chain_spec = inner.chain_spec().clone();
+        let executor_factory = BbBlockExecutorFactory::new(
+            RethReceiptBuilder::default(),
+            chain_spec,
+            EthEvmFactory::default(),
+        );
+        let block_assembler = inner.block_assembler.clone();
+
+        Self { inner, pending, executor_factory, block_assembler }
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Block hash seeder for State<DB>
+// ---------------------------------------------------------------------------
+
+/// Reseeds `State::block_hashes` with the given hashes.
+///
+/// This is used as a [`BlockHashSeeder`](crate::evm::BlockHashSeeder) callback,
+/// injected into [`BbBlockExecutor`](crate::evm::BbBlockExecutor) from
+/// `ConfigureEvm::create_executor` where the concrete `State<DB>` type is known.
+/// At each segment boundary the executor calls this to populate the ring buffer
+/// with the 256 block hashes relevant to the new segment's block number window.
+fn seed_state_block_hashes<DB>(state: &mut &mut revm::database::State<DB>, hashes: &[(u64, B256)]) {
+    for &(number, hash) in hashes {
+        state.block_hashes.insert(number, hash);
+    }
+}
+
+// ---------------------------------------------------------------------------
+// ConfigureEvm
+// ---------------------------------------------------------------------------
+
+impl<C> ConfigureEvm for BbEvmConfig<C>
+where
+    C: EthExecutorSpec + EthChainSpec<Header = Header> + Hardforks + 'static,
+{
+    type Primitives = EthPrimitives;
+    type Error = Infallible;
+    type NextBlockEnvCtx = NextBlockEnvAttributes;
+    type BlockExecutorFactory = BbBlockExecutorFactory<Arc<C>>;
+    type BlockAssembler = EthBlockAssembler<C>;
+
+    fn block_executor_factory(&self) -> &Self::BlockExecutorFactory {
+        &self.executor_factory
+    }
+
+    fn block_assembler(&self) -> &Self::BlockAssembler {
+        &self.block_assembler
+    }
+
+    fn evm_env(&self, header: &Header) -> Result<EvmEnv<SpecId>, Self::Error> {
+        self.inner.evm_env(header)
+    }
+
+    fn next_evm_env(
+        &self,
+        parent: &Header,
+        attributes: &NextBlockEnvAttributes,
+    ) -> Result<EvmEnv, Self::Error> {
+        self.inner.next_evm_env(parent, attributes)
+    }
+
+    fn context_for_block<'a>(
+        &self,
+        block: &'a SealedBlock<reth_ethereum_primitives::Block>,
+    ) -> Result<EthBlockExecutionCtx<'a>, Self::Error> {
+        self.inner.context_for_block(block)
+    }
+
+    fn context_for_next_block(
+        &self,
+        parent: &SealedHeader,
+        attributes: Self::NextBlockEnvCtx,
+    ) -> Result<EthBlockExecutionCtx<'_>, Self::Error> {
+        self.inner.context_for_next_block(parent, attributes)
+    }
+
+    fn create_executor<'a, DB, I>(
+        &'a self,
+        evm: reth_evm::EvmFor<Self, &'a mut revm::database::State<DB>, I>,
+        ctx: EthBlockExecutionCtx<'a>,
+    ) -> impl alloy_evm::block::BlockExecutorFor<
+        'a,
+        Self::BlockExecutorFactory,
+        &'a mut revm::database::State<DB>,
+        I,
+    >
+    where
+        DB: Database,
+        I: reth_evm::InspectorFor<Self, &'a mut revm::database::State<DB>> + 'a,
+    {
+        // Use create_executor_with_seeder to inject a concrete seeder that
+        // can reseed State::block_hashes at segment boundaries. The seeder
+        // is a function pointer that knows the concrete State<DB> type,
+        // allowing the generic BbBlockExecutor to reseed without additional
+        // trait bounds on DB.
+        self.executor_factory.create_executor_with_seeder(
+            evm,
+            ctx,
+            Some(seed_state_block_hashes::<DB>),
+        )
+    }
+}
+
+// ---------------------------------------------------------------------------
+// ConfigureEngineEvm — intercepts payload methods for big blocks
+// ---------------------------------------------------------------------------
+
+impl<C> ConfigureEngineEvm<ExecutionData> for BbEvmConfig<C>
+where
+    C: EthExecutorSpec + EthChainSpec<Header = Header> + Hardforks + 'static,
+{
+    fn evm_env_for_payload(&self, payload: &ExecutionData) -> Result<EvmEnvFor<Self>, Self::Error> {
+        let payload_hash = payload.block_hash();
+        let has_plan = self.pending.lock().unwrap().contains_key(&payload_hash);
+
+        if has_plan {
+            // Compute the env from the first segment BEFORE removing the
+            // entry (stage_plan_for_payload removes it).
+            let first_exec_data = {
+                let pending = self.pending.lock().unwrap();
+                let bb_data = pending.get(&payload_hash).unwrap();
+                bb_data.env_switches[0].1.clone()
+            };
+            let mut env = self.inner.evm_env_for_payload(&first_exec_data)?;
+
+            // Disable basefee validation: transactions from different
+            // original blocks may have gas prices below the big block's
+            // effective basefee.
+            env.cfg_env.disable_base_fee = true;
+
+            // Now stage the plan on the factory (removes the entry).
+            self.stage_plan_for_payload(&payload_hash);
+
+            Ok(env)
+        } else {
+            self.inner.evm_env_for_payload(payload)
+        }
+    }
+
+    fn context_for_payload<'a>(
+        &self,
+        payload: &'a ExecutionData,
+    ) -> Result<ExecutionCtxFor<'a, Self>, Self::Error> {
+        self.inner.context_for_payload(payload)
+    }
+
+    fn tx_iterator_for_payload(
+        &self,
+        payload: &ExecutionData,
+    ) -> Result<impl ExecutableTxIterator<Self>, Self::Error> {
+        self.inner.tx_iterator_for_payload(payload)
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Plan construction and staging
+// ---------------------------------------------------------------------------
+
+impl<C> BbEvmConfig<C>
+where
+    C: EthExecutorSpec + EthChainSpec<Header = Header> + Hardforks + 'static,
+{
+    /// Takes the big-block plan for a payload hash, builds a [`BbEvmPlan`],
+    /// and stages it on the factory.
+    ///
+    /// Must be called before `evm_with_env` is invoked for this payload.
+    /// In practice, this is called from `evm_env_for_payload` in the
+    /// engine pipeline.
+    pub fn stage_plan_for_payload(&self, payload_hash: &B256) {
+        let bb = match self.pending.lock().unwrap().remove(payload_hash) {
+            Some(bb) => bb,
+            None => return,
+        };
+
+        let segments: Vec<_> = bb
+            .env_switches
+            .into_iter()
+            .map(|(start_tx, exec_data)| {
+                let evm_env = self.inner.evm_env_for_payload(&exec_data).unwrap();
+                let ctx = self.inner.context_for_payload(&exec_data).unwrap();
+                let ctx = EthBlockExecutionCtx {
+                    tx_count_hint: ctx.tx_count_hint,
+                    parent_hash: ctx.parent_hash,
+                    parent_beacon_block_root: ctx.parent_beacon_block_root,
+                    ommers: &[],
+                    withdrawals: ctx.withdrawals.map(|w| std::borrow::Cow::Owned(w.into_owned())),
+                    extra_data: ctx.extra_data,
+                };
+                BigBlockSegment { start_tx, evm_env, ctx }
+            })
+            .collect();
+
+        debug!(
+            target: "engine::bb",
+            ?payload_hash,
+            segments = segments.len(),
+            seed_hashes = bb.prior_block_hashes.len(),
+            "Staging multi-segment plan"
+        );
+
+        let mut plan = BbEvmPlan::new(segments);
+
+        // Add prior block hashes to the seeding list.
+        plan.block_hashes_to_seed.extend(bb.prior_block_hashes);
+
+        plan.block_hashes_to_seed.sort_unstable_by_key(|(n, _)| *n);
+
+        self.executor_factory.stage_plan(plan);
+    }
+}

bin/reth-bb/src/main.rs

@@ -0,0 +1,374 @@
+//! reth-bb: a modified reth node for benchmarking big block execution.
+#![allow(missing_docs)]
+
+#[global_allocator]
+static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
+
+mod evm;
+mod evm_config;
+
+use alloy_primitives::B256;
+
+use alloy_rpc_types::engine::{ExecutionData, ForkchoiceState, ForkchoiceUpdated};
+use async_trait::async_trait;
+use clap::Parser;
+use evm_config::{BbEvmConfig, BigBlockData};
+use jsonrpsee::core::RpcResult;
+use reth_chainspec::{ChainSpec, EthChainSpec, EthereumHardforks, Hardforks};
+use reth_engine_primitives::ConsensusEngineHandle;
+use reth_ethereum_cli::{chainspec::EthereumChainSpecParser, interface::Cli};
+use reth_ethereum_consensus::EthBeaconConsensus;
+use reth_ethereum_primitives::EthPrimitives;
+use reth_evm_ethereum::EthEvmConfig;
+use reth_node_api::{AddOnsContext, FullNodeComponents, NodeTypes, PayloadTypes};
+use reth_node_builder::{
+    components::{
+        BasicPayloadServiceBuilder, ComponentsBuilder, ConsensusBuilder, ExecutorBuilder,
+    },
+    node::FullNodeTypes,
+    rpc::{
+        BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn,
+        EngineValidatorBuilder, PayloadValidatorBuilder, RethRpcAddOns, RpcAddOns, RpcHandle,
+        RpcHooks,
+    },
+    BuilderContext, Node,
+};
+use reth_node_ethereum::{
+    EthEngineTypes, EthereumEngineValidatorBuilder, EthereumEthApiBuilder, EthereumNetworkBuilder,
+    EthereumNode, EthereumPayloadBuilder, EthereumPoolBuilder,
+};
+use reth_payload_primitives::ExecutionPayload;
+use reth_primitives_traits::SealedBlock;
+use reth_provider::EthStorage;
+use reth_rpc_api::{RethNewPayloadInput, RethPayloadStatus};
+use reth_rpc_engine_api::EngineApiError;
+use std::{
+    collections::HashMap,
+    sync::{Arc, Mutex},
+};
+use tracing::{info, trace};
+
+/// Shared map for big block data, keyed by payload hash.
+pub type BigBlockMap = Arc<Mutex<HashMap<B256, BigBlockData<ExecutionData>>>>;
+
+// ---------------------------------------------------------------------------
+// Custom RPC trait for big-block payloads
+// ---------------------------------------------------------------------------
+
+/// Big-block extension of the `reth_` engine API.
+#[jsonrpsee::proc_macros::rpc(server, namespace = "reth")]
+pub trait BbRethEngineApi {
+    /// `reth_newPayload` with optional big-block data.
+    #[method(name = "newPayload")]
+    async fn reth_new_payload(
+        &self,
+        payload: RethNewPayloadInput<ExecutionData>,
+        wait_for_persistence: Option<bool>,
+        wait_for_caches: Option<bool>,
+        big_block_data: Option<BigBlockData<ExecutionData>>,
+    ) -> RpcResult<RethPayloadStatus>;
+
+    /// `reth_forkchoiceUpdated` – pass-through.
+    #[method(name = "forkchoiceUpdated")]
+    async fn reth_forkchoice_updated(
+        &self,
+        forkchoice_state: ForkchoiceState,
+    ) -> RpcResult<ForkchoiceUpdated>;
+}
+
+/// Server-side implementation of `BbRethEngineApi`.
+#[derive(Debug)]
+struct BbRethEngineApiHandler {
+    pending: BigBlockMap,
+    engine: ConsensusEngineHandle<EthEngineTypes>,
+}
+
+#[async_trait]
+impl BbRethEngineApiServer for BbRethEngineApiHandler {
+    async fn reth_new_payload(
+        &self,
+        input: RethNewPayloadInput<ExecutionData>,
+        wait_for_persistence: Option<bool>,
+        wait_for_caches: Option<bool>,
+        big_block_data: Option<BigBlockData<ExecutionData>>,
+    ) -> RpcResult<RethPayloadStatus> {
+        let wait_for_persistence = wait_for_persistence.unwrap_or(true);
+        let wait_for_caches = wait_for_caches.unwrap_or(true);
+        trace!(
+            target: "rpc::engine",
+            wait_for_persistence,
+            wait_for_caches,
+            has_big_block_data = big_block_data.is_some(),
+            "Serving bb reth_newPayload"
+        );
+
+        let payload = match input {
+            RethNewPayloadInput::ExecutionData(data) => data,
+            RethNewPayloadInput::BlockRlp(rlp) => {
+                let block = alloy_rlp::Decodable::decode(&mut rlp.as_ref())
+                    .map_err(|err| EngineApiError::Internal(Box::new(err)))?;
+                <EthEngineTypes as PayloadTypes>::block_to_payload(SealedBlock::new_unhashed(block))
+            }
+        };
+
+        if let Some(data) = big_block_data {
+            let hash = ExecutionPayload::block_hash(&payload);
+            self.pending.lock().unwrap().insert(hash, data);
+        }
+
+        let (status, timings) = self
+            .engine
+            .reth_new_payload(payload, wait_for_persistence, wait_for_caches)
+            .await
+            .map_err(EngineApiError::from)?;
+
+        Ok(RethPayloadStatus {
+            status,
+            latency_us: timings.latency.as_micros() as u64,
+            persistence_wait_us: timings.persistence_wait.map(|d| d.as_micros() as u64),
+            execution_cache_wait_us: timings.execution_cache_wait.map(|d| d.as_micros() as u64),
+            sparse_trie_wait_us: timings.sparse_trie_wait.map(|d| d.as_micros() as u64),
+        })
+    }
+
+    async fn reth_forkchoice_updated(
+        &self,
+        forkchoice_state: ForkchoiceState,
+    ) -> RpcResult<ForkchoiceUpdated> {
+        trace!(target: "rpc::engine", "Serving reth_forkchoiceUpdated");
+        self.engine
+            .fork_choice_updated(forkchoice_state, None)
+            .await
+            .map_err(|e| EngineApiError::from(e).into())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Node add-ons wrapper
+// ---------------------------------------------------------------------------
+
+/// Add-ons for the big-block node.
+#[derive(Debug)]
+pub struct BbAddOns {
+    pending: BigBlockMap,
+}
+
+impl BbAddOns {
+    const fn new(pending: BigBlockMap) -> Self {
+        Self { pending }
+    }
+
+    fn make_rpc_add_ons<N: FullNodeComponents>(
+        &self,
+    ) -> RpcAddOns<
+        N,
+        EthereumEthApiBuilder,
+        EthereumEngineValidatorBuilder,
+        BasicEngineApiBuilder<EthereumEngineValidatorBuilder>,
+        BasicEngineValidatorBuilder<EthereumEngineValidatorBuilder>,
+    >
+    where
+        EthereumEthApiBuilder: reth_node_builder::rpc::EthApiBuilder<N>,
+    {
+        RpcAddOns::new(
+            EthereumEthApiBuilder::default(),
+            EthereumEngineValidatorBuilder::default(),
+            BasicEngineApiBuilder::default(),
+            BasicEngineValidatorBuilder::default(),
+            Default::default(),
+        )
+    }
+}
+
+impl<N> reth_node_api::NodeAddOns<N> for BbAddOns
+where
+    N: FullNodeComponents<
+        Types: NodeTypes<
+            ChainSpec: EthereumHardforks + Hardforks + Clone + 'static,
+            Payload = EthEngineTypes,
+            Primitives = EthPrimitives,
+        >,
+    >,
+    EthereumEthApiBuilder: reth_node_builder::rpc::EthApiBuilder<N>,
+    EthereumEngineValidatorBuilder: PayloadValidatorBuilder<N>,
+    BasicEngineApiBuilder<EthereumEngineValidatorBuilder>: EngineApiBuilder<N>,
+    BasicEngineValidatorBuilder<EthereumEngineValidatorBuilder>: EngineValidatorBuilder<N>,
+{
+    type Handle =
+        RpcHandle<N, <EthereumEthApiBuilder as reth_node_builder::rpc::EthApiBuilder<N>>::EthApi>;
+
+    async fn launch_add_ons(self, ctx: AddOnsContext<'_, N>) -> eyre::Result<Self::Handle> {
+        let engine_handle = ctx.beacon_engine_handle.clone();
+        let pending = self.pending.clone();
+        let rpc_add_ons = self.make_rpc_add_ons::<N>();
+
+        rpc_add_ons
+            .launch_add_ons_with(ctx, move |container| {
+                let handler = BbRethEngineApiHandler { pending, engine: engine_handle };
+                let bb_module = BbRethEngineApiServer::into_rpc(handler);
+                container.auth_module.replace_auth_methods(bb_module.remove_context())?;
+                Ok(())
+            })
+            .await
+    }
+}
+
+impl<N> RethRpcAddOns<N> for BbAddOns
+where
+    N: FullNodeComponents<
+        Types: NodeTypes<
+            ChainSpec: EthereumHardforks + Hardforks + Clone + 'static,
+            Payload = EthEngineTypes,
+            Primitives = EthPrimitives,
+        >,
+    >,
+    EthereumEthApiBuilder: reth_node_builder::rpc::EthApiBuilder<N>,
+    EthereumEngineValidatorBuilder: PayloadValidatorBuilder<N>,
+    BasicEngineApiBuilder<EthereumEngineValidatorBuilder>: EngineApiBuilder<N>,
+    BasicEngineValidatorBuilder<EthereumEngineValidatorBuilder>: EngineValidatorBuilder<N>,
+{
+    type EthApi = <EthereumEthApiBuilder as reth_node_builder::rpc::EthApiBuilder<N>>::EthApi;
+
+    fn hooks_mut(&mut self) -> &mut RpcHooks<N, Self::EthApi> {
+        unimplemented!("BbAddOns does not support dynamic hook mutation")
+    }
+}
+
+impl<N> EngineValidatorAddOn<N> for BbAddOns
+where
+    N: FullNodeComponents,
+    BasicEngineValidatorBuilder<EthereumEngineValidatorBuilder>: EngineValidatorBuilder<N>,
+{
+    type ValidatorBuilder = BasicEngineValidatorBuilder<EthereumEngineValidatorBuilder>;
+
+    fn engine_validator_builder(&self) -> Self::ValidatorBuilder {
+        BasicEngineValidatorBuilder::default()
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Custom executor builder
+// ---------------------------------------------------------------------------
+
+/// Executor builder that creates a [`BbEvmConfig`].
+#[derive(Debug)]
+pub struct BbExecutorBuilder {
+    pending: BigBlockMap,
+}
+
+impl<Node> ExecutorBuilder<Node> for BbExecutorBuilder
+where
+    Node: FullNodeTypes<
+        Types: NodeTypes<
+            ChainSpec: reth_ethereum_forks::Hardforks
+                           + alloy_evm::eth::spec::EthExecutorSpec
+                           + EthereumHardforks,
+            Primitives = EthPrimitives,
+        >,
+    >,
+{
+    type EVM = BbEvmConfig<<Node::Types as NodeTypes>::ChainSpec>;
+
+    async fn build_evm(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::EVM> {
+        Ok(BbEvmConfig::new(EthEvmConfig::new(ctx.chain_spec()), self.pending))
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Node type
+// ---------------------------------------------------------------------------
+
+/// Node type for big block execution.
+#[derive(Debug, Clone)]
+pub struct BbNode {
+    pending: BigBlockMap,
+}
+
+impl BbNode {
+    const fn new(pending: BigBlockMap) -> Self {
+        Self { pending }
+    }
+}
+
+impl NodeTypes for BbNode {
+    type Primitives = EthPrimitives;
+    type ChainSpec = ChainSpec;
+    type Storage = EthStorage;
+    type Payload = EthEngineTypes;
+}
+
+impl<N> Node<N> for BbNode
+where
+    N: FullNodeTypes<Types = Self>,
+{
+    type ComponentsBuilder = ComponentsBuilder<
+        N,
+        EthereumPoolBuilder,
+        BasicPayloadServiceBuilder<EthereumPayloadBuilder>,
+        EthereumNetworkBuilder,
+        BbExecutorBuilder,
+        BbConsensusBuilder,
+    >;
+
+    type AddOns = BbAddOns;
+
+    fn components_builder(&self) -> Self::ComponentsBuilder {
+        EthereumNode::components()
+            .executor(BbExecutorBuilder { pending: self.pending.clone() })
+            .consensus(BbConsensusBuilder)
+    }
+
+    fn add_ons(&self) -> Self::AddOns {
+        BbAddOns::new(self.pending.clone())
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Consensus builder
+// ---------------------------------------------------------------------------
+
+/// Consensus builder for big block execution.
+#[derive(Debug, Default, Clone, Copy)]
+pub struct BbConsensusBuilder;
+
+impl<Node> ConsensusBuilder<Node> for BbConsensusBuilder
+where
+    Node: FullNodeTypes<
+        Types: NodeTypes<ChainSpec: EthChainSpec + EthereumHardforks, Primitives = EthPrimitives>,
+    >,
+{
+    type Consensus = Arc<EthBeaconConsensus<<Node::Types as NodeTypes>::ChainSpec>>;
+
+    async fn build_consensus(self, ctx: &BuilderContext<Node>) -> eyre::Result<Self::Consensus> {
+        Ok(Arc::new(
+            EthBeaconConsensus::new(ctx.chain_spec())
+                .with_skip_gas_limit_ramp_check(true)
+                .with_skip_blob_gas_used_check(true)
+                .with_skip_requests_hash_check(true),
+        ))
+    }
+}
+
+// ---------------------------------------------------------------------------
+// Main
+// ---------------------------------------------------------------------------
+
+fn main() {
+    reth_cli_util::sigsegv_handler::install();
+
+    if std::env::var_os("RUST_BACKTRACE").is_none() {
+        unsafe { std::env::set_var("RUST_BACKTRACE", "1") };
+    }
+
+    let pending: BigBlockMap = Arc::new(Mutex::new(HashMap::new()));
+
+    if let Err(err) = Cli::<EthereumChainSpecParser>::parse().run(async move |builder, _| {
+        info!(target: "reth::cli", "Launching big block node");
+        let handle = builder.launch_node(BbNode::new(pending.clone())).await?;
+
+        handle.wait_for_node_exit().await
+    }) {
+        eprintln!("Error: {err:?}");
+        std::process::exit(1);
+    }
+}

bin/reth-bench-compare/Cargo.toml

@@ -1,99 +0,0 @@
-[package]
-name = "reth-bench-compare"
-version.workspace = true
-edition.workspace = true
-rust-version.workspace = true
-license.workspace = true
-homepage.workspace = true
-repository.workspace = true
-description = "Automated reth benchmark comparison between git references"
-
-[lints]
-workspace = true
-
-[[bin]]
-name = "reth-bench-compare"
-path = "src/main.rs"
-
-[dependencies]
-# reth
-reth-cli-runner.workspace = true
-reth-cli-util.workspace = true
-reth-node-core.workspace = true
-reth-tracing.workspace = true
-reth-chainspec.workspace = true
-
-# alloy
-alloy-provider = { workspace = true, features = ["reqwest-rustls-tls"], default-features = false }
-alloy-rpc-client = { workspace = true, features = ["pubsub"] }
-alloy-rpc-types-eth.workspace = true
-alloy-transport-ws.workspace = true
-alloy-primitives.workspace = true
-
-# CLI and argument parsing
-clap = { workspace = true, features = ["derive", "env"] }
-eyre.workspace = true
-
-# Async runtime
-tokio = { workspace = true, features = ["full"] }
-tracing.workspace = true
-
-# Serialization
-serde = { workspace = true, features = ["derive"] }
-serde_json.workspace = true
-
-# Time handling
-chrono = { workspace = true, features = ["serde"] }
-
-# CSV handling
-csv.workspace = true
-
-# Process management
-ctrlc.workspace = true
-shlex.workspace = true
-
-[target.'cfg(unix)'.dependencies]
-nix = { version = "0.31", features = ["signal", "process"] }
-
-[features]
-default = ["jemalloc"]
-
-asm-keccak = [
-    "reth-node-core/asm-keccak",
-    "alloy-primitives/asm-keccak",
-]
-
-jemalloc = [
-    "reth-cli-util/jemalloc",
-    "reth-node-core/jemalloc",
-]
-jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
-tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
-tracy = [
-    "reth-node-core/tracy",
-    "reth-tracing/tracy",
-]
-
-min-error-logs = [
-    "tracing/release_max_level_error",
-    "reth-node-core/min-error-logs",
-]
-min-warn-logs = [
-    "tracing/release_max_level_warn",
-    "reth-node-core/min-warn-logs",
-]
-min-info-logs = [
-    "tracing/release_max_level_info",
-    "reth-node-core/min-info-logs",
-]
-min-debug-logs = [
-    "tracing/release_max_level_debug",
-    "reth-node-core/min-debug-logs",
-]
-min-trace-logs = [
-    "tracing/release_max_level_trace",
-    "reth-node-core/min-trace-logs",
-]
-
-# no-op feature flag for CI matrices
-ethereum = []

bin/reth-bench-compare/README.md

@@ -1,50 +0,0 @@
-# reth-bench-compare
-
-Compare reth performance between two git references.
-
-## Usage
-
-```bash
-reth-bench-compare \
-  --baseline-ref main \
-  --feature-ref my-feature \
-  --blocks 100 \
-  --wait-for-persistence
-```
-
-## Arguments
-
-| Argument | Description | Default | Required |
-|----------|-------------|---------|----------|
-| `--baseline-ref <REF>` | Git reference for baseline | - | Yes |
-| `--feature-ref <REF>` | Git reference to compare | - | Yes |
-| `--blocks <N>` | Number of blocks to benchmark | `100` | No |
-| `--chain <CHAIN>` | Chain to benchmark | `mainnet` | No |
-| `--datadir <PATH>` | Data directory path | OS-specific | No |
-| `--rpc-url <URL>` | RPC endpoint for block data | Chain default | No |
-| `--output-dir <PATH>` | Output directory | `./reth-bench-compare` | No |
-| `--wait-for-persistence` | Wait for block persistence | `false` | No |
-| `--persistence-threshold <N>` | Wait after every N+1 blocks | `2` | No |
-| `--wait-time <DURATION>` | Fixed delay (legacy) | - | No |
-| `--warmup-blocks <N>` | Cache warmup blocks | Same as `--blocks` | No |
-| `--draw` | Generate charts (needs Python/uv) | `false` | No |
-| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
-| `-vvvv` | Debug logging | Info | No |
-| `--features <FEATURES>` | Extra Rust features for both builds | - | No |
-| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
-| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
-| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |
-| `--baseline-rustflags <FLAGS>` | RUSTFLAGS for baseline only | Inherits `--rustflags` | No |
-| `--feature-rustflags <FLAGS>` | RUSTFLAGS for feature only | Inherits `--rustflags` | No |
-| `--baseline-args <ARGS>` | Extra args for baseline node | - | No |
-| `--feature-args <ARGS>` | Extra args for feature node | - | No |
-| `--metrics-port <PORT>` | Metrics endpoint port | `5005` | No |
-| `--sudo` | Run with elevated privileges | `false` | No |
-
-## Output
-
-Results in `./reth-bench-compare/results/<timestamp>/`:
-- `comparison_report.json` - Metrics comparison
-- `per_block_comparison.csv` - Per-block statistics
-- `baseline/` and `feature/` - Individual run results
-- `latency_comparison.png` - Chart (if `--draw` used)

bin/reth-bench-compare/src/benchmark.rs

@@ -1,307 +0,0 @@
-//! Benchmark execution using reth-bench.
-
-use crate::cli::Args;
-use eyre::{eyre, Result, WrapErr};
-use std::{
-    path::Path,
-    sync::{Arc, Mutex},
-};
-use tokio::{
-    fs::File as AsyncFile,
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader},
-    process::Command,
-};
-use tracing::{debug, error, info, warn};
-
-/// Manages benchmark execution using reth-bench
-pub(crate) struct BenchmarkRunner {
-    rpc_url: String,
-    jwt_secret: String,
-    wait_time: Option<String>,
-    wait_for_persistence: bool,
-    persistence_threshold: Option<u64>,
-    warmup_blocks: u64,
-}
-
-impl BenchmarkRunner {
-    /// Create a new `BenchmarkRunner` from CLI arguments
-    pub(crate) fn new(args: &Args) -> Self {
-        Self {
-            rpc_url: args.get_rpc_url(),
-            jwt_secret: args.jwt_secret_path().to_string_lossy().to_string(),
-            wait_time: args.wait_time.clone(),
-            wait_for_persistence: args.wait_for_persistence,
-            persistence_threshold: args.persistence_threshold,
-            warmup_blocks: args.get_warmup_blocks(),
-        }
-    }
-
-    /// Clear filesystem caches (page cache, dentries, and inodes)
-    pub(crate) async fn clear_fs_caches() -> Result<()> {
-        info!("Clearing filesystem caches...");
-
-        // First sync to ensure all pending writes are flushed
-        let sync_output =
-            Command::new("sync").output().await.wrap_err("Failed to execute sync command")?;
-
-        if !sync_output.status.success() {
-            return Err(eyre!("sync command failed"));
-        }
-
-        // Drop caches - requires sudo/root permissions
-        // 3 = drop pagecache, dentries, and inodes
-        let drop_caches_cmd = Command::new("sudo")
-            .args(["-n", "sh", "-c", "echo 3 > /proc/sys/vm/drop_caches"])
-            .output()
-            .await;
-
-        match drop_caches_cmd {
-            Ok(output) if output.status.success() => {
-                info!("Successfully cleared filesystem caches");
-                Ok(())
-            }
-            Ok(output) => {
-                let stderr = String::from_utf8_lossy(&output.stderr);
-                if stderr.contains("sudo: a password is required") {
-                    warn!("Unable to clear filesystem caches: sudo password required");
-                    warn!(
-                        "For optimal benchmarking, configure passwordless sudo for cache clearing:"
-                    );
-                    warn!("  echo '$USER ALL=(ALL) NOPASSWD: /bin/sh -c echo\\\\ [0-9]\\\\ \\\\>\\\\ /proc/sys/vm/drop_caches' | sudo tee /etc/sudoers.d/drop_caches");
-                    Ok(())
-                } else {
-                    Err(eyre!("Failed to clear filesystem caches: {}", stderr))
-                }
-            }
-            Err(e) => {
-                warn!("Unable to clear filesystem caches: {}", e);
-                Ok(())
-            }
-        }
-    }
-
-    /// Run a warmup benchmark for cache warming
-    pub(crate) async fn run_warmup(&self, from_block: u64) -> Result<()> {
-        let to_block = from_block + self.warmup_blocks;
-        info!(
-            "Running warmup benchmark from block {} to {} ({} blocks)",
-            from_block, to_block, self.warmup_blocks
-        );
-
-        // Build the reth-bench command for warmup (no output flag)
-        let mut cmd = Command::new("reth-bench");
-        cmd.args([
-            "new-payload-fcu",
-            "--rpc-url",
-            &self.rpc_url,
-            "--jwt-secret",
-            &self.jwt_secret,
-            "--from",
-            &from_block.to_string(),
-            "--to",
-            &to_block.to_string(),
-            "--wait-time=0ms", // Warmup should avoid persistence waits.
-        ]);
-
-        cmd.env("RUST_LOG_STYLE", "never")
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true);
-
-        // Set process group for consistent signal handling
-        #[cfg(unix)]
-        {
-            cmd.process_group(0);
-        }
-
-        debug!("Executing warmup reth-bench command: {:?}", cmd);
-
-        // Execute the warmup benchmark
-        let mut child = cmd.spawn().wrap_err("Failed to start warmup reth-bench process")?;
-
-        // Stream output at debug level
-        if let Some(stdout) = child.stdout.take() {
-            tokio::spawn(async move {
-                let reader = BufReader::new(stdout);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[WARMUP] {}", line);
-                }
-            });
-        }
-
-        if let Some(stderr) = child.stderr.take() {
-            tokio::spawn(async move {
-                let reader = BufReader::new(stderr);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[WARMUP] {}", line);
-                }
-            });
-        }
-
-        let status = child.wait().await.wrap_err("Failed to wait for warmup reth-bench")?;
-
-        if !status.success() {
-            return Err(eyre!("Warmup reth-bench failed with exit code: {:?}", status.code()));
-        }
-
-        info!("Warmup completed successfully");
-        Ok(())
-    }
-
-    /// Run a benchmark for the specified block range
-    pub(crate) async fn run_benchmark(
-        &self,
-        from_block: u64,
-        to_block: u64,
-        output_dir: &Path,
-    ) -> Result<()> {
-        info!(
-            "Running benchmark from block {} to {} (output: {:?})",
-            from_block, to_block, output_dir
-        );
-
-        // Ensure output directory exists
-        std::fs::create_dir_all(output_dir)
-            .wrap_err_with(|| format!("Failed to create output directory: {output_dir:?}"))?;
-
-        // Create log file path for reth-bench output
-        let log_file_path = output_dir.join("reth_bench.log");
-        info!("reth-bench logs will be saved to: {:?}", log_file_path);
-
-        // Build the reth-bench command
-        let mut cmd = Command::new("reth-bench");
-        cmd.args([
-            "new-payload-fcu",
-            "--rpc-url",
-            &self.rpc_url,
-            "--jwt-secret",
-            &self.jwt_secret,
-            "--from",
-            &from_block.to_string(),
-            "--to",
-            &to_block.to_string(),
-            "--output",
-            &output_dir.to_string_lossy(),
-        ]);
-
-        // Configure wait mode: both can be used together
-        // When both are set: wait at least wait_time, and also wait for persistence if needed
-        if let Some(ref wait_time) = self.wait_time {
-            cmd.args(["--wait-time", wait_time]);
-        }
-        if self.wait_for_persistence {
-            cmd.arg("--wait-for-persistence");
-
-            // Add persistence threshold if specified
-            if let Some(threshold) = self.persistence_threshold {
-                cmd.args(["--persistence-threshold", &threshold.to_string()]);
-            }
-        }
-
-        cmd.env("RUST_LOG_STYLE", "never")
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true);
-
-        // Set process group for consistent signal handling
-        #[cfg(unix)]
-        {
-            cmd.process_group(0);
-        }
-
-        // Debug log the command
-        debug!("Executing reth-bench command: {:?}", cmd);
-
-        // Execute the benchmark
-        let mut child = cmd.spawn().wrap_err("Failed to start reth-bench process")?;
-
-        // Capture stdout and stderr for error reporting
-        let stdout_lines = Arc::new(Mutex::new(Vec::new()));
-        let stderr_lines = Arc::new(Mutex::new(Vec::new()));
-
-        // Stream stdout with prefix at debug level, capture for error reporting, and write to log
-        // file
-        if let Some(stdout) = child.stdout.take() {
-            let stdout_lines_clone = stdout_lines.clone();
-            let log_file = AsyncFile::create(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = BufReader::new(stdout);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-BENCH] {}", line);
-                    if let Ok(mut captured) = stdout_lines_clone.lock() {
-                        captured.push(line.clone());
-                    }
-                    // Write to log file (reth-bench output already has timestamps if needed)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        // Stream stderr with prefix at debug level, capture for error reporting, and write to log
-        // file
-        if let Some(stderr) = child.stderr.take() {
-            let stderr_lines_clone = stderr_lines.clone();
-            let log_file = AsyncFile::options()
-                .create(true)
-                .append(true)
-                .open(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = BufReader::new(stderr);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-BENCH] {}", line);
-                    if let Ok(mut captured) = stderr_lines_clone.lock() {
-                        captured.push(line.clone());
-                    }
-                    // Write to log file (reth-bench output already has timestamps if needed)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        let status = child.wait().await.wrap_err("Failed to wait for reth-bench")?;
-
-        if !status.success() {
-            // Print all captured output when command fails
-            error!("reth-bench failed with exit code: {:?}", status.code());
-
-            if let Ok(stdout) = stdout_lines.lock() &&
-                !stdout.is_empty()
-            {
-                error!("reth-bench stdout:");
-                for line in stdout.iter() {
-                    error!("  {}", line);
-                }
-            }
-
-            if let Ok(stderr) = stderr_lines.lock() &&
-                !stderr.is_empty()
-            {
-                error!("reth-bench stderr:");
-                for line in stderr.iter() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!("reth-bench failed with exit code: {:?}", status.code()));
-        }
-
-        info!("Benchmark completed");
-        Ok(())
-    }
-}

bin/reth-bench-compare/src/comparison.rs

@@ -1,723 +0,0 @@
-//! Results comparison and report generation.
-
-use crate::cli::Args;
-use chrono::{DateTime, Utc};
-use csv::Reader;
-use eyre::{eyre, Result, WrapErr};
-use serde::{Deserialize, Serialize};
-use std::{
-    cmp::Ordering,
-    collections::HashMap,
-    fs,
-    path::{Path, PathBuf},
-};
-use tracing::{info, warn};
-
-/// Manages comparison between baseline and feature reference results
-pub(crate) struct ComparisonGenerator {
-    output_dir: PathBuf,
-    timestamp: String,
-    baseline_ref_name: String,
-    feature_ref_name: String,
-    baseline_results: Option<BenchmarkResults>,
-    feature_results: Option<BenchmarkResults>,
-    baseline_command: Option<String>,
-    feature_command: Option<String>,
-}
-
-/// Represents the results from a single benchmark run
-#[derive(Debug, Clone)]
-pub(crate) struct BenchmarkResults {
-    pub ref_name: String,
-    pub combined_latency_data: Vec<CombinedLatencyRow>,
-    pub summary: BenchmarkSummary,
-    pub start_timestamp: Option<DateTime<Utc>>,
-    pub end_timestamp: Option<DateTime<Utc>>,
-}
-
-/// Combined latency CSV row structure
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub(crate) struct CombinedLatencyRow {
-    pub block_number: u64,
-    #[serde(default)]
-    pub transaction_count: Option<u64>,
-    pub gas_used: u64,
-    pub new_payload_latency: u128,
-}
-
-/// Total gas CSV row structure
-#[derive(Debug, Clone, Deserialize, Serialize)]
-pub(crate) struct TotalGasRow {
-    pub block_number: u64,
-    #[serde(default)]
-    pub transaction_count: Option<u64>,
-    pub gas_used: u64,
-    pub time: u128,
-}
-
-/// Summary statistics for a benchmark run.
-///
-/// Latencies are derived from per-block `engine_newPayload` timings (converted from µs to ms):
-/// - `mean_new_payload_latency_ms`: arithmetic mean latency across blocks.
-/// - `median_new_payload_latency_ms`: p50 latency across blocks.
-/// - `p90_new_payload_latency_ms` / `p99_new_payload_latency_ms`: tail latencies across blocks.
-#[derive(Debug, Clone, Serialize)]
-pub(crate) struct BenchmarkSummary {
-    pub total_blocks: u64,
-    pub total_gas_used: u64,
-    pub total_duration_ms: u128,
-    pub mean_new_payload_latency_ms: f64,
-    pub median_new_payload_latency_ms: f64,
-    pub p90_new_payload_latency_ms: f64,
-    pub p99_new_payload_latency_ms: f64,
-    pub gas_per_second: f64,
-    pub blocks_per_second: f64,
-    pub min_block_number: u64,
-    pub max_block_number: u64,
-}
-
-/// Comparison report between two benchmark runs
-#[derive(Debug, Serialize)]
-pub(crate) struct ComparisonReport {
-    pub timestamp: String,
-    pub baseline: RefInfo,
-    pub feature: RefInfo,
-    pub comparison_summary: ComparisonSummary,
-    pub per_block_comparisons: Vec<BlockComparison>,
-}
-
-/// Information about a reference in the comparison
-#[derive(Debug, Serialize)]
-pub(crate) struct RefInfo {
-    pub ref_name: String,
-    pub summary: BenchmarkSummary,
-    pub start_timestamp: Option<DateTime<Utc>>,
-    pub end_timestamp: Option<DateTime<Utc>>,
-    pub reth_command: Option<String>,
-}
-
-/// Summary of the comparison between references.
-///
-/// Percent deltas are `(feature - baseline) / baseline * 100`:
-/// - `new_payload_latency_mean_change_percent`: percent changes of the per-block means.
-/// - `new_payload_latency_p50_change_percent` / p90 / p99: percent changes of the respective
-///   per-block percentiles.
-/// - `per_block_latency_change_mean_percent` / `per_block_latency_change_median_percent` are the
-///   mean and median of per-block percent deltas (feature vs baseline), capturing block-level
-///   drift.
-/// - `per_block_latency_change_std_dev_percent`: standard deviation of per-block percent changes,
-///   measuring consistency of performance changes across blocks.
-/// - `new_payload_total_latency_change_percent` is the percent change of the total newPayload time
-///   across the run.
-///
-/// Positive means slower/higher; negative means faster/lower.
-#[derive(Debug, Serialize)]
-pub(crate) struct ComparisonSummary {
-    pub per_block_latency_change_mean_percent: f64,
-    pub per_block_latency_change_median_percent: f64,
-    pub per_block_latency_change_std_dev_percent: f64,
-    pub new_payload_total_latency_change_percent: f64,
-    pub new_payload_latency_mean_change_percent: f64,
-    pub new_payload_latency_p50_change_percent: f64,
-    pub new_payload_latency_p90_change_percent: f64,
-    pub new_payload_latency_p99_change_percent: f64,
-    pub gas_per_second_change_percent: f64,
-    pub blocks_per_second_change_percent: f64,
-}
-
-/// Per-block comparison data
-#[derive(Debug, Serialize)]
-pub(crate) struct BlockComparison {
-    pub block_number: u64,
-    #[serde(skip_serializing_if = "Option::is_none")]
-    pub transaction_count: Option<u64>,
-    pub gas_used: u64,
-    pub baseline_new_payload_latency: u128,
-    pub feature_new_payload_latency: u128,
-    pub new_payload_latency_change_percent: f64,
-}
-
-impl ComparisonGenerator {
-    /// Create a new comparison generator
-    pub(crate) fn new(args: &Args) -> Self {
-        let now: DateTime<Utc> = Utc::now();
-        let timestamp = now.format("%Y%m%d_%H%M%S").to_string();
-
-        Self {
-            output_dir: args.output_dir_path(),
-            timestamp,
-            baseline_ref_name: args.baseline_ref.clone(),
-            feature_ref_name: args.feature_ref.clone(),
-            baseline_results: None,
-            feature_results: None,
-            baseline_command: None,
-            feature_command: None,
-        }
-    }
-
-    /// Get the output directory for a specific reference
-    pub(crate) fn get_ref_output_dir(&self, ref_type: &str) -> PathBuf {
-        self.output_dir.join("results").join(&self.timestamp).join(ref_type)
-    }
-
-    /// Get the main output directory for this comparison run
-    pub(crate) fn get_output_dir(&self) -> PathBuf {
-        self.output_dir.join("results").join(&self.timestamp)
-    }
-
-    /// Add benchmark results for a reference
-    pub(crate) fn add_ref_results(&mut self, ref_type: &str, output_path: &Path) -> Result<()> {
-        let ref_name = match ref_type {
-            "baseline" => &self.baseline_ref_name,
-            "feature" => &self.feature_ref_name,
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        };
-
-        let results = self.load_benchmark_results(ref_name, output_path)?;
-
-        match ref_type {
-            "baseline" => self.baseline_results = Some(results),
-            "feature" => self.feature_results = Some(results),
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        }
-
-        info!("Loaded benchmark results for {} reference", ref_type);
-
-        Ok(())
-    }
-
-    /// Set the benchmark run timestamps for a reference
-    pub(crate) fn set_ref_timestamps(
-        &mut self,
-        ref_type: &str,
-        start: DateTime<Utc>,
-        end: DateTime<Utc>,
-    ) -> Result<()> {
-        match ref_type {
-            "baseline" => {
-                if let Some(ref mut results) = self.baseline_results {
-                    results.start_timestamp = Some(start);
-                    results.end_timestamp = Some(end);
-                } else {
-                    return Err(eyre!("Baseline results not loaded yet"));
-                }
-            }
-            "feature" => {
-                if let Some(ref mut results) = self.feature_results {
-                    results.start_timestamp = Some(start);
-                    results.end_timestamp = Some(end);
-                } else {
-                    return Err(eyre!("Feature results not loaded yet"));
-                }
-            }
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        }
-
-        Ok(())
-    }
-
-    /// Set the reth command for a reference
-    pub(crate) fn set_ref_command(&mut self, ref_type: &str, command: String) -> Result<()> {
-        match ref_type {
-            "baseline" => {
-                self.baseline_command = Some(command);
-            }
-            "feature" => {
-                self.feature_command = Some(command);
-            }
-            _ => return Err(eyre!("Unknown reference type: {}", ref_type)),
-        }
-
-        Ok(())
-    }
-
-    /// Generate the final comparison report
-    pub(crate) async fn generate_comparison_report(&self) -> Result<()> {
-        info!("Generating comparison report...");
-
-        let baseline =
-            self.baseline_results.as_ref().ok_or_else(|| eyre!("Baseline results not loaded"))?;
-
-        let feature =
-            self.feature_results.as_ref().ok_or_else(|| eyre!("Feature results not loaded"))?;
-
-        let per_block_comparisons = self.calculate_per_block_comparisons(baseline, feature)?;
-        let comparison_summary = self.calculate_comparison_summary(
-            &baseline.summary,
-            &feature.summary,
-            &per_block_comparisons,
-        )?;
-
-        let report = ComparisonReport {
-            timestamp: self.timestamp.clone(),
-            baseline: RefInfo {
-                ref_name: baseline.ref_name.clone(),
-                summary: baseline.summary.clone(),
-                start_timestamp: baseline.start_timestamp,
-                end_timestamp: baseline.end_timestamp,
-                reth_command: self.baseline_command.clone(),
-            },
-            feature: RefInfo {
-                ref_name: feature.ref_name.clone(),
-                summary: feature.summary.clone(),
-                start_timestamp: feature.start_timestamp,
-                end_timestamp: feature.end_timestamp,
-                reth_command: self.feature_command.clone(),
-            },
-            comparison_summary,
-            per_block_comparisons,
-        };
-
-        // Write reports
-        self.write_comparison_reports(&report).await?;
-
-        // Print summary to console
-        self.print_comparison_summary(&report);
-
-        Ok(())
-    }
-
-    /// Load benchmark results from CSV files
-    fn load_benchmark_results(
-        &self,
-        ref_name: &str,
-        output_path: &Path,
-    ) -> Result<BenchmarkResults> {
-        let combined_latency_path = output_path.join("combined_latency.csv");
-        let total_gas_path = output_path.join("total_gas.csv");
-
-        let combined_latency_data = self.load_combined_latency_csv(&combined_latency_path)?;
-        let total_gas_data = self.load_total_gas_csv(&total_gas_path)?;
-
-        let summary = self.calculate_summary(&combined_latency_data, &total_gas_data)?;
-
-        Ok(BenchmarkResults {
-            ref_name: ref_name.to_string(),
-            combined_latency_data,
-            summary,
-            start_timestamp: None,
-            end_timestamp: None,
-        })
-    }
-
-    /// Load combined latency CSV data
-    fn load_combined_latency_csv(&self, path: &Path) -> Result<Vec<CombinedLatencyRow>> {
-        let mut reader = Reader::from_path(path)
-            .wrap_err_with(|| format!("Failed to open combined latency CSV: {path:?}"))?;
-
-        let mut rows = Vec::new();
-        for result in reader.deserialize() {
-            let row: CombinedLatencyRow = result
-                .wrap_err_with(|| format!("Failed to parse combined latency row in {path:?}"))?;
-            rows.push(row);
-        }
-
-        if rows.is_empty() {
-            return Err(eyre!("No data found in combined latency CSV: {:?}", path));
-        }
-
-        Ok(rows)
-    }
-
-    /// Load total gas CSV data
-    fn load_total_gas_csv(&self, path: &Path) -> Result<Vec<TotalGasRow>> {
-        let mut reader = Reader::from_path(path)
-            .wrap_err_with(|| format!("Failed to open total gas CSV: {path:?}"))?;
-
-        let mut rows = Vec::new();
-        for result in reader.deserialize() {
-            let row: TotalGasRow =
-                result.wrap_err_with(|| format!("Failed to parse total gas row in {path:?}"))?;
-            rows.push(row);
-        }
-
-        if rows.is_empty() {
-            return Err(eyre!("No data found in total gas CSV: {:?}", path));
-        }
-
-        Ok(rows)
-    }
-
-    /// Calculate summary statistics for a benchmark run.
-    ///
-    /// Computes latency statistics from per-block `new_payload_latency` values in `combined_data`
-    /// (converting from µs to ms), and throughput metrics using the total run duration from
-    /// `total_gas_data`. Percentiles (p50/p90/p99) use linear interpolation on sorted latencies.
-    fn calculate_summary(
-        &self,
-        combined_data: &[CombinedLatencyRow],
-        total_gas_data: &[TotalGasRow],
-    ) -> Result<BenchmarkSummary> {
-        if combined_data.is_empty() || total_gas_data.is_empty() {
-            return Err(eyre!("Cannot calculate summary for empty data"));
-        }
-
-        let total_blocks = combined_data.len() as u64;
-        let total_gas_used: u64 = combined_data.iter().map(|r| r.gas_used).sum();
-
-        let total_duration_ms = total_gas_data.last().unwrap().time / 1000; // Convert microseconds to milliseconds
-
-        let latencies_ms: Vec<f64> =
-            combined_data.iter().map(|r| r.new_payload_latency as f64 / 1000.0).collect();
-        let mean_new_payload_latency_ms: f64 =
-            latencies_ms.iter().sum::<f64>() / total_blocks as f64;
-
-        let mut sorted_latencies_ms = latencies_ms;
-        sorted_latencies_ms.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
-        let median_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.5);
-        let p90_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.9);
-        let p99_new_payload_latency_ms = percentile(&sorted_latencies_ms, 0.99);
-
-        let total_duration_seconds = total_duration_ms as f64 / 1000.0;
-        let gas_per_second = if total_duration_seconds > f64::EPSILON {
-            total_gas_used as f64 / total_duration_seconds
-        } else {
-            0.0
-        };
-
-        let blocks_per_second = if total_duration_seconds > f64::EPSILON {
-            total_blocks as f64 / total_duration_seconds
-        } else {
-            0.0
-        };
-
-        let min_block_number = combined_data.first().unwrap().block_number;
-        let max_block_number = combined_data.last().unwrap().block_number;
-
-        Ok(BenchmarkSummary {
-            total_blocks,
-            total_gas_used,
-            total_duration_ms,
-            mean_new_payload_latency_ms,
-            median_new_payload_latency_ms,
-            p90_new_payload_latency_ms,
-            p99_new_payload_latency_ms,
-            gas_per_second,
-            blocks_per_second,
-            min_block_number,
-            max_block_number,
-        })
-    }
-
-    /// Calculate comparison summary between baseline and feature
-    fn calculate_comparison_summary(
-        &self,
-        baseline: &BenchmarkSummary,
-        feature: &BenchmarkSummary,
-        per_block_comparisons: &[BlockComparison],
-    ) -> Result<ComparisonSummary> {
-        let calc_percent_change = |baseline: f64, feature: f64| -> f64 {
-            if baseline.abs() > f64::EPSILON {
-                ((feature - baseline) / baseline) * 100.0
-            } else {
-                0.0
-            }
-        };
-
-        // Calculate per-block statistics. "Per-block" means: for each block, compute the percent
-        // change (feature - baseline) / baseline * 100, then calculate statistics across those
-        // per-block percent changes. This captures how consistently the feature performs relative
-        // to baseline across all blocks.
-        let per_block_percent_changes: Vec<f64> =
-            per_block_comparisons.iter().map(|c| c.new_payload_latency_change_percent).collect();
-        let per_block_latency_change_mean_percent = if per_block_percent_changes.is_empty() {
-            0.0
-        } else {
-            per_block_percent_changes.iter().sum::<f64>() / per_block_percent_changes.len() as f64
-        };
-        let per_block_latency_change_median_percent = if per_block_percent_changes.is_empty() {
-            0.0
-        } else {
-            let mut sorted = per_block_percent_changes.clone();
-            sorted.sort_by(|a, b| a.partial_cmp(b).unwrap_or(Ordering::Equal));
-            percentile(&sorted, 0.5)
-        };
-        let per_block_latency_change_std_dev_percent =
-            calculate_std_dev(&per_block_percent_changes, per_block_latency_change_mean_percent);
-
-        let baseline_total_latency_ms =
-            baseline.mean_new_payload_latency_ms * baseline.total_blocks as f64;
-        let feature_total_latency_ms =
-            feature.mean_new_payload_latency_ms * feature.total_blocks as f64;
-        let new_payload_total_latency_change_percent =
-            calc_percent_change(baseline_total_latency_ms, feature_total_latency_ms);
-
-        Ok(ComparisonSummary {
-            per_block_latency_change_mean_percent,
-            per_block_latency_change_median_percent,
-            per_block_latency_change_std_dev_percent,
-            new_payload_total_latency_change_percent,
-            new_payload_latency_mean_change_percent: calc_percent_change(
-                baseline.mean_new_payload_latency_ms,
-                feature.mean_new_payload_latency_ms,
-            ),
-            new_payload_latency_p50_change_percent: calc_percent_change(
-                baseline.median_new_payload_latency_ms,
-                feature.median_new_payload_latency_ms,
-            ),
-            new_payload_latency_p90_change_percent: calc_percent_change(
-                baseline.p90_new_payload_latency_ms,
-                feature.p90_new_payload_latency_ms,
-            ),
-            new_payload_latency_p99_change_percent: calc_percent_change(
-                baseline.p99_new_payload_latency_ms,
-                feature.p99_new_payload_latency_ms,
-            ),
-            gas_per_second_change_percent: calc_percent_change(
-                baseline.gas_per_second,
-                feature.gas_per_second,
-            ),
-            blocks_per_second_change_percent: calc_percent_change(
-                baseline.blocks_per_second,
-                feature.blocks_per_second,
-            ),
-        })
-    }
-
-    /// Calculate per-block comparisons
-    fn calculate_per_block_comparisons(
-        &self,
-        baseline: &BenchmarkResults,
-        feature: &BenchmarkResults,
-    ) -> Result<Vec<BlockComparison>> {
-        let mut baseline_map: HashMap<u64, &CombinedLatencyRow> = HashMap::new();
-        for row in &baseline.combined_latency_data {
-            baseline_map.insert(row.block_number, row);
-        }
-
-        let mut comparisons = Vec::new();
-        for feature_row in &feature.combined_latency_data {
-            if let Some(baseline_row) = baseline_map.get(&feature_row.block_number) {
-                let calc_percent_change = |baseline: u128, feature: u128| -> f64 {
-                    if baseline > 0 {
-                        ((feature as f64 - baseline as f64) / baseline as f64) * 100.0
-                    } else {
-                        0.0
-                    }
-                };
-
-                let comparison = BlockComparison {
-                    block_number: feature_row.block_number,
-                    transaction_count: feature_row.transaction_count,
-                    gas_used: feature_row.gas_used,
-                    baseline_new_payload_latency: baseline_row.new_payload_latency,
-                    feature_new_payload_latency: feature_row.new_payload_latency,
-                    new_payload_latency_change_percent: calc_percent_change(
-                        baseline_row.new_payload_latency,
-                        feature_row.new_payload_latency,
-                    ),
-                };
-                comparisons.push(comparison);
-            } else {
-                warn!("Block {} not found in baseline data", feature_row.block_number);
-            }
-        }
-
-        Ok(comparisons)
-    }
-
-    /// Write comparison reports to files
-    async fn write_comparison_reports(&self, report: &ComparisonReport) -> Result<()> {
-        let report_dir = self.output_dir.join("results").join(&self.timestamp);
-        fs::create_dir_all(&report_dir)
-            .wrap_err_with(|| format!("Failed to create report directory: {report_dir:?}"))?;
-
-        // Write JSON report
-        let json_path = report_dir.join("comparison_report.json");
-        let json_content = serde_json::to_string_pretty(report)
-            .wrap_err("Failed to serialize comparison report to JSON")?;
-        fs::write(&json_path, json_content)
-            .wrap_err_with(|| format!("Failed to write JSON report: {json_path:?}"))?;
-
-        // Write CSV report for per-block comparisons
-        let csv_path = report_dir.join("per_block_comparison.csv");
-        let mut writer = csv::Writer::from_path(&csv_path)
-            .wrap_err_with(|| format!("Failed to create CSV writer: {csv_path:?}"))?;
-
-        for comparison in &report.per_block_comparisons {
-            writer.serialize(comparison).wrap_err("Failed to write comparison row to CSV")?;
-        }
-        writer.flush().wrap_err("Failed to flush CSV writer")?;
-
-        info!("Comparison reports written to: {:?}", report_dir);
-        Ok(())
-    }
-
-    /// Print comparison summary to console
-    fn print_comparison_summary(&self, report: &ComparisonReport) {
-        // Parse and format timestamp nicely
-        let formatted_timestamp = if let Ok(dt) = chrono::DateTime::parse_from_str(
-            &format!("{} +0000", report.timestamp.replace('_', " ")),
-            "%Y%m%d %H%M%S %z",
-        ) {
-            dt.format("%Y-%m-%d %H:%M:%S UTC").to_string()
-        } else {
-            // Fallback to original if parsing fails
-            report.timestamp.clone()
-        };
-
-        println!("\n=== BENCHMARK COMPARISON SUMMARY ===");
-        println!("Timestamp: {formatted_timestamp}");
-        println!("Baseline: {}", report.baseline.ref_name);
-        println!("Feature:  {}", report.feature.ref_name);
-        println!();
-
-        let summary = &report.comparison_summary;
-
-        println!("Performance Changes:");
-        println!(
-            "  NewPayload Latency per-block mean change:   {:+.2}%",
-            summary.per_block_latency_change_mean_percent
-        );
-        println!(
-            "  NewPayload Latency per-block median change: {:+.2}%",
-            summary.per_block_latency_change_median_percent
-        );
-        println!(
-            "  NewPayload Latency per-block std dev:       {:.2}%",
-            summary.per_block_latency_change_std_dev_percent
-        );
-        println!(
-            "  Total newPayload time change:                {:+.2}%",
-            summary.new_payload_total_latency_change_percent
-        );
-        println!(
-            "  NewPayload Latency mean:           {:+.2}%",
-            summary.new_payload_latency_mean_change_percent
-        );
-        println!(
-            "  NewPayload Latency p50:           {:+.2}%",
-            summary.new_payload_latency_p50_change_percent
-        );
-        println!(
-            "  NewPayload Latency p90:           {:+.2}%",
-            summary.new_payload_latency_p90_change_percent
-        );
-        println!(
-            "  NewPayload Latency p99:           {:+.2}%",
-            summary.new_payload_latency_p99_change_percent
-        );
-        println!(
-            "  Gas/Second:                      {:+.2}%",
-            summary.gas_per_second_change_percent
-        );
-        println!(
-            "  Blocks/Second:                   {:+.2}%",
-            summary.blocks_per_second_change_percent
-        );
-        println!();
-
-        println!("Baseline Summary:");
-        let baseline = &report.baseline.summary;
-        println!(
-            "  Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
-            baseline.total_blocks,
-            baseline.min_block_number,
-            baseline.max_block_number,
-            baseline.total_gas_used,
-            baseline.total_duration_ms as f64 / 1000.0
-        );
-        println!("  NewPayload latency (ms):");
-        println!(
-            "    mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
-            baseline.mean_new_payload_latency_ms,
-            baseline.median_new_payload_latency_ms,
-            baseline.p90_new_payload_latency_ms,
-            baseline.p99_new_payload_latency_ms
-        );
-        if let (Some(start), Some(end)) =
-            (&report.baseline.start_timestamp, &report.baseline.end_timestamp)
-        {
-            println!(
-                "  Started: {}, Ended: {}",
-                start.format("%Y-%m-%d %H:%M:%S UTC"),
-                end.format("%Y-%m-%d %H:%M:%S UTC")
-            );
-        }
-        if let Some(ref cmd) = report.baseline.reth_command {
-            println!("  Command: {}", cmd);
-        }
-        println!();
-
-        println!("Feature Summary:");
-        let feature = &report.feature.summary;
-        println!(
-            "  Blocks: {} (blocks {} to {}), Gas: {}, Duration: {:.2}s",
-            feature.total_blocks,
-            feature.min_block_number,
-            feature.max_block_number,
-            feature.total_gas_used,
-            feature.total_duration_ms as f64 / 1000.0
-        );
-        println!("  NewPayload latency (ms):");
-        println!(
-            "    mean: {:.2}, p50: {:.2}, p90: {:.2}, p99: {:.2}",
-            feature.mean_new_payload_latency_ms,
-            feature.median_new_payload_latency_ms,
-            feature.p90_new_payload_latency_ms,
-            feature.p99_new_payload_latency_ms
-        );
-        if let (Some(start), Some(end)) =
-            (&report.feature.start_timestamp, &report.feature.end_timestamp)
-        {
-            println!(
-                "  Started: {}, Ended: {}",
-                start.format("%Y-%m-%d %H:%M:%S UTC"),
-                end.format("%Y-%m-%d %H:%M:%S UTC")
-            );
-        }
-        if let Some(ref cmd) = report.feature.reth_command {
-            println!("  Command: {}", cmd);
-        }
-        println!();
-    }
-}
-
-/// Calculate standard deviation from a set of values and their mean.
-///
-/// Computes the population standard deviation using the formula:
-/// `sqrt(sum((x - mean)²) / n)`
-///
-/// Returns 0.0 for empty input.
-fn calculate_std_dev(values: &[f64], mean: f64) -> f64 {
-    if values.is_empty() {
-        return 0.0;
-    }
-
-    let variance = values
-        .iter()
-        .map(|x| {
-            let diff = x - mean;
-            diff * diff
-        })
-        .sum::<f64>() /
-        values.len() as f64;
-
-    variance.sqrt()
-}
-
-/// Calculate percentile using linear interpolation on a sorted slice.
-///
-/// Computes `rank = percentile × (n - 1)` where n is the array length. If the rank falls
-/// between two indices, linearly interpolates between those values. For example, with 100 values,
-/// p90 computes rank = 0.9 × 99 = 89.1, then returns `values[89] × 0.9 + values[90] × 0.1`.
-///
-/// Returns 0.0 for empty input.
-fn percentile(sorted_values: &[f64], percentile: f64) -> f64 {
-    if sorted_values.is_empty() {
-        return 0.0;
-    }
-
-    let clamped = percentile.clamp(0.0, 1.0);
-    let max_index = sorted_values.len() - 1;
-    let rank = clamped * max_index as f64;
-    let lower = rank.floor() as usize;
-    let upper = rank.ceil() as usize;
-
-    if lower == upper {
-        sorted_values[lower]
-    } else {
-        let weight = rank - lower as f64;
-        sorted_values[lower].mul_add(1.0 - weight, sorted_values[upper] * weight)
-    }
-}

bin/reth-bench-compare/src/compilation.rs

@@ -1,305 +0,0 @@
-//! Compilation operations for reth and reth-bench.
-
-use crate::git::GitManager;
-use eyre::{eyre, Result, WrapErr};
-use std::{fs, path::PathBuf, process::Command};
-use tracing::{debug, error, info, warn};
-
-/// Manages compilation operations for reth components
-#[derive(Debug)]
-pub(crate) struct CompilationManager {
-    repo_root: String,
-    output_dir: PathBuf,
-    git_manager: GitManager,
-}
-
-impl CompilationManager {
-    /// Create a new `CompilationManager`
-    pub(crate) const fn new(
-        repo_root: String,
-        output_dir: PathBuf,
-        git_manager: GitManager,
-    ) -> Result<Self> {
-        Ok(Self { repo_root, output_dir, git_manager })
-    }
-
-    /// Get the path to the cached binary using explicit commit hash
-    pub(crate) fn get_cached_binary_path_for_commit(&self, commit: &str) -> PathBuf {
-        let identifier = &commit[..8]; // Use first 8 chars of commit
-        self.output_dir.join("bin").join(format!("reth_{identifier}"))
-    }
-
-    /// Compile reth using cargo build and cache the binary
-    pub(crate) fn compile_reth(&self, commit: &str, features: &str, rustflags: &str) -> Result<()> {
-        // Validate that current git commit matches the expected commit
-        let current_commit = self.git_manager.get_current_commit()?;
-        if current_commit != commit {
-            return Err(eyre!(
-                "Git commit mismatch! Expected: {}, but currently at: {}",
-                &commit[..8],
-                &current_commit[..8]
-            ));
-        }
-
-        let cached_path = self.get_cached_binary_path_for_commit(commit);
-
-        // Check if cached binary already exists (since path contains commit hash, it's valid)
-        if cached_path.exists() {
-            info!("Using cached binary (commit: {})", &commit[..8]);
-            return Ok(());
-        }
-
-        info!("No cached binary found, compiling (commit: {})...", &commit[..8]);
-
-        let binary_name = "reth";
-
-        info!(
-            "Compiling {} with profiling configuration (commit: {})...",
-            binary_name,
-            &commit[..8]
-        );
-
-        let mut cmd = Command::new("cargo");
-        cmd.arg("build").arg("--profile").arg("profiling");
-
-        cmd.arg("--features").arg(features);
-        info!("Using features: {features}");
-
-        cmd.current_dir(&self.repo_root);
-
-        // Set RUSTFLAGS
-        cmd.env("RUSTFLAGS", rustflags);
-        info!("Using RUSTFLAGS: {rustflags}");
-
-        info!("Compiling {binary_name} with {cmd:?}");
-
-        let output = cmd.output().wrap_err("Failed to execute cargo build command")?;
-
-        // Print stdout and stderr with prefixes at debug level
-        let stdout = String::from_utf8_lossy(&output.stdout);
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        for line in stdout.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO] {}", line);
-            }
-        }
-
-        for line in stderr.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO] {}", line);
-            }
-        }
-
-        if !output.status.success() {
-            // Print all output when compilation fails
-            error!("Cargo build failed with exit code: {:?}", output.status.code());
-
-            if !stdout.trim().is_empty() {
-                error!("Cargo stdout:");
-                for line in stdout.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            if !stderr.trim().is_empty() {
-                error!("Cargo stderr:");
-                for line in stderr.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!("Compilation failed with exit code: {:?}", output.status.code()));
-        }
-
-        info!("{} compilation completed", binary_name);
-
-        // Copy the compiled binary to cache
-        let source_path =
-            PathBuf::from(&self.repo_root).join(format!("target/profiling/{}", binary_name));
-        if !source_path.exists() {
-            return Err(eyre!("Compiled binary not found at {:?}", source_path));
-        }
-
-        // Create bin directory if it doesn't exist
-        let bin_dir = self.output_dir.join("bin");
-        fs::create_dir_all(&bin_dir).wrap_err("Failed to create bin directory")?;
-
-        // Copy binary to cache
-        fs::copy(&source_path, &cached_path).wrap_err("Failed to copy binary to cache")?;
-
-        // Make the cached binary executable
-        #[cfg(unix)]
-        {
-            use std::os::unix::fs::PermissionsExt;
-            let mut perms = fs::metadata(&cached_path)?.permissions();
-            perms.set_mode(0o755);
-            fs::set_permissions(&cached_path, perms)?;
-        }
-
-        info!("Cached compiled binary at: {:?}", cached_path);
-        Ok(())
-    }
-
-    /// Check if reth-bench is available in PATH
-    pub(crate) fn is_reth_bench_available(&self) -> bool {
-        match Command::new("which").arg("reth-bench").output() {
-            Ok(output) => {
-                if output.status.success() {
-                    let path = String::from_utf8_lossy(&output.stdout);
-                    info!("Found reth-bench: {}", path.trim());
-                    true
-                } else {
-                    false
-                }
-            }
-            Err(_) => false,
-        }
-    }
-
-    /// Check if samply is available in PATH
-    pub(crate) fn is_samply_available(&self) -> bool {
-        match Command::new("which").arg("samply").output() {
-            Ok(output) => {
-                if output.status.success() {
-                    let path = String::from_utf8_lossy(&output.stdout);
-                    info!("Found samply: {}", path.trim());
-                    true
-                } else {
-                    false
-                }
-            }
-            Err(_) => false,
-        }
-    }
-
-    /// Install samply using cargo
-    pub(crate) fn install_samply(&self) -> Result<()> {
-        info!("Installing samply via cargo...");
-
-        let mut cmd = Command::new("cargo");
-        cmd.args(["install", "--locked", "samply"]);
-
-        info!("Installing samply with {cmd:?}");
-
-        let output = cmd.output().wrap_err("Failed to execute cargo install samply command")?;
-
-        // Print stdout and stderr with prefixes at debug level
-        let stdout = String::from_utf8_lossy(&output.stdout);
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        for line in stdout.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO-SAMPLY] {}", line);
-            }
-        }
-
-        for line in stderr.lines() {
-            if !line.trim().is_empty() {
-                debug!("[CARGO-SAMPLY] {}", line);
-            }
-        }
-
-        if !output.status.success() {
-            // Print all output when installation fails
-            error!("Cargo install samply failed with exit code: {:?}", output.status.code());
-
-            if !stdout.trim().is_empty() {
-                error!("Cargo stdout:");
-                for line in stdout.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            if !stderr.trim().is_empty() {
-                error!("Cargo stderr:");
-                for line in stderr.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!(
-                "samply installation failed with exit code: {:?}",
-                output.status.code()
-            ));
-        }
-
-        info!("Samply installation completed");
-        Ok(())
-    }
-
-    /// Ensure samply is available, installing if necessary
-    pub(crate) fn ensure_samply_available(&self) -> Result<()> {
-        if self.is_samply_available() {
-            Ok(())
-        } else {
-            warn!("samply not found in PATH, installing...");
-            self.install_samply()
-        }
-    }
-
-    /// Ensure reth-bench is available, compiling if necessary
-    pub(crate) fn ensure_reth_bench_available(&self) -> Result<()> {
-        if self.is_reth_bench_available() {
-            Ok(())
-        } else {
-            warn!("reth-bench not found in PATH, compiling and installing...");
-            self.compile_reth_bench()
-        }
-    }
-
-    /// Compile and install reth-bench using `make install-reth-bench`
-    pub(crate) fn compile_reth_bench(&self) -> Result<()> {
-        info!("Compiling and installing reth-bench...");
-
-        let mut cmd = Command::new("make");
-        cmd.arg("install-reth-bench").current_dir(&self.repo_root);
-
-        info!("Compiling reth-bench with {cmd:?}");
-
-        let output = cmd.output().wrap_err("Failed to execute make install-reth-bench command")?;
-
-        // Print stdout and stderr with prefixes at debug level
-        let stdout = String::from_utf8_lossy(&output.stdout);
-        let stderr = String::from_utf8_lossy(&output.stderr);
-
-        for line in stdout.lines() {
-            if !line.trim().is_empty() {
-                debug!("[MAKE-BENCH] {}", line);
-            }
-        }
-
-        for line in stderr.lines() {
-            if !line.trim().is_empty() {
-                debug!("[MAKE-BENCH] {}", line);
-            }
-        }
-
-        if !output.status.success() {
-            // Print all output when compilation fails
-            error!("Make install-reth-bench failed with exit code: {:?}", output.status.code());
-
-            if !stdout.trim().is_empty() {
-                error!("Make stdout:");
-                for line in stdout.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            if !stderr.trim().is_empty() {
-                error!("Make stderr:");
-                for line in stderr.lines() {
-                    error!("  {}", line);
-                }
-            }
-
-            return Err(eyre!(
-                "reth-bench compilation failed with exit code: {:?}",
-                output.status.code()
-            ));
-        }
-
-        info!("Reth-bench compilation completed");
-        Ok(())
-    }
-}

bin/reth-bench-compare/src/git.rs

@@ -1,328 +0,0 @@
-//! Git operations for branch management.
-
-use eyre::{eyre, Result, WrapErr};
-use std::process::Command;
-use tracing::{info, warn};
-
-/// Manages git operations for branch switching
-#[derive(Debug, Clone)]
-pub(crate) struct GitManager {
-    repo_root: String,
-}
-
-impl GitManager {
-    /// Create a new `GitManager`, detecting the repository root
-    pub(crate) fn new() -> Result<Self> {
-        let output = Command::new("git")
-            .args(["rev-parse", "--show-toplevel"])
-            .output()
-            .wrap_err("Failed to execute git command - is git installed?")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Not in a git repository or git command failed"));
-        }
-
-        let repo_root = String::from_utf8(output.stdout)
-            .wrap_err("Git output is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        let manager = Self { repo_root };
-        info!(
-            "Detected git repository at: {}, current reference: {}",
-            manager.repo_root(),
-            manager.get_current_ref()?
-        );
-
-        Ok(manager)
-    }
-
-    /// Get the current git branch name
-    pub(crate) fn get_current_branch(&self) -> Result<String> {
-        let output = Command::new("git")
-            .args(["branch", "--show-current"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current branch")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Failed to determine current branch"));
-        }
-
-        let branch = String::from_utf8(output.stdout)
-            .wrap_err("Branch name is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        if branch.is_empty() {
-            return Err(eyre!("Not on a named branch (detached HEAD?)"));
-        }
-
-        Ok(branch)
-    }
-
-    /// Get the current git reference (branch name, tag, or commit hash)
-    pub(crate) fn get_current_ref(&self) -> Result<String> {
-        // First try to get branch name
-        if let Ok(branch) = self.get_current_branch() {
-            return Ok(branch);
-        }
-
-        // If not on a branch, check if we're on a tag
-        let tag_output = Command::new("git")
-            .args(["describe", "--exact-match", "--tags", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to check for tag")?;
-
-        if tag_output.status.success() {
-            let tag = String::from_utf8(tag_output.stdout)
-                .wrap_err("Tag name is not valid UTF-8")?
-                .trim()
-                .to_string();
-            return Ok(tag);
-        }
-
-        // If not on a branch or tag, return the commit hash
-        let commit_output = Command::new("git")
-            .args(["rev-parse", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current commit")?;
-
-        if !commit_output.status.success() {
-            return Err(eyre!("Failed to get current commit hash"));
-        }
-
-        let commit_hash = String::from_utf8(commit_output.stdout)
-            .wrap_err("Commit hash is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        Ok(commit_hash)
-    }
-
-    /// Check if the git working directory has uncommitted changes to tracked files
-    pub(crate) fn validate_clean_state(&self) -> Result<()> {
-        let output = Command::new("git")
-            .args(["status", "--porcelain"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to check git status")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Git status command failed"));
-        }
-
-        let status_output =
-            String::from_utf8(output.stdout).wrap_err("Git status output is not valid UTF-8")?;
-
-        // Check for uncommitted changes to tracked files
-        // Status codes: M = modified, A = added, D = deleted, R = renamed, C = copied, U = updated
-        // ?? = untracked files (we want to ignore these)
-        let has_uncommitted_changes = status_output.lines().any(|line| {
-            if line.len() >= 2 {
-                let status = &line[0..2];
-                // Ignore untracked files (??) and ignored files (!!)
-                !matches!(status, "??" | "!!")
-            } else {
-                false
-            }
-        });
-
-        if has_uncommitted_changes {
-            warn!("Git working directory has uncommitted changes to tracked files:");
-            for line in status_output.lines() {
-                if line.len() >= 2 && !matches!(&line[0..2], "??" | "!!") {
-                    warn!("  {}", line);
-                }
-            }
-            return Err(eyre!(
-                "Git working directory has uncommitted changes to tracked files. Please commit or stash changes before running benchmark comparison."
-            ));
-        }
-
-        // Check if there are untracked files and log them as info
-        let untracked_files: Vec<&str> =
-            status_output.lines().filter(|line| line.starts_with("??")).collect();
-
-        if !untracked_files.is_empty() {
-            info!(
-                "Git working directory has {} untracked files (this is OK)",
-                untracked_files.len()
-            );
-        }
-
-        info!("Git working directory is clean (no uncommitted changes to tracked files)");
-        Ok(())
-    }
-
-    /// Fetch all refs from remote to ensure we have latest branches and tags
-    pub(crate) fn fetch_all(&self) -> Result<()> {
-        let output = Command::new("git")
-            .args(["fetch", "--all", "--tags", "--quiet", "--force"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to fetch latest refs")?;
-
-        if output.status.success() {
-            info!("Fetched latest refs");
-        } else {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            // Only warn if there's actual error content, not just fetch progress
-            if !stderr.trim().is_empty() && !stderr.contains("-> origin/") {
-                warn!("Git fetch encountered issues (continuing anyway): {}", stderr);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Validate that the specified git references exist (branches, tags, or commits)
-    pub(crate) fn validate_refs(&self, refs: &[&str]) -> Result<()> {
-        for &git_ref in refs {
-            // Try to resolve the ref similar to `git checkout` by peeling to a commit.
-            // First try the ref as-is with ^{commit}, then fall back to origin/{ref}^{commit}.
-            let as_is = format!("{git_ref}^{{commit}}");
-            let ref_check = Command::new("git")
-                .args(["rev-parse", "--verify", &as_is])
-                .current_dir(&self.repo_root)
-                .output();
-
-            let found = if let Ok(output) = ref_check &&
-                output.status.success()
-            {
-                info!("Validated reference exists: {}", git_ref);
-                true
-            } else {
-                // Try remote-only branches via origin/{ref}
-                let origin_ref = format!("origin/{git_ref}^{{commit}}");
-                let origin_check = Command::new("git")
-                    .args(["rev-parse", "--verify", &origin_ref])
-                    .current_dir(&self.repo_root)
-                    .output();
-
-                if let Ok(output) = origin_check &&
-                    output.status.success()
-                {
-                    info!("Validated remote reference exists: origin/{}", git_ref);
-                    true
-                } else {
-                    false
-                }
-            };
-
-            if !found {
-                return Err(eyre!(
-                    "Git reference '{}' does not exist as branch, tag, or commit (tried '{}' and 'origin/{}^{{commit}}')",
-                    git_ref,
-                    format!("{git_ref}^{{commit}}"),
-                    git_ref,
-                ));
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Switch to the specified git reference (branch, tag, or commit)
-    pub(crate) fn switch_ref(&self, git_ref: &str) -> Result<()> {
-        // First checkout the reference
-        let output = Command::new("git")
-            .args(["checkout", git_ref])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err_with(|| format!("Failed to switch to reference '{git_ref}'"))?;
-
-        if !output.status.success() {
-            let stderr = String::from_utf8_lossy(&output.stderr);
-            return Err(eyre!("Failed to switch to reference '{}': {}", git_ref, stderr));
-        }
-
-        // Check if this is a branch that tracks a remote and pull latest changes
-        let is_branch = Command::new("git")
-            .args(["show-ref", "--verify", "--quiet", &format!("refs/heads/{git_ref}")])
-            .current_dir(&self.repo_root)
-            .status()
-            .map(|s| s.success())
-            .unwrap_or(false);
-
-        if is_branch {
-            // Check if the branch tracks a remote
-            let tracking_output = Command::new("git")
-                .args([
-                    "rev-parse",
-                    "--abbrev-ref",
-                    "--symbolic-full-name",
-                    &format!("{git_ref}@{{upstream}}"),
-                ])
-                .current_dir(&self.repo_root)
-                .output();
-
-            if let Ok(output) = tracking_output &&
-                output.status.success()
-            {
-                let upstream = String::from_utf8_lossy(&output.stdout).trim().to_string();
-                if !upstream.is_empty() && upstream != format!("{git_ref}@{{upstream}}") {
-                    // Branch tracks a remote, pull latest changes
-                    info!("Pulling latest changes for branch: {}", git_ref);
-
-                    let pull_output = Command::new("git")
-                        .args(["pull", "--ff-only"])
-                        .current_dir(&self.repo_root)
-                        .output()
-                        .wrap_err_with(|| {
-                            format!("Failed to pull latest changes for branch '{git_ref}'")
-                        })?;
-
-                    if pull_output.status.success() {
-                        info!("Successfully pulled latest changes for branch: {}", git_ref);
-                    } else {
-                        let stderr = String::from_utf8_lossy(&pull_output.stderr);
-                        warn!("Failed to pull latest changes for branch '{}': {}", git_ref, stderr);
-                        // Continue anyway, we'll use whatever version we have
-                    }
-                }
-            }
-        }
-
-        // Verify the checkout succeeded by checking the current commit
-        let current_commit_output = Command::new("git")
-            .args(["rev-parse", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current commit")?;
-
-        if !current_commit_output.status.success() {
-            return Err(eyre!("Failed to verify git checkout"));
-        }
-
-        info!("Switched to reference: {}", git_ref);
-        Ok(())
-    }
-
-    /// Get the current commit hash
-    pub(crate) fn get_current_commit(&self) -> Result<String> {
-        let output = Command::new("git")
-            .args(["rev-parse", "HEAD"])
-            .current_dir(&self.repo_root)
-            .output()
-            .wrap_err("Failed to get current commit")?;
-
-        if !output.status.success() {
-            return Err(eyre!("Failed to get current commit hash"));
-        }
-
-        let commit_hash = String::from_utf8(output.stdout)
-            .wrap_err("Commit hash is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        Ok(commit_hash)
-    }
-
-    /// Get the repository root path
-    pub(crate) fn repo_root(&self) -> &str {
-        &self.repo_root
-    }
-}

bin/reth-bench-compare/src/main.rs

@@ -1,47 +0,0 @@
-//! # reth-bench-compare
-//!
-//! Automated tool for comparing reth performance between two git branches.
-//! This tool automates the complete workflow of compiling, running, and benchmarking
-//! reth on different branches to provide meaningful performance comparisons.
-
-#![doc(
-    html_logo_url = "https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-docs.png",
-    html_favicon_url = "https://avatars0.githubusercontent.com/u/97369466?s=256",
-    issue_tracker_base_url = "https://github.com/paradigmxyz/reth/issues/"
-)]
-#![cfg_attr(not(test), warn(unused_crate_dependencies))]
-
-#[global_allocator]
-static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
-
-use alloy_primitives as _;
-
-mod benchmark;
-mod cli;
-mod comparison;
-mod compilation;
-mod git;
-mod node;
-
-use clap::Parser;
-use cli::{run_comparison, Args};
-use eyre::Result;
-use reth_cli_runner::CliRunner;
-
-fn main() -> Result<()> {
-    // Enable backtraces unless a RUST_BACKTRACE value has already been explicitly provided.
-    if std::env::var_os("RUST_BACKTRACE").is_none() {
-        unsafe {
-            std::env::set_var("RUST_BACKTRACE", "1");
-        }
-    }
-
-    let args = Args::parse();
-
-    // Initialize tracing
-    let _guard = args.init_tracing()?;
-
-    // Run until either exit or sigint or sigterm
-    let runner = CliRunner::try_default_runtime()?;
-    runner.run_command_until_exit(|ctx| run_comparison(args, ctx))
-}

bin/reth-bench-compare/src/node.rs

@@ -1,695 +0,0 @@
-//! Node management for starting, stopping, and controlling reth instances.
-
-use crate::cli::Args;
-use alloy_provider::{Provider, ProviderBuilder};
-use alloy_rpc_client::RpcClient;
-use alloy_rpc_types_eth::SyncStatus;
-use alloy_transport_ws::WsConnect;
-use eyre::{eyre, OptionExt, Result, WrapErr};
-#[cfg(unix)]
-use nix::sys::signal::{killpg, Signal};
-#[cfg(unix)]
-use nix::unistd::Pid;
-use reth_chainspec::Chain;
-use std::{fs, path::PathBuf, time::Duration};
-use tokio::{
-    fs::File as AsyncFile,
-    io::{AsyncBufReadExt, AsyncWriteExt, BufReader as AsyncBufReader},
-    process::Command,
-    time::{sleep, timeout},
-};
-use tracing::{debug, info, warn};
-
-/// Default websocket RPC port used by reth
-const DEFAULT_WS_RPC_PORT: u16 = 8546;
-
-/// Manages reth node lifecycle and operations
-pub(crate) struct NodeManager {
-    datadir: Option<String>,
-    metrics_port: u16,
-    chain: Chain,
-    use_sudo: bool,
-    binary_path: Option<std::path::PathBuf>,
-    enable_profiling: bool,
-    output_dir: PathBuf,
-    additional_reth_args: Vec<String>,
-    comparison_dir: Option<PathBuf>,
-    tracing_endpoint: Option<String>,
-    otlp_max_queue_size: usize,
-}
-
-impl NodeManager {
-    /// Create a new `NodeManager` with configuration from CLI args
-    pub(crate) fn new(args: &Args) -> Self {
-        Self {
-            datadir: Some(args.datadir_path().to_string_lossy().to_string()),
-            metrics_port: args.metrics_port,
-            chain: args.chain,
-            use_sudo: args.sudo,
-            binary_path: None,
-            enable_profiling: args.profile,
-            output_dir: args.output_dir_path(),
-            // Filter out empty strings to prevent invalid arguments being passed to reth node
-            additional_reth_args: args
-                .reth_args
-                .iter()
-                .filter(|s| !s.is_empty())
-                .cloned()
-                .collect(),
-            comparison_dir: None,
-            tracing_endpoint: args.traces.otlp.as_ref().map(|u| u.to_string()),
-            otlp_max_queue_size: args.otlp_max_queue_size,
-        }
-    }
-
-    /// Set the comparison directory path for logging
-    pub(crate) fn set_comparison_dir(&mut self, dir: PathBuf) {
-        self.comparison_dir = Some(dir);
-    }
-
-    /// Get the log file path for a given reference type
-    fn get_log_file_path(&self, ref_type: &str) -> Result<PathBuf> {
-        let comparison_dir = self
-            .comparison_dir
-            .as_ref()
-            .ok_or_eyre("Comparison directory not set. Call set_comparison_dir first.")?;
-
-        // The comparison directory already contains the full path to results/<timestamp>
-        let log_dir = comparison_dir.join(ref_type);
-
-        // Create the directory if it doesn't exist
-        fs::create_dir_all(&log_dir)
-            .wrap_err(format!("Failed to create log directory: {:?}", log_dir))?;
-
-        let log_file = log_dir.join("reth_node.log");
-        Ok(log_file)
-    }
-
-    /// Get the perf event max sample rate from the system, capped at 10000
-    fn get_perf_sample_rate(&self) -> Option<String> {
-        let perf_rate_file = "/proc/sys/kernel/perf_event_max_sample_rate";
-        if let Ok(content) = fs::read_to_string(perf_rate_file) {
-            let rate_str = content.trim();
-            if !rate_str.is_empty() {
-                if let Ok(system_rate) = rate_str.parse::<u32>() {
-                    let capped_rate = std::cmp::min(system_rate, 10000);
-                    info!(
-                        "Detected perf_event_max_sample_rate: {}, using: {}",
-                        system_rate, capped_rate
-                    );
-                    return Some(capped_rate.to_string());
-                }
-                warn!("Failed to parse perf_event_max_sample_rate: {}", rate_str);
-            }
-        }
-        None
-    }
-
-    /// Get the absolute path to samply using 'which' command
-    async fn get_samply_path(&self) -> Result<String> {
-        let output = Command::new("which")
-            .arg("samply")
-            .output()
-            .await
-            .wrap_err("Failed to execute 'which samply' command")?;
-
-        if !output.status.success() {
-            return Err(eyre!("samply not found in PATH"));
-        }
-
-        let samply_path = String::from_utf8(output.stdout)
-            .wrap_err("samply path is not valid UTF-8")?
-            .trim()
-            .to_string();
-
-        if samply_path.is_empty() {
-            return Err(eyre!("which samply returned empty path"));
-        }
-
-        Ok(samply_path)
-    }
-
-    /// Build reth arguments as a vector of strings
-    fn build_reth_args(
-        &self,
-        binary_path_str: &str,
-        additional_args: &[String],
-        ref_type: &str,
-    ) -> (Vec<String>, String) {
-        let mut reth_args = vec![binary_path_str.to_string(), "node".to_string()];
-
-        // Add chain argument (skip for mainnet as it's the default)
-        let chain_str = self.chain.to_string();
-        if chain_str != "mainnet" {
-            reth_args.extend_from_slice(&["--chain".to_string(), chain_str.clone()]);
-        }
-
-        // Add datadir if specified
-        if let Some(ref datadir) = self.datadir {
-            reth_args.extend_from_slice(&["--datadir".to_string(), datadir.clone()]);
-        }
-
-        // Add reth-specific arguments
-        let metrics_arg = format!("0.0.0.0:{}", self.metrics_port);
-        reth_args.extend_from_slice(&[
-            "--engine.accept-execution-requests-hash".to_string(),
-            "--metrics".to_string(),
-            metrics_arg,
-            "--http".to_string(),
-            "--http.api".to_string(),
-            "eth,reth".to_string(),
-            "--ws".to_string(),
-            "--ws.api".to_string(),
-            "eth,reth".to_string(),
-            "--disable-discovery".to_string(),
-            "--trusted-only".to_string(),
-            "--disable-tx-gossip".to_string(),
-        ]);
-
-        // Add tracing arguments if OTLP endpoint is configured
-        if let Some(ref endpoint) = self.tracing_endpoint {
-            info!("Enabling OTLP tracing export to: {} (service: reth-{})", endpoint, ref_type);
-            // Endpoint requires equals per clap settings in reth
-            reth_args.push(format!("--tracing-otlp={}", endpoint));
-        }
-
-        // Add any additional arguments passed via command line (common to both baseline and
-        // feature)
-        reth_args.extend_from_slice(&self.additional_reth_args);
-
-        // Add reference-specific additional arguments
-        reth_args.extend_from_slice(additional_args);
-
-        (reth_args, chain_str)
-    }
-
-    /// Create a command for profiling mode
-    async fn create_profiling_command(
-        &self,
-        ref_type: &str,
-        reth_args: &[String],
-    ) -> Result<Command> {
-        // Create profiles directory if it doesn't exist
-        let profile_dir = self.output_dir.join("profiles");
-        fs::create_dir_all(&profile_dir).wrap_err("Failed to create profiles directory")?;
-
-        let profile_path = profile_dir.join(format!("{}.json.gz", ref_type));
-        info!("Starting reth node with samply profiling...");
-        info!("Profile output: {:?}", profile_path);
-
-        // Get absolute path to samply
-        let samply_path = self.get_samply_path().await?;
-
-        let mut cmd = if self.use_sudo {
-            let mut sudo_cmd = Command::new("sudo");
-            sudo_cmd.arg(&samply_path);
-            sudo_cmd
-        } else {
-            Command::new(&samply_path)
-        };
-
-        // Add samply arguments
-        cmd.args(["record", "--save-only", "-o", &profile_path.to_string_lossy()]);
-
-        // Add rate argument if available
-        if let Some(rate) = self.get_perf_sample_rate() {
-            cmd.args(["--rate", &rate]);
-        }
-
-        // Add separator and complete reth command
-        cmd.arg("--");
-        cmd.args(reth_args);
-
-        // Enable tracing-samply
-        if supports_samply_flags(&reth_args[0]) {
-            cmd.arg("--log.samply");
-        }
-
-        // Set environment variable to disable log styling
-        cmd.env("RUST_LOG_STYLE", "never");
-
-        Ok(cmd)
-    }
-
-    /// Create a command for direct reth execution
-    fn create_direct_command(&self, reth_args: &[String]) -> Command {
-        let binary_path = &reth_args[0];
-
-        let mut cmd = if self.use_sudo {
-            info!("Starting reth node with sudo...");
-            let mut sudo_cmd = Command::new("sudo");
-            sudo_cmd.args(reth_args);
-            sudo_cmd
-        } else {
-            info!("Starting reth node...");
-            let mut reth_cmd = Command::new(binary_path);
-            reth_cmd.args(&reth_args[1..]); // Skip the binary path since it's the command
-            reth_cmd
-        };
-
-        // Set environment variable to disable log styling
-        cmd.env("RUST_LOG_STYLE", "never");
-
-        cmd
-    }
-
-    /// Start a reth node using the specified binary path and return the process handle
-    /// along with the formatted reth command string for reporting.
-    pub(crate) async fn start_node(
-        &mut self,
-        binary_path: &std::path::Path,
-        _git_ref: &str,
-        ref_type: &str,
-        additional_args: &[String],
-    ) -> Result<(tokio::process::Child, String)> {
-        // Store the binary path for later use (e.g., in unwind_to_block)
-        self.binary_path = Some(binary_path.to_path_buf());
-
-        let binary_path_str = binary_path.to_string_lossy();
-        let (reth_args, _) = self.build_reth_args(&binary_path_str, additional_args, ref_type);
-
-        // Format the reth command string for reporting
-        let reth_command = shlex::try_join(reth_args.iter().map(|s| s.as_str()))
-            .wrap_err("Failed to format reth command string")?;
-
-        // Log additional arguments if any
-        if !self.additional_reth_args.is_empty() {
-            info!("Using common additional reth arguments: {:?}", self.additional_reth_args);
-        }
-        if !additional_args.is_empty() {
-            info!("Using reference-specific additional reth arguments: {:?}", additional_args);
-        }
-
-        let mut cmd = if self.enable_profiling {
-            self.create_profiling_command(ref_type, &reth_args).await?
-        } else {
-            self.create_direct_command(&reth_args)
-        };
-
-        // Set process group for better signal handling
-        #[cfg(unix)]
-        {
-            cmd.process_group(0);
-        }
-
-        // Set high queue size to prevent trace dropping during benchmarks
-        if self.tracing_endpoint.is_some() {
-            cmd.env("OTEL_BSP_MAX_QUEUE_SIZE", self.otlp_max_queue_size.to_string()); // Traces
-            cmd.env("OTEL_BLRP_MAX_QUEUE_SIZE", "10000"); // Logs
-
-            // Set service name to differentiate baseline vs feature runs in Jaeger
-            cmd.env("OTEL_SERVICE_NAME", format!("reth-{}", ref_type));
-        }
-
-        debug!("Executing reth command: {cmd:?}");
-
-        let mut child = cmd
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .kill_on_drop(true) // Kill on drop so that on Ctrl-C for parent process we stop all child processes
-            .spawn()
-            .wrap_err("Failed to start reth node")?;
-
-        info!(
-            "Reth node started with PID: {:?} (binary: {})",
-            child.id().ok_or_eyre("Reth node is not running")?,
-            binary_path_str
-        );
-
-        // Prepare log file path
-        let log_file_path = self.get_log_file_path(ref_type)?;
-        info!("Reth node logs will be saved to: {:?}", log_file_path);
-
-        // Stream stdout and stderr with prefixes at debug level and to log file
-        if let Some(stdout) = child.stdout.take() {
-            let log_file = AsyncFile::create(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to create log file: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stdout);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH] {}", line);
-                    // Write to log file (reth already includes timestamps)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        if let Some(stderr) = child.stderr.take() {
-            let log_file = AsyncFile::options()
-                .create(true)
-                .append(true)
-                .open(&log_file_path)
-                .await
-                .wrap_err(format!("Failed to open log file for stderr: {:?}", log_file_path))?;
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stderr);
-                let mut lines = reader.lines();
-                let mut log_file = log_file;
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH] {}", line);
-                    // Write to log file (reth already includes timestamps)
-                    let log_line = format!("{}\n", line);
-                    if let Err(e) = log_file.write_all(log_line.as_bytes()).await {
-                        debug!("Failed to write to log file: {}", e);
-                    }
-                }
-            });
-        }
-
-        // Give the node a moment to start up
-        sleep(Duration::from_secs(5)).await;
-
-        Ok((child, reth_command))
-    }
-
-    /// Wait for the node to be ready and return its current tip.
-    ///
-    /// Fails early if the node process exits before becoming ready.
-    pub(crate) async fn wait_for_node_ready_and_get_tip(
-        &self,
-        child: &mut tokio::process::Child,
-    ) -> Result<u64> {
-        info!("Waiting for node to be ready and synced...");
-
-        let max_wait = Duration::from_secs(120); // 2 minutes to allow for sync
-        let check_interval = Duration::from_secs(2);
-        let rpc_url = "http://localhost:8545";
-
-        // Create Alloy provider
-        let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
-        let provider = ProviderBuilder::new().connect_http(url);
-
-        let start_time = tokio::time::Instant::now();
-        let mut iteration = 0;
-
-        timeout(max_wait, async {
-            loop {
-                iteration += 1;
-                debug!(
-                    "Readiness check iteration {} (elapsed: {:?})",
-                    iteration,
-                    start_time.elapsed()
-                );
-
-                // Check if the node process has exited.
-                if let Some(status) = child.try_wait()? {
-                    return Err(eyre!("Node process exited unexpectedly with {status}"));
-                }
-
-                // First check if RPC is up and node is not syncing
-                match provider.syncing().await {
-                    Ok(sync_result) => {
-                        match sync_result {
-                            SyncStatus::Info(sync_info) => {
-                                debug!("Node is still syncing {sync_info:?}, waiting...");
-                            }
-                            _ => {
-                                debug!("HTTP RPC is up and node is not syncing, checking block number...");
-                                // Node is not syncing, now get the tip
-                                match provider.get_block_number().await {
-                                    Ok(tip) => {
-                                        debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
-
-                                        // Verify WebSocket RPC is ready (public endpoint, no JWT required)
-                                        let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
-                                        debug!("Attempting WebSocket connection to {} (public endpoint)", ws_url);
-                                        let ws_connect = WsConnect::new(&ws_url);
-
-                                        match RpcClient::connect_pubsub(ws_connect).await
-                                        {
-                                            Ok(_) => {
-                                                info!(
-                                                    "Node is ready (HTTP and WebSocket) at block: {} (took {:?}, {} iterations)",
-                                                    tip, start_time.elapsed(), iteration
-                                                );
-                                                return Ok(tip);
-                                            }
-                                            Err(e) => {
-                                                debug!(
-                                                    "HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
-                                                    iteration, e
-                                                );
-                                                debug!("WebSocket error details: {}", e);
-                                            }
-                                        }
-                                    }
-                                    Err(e) => {
-                                        debug!("Failed to get block number (iteration {}): {:?}", iteration, e);
-                                    }
-                                }
-                            }
-                        }
-                    }
-                    Err(e) => {
-                        debug!("Node RPC not ready yet or failed to check sync status (iteration {}): {:?}", iteration, e);
-                    }
-                }
-
-                debug!("Sleeping for {:?} before next check", check_interval);
-                sleep(check_interval).await;
-            }
-        })
-        .await
-        .wrap_err("Timed out waiting for node to be ready and synced")?
-    }
-
-    /// Wait for the node RPC to be ready and return its current tip, without waiting for sync.
-    ///
-    /// This is faster than `wait_for_node_ready_and_get_tip` but may return a tip while
-    /// the node is still syncing.
-    pub(crate) async fn wait_for_rpc_and_get_tip(
-        &self,
-        child: &mut tokio::process::Child,
-    ) -> Result<u64> {
-        info!("Waiting for node RPC to be ready (skipping sync wait)...");
-
-        let max_wait = Duration::from_secs(60);
-        let check_interval = Duration::from_secs(2);
-        let rpc_url = "http://localhost:8545";
-
-        let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
-        let provider = ProviderBuilder::new().connect_http(url);
-
-        let start_time = tokio::time::Instant::now();
-        let mut iteration = 0;
-
-        timeout(max_wait, async {
-            loop {
-                iteration += 1;
-                debug!(
-                    "RPC readiness check iteration {} (elapsed: {:?})",
-                    iteration,
-                    start_time.elapsed()
-                );
-
-                if let Some(status) = child.try_wait()? {
-                    return Err(eyre!("Node process exited unexpectedly with {status}"));
-                }
-
-                match provider.get_block_number().await {
-                    Ok(tip) => {
-                        debug!("HTTP RPC ready at block: {}, checking WebSocket...", tip);
-
-                        let ws_url = format!("ws://localhost:{}", DEFAULT_WS_RPC_PORT);
-                        let ws_connect = WsConnect::new(&ws_url);
-
-                        match RpcClient::connect_pubsub(ws_connect).await {
-                            Ok(_) => {
-                                info!(
-                                    "Node RPC is ready at block: {} (took {:?}, {} iterations)",
-                                    tip,
-                                    start_time.elapsed(),
-                                    iteration
-                                );
-                                return Ok(tip);
-                            }
-                            Err(e) => {
-                                debug!(
-                                    "HTTP RPC ready but WebSocket not ready yet (iteration {}): {:?}",
-                                    iteration, e
-                                );
-                            }
-                        }
-                    }
-                    Err(e) => {
-                        debug!("RPC not ready yet (iteration {}): {:?}", iteration, e);
-                    }
-                }
-
-                sleep(check_interval).await;
-            }
-        })
-        .await
-        .wrap_err("Timed out waiting for node RPC to be ready")?
-    }
-
-    /// Stop the reth node gracefully
-    pub(crate) async fn stop_node(&self, child: &mut tokio::process::Child) -> Result<()> {
-        let pid = child.id().ok_or_eyre("Child process ID should be available")?;
-
-        // Check if the process has already exited
-        match child.try_wait() {
-            Ok(Some(status)) => {
-                info!("Reth node (PID: {}) has already exited with status: {:?}", pid, status);
-                return Ok(());
-            }
-            Ok(None) => {
-                // Process is still running, proceed to stop it
-                info!("Stopping process gracefully with SIGINT (PID: {})...", pid);
-            }
-            Err(e) => {
-                return Err(eyre!("Failed to check process status: {}", e));
-            }
-        }
-
-        #[cfg(unix)]
-        {
-            // Send SIGINT to process group to mimic Ctrl-C behavior
-            let nix_pgid = Pid::from_raw(pid as i32);
-
-            match killpg(nix_pgid, Signal::SIGINT) {
-                Ok(()) => {}
-                Err(nix::errno::Errno::ESRCH) => {
-                    info!("Process group {} has already exited", pid);
-                }
-                Err(e) => {
-                    return Err(eyre!("Failed to send SIGINT to process group {}: {}", pid, e));
-                }
-            }
-        }
-
-        #[cfg(not(unix))]
-        {
-            // On non-Unix systems, fall back to using external kill command
-            let output = Command::new("taskkill")
-                .args(["/PID", &pid.to_string(), "/F"])
-                .output()
-                .await
-                .wrap_err("Failed to execute taskkill command")?;
-
-            if !output.status.success() {
-                let stderr = String::from_utf8_lossy(&output.stderr);
-                // Check if the error is because the process doesn't exist
-                if stderr.contains("not found") || stderr.contains("not exist") {
-                    info!("Process {} has already exited", pid);
-                } else {
-                    return Err(eyre!("Failed to kill process {}: {}", pid, stderr));
-                }
-            }
-        }
-
-        // Wait for the process to exit
-        match child.wait().await {
-            Ok(status) => {
-                info!("Reth node (PID: {}) exited with status: {:?}", pid, status);
-            }
-            Err(e) => {
-                // If we get an error here, it might be because the process already exited
-                debug!("Error waiting for process exit (may have already exited): {}", e);
-            }
-        }
-
-        Ok(())
-    }
-
-    /// Unwind the node to a specific block
-    pub(crate) async fn unwind_to_block(&self, block_number: u64) -> Result<()> {
-        if self.use_sudo {
-            info!("Unwinding node to block: {} (with sudo)", block_number);
-        } else {
-            info!("Unwinding node to block: {}", block_number);
-        }
-
-        // Use the binary path from the last start_node call, or fallback to default
-        let binary_path = self
-            .binary_path
-            .as_ref()
-            .map(|p| p.to_string_lossy().to_string())
-            .unwrap_or_else(|| "./target/profiling/reth".to_string());
-
-        let mut cmd = if self.use_sudo {
-            let mut sudo_cmd = Command::new("sudo");
-            sudo_cmd.args([&binary_path, "stage", "unwind"]);
-            sudo_cmd
-        } else {
-            let mut reth_cmd = Command::new(&binary_path);
-            reth_cmd.args(["stage", "unwind"]);
-            reth_cmd
-        };
-
-        // Add chain argument (skip for mainnet as it's the default)
-        let chain_str = self.chain.to_string();
-        if chain_str != "mainnet" {
-            cmd.args(["--chain", &chain_str]);
-        }
-
-        // Add datadir if specified
-        if let Some(ref datadir) = self.datadir {
-            cmd.args(["--datadir", datadir]);
-        }
-
-        cmd.args(["to-block", &block_number.to_string()]);
-
-        // Set environment variable to disable log styling
-        cmd.env("RUST_LOG_STYLE", "never");
-
-        // Debug log the command
-        debug!("Executing reth unwind command: {:?}", cmd);
-
-        let mut child = cmd
-            .stdout(std::process::Stdio::piped())
-            .stderr(std::process::Stdio::piped())
-            .spawn()
-            .wrap_err("Failed to start unwind command")?;
-
-        // Stream stdout and stderr with prefixes in real-time
-        if let Some(stdout) = child.stdout.take() {
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stdout);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-UNWIND] {}", line);
-                }
-            });
-        }
-
-        if let Some(stderr) = child.stderr.take() {
-            tokio::spawn(async move {
-                let reader = AsyncBufReader::new(stderr);
-                let mut lines = reader.lines();
-                while let Ok(Some(line)) = lines.next_line().await {
-                    debug!("[RETH-UNWIND] {}", line);
-                }
-            });
-        }
-
-        // Wait for the command to complete
-        let status = child.wait().await.wrap_err("Failed to wait for unwind command")?;
-
-        if !status.success() {
-            return Err(eyre!("Unwind command failed with exit code: {:?}", status.code()));
-        }
-
-        info!("Unwound to block: {}", block_number);
-        Ok(())
-    }
-}
-
-fn supports_samply_flags(bin: &str) -> bool {
-    let mut cmd = std::process::Command::new(bin);
-    // NOTE: The flag to check must come before --help.
-    // We pass --help as a shortcut to not execute any command.
-    cmd.args(["--log.samply", "--help"]);
-    debug!(?cmd, "Checking samply flags support");
-    let Ok(output) = cmd.output() else {
-        return false;
-    };
-    debug!(?output, "Samply flags support check");
-    output.status.success()
-}

bin/reth-bench/Cargo.toml

@@ -14,9 +14,12 @@ workspace = true
 
 [dependencies]
 # reth
+reth-chainspec.workspace = true
+reth-cli.workspace = true
 reth-cli-runner.workspace = true
 reth-cli-util.workspace = true
 reth-engine-primitives.workspace = true
+reth-ethereum-cli.workspace = true
 reth-ethereum-primitives.workspace = true
 reth-fs-util.workspace = true
 reth-node-api.workspace = true
@@ -25,13 +28,12 @@ reth-primitives-traits.workspace = true
 reth-rpc-api.workspace = true
 
 reth-tracing.workspace = true
-reth-chainspec.workspace = true
 
 # alloy
+alloy-consensus.workspace = true
 alloy-eips.workspace = true
 alloy-json-rpc.workspace = true
-alloy-consensus.workspace = true
-alloy-network.workspace = true
+
 alloy-primitives = { workspace = true, features = ["rand"] }
 alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
 alloy-pubsub.workspace = true
@@ -82,39 +84,54 @@ default = ["jemalloc"]
 
 asm-keccak = [
     "reth-node-core/asm-keccak",
+    "reth-ethereum-cli/asm-keccak",
     "alloy-primitives/asm-keccak",
 ]
 
 jemalloc = [
     "reth-cli-util/jemalloc",
     "reth-node-core/jemalloc",
+    "reth-ethereum-cli/jemalloc",
+]
+jemalloc-prof = [
+    "reth-cli-util/jemalloc-prof",
+    "reth-ethereum-cli/jemalloc-prof",
+]
+tracy-allocator = [
+    "reth-cli-util/tracy-allocator",
+    "tracy",
+    "reth-ethereum-cli/tracy-allocator",
 ]
-jemalloc-prof = ["reth-cli-util/jemalloc-prof"]
-tracy-allocator = ["reth-cli-util/tracy-allocator", "tracy"]
 tracy = [
     "reth-node-core/tracy",
     "reth-tracing/tracy",
+    "reth-ethereum-cli/tracy",
 ]
 
 min-error-logs = [
     "tracing/release_max_level_error",
     "reth-node-core/min-error-logs",
+    "reth-ethereum-cli/min-error-logs",
 ]
 min-warn-logs = [
     "tracing/release_max_level_warn",
     "reth-node-core/min-warn-logs",
+    "reth-ethereum-cli/min-warn-logs",
 ]
 min-info-logs = [
     "tracing/release_max_level_info",
     "reth-node-core/min-info-logs",
+    "reth-ethereum-cli/min-info-logs",
 ]
 min-debug-logs = [
     "tracing/release_max_level_debug",
     "reth-node-core/min-debug-logs",
+    "reth-ethereum-cli/min-debug-logs",
 ]
 min-trace-logs = [
     "tracing/release_max_level_trace",
     "reth-node-core/min-trace-logs",
+    "reth-ethereum-cli/min-trace-logs",
 ]
 
 # no-op feature flag for CI matrices

bin/reth-bench/README.md

@@ -35,6 +35,10 @@ The `new-payload-fcu` command supports two optional waiting modes that can be us
 - `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms` or `--wait-time 400` for 400ms)
 - `--wait-for-persistence`: Waits for blocks to be persisted using the `reth_subscribePersistedBlock` subscription
 
+Both `new-payload-fcu` and `new-payload-only` support `--rpc-block-fetch-retries <RETRIES>`
+to control how many times block fetches are retried after an RPC failure. The default is `10`.
+Use `--rpc-block-fetch-retries forever` to keep retrying indefinitely.
+
 When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
 
 By default, the WebSocket URL for persistence subscriptions is derived from `--engine-rpc-url` (converting to ws:// on port 8546). Use `--ws-rpc-url` to override this.

bin/reth-bench/src/authenticated_transport.rs

@@ -52,8 +52,7 @@ impl InnerTransport {
         url: Url,
         jwt: JwtSecret,
     ) -> Result<(Self, Claims), AuthenticatedTransportError> {
-        let mut client_builder =
-            reqwest::Client::builder().tls_built_in_root_certs(url.scheme() == "https");
+        let mut client_builder = reqwest::Client::builder();
         let mut headers = reqwest::header::HeaderMap::new();
 
         // Add the JWT to the headers if we can decode it.

bin/reth-bench/src/bench/context.rs

@@ -9,7 +9,7 @@ use alloy_rpc_client::ClientBuilder;
 use alloy_rpc_types_engine::JwtSecret;
 use alloy_transport::layers::{RateLimitRetryPolicy, RetryBackoffLayer};
 use reqwest::Url;
-use reth_node_core::args::BenchmarkArgs;
+use reth_node_core::args::{BenchmarkArgs, WaitForPersistence};
 use tracing::info;
 
 /// This is intended to be used by benchmarks that replay blocks from an RPC.
@@ -33,6 +33,10 @@ pub(crate) struct BenchContext {
     pub(crate) use_reth_namespace: bool,
     /// Whether to fetch and replay RLP-encoded blocks.
     pub(crate) rlp_blocks: bool,
+    /// Controls when `reth_newPayload` waits for persistence.
+    pub(crate) wait_for_persistence: WaitForPersistence,
+    /// Whether to skip waiting for caches (pass `wait_for_caches: false`).
+    pub(crate) no_wait_for_caches: bool,
 }
 
 impl BenchContext {
@@ -60,8 +64,9 @@ impl BenchContext {
                     .and_then(|t| t.as_http_error())
                     .is_some_and(|e| e.status == 502)
             });
+        let max_retries = bench_args.rpc_block_fetch_retries.as_max_retries();
         let client = ClientBuilder::default()
-            .layer(RetryBackoffLayer::new_with_policy(10, 800, u64::MAX, retry_policy))
+            .layer(RetryBackoffLayer::new_with_policy(max_retries, 800, u64::MAX, retry_policy))
             .http(rpc_url.parse()?);
         let block_provider = RootProvider::<AnyNetwork>::new(client);
 
@@ -161,7 +166,10 @@ impl BenchContext {
 
         let next_block = first_block.header.number + 1;
         let rlp_blocks = bench_args.rlp_blocks;
+        let wait_for_persistence =
+            bench_args.wait_for_persistence.unwrap_or(WaitForPersistence::Never);
         let use_reth_namespace = bench_args.reth_new_payload || rlp_blocks;
+        let no_wait_for_caches = bench_args.no_wait_for_caches;
         Ok(Self {
             auth_provider,
             block_provider,
@@ -170,6 +178,8 @@ impl BenchContext {
             is_optimism,
             use_reth_namespace,
             rlp_blocks,
+            wait_for_persistence,
+            no_wait_for_caches,
         })
     }
 }

bin/reth-bench/src/bench/gas_limit_ramp.rs

@@ -1,251 +0,0 @@
-//! Benchmarks empty block processing by ramping the block gas limit.
-
-use crate::{
-    authenticated_transport::AuthenticatedTransportConnect,
-    bench::{
-        helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
-        output::GasRampPayloadFile,
-    },
-    valid_payload::{
-        call_forkchoice_updated_with_reth, call_new_payload_with_reth, payload_to_new_payload,
-    },
-};
-use alloy_eips::BlockNumberOrTag;
-use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
-use alloy_rpc_client::ClientBuilder;
-use alloy_rpc_types_engine::{ExecutionPayload, ForkchoiceState, JwtSecret};
-
-use clap::Parser;
-use reqwest::Url;
-use reth_chainspec::ChainSpec;
-use reth_cli_runner::CliContext;
-use reth_ethereum_primitives::TransactionSigned;
-use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIMIT_BLOCK};
-use reth_rpc_api::RethNewPayloadInput;
-use std::{path::PathBuf, time::Instant};
-use tracing::info;
-
-/// `reth benchmark gas-limit-ramp` command.
-#[derive(Debug, Parser)]
-pub struct Command {
-    /// Number of blocks to generate. Mutually exclusive with --target-gas-limit.
-    #[arg(long, value_name = "BLOCKS", conflicts_with = "target_gas_limit")]
-    blocks: Option<u64>,
-
-    /// Target gas limit to ramp up to. The benchmark will generate blocks until the gas limit
-    /// reaches or exceeds this value. Mutually exclusive with --blocks.
-    /// Accepts short notation: K for thousand, M for million, G for billion (e.g., 2G = 2
-    /// billion).
-    #[arg(long, value_name = "TARGET_GAS_LIMIT", conflicts_with = "blocks", value_parser = parse_gas_limit)]
-    target_gas_limit: Option<u64>,
-
-    /// The Engine API RPC URL.
-    #[arg(long = "engine-rpc-url", value_name = "ENGINE_RPC_URL")]
-    engine_rpc_url: String,
-
-    /// Path to the JWT secret for Engine API authentication.
-    #[arg(long = "jwt-secret", value_name = "JWT_SECRET")]
-    jwt_secret: PathBuf,
-
-    /// Output directory for benchmark results and generated payloads.
-    #[arg(long, value_name = "OUTPUT")]
-    output: PathBuf,
-
-    /// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
-    ///
-    /// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
-    /// directly, waits for persistence and cache updates to complete before processing,
-    /// and returns server-side timing breakdowns (latency, persistence wait, cache wait).
-    #[arg(long, default_value = "false", verbatim_doc_comment)]
-    reth_new_payload: bool,
-}
-
-/// Mode for determining when to stop ramping.
-#[derive(Debug, Clone, Copy)]
-enum RampMode {
-    /// Ramp for a fixed number of blocks.
-    Blocks(u64),
-    /// Ramp until reaching or exceeding target gas limit.
-    TargetGasLimit(u64),
-}
-
-impl Command {
-    /// Execute `benchmark gas-limit-ramp` command.
-    pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
-        let mode = match (self.blocks, self.target_gas_limit) {
-            (Some(blocks), None) => {
-                if blocks == 0 {
-                    return Err(eyre::eyre!("--blocks must be greater than 0"));
-                }
-                RampMode::Blocks(blocks)
-            }
-            (None, Some(target)) => {
-                if target == 0 {
-                    return Err(eyre::eyre!("--target-gas-limit must be greater than 0"));
-                }
-                RampMode::TargetGasLimit(target)
-            }
-            _ => {
-                return Err(eyre::eyre!(
-                    "Exactly one of --blocks or --target-gas-limit must be specified"
-                ));
-            }
-        };
-
-        // Ensure output directory exists
-        if self.output.is_file() {
-            return Err(eyre::eyre!("Output path must be a directory"));
-        }
-        if !self.output.exists() {
-            std::fs::create_dir_all(&self.output)?;
-            info!(target: "reth-bench", "Created output directory: {:?}", self.output);
-        }
-
-        // Set up authenticated provider (used for both Engine API and eth_ methods)
-        let jwt = std::fs::read_to_string(&self.jwt_secret)?;
-        let jwt = JwtSecret::from_hex(jwt)?;
-        let auth_url = Url::parse(&self.engine_rpc_url)?;
-
-        info!(target: "reth-bench", "Connecting to Engine RPC at {}", auth_url);
-        let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
-        let client = ClientBuilder::default().connect_with(auth_transport).await?;
-        let provider = RootProvider::<AnyNetwork>::new(client);
-
-        // Get chain spec - required for fork detection
-        let chain_id = provider.get_chain_id().await?;
-        let chain_spec = ChainSpec::from_chain_id(chain_id)
-            .ok_or_else(|| eyre::eyre!("Unsupported chain id: {chain_id}"))?;
-
-        // Fetch the current head block as parent
-        let parent_block = provider
-            .get_block_by_number(BlockNumberOrTag::Latest)
-            .full()
-            .await?
-            .ok_or_else(|| eyre::eyre!("Failed to fetch latest block"))?;
-
-        let (mut parent_header, mut parent_hash) = rpc_block_to_header(parent_block);
-
-        let canonical_parent = parent_header.number;
-        let start_block = canonical_parent + 1;
-
-        match mode {
-            RampMode::Blocks(blocks) => {
-                info!(
-                    target: "reth-bench",
-                    canonical_parent,
-                    start_block,
-                    end_block = start_block + blocks - 1,
-                    "Starting gas limit ramp benchmark (block count mode)"
-                );
-            }
-            RampMode::TargetGasLimit(target) => {
-                info!(
-                    target: "reth-bench",
-                    canonical_parent,
-                    start_block,
-                    current_gas_limit = parent_header.gas_limit,
-                    target_gas_limit = target,
-                    "Starting gas limit ramp benchmark (target gas limit mode)"
-                );
-            }
-        }
-        if self.reth_new_payload {
-            info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
-        }
-
-        let mut blocks_processed = 0u64;
-        let total_benchmark_duration = Instant::now();
-
-        while !should_stop(mode, blocks_processed, parent_header.gas_limit) {
-            let timestamp = parent_header.timestamp.saturating_add(1);
-
-            let request = prepare_payload_request(&chain_spec, timestamp, parent_hash);
-            let new_payload_version = request.new_payload_version;
-
-            let (payload, sidecar) = build_payload(&provider, request).await?;
-
-            let mut block =
-                payload.clone().try_into_block_with_sidecar::<TransactionSigned>(&sidecar)?;
-
-            let max_increase = max_gas_limit_increase(parent_header.gas_limit);
-            let gas_limit =
-                parent_header.gas_limit.saturating_add(max_increase).min(MAXIMUM_GAS_LIMIT_BLOCK);
-
-            block.header.gas_limit = gas_limit;
-
-            let block_hash = block.header.hash_slow();
-            // Regenerate the payload from the modified block, but keep the original sidecar
-            // which contains the actual execution requests data (not just the hash)
-            let (payload, _) = ExecutionPayload::from_block_unchecked(block_hash, &block);
-            let (version, params, execution_data) = payload_to_new_payload(
-                payload,
-                sidecar,
-                false,
-                block.header.withdrawals_root,
-                Some(new_payload_version),
-            )?;
-
-            let (version, params) = if self.reth_new_payload {
-                (None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?)
-            } else {
-                (Some(version), params)
-            };
-
-            // Save payload to file with version info for replay
-            let payload_path =
-                self.output.join(format!("payload_block_{}.json", block.header.number));
-            let file = GasRampPayloadFile {
-                version: version.map(|v| v as u8),
-                block_hash,
-                params: params.clone(),
-            };
-            let payload_json = serde_json::to_string_pretty(&file)?;
-            std::fs::write(&payload_path, &payload_json)?;
-            info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
-
-            let _ = call_new_payload_with_reth(&provider, version, params).await?;
-
-            let forkchoice_state = ForkchoiceState {
-                head_block_hash: block_hash,
-                safe_block_hash: block_hash,
-                finalized_block_hash: block_hash,
-            };
-            call_forkchoice_updated_with_reth(&provider, version, forkchoice_state).await?;
-
-            parent_header = block.header;
-            parent_hash = block_hash;
-            blocks_processed += 1;
-
-            let progress = match mode {
-                RampMode::Blocks(total) => format!("{blocks_processed}/{total}"),
-                RampMode::TargetGasLimit(target) => {
-                    let pct = (parent_header.gas_limit as f64 / target as f64 * 100.0).min(100.0);
-                    format!("{pct:.1}%")
-                }
-            };
-            info!(target: "reth-bench", progress, block_number = parent_header.number, gas_limit = parent_header.gas_limit, "Block processed");
-        }
-
-        let final_gas_limit = parent_header.gas_limit;
-        info!(
-            target: "reth-bench",
-            total_duration=?total_benchmark_duration.elapsed(),
-            blocks_processed,
-            final_gas_limit,
-            "Benchmark complete"
-        );
-
-        Ok(())
-    }
-}
-
-const fn max_gas_limit_increase(parent_gas_limit: u64) -> u64 {
-    (parent_gas_limit / GAS_LIMIT_BOUND_DIVISOR).saturating_sub(1)
-}
-
-const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u64) -> bool {
-    match mode {
-        RampMode::Blocks(target_blocks) => blocks_processed >= target_blocks,
-        RampMode::TargetGasLimit(target) => current_gas_limit >= target,
-    }
-}

bin/reth-bench/src/bench/helpers.rs

@@ -1,6 +1,5 @@
 //! Common helpers for reth-bench commands.
 
-use crate::valid_payload::call_forkchoice_updated;
 use eyre::Result;
 use std::{
     io::{BufReader, Read},
@@ -70,180 +69,6 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
     }
 }
 
-use alloy_consensus::Header;
-use alloy_eips::eip4844::kzg_to_versioned_hash;
-use alloy_primitives::{Address, B256};
-use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
-use alloy_rpc_types_engine::{
-    CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
-    PayloadAttributes, PayloadId,
-};
-use eyre::OptionExt;
-use reth_chainspec::{ChainSpec, EthereumHardforks};
-use reth_node_api::EngineApiMessageVersion;
-use tracing::debug;
-
-/// Prepared payload request data for triggering block building.
-pub(crate) struct PayloadRequest {
-    /// The payload attributes for the new block.
-    pub(crate) attributes: PayloadAttributes,
-    /// The forkchoice state pointing to the parent block.
-    pub(crate) forkchoice_state: ForkchoiceState,
-    /// The engine API version for FCU calls.
-    pub(crate) fcu_version: EngineApiMessageVersion,
-    /// The getPayload version to use (1-5).
-    pub(crate) get_payload_version: u8,
-    /// The newPayload version to use.
-    pub(crate) new_payload_version: EngineApiMessageVersion,
-}
-
-/// Prepare payload attributes and forkchoice state for a new block.
-pub(crate) fn prepare_payload_request(
-    chain_spec: &ChainSpec,
-    timestamp: u64,
-    parent_hash: B256,
-) -> PayloadRequest {
-    let shanghai_active = chain_spec.is_shanghai_active_at_timestamp(timestamp);
-    let cancun_active = chain_spec.is_cancun_active_at_timestamp(timestamp);
-    let prague_active = chain_spec.is_prague_active_at_timestamp(timestamp);
-    let osaka_active = chain_spec.is_osaka_active_at_timestamp(timestamp);
-
-    // FCU version: V3 for Cancun+Prague+Osaka, V2 for Shanghai, V1 otherwise
-    let fcu_version = if cancun_active {
-        EngineApiMessageVersion::V3
-    } else if shanghai_active {
-        EngineApiMessageVersion::V2
-    } else {
-        EngineApiMessageVersion::V1
-    };
-
-    // getPayload version: 5 for Osaka, 4 for Prague, 3 for Cancun, 2 for Shanghai, 1 otherwise
-    // newPayload version: 4 for Prague+Osaka (no V5), 3 for Cancun, 2 for Shanghai, 1 otherwise
-    let (get_payload_version, new_payload_version) = if osaka_active {
-        (5, EngineApiMessageVersion::V4) // Osaka uses getPayloadV5 but newPayloadV4
-    } else if prague_active {
-        (4, EngineApiMessageVersion::V4)
-    } else if cancun_active {
-        (3, EngineApiMessageVersion::V3)
-    } else if shanghai_active {
-        (2, EngineApiMessageVersion::V2)
-    } else {
-        (1, EngineApiMessageVersion::V1)
-    };
-
-    PayloadRequest {
-        attributes: PayloadAttributes {
-            timestamp,
-            prev_randao: B256::ZERO,
-            suggested_fee_recipient: Address::ZERO,
-            withdrawals: shanghai_active.then(Vec::new),
-            parent_beacon_block_root: cancun_active.then_some(B256::ZERO),
-        },
-        forkchoice_state: ForkchoiceState {
-            head_block_hash: parent_hash,
-            safe_block_hash: parent_hash,
-            finalized_block_hash: parent_hash,
-        },
-        fcu_version,
-        get_payload_version,
-        new_payload_version,
-    }
-}
-
-/// Trigger payload building via FCU and retrieve the built payload.
-///
-/// This sends a forkchoiceUpdated with payload attributes to start building,
-/// then calls getPayload to retrieve the result.
-pub(crate) async fn build_payload(
-    provider: &RootProvider<AnyNetwork>,
-    request: PayloadRequest,
-) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
-    let fcu_result = call_forkchoice_updated(
-        provider,
-        request.fcu_version,
-        request.forkchoice_state,
-        Some(request.attributes.clone()),
-    )
-    .await?;
-
-    let payload_id =
-        fcu_result.payload_id.ok_or_eyre("Payload builder did not return a payload id")?;
-
-    get_payload_with_sidecar(
-        provider,
-        request.get_payload_version,
-        payload_id,
-        request.attributes.parent_beacon_block_root,
-    )
-    .await
-}
-
-/// Convert an RPC block to a consensus header and block hash.
-pub(crate) fn rpc_block_to_header(block: alloy_provider::network::AnyRpcBlock) -> (Header, B256) {
-    let block_hash = block.header.hash;
-    let header = block.header.inner.clone().into_header_with_defaults();
-    (header, block_hash)
-}
-
-/// Compute versioned hashes from KZG commitments.
-fn versioned_hashes_from_commitments(
-    commitments: &[alloy_primitives::FixedBytes<48>],
-) -> Vec<B256> {
-    commitments.iter().map(|c| kzg_to_versioned_hash(c.as_ref())).collect()
-}
-
-/// Fetch an execution payload using the appropriate engine API version.
-pub(crate) async fn get_payload_with_sidecar(
-    provider: &RootProvider<AnyNetwork>,
-    version: u8,
-    payload_id: PayloadId,
-    parent_beacon_block_root: Option<B256>,
-) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
-    debug!(target: "reth-bench", get_payload_version = ?version, ?payload_id, "Sending getPayload");
-
-    match version {
-        1 => {
-            let payload = provider.get_payload_v1(payload_id).await?;
-            Ok((ExecutionPayload::V1(payload), ExecutionPayloadSidecar::none()))
-        }
-        2 => {
-            let envelope = provider.get_payload_v2(payload_id).await?;
-            let payload = match envelope.execution_payload {
-                alloy_rpc_types_engine::ExecutionPayloadFieldV2::V1(p) => ExecutionPayload::V1(p),
-                alloy_rpc_types_engine::ExecutionPayloadFieldV2::V2(p) => ExecutionPayload::V2(p),
-            };
-            Ok((payload, ExecutionPayloadSidecar::none()))
-        }
-        3 => {
-            let envelope = provider.get_payload_v3(payload_id).await?;
-            let versioned_hashes =
-                versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
-            let cancun_fields = CancunPayloadFields {
-                parent_beacon_block_root: parent_beacon_block_root
-                    .ok_or_eyre("parent_beacon_block_root required for V3")?,
-                versioned_hashes,
-            };
-            Ok((
-                ExecutionPayload::V3(envelope.execution_payload),
-                ExecutionPayloadSidecar::v3(cancun_fields),
-            ))
-        }
-        4 => {
-            let envelope = provider.get_payload_v4(payload_id).await?;
-            Ok(envelope.into_payload_and_sidecar(
-                parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V4")?,
-            ))
-        }
-        5 => {
-            let envelope = provider.get_payload_v5(payload_id).await?;
-            Ok(envelope.into_payload_and_sidecar(
-                parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V5")?,
-            ))
-        }
-        _ => panic!("This tool does not support getPayload versions past v5"),
-    }
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

bin/reth-bench/src/bench/mod.rs

@@ -6,17 +6,16 @@ use reth_node_core::args::LogArgs;
 use reth_tracing::FileWorkerGuard;
 
 mod context;
-mod gas_limit_ramp;
 mod generate_big_block;
 pub(crate) mod helpers;
 pub use generate_big_block::{
-    RawTransaction, RpcTransactionSource, TransactionCollector, TransactionSource,
+    compute_payload_block_hash, BigBlockPayload, RawTransaction, RpcTransactionSource,
+    TransactionCollector, TransactionSource,
 };
 pub(crate) mod metrics_scraper;
 mod new_payload_fcu;
 mod new_payload_only;
 mod output;
-mod persistence_waiter;
 mod replay_payloads;
 mod send_invalid_payload;
 mod send_payload;
@@ -37,9 +36,6 @@ pub enum Subcommands {
     /// Benchmark which calls `newPayload`, then `forkchoiceUpdated`.
     NewPayloadFcu(new_payload_fcu::Command),
 
-    /// Benchmark which builds empty blocks with a ramped gas limit.
-    GasLimitRamp(gas_limit_ramp::Command),
-
     /// Benchmark which only calls subsequent `newPayload` calls.
     NewPayloadOnly(new_payload_only::Command),
 
@@ -55,16 +51,16 @@ pub enum Subcommands {
     /// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex)`
     SendPayload(send_payload::Command),
 
-    /// Generate a large block by packing transactions from existing blocks.
+    /// Generate a large block by merging consecutive blocks from an RPC.
     ///
-    /// This command fetches transactions from real blocks and packs them into a single
-    /// block using the `testing_buildBlockV1` RPC endpoint.
+    /// Fetches N consecutive blocks, takes block 0 as the base payload, concatenates
+    /// transactions from blocks 1..N-1, and saves the result to disk as a JSON file
+    /// containing the merged execution data and environment switches at block boundaries.
     ///
     /// Example:
     ///
-    /// `reth-bench generate-big-block --rpc-url http://localhost:8545 --engine-rpc-url
-    /// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex --target-gas
-    /// 30000000`
+    /// `reth-bench generate-big-block --rpc-url http://localhost:8545 --from-block 20000000
+    /// --count 10 --output-dir ./payloads`
     GenerateBigBlock(generate_big_block::Command),
 
     /// Replay pre-generated payloads from a directory.
@@ -99,7 +95,6 @@ impl BenchmarkCommand {
 
         match self.command {
             Subcommands::NewPayloadFcu(command) => command.execute(ctx).await,
-            Subcommands::GasLimitRamp(command) => command.execute(ctx).await,
             Subcommands::NewPayloadOnly(command) => command.execute(ctx).await,
             Subcommands::SendPayload(command) => command.execute(ctx).await,
             Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,

bin/reth-bench/src/bench/new_payload_fcu.rs

@@ -1,13 +1,5 @@
 //! Runs the `reth bench` command, calling first newPayload for each block, then calling
 //! forkchoiceUpdated.
-//!
-//! Supports configurable waiting behavior:
-//! - **`--wait-time`**: Fixed sleep interval between blocks.
-//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
-//!   `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
-//!   threshold. This ensures the benchmark doesn't outpace persistence.
-//!
-//! Both options can be used together or independently.
 
 use crate::{
     bench::{
@@ -17,9 +9,6 @@ use crate::{
         output::{
             write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
         },
-        persistence_waiter::{
-            derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
-        },
     },
     valid_payload::{
         block_to_new_payload, call_forkchoice_updated_with_reth, call_new_payload_with_reth,
@@ -29,6 +18,7 @@ use alloy_provider::{ext::DebugApi, Provider};
 use alloy_rpc_types_engine::ForkchoiceState;
 use clap::Parser;
 use eyre::{Context, OptionExt};
+use futures::{stream, StreamExt, TryStreamExt};
 use reth_cli_runner::CliContext;
 use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
 use reth_node_core::args::BenchmarkArgs;
@@ -49,16 +39,6 @@ pub struct Command {
     #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
     wait_time: Option<Duration>,
 
-    /// Wait for blocks to be persisted before sending the next batch.
-    ///
-    /// When enabled, waits for every Nth block to be persisted using the
-    /// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
-    /// doesn't outpace persistence.
-    ///
-    /// The subscription uses the regular RPC websocket endpoint (no JWT required).
-    #[arg(long, default_value = "false", verbatim_doc_comment)]
-    wait_for_persistence: bool,
-
     /// Engine persistence threshold used for deciding when to wait for persistence.
     ///
     /// The benchmark waits after every `(threshold + 1)` blocks. By default this
@@ -106,55 +86,17 @@ impl Command {
         if let Some(duration) = self.wait_time {
             info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
         }
-        if self.wait_for_persistence {
-            info!(
-                target: "reth-bench",
-                "Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
-                self.persistence_threshold + 1,
-                self.persistence_threshold
-            );
-        }
-
-        // Set up waiter based on configured options
-        // When both are set: wait at least wait_time, and also wait for persistence if needed
-        let mut waiter = match (self.wait_time, self.wait_for_persistence) {
-            (Some(duration), true) => {
-                let ws_url = derive_ws_rpc_url(
-                    self.benchmark.ws_rpc_url.as_deref(),
-                    &self.benchmark.engine_rpc_url,
-                )?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_duration_and_subscription(
-                    duration,
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
-            (None, true) => {
-                let ws_url = derive_ws_rpc_url(
-                    self.benchmark.ws_rpc_url.as_deref(),
-                    &self.benchmark.engine_rpc_url,
-                )?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_subscription(
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (None, false) => None,
-        };
 
         let BenchContext {
             benchmark_mode,
             block_provider,
             auth_provider,
-            mut next_block,
+            next_block,
             is_optimism,
             use_reth_namespace,
             rlp_blocks,
+            wait_for_persistence,
+            no_wait_for_caches,
         } = BenchContext::new(&self.benchmark, self.rpc_url).await?;
 
         let total_blocks = benchmark_mode.total_blocks();
@@ -167,70 +109,71 @@ impl Command {
 
         let buffer_size = self.rpc_block_buffer_size;
 
-        // Use a oneshot channel to propagate errors from the spawned task
-        let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
-        let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
+        let mut blocks = Box::pin(
+            stream::iter((next_block..)
+                .take_while(|next_block| {
+                    benchmark_mode.contains(*next_block)
+                }))
+                .map(|next_block| {
+                    let block_provider = block_provider.clone();
+                    async move {
+                        let block_res = block_provider
+                            .get_block_by_number(next_block.into())
+                            .full()
+                            .await
+                            .wrap_err_with(|| {
+                                format!("Failed to fetch block by number {next_block}")
+                            });
+                        let block =
+                            match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
+                                Ok(block) => block,
+                                Err(e) => {
+                                    tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
+                                    return Err(e)
+                                }
+                            };
 
-        tokio::task::spawn(async move {
-            while benchmark_mode.contains(next_block) {
-                let block_res = block_provider
-                    .get_block_by_number(next_block.into())
-                    .full()
-                    .await
-                    .wrap_err_with(|| format!("Failed to fetch block by number {next_block}"));
-                let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
-                    Ok(block) => block,
-                    Err(e) => {
-                        tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
-                        let _ = error_sender.send(e);
-                        break;
+                        let rlp = if rlp_blocks {
+                            let rlp = match block_provider
+                                .debug_get_raw_block(next_block.into())
+                                .await
+                            {
+                                Ok(rlp) => rlp,
+                                Err(e) => {
+                                    tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
+                                    return Err(e.into())
+                                }
+                            };
+                            Some(rlp)
+                        } else {
+                            None
+                        };
+
+                        let head_block_hash = block.header.hash;
+                        let safe_block_hash = block_provider
+                            .get_block_by_number(block.header.number.saturating_sub(32).into());
+
+                        let finalized_block_hash = block_provider
+                            .get_block_by_number(block.header.number.saturating_sub(64).into());
+
+                        let (safe, finalized) =
+                            tokio::join!(safe_block_hash, finalized_block_hash);
+
+                        let safe_block_hash = match safe {
+                            Ok(Some(block)) => block.header.hash,
+                            Ok(None) | Err(_) => head_block_hash,
+                        };
+
+                        let finalized_block_hash = match finalized {
+                            Ok(Some(block)) => block.header.hash,
+                            Ok(None) | Err(_) => head_block_hash,
+                        };
+
+                        Ok((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
                     }
-                };
-
-                let rlp = if rlp_blocks {
-                    let rlp = match block_provider.debug_get_raw_block(next_block.into()).await {
-                        Ok(rlp) => rlp,
-                        Err(e) => {
-                            tracing::error!(target: "reth-bench", "Failed to fetch raw block {next_block}: {e}");
-                            let _ = error_sender
-                                .send(eyre::eyre!("Failed to fetch raw block {next_block}: {e}"));
-                            break;
-                        }
-                    };
-                    Some(rlp)
-                } else {
-                    None
-                };
-
-                let head_block_hash = block.header.hash;
-                let safe_block_hash = block_provider
-                    .get_block_by_number(block.header.number.saturating_sub(32).into());
-
-                let finalized_block_hash = block_provider
-                    .get_block_by_number(block.header.number.saturating_sub(64).into());
-
-                let (safe, finalized) = tokio::join!(safe_block_hash, finalized_block_hash,);
-
-                let safe_block_hash = match safe {
-                    Ok(Some(block)) => block.header.hash,
-                    Ok(None) | Err(_) => head_block_hash,
-                };
-
-                let finalized_block_hash = match finalized {
-                    Ok(Some(block)) => block.header.hash,
-                    Ok(None) | Err(_) => head_block_hash,
-                };
-
-                next_block += 1;
-                if let Err(e) = sender
-                    .send((block, head_block_hash, safe_block_hash, finalized_block_hash, rlp))
-                    .await
-                {
-                    tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
-                    break;
-                }
-            }
-        });
+                })
+                .buffered(buffer_size),
+        );
 
         let mut results = Vec::new();
         let mut blocks_processed = 0u64;
@@ -239,7 +182,7 @@ impl Command {
 
         while let Some((block, head, safe, finalized, rlp)) = {
             let wait_start = Instant::now();
-            let result = receiver.recv().await;
+            let result = blocks.try_next().await?;
             total_wait_time += wait_start.elapsed();
             result
         } {
@@ -256,8 +199,14 @@ impl Command {
                 finalized_block_hash: finalized,
             };
 
-            let (version, params) =
-                block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
+            let (version, params) = block_to_new_payload(
+                block,
+                is_optimism,
+                rlp,
+                use_reth_namespace,
+                wait_for_persistence,
+                no_wait_for_caches,
+            )?;
             let start = Instant::now();
             let server_timings =
                 call_new_payload_with_reth(&auth_provider, version, params).await?;
@@ -315,8 +264,8 @@ impl Command {
                 warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
             }
 
-            if let Some(w) = &mut waiter {
-                w.on_block(block_number).await?;
+            if let Some(wait_time) = self.wait_time {
+                tokio::time::sleep(wait_time).await;
             }
 
             let gas_row =
@@ -324,15 +273,6 @@ impl Command {
             results.push((gas_row, combined_result));
         }
 
-        // Check if the spawned task encountered an error
-        if let Ok(error) = error_receiver.try_recv() {
-            return Err(error);
-        }
-
-        // Drop waiter - we don't need to wait for final blocks to persist
-        // since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
-        drop(waiter);
-
         let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
             results.into_iter().unzip();
 

bin/reth-bench/src/bench/new_payload_only.rs

@@ -52,6 +52,8 @@ impl Command {
             is_optimism,
             use_reth_namespace,
             rlp_blocks,
+            wait_for_persistence,
+            no_wait_for_caches,
         } = BenchContext::new(&self.benchmark, self.rpc_url).await?;
 
         let total_blocks = benchmark_mode.total_blocks();
@@ -122,8 +124,14 @@ impl Command {
 
             debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
 
-            let (version, params) =
-                block_to_new_payload(block, is_optimism, rlp, use_reth_namespace)?;
+            let (version, params) = block_to_new_payload(
+                block,
+                is_optimism,
+                rlp,
+                use_reth_namespace,
+                wait_for_persistence,
+                no_wait_for_caches,
+            )?;
 
             let start = Instant::now();
             let server_timings =

bin/reth-bench/src/bench/output.rs

@@ -1,11 +1,10 @@
 //! Contains various benchmark output formats, either for logging or for
 //! serialization to / from files.
 
-use alloy_primitives::B256;
 use csv::Writer;
 use eyre::OptionExt;
 use reth_primitives_traits::constants::GIGAGAS;
-use serde::{ser::SerializeStruct, Deserialize, Serialize};
+use serde::{ser::SerializeStruct, Serialize};
 use std::{fs, path::Path, time::Duration};
 use tracing::info;
 
@@ -18,20 +17,6 @@ pub(crate) const COMBINED_OUTPUT_SUFFIX: &str = "combined_latency.csv";
 /// This is the suffix for new payload output csv files.
 pub(crate) const NEW_PAYLOAD_OUTPUT_SUFFIX: &str = "new_payload_latency.csv";
 
-/// Serialized format for gas ramp payloads on disk.
-#[derive(Debug, Serialize, Deserialize)]
-pub(crate) struct GasRampPayloadFile {
-    /// Engine API version (1-5).
-    ///
-    /// `None` indicates that `reth_newPayload` should be used.
-    #[serde(default, skip_serializing_if = "Option::is_none")]
-    pub(crate) version: Option<u8>,
-    /// The block hash for FCU.
-    pub(crate) block_hash: B256,
-    /// The params to pass to newPayload.
-    pub(crate) params: serde_json::Value,
-}
-
 /// This represents the results of a single `newPayload` call in the benchmark, containing the gas
 /// used and the `newPayload` latency.
 #[derive(Debug)]

bin/reth-bench/src/bench/persistence_waiter.rs

@@ -1,334 +0,0 @@
-//! Persistence waiting utilities for benchmarks.
-//!
-//! Provides waiting behavior to control benchmark pacing:
-//! - **Fixed duration waits**: Sleep for a fixed time between blocks
-//! - **Persistence-based waits**: Wait for blocks to be persisted using
-//!   `reth_subscribePersistedBlock` subscription
-//! - **Combined mode**: Wait at least the fixed duration, and also wait for persistence if the
-//!   block hasn't been persisted yet (whichever takes longer)
-
-use alloy_eips::BlockNumHash;
-use alloy_network::Ethereum;
-use alloy_provider::{Provider, RootProvider};
-use alloy_pubsub::SubscriptionStream;
-use alloy_rpc_client::RpcClient;
-use alloy_transport_ws::WsConnect;
-use eyre::Context;
-use futures::StreamExt;
-use std::time::Duration;
-use tracing::{debug, info};
-
-/// Default `WebSocket` RPC port for reth.
-const DEFAULT_WS_RPC_PORT: u16 = 8546;
-use url::Url;
-
-/// Returns the websocket RPC URL used for the persistence subscription.
-///
-/// Preference:
-/// - If `ws_rpc_url` is provided, use it directly.
-/// - Otherwise, derive a WS RPC URL from `engine_rpc_url`.
-///
-/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
-/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
-/// Since we may only have the engine URL by default, we convert the scheme
-/// (http→ws, https→wss) and force the port to 8546.
-pub(crate) fn derive_ws_rpc_url(
-    ws_rpc_url: Option<&str>,
-    engine_rpc_url: &str,
-) -> eyre::Result<Url> {
-    if let Some(ws_url) = ws_rpc_url {
-        let parsed: Url = ws_url
-            .parse()
-            .wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
-        info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
-        Ok(parsed)
-    } else {
-        let derived = engine_url_to_ws_url(engine_rpc_url)?;
-        debug!(
-            target: "reth-bench",
-            engine_url = %engine_rpc_url,
-            %derived,
-            "Derived WebSocket RPC URL from engine RPC URL"
-        );
-        Ok(derived)
-    }
-}
-
-/// Converts an engine API URL to the default RPC websocket URL.
-///
-/// Transformations:
-/// - `http`  → `ws`
-/// - `https` → `wss`
-/// - `ws` / `wss` keep their scheme
-/// - Port is always set to `8546`, reth's default RPC websocket port.
-///
-/// This is used when we only know the engine API URL (typically `:8551`) but
-/// need to connect to the node's WS RPC endpoint for persistence events.
-fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
-    let url: Url = engine_url
-        .parse()
-        .wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
-
-    let mut ws_url = url.clone();
-
-    match ws_url.scheme() {
-        "http" => ws_url
-            .set_scheme("ws")
-            .map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
-        "https" => ws_url
-            .set_scheme("wss")
-            .map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
-        "ws" | "wss" => {}
-        scheme => {
-            return Err(eyre::eyre!(
-            "Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
-        ))
-        }
-    }
-
-    ws_url
-        .set_port(Some(DEFAULT_WS_RPC_PORT))
-        .map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
-
-    Ok(ws_url)
-}
-
-/// Waits until the persistence subscription reports that `target` has been persisted.
-///
-/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
-/// - the subscription stream ends unexpectedly, or
-/// - `timeout` elapses before `target` is observed.
-async fn wait_for_persistence(
-    stream: &mut SubscriptionStream<BlockNumHash>,
-    target: u64,
-    last_persisted: &mut u64,
-    timeout: Duration,
-) -> eyre::Result<()> {
-    tokio::time::timeout(timeout, async {
-        while *last_persisted < target {
-            match stream.next().await {
-                Some(persisted) => {
-                    *last_persisted = persisted.number;
-                    debug!(
-                        target: "reth-bench",
-                        persisted_block = ?last_persisted,
-                        "Received persistence notification"
-                    );
-                }
-                None => {
-                    return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
-                }
-            }
-        }
-        Ok(())
-    })
-    .await
-    .map_err(|_| {
-        eyre::eyre!(
-            "Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
-            target,
-            timeout,
-            last_persisted
-        )
-    })?
-}
-
-/// Wrapper that keeps both the subscription stream and the underlying provider alive.
-/// The provider must be kept alive for the subscription to continue receiving events.
-pub(crate) struct PersistenceSubscription {
-    _provider: RootProvider<Ethereum>,
-    stream: SubscriptionStream<BlockNumHash>,
-}
-
-impl PersistenceSubscription {
-    const fn new(
-        provider: RootProvider<Ethereum>,
-        stream: SubscriptionStream<BlockNumHash>,
-    ) -> Self {
-        Self { _provider: provider, stream }
-    }
-
-    const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
-        &mut self.stream
-    }
-}
-
-/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
-///
-/// The `keepalive_interval` is set to match `persistence_timeout` so that the `WebSocket`
-/// connection is not dropped during long MDBX commits that block the server from responding
-/// to pings.
-pub(crate) async fn setup_persistence_subscription(
-    ws_url: Url,
-    persistence_timeout: Duration,
-) -> eyre::Result<PersistenceSubscription> {
-    info!(target: "reth-bench", "Connecting to WebSocket at {} for persistence subscription", ws_url);
-
-    let ws_connect =
-        WsConnect::new(ws_url.to_string()).with_keepalive_interval(persistence_timeout);
-    let client = RpcClient::connect_pubsub(ws_connect)
-        .await
-        .wrap_err("Failed to connect to WebSocket RPC endpoint")?;
-    let provider: RootProvider<Ethereum> = RootProvider::new(client);
-
-    let subscription = provider
-        .subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
-        .await
-        .wrap_err("Failed to subscribe to persistence notifications")?;
-
-    info!(target: "reth-bench", "Subscribed to persistence notifications");
-
-    Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
-}
-
-/// Encapsulates the block waiting logic.
-///
-/// Provides a simple `on_block()` interface that handles both:
-/// - Fixed duration waits (when `wait_time` is set)
-/// - Persistence-based waits (when `subscription` is set)
-///
-/// For persistence mode, waits after every `(threshold + 1)` blocks.
-pub(crate) struct PersistenceWaiter {
-    wait_time: Option<Duration>,
-    subscription: Option<PersistenceSubscription>,
-    blocks_sent: u64,
-    last_persisted: u64,
-    threshold: u64,
-    timeout: Duration,
-}
-
-impl PersistenceWaiter {
-    pub(crate) const fn with_duration(wait_time: Duration) -> Self {
-        Self {
-            wait_time: Some(wait_time),
-            subscription: None,
-            blocks_sent: 0,
-            last_persisted: 0,
-            threshold: 0,
-            timeout: Duration::ZERO,
-        }
-    }
-
-    pub(crate) const fn with_subscription(
-        subscription: PersistenceSubscription,
-        threshold: u64,
-        timeout: Duration,
-    ) -> Self {
-        Self {
-            wait_time: None,
-            subscription: Some(subscription),
-            blocks_sent: 0,
-            last_persisted: 0,
-            threshold,
-            timeout,
-        }
-    }
-
-    /// Creates a waiter that combines both duration and persistence waiting.
-    ///
-    /// Waits at least `wait_time` between blocks, and also waits for persistence
-    /// if the block hasn't been persisted yet (whichever takes longer).
-    pub(crate) const fn with_duration_and_subscription(
-        wait_time: Duration,
-        subscription: PersistenceSubscription,
-        threshold: u64,
-        timeout: Duration,
-    ) -> Self {
-        Self {
-            wait_time: Some(wait_time),
-            subscription: Some(subscription),
-            blocks_sent: 0,
-            last_persisted: 0,
-            threshold,
-            timeout,
-        }
-    }
-
-    /// Called once per block. Waits based on the configured mode.
-    ///
-    /// When both `wait_time` and `subscription` are set (combined mode):
-    /// - Always waits at least `wait_time`
-    /// - Additionally waits for persistence if we're at a persistence checkpoint
-    #[allow(clippy::manual_is_multiple_of)]
-    pub(crate) async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
-        // Always wait for the fixed duration if configured
-        if let Some(wait_time) = self.wait_time {
-            tokio::time::sleep(wait_time).await;
-        }
-
-        // Check persistence if subscription is configured
-        let Some(ref mut subscription) = self.subscription else {
-            return Ok(());
-        };
-
-        self.blocks_sent += 1;
-
-        if self.blocks_sent % (self.threshold + 1) == 0 {
-            debug!(
-                target: "reth-bench",
-                target_block = ?block_number,
-                last_persisted = self.last_persisted,
-                blocks_sent = self.blocks_sent,
-                "Waiting for persistence"
-            );
-
-            wait_for_persistence(
-                subscription.stream_mut(),
-                block_number,
-                &mut self.last_persisted,
-                self.timeout,
-            )
-            .await?;
-
-            debug!(
-                target: "reth-bench",
-                persisted = self.last_persisted,
-                "Persistence caught up"
-            );
-        }
-
-        Ok(())
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use super::*;
-    use std::time::Instant;
-
-    #[test]
-    fn test_engine_url_to_ws_url() {
-        // http -> ws, always uses port 8546
-        let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
-        assert_eq!(result.as_str(), "ws://localhost:8546/");
-
-        // https -> wss
-        let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
-        assert_eq!(result.as_str(), "wss://localhost:8546/");
-
-        // Custom engine port still maps to 8546
-        let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
-        assert_eq!(result.port(), Some(8546));
-
-        // Already ws passthrough
-        let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
-        assert_eq!(result.scheme(), "ws");
-
-        // Invalid inputs
-        assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
-        assert!(engine_url_to_ws_url("not a valid url").is_err());
-    }
-
-    #[tokio::test]
-    async fn test_waiter_with_duration() {
-        let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
-
-        let start = Instant::now();
-        waiter.on_block(1).await.unwrap();
-        waiter.on_block(2).await.unwrap();
-        waiter.on_block(3).await.unwrap();
-
-        // Should have waited ~3ms total
-        assert!(start.elapsed() >= Duration::from_millis(3));
-    }
-}

bin/reth-bench/src/bench/replay_payloads.rs

@@ -1,27 +1,13 @@
 //! Command for replaying pre-generated payloads from disk.
-//!
-//! This command reads `ExecutionPayloadEnvelopeV4` files from a directory and replays them
-//! in sequence using `newPayload` followed by `forkchoiceUpdated`.
-//!
-//! Supports configurable waiting behavior:
-//! - **`--wait-time`**: Fixed sleep interval between blocks.
-//! - **`--wait-for-persistence`**: Waits for every Nth block to be persisted using the
-//!   `reth_subscribePersistedBlock` subscription, where N matches the engine's persistence
-//!   threshold. This ensures the benchmark doesn't outpace persistence.
-//!
-//! Both options can be used together or independently.
 
 use crate::{
     authenticated_transport::AuthenticatedTransportConnect,
     bench::{
+        generate_big_block::BigBlockPayload,
         helpers::parse_duration,
         metrics_scraper::MetricsScraper,
         output::{
-            write_benchmark_results, CombinedResult, GasRampPayloadFile, NewPayloadResult,
-            TotalGasOutput, TotalGasRow,
-        },
-        persistence_waiter::{
-            derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
+            write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
         },
     },
     valid_payload::{call_forkchoice_updated_with_reth, call_new_payload_with_reth},
@@ -36,14 +22,15 @@ use alloy_rpc_types_engine::{
 use clap::Parser;
 use eyre::Context;
 use reth_cli_runner::CliContext;
-use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
+use reth_engine_primitives::BigBlockData;
 use reth_node_api::EngineApiMessageVersion;
+use reth_node_core::args::WaitForPersistence;
 use reth_rpc_api::RethNewPayloadInput;
 use std::{
     path::PathBuf,
     time::{Duration, Instant},
 };
-use tracing::{debug, info};
+use tracing::{debug, info, warn};
 use url::Url;
 
 /// `reth bench replay-payloads` command
@@ -73,9 +60,8 @@ pub struct Command {
     #[arg(long, value_name = "SKIP", default_value = "0")]
     skip: usize,
 
-    /// Optional directory containing gas ramp payloads to replay first.
-    /// These are replayed before the main payloads to warm up the gas limit.
-    #[arg(long, value_name = "GAS_RAMP_DIR")]
+    /// Deprecated: gas ramp is no longer needed. This flag is accepted but ignored.
+    #[arg(long, value_name = "GAS_RAMP_DIR", hide = true)]
     gas_ramp_dir: Option<PathBuf>,
 
     /// Optional output directory for benchmark results (CSV files).
@@ -89,47 +75,6 @@ pub struct Command {
     #[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
     wait_time: Option<Duration>,
 
-    /// Wait for blocks to be persisted before sending the next batch.
-    ///
-    /// When enabled, waits for every Nth block to be persisted using the
-    /// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
-    /// doesn't outpace persistence.
-    ///
-    /// The subscription uses the regular RPC websocket endpoint (no JWT required).
-    #[arg(long, default_value = "false", verbatim_doc_comment)]
-    wait_for_persistence: bool,
-
-    /// Engine persistence threshold used for deciding when to wait for persistence.
-    ///
-    /// The benchmark waits after every `(threshold + 1)` blocks. By default this
-    /// matches the engine's `DEFAULT_PERSISTENCE_THRESHOLD` (2), so waits occur
-    /// at blocks 3, 6, 9, etc.
-    #[arg(
-        long = "persistence-threshold",
-        value_name = "PERSISTENCE_THRESHOLD",
-        default_value_t = DEFAULT_PERSISTENCE_THRESHOLD,
-        verbatim_doc_comment
-    )]
-    persistence_threshold: u64,
-
-    /// Timeout for waiting on persistence at each checkpoint.
-    ///
-    /// Must be long enough to account for the persistence thread being blocked
-    /// by pruning after the previous save.
-    #[arg(
-        long = "persistence-timeout",
-        value_name = "PERSISTENCE_TIMEOUT",
-        value_parser = parse_duration,
-        default_value = "120s",
-        verbatim_doc_comment
-    )]
-    persistence_timeout: Duration,
-
-    /// Optional `WebSocket` RPC URL for persistence subscription.
-    /// If not provided, derives from engine RPC URL by changing scheme to ws and port to 8546.
-    #[arg(long, value_name = "WS_RPC_URL", verbatim_doc_comment)]
-    ws_rpc_url: Option<String>,
-
     /// Use `reth_newPayload` endpoint instead of `engine_newPayload*`.
     ///
     /// The `reth_newPayload` endpoint is a reth-specific extension that takes `ExecutionData`
@@ -138,6 +83,30 @@ pub struct Command {
     #[arg(long, default_value = "false", verbatim_doc_comment)]
     reth_new_payload: bool,
 
+    /// Control when `reth_newPayload` waits for in-flight persistence.
+    ///
+    /// Accepts `always` (default — wait on every block), `never`, or a number N
+    /// to wait every N blocks and skip the rest.
+    ///
+    /// Requires `--reth-new-payload`.
+    #[arg(
+        long = "wait-for-persistence",
+        value_name = "MODE",
+        num_args = 0..=1,
+        default_missing_value = "always",
+        value_parser = clap::value_parser!(WaitForPersistence),
+        requires = "reth_new_payload",
+        verbatim_doc_comment
+    )]
+    wait_for_persistence: Option<WaitForPersistence>,
+
+    /// Skip waiting for execution cache and sparse trie locks before processing.
+    ///
+    /// Only works with `--reth-new-payload`. When set, passes `wait_for_caches: false`
+    /// to the `reth_newPayload` endpoint.
+    #[arg(long, default_value = "false", verbatim_doc_comment, requires = "reth_new_payload")]
+    no_wait_for_caches: bool,
+
     /// Optional Prometheus metrics endpoint to scrape after each block.
     ///
     /// When provided, reth-bench will fetch metrics from this URL after each
@@ -151,22 +120,12 @@ pub struct Command {
 struct LoadedPayload {
     /// The index (from filename).
     index: u64,
-    /// The payload envelope.
-    envelope: ExecutionPayloadEnvelopeV4,
+    /// The execution data for the block.
+    execution_data: ExecutionData,
     /// The block hash.
     block_hash: B256,
-}
-
-/// A gas ramp payload loaded from disk.
-struct GasRampPayload {
-    /// Block number from filename.
-    block_number: u64,
-    /// Engine API version for newPayload.
-    ///
-    /// `None` indicates that `reth_newPayload` should be used.
-    version: Option<EngineApiMessageVersion>,
-    /// The file contents.
-    file: GasRampPayloadFile,
+    /// Big block data containing environment switches and prior block hashes.
+    big_block_data: BigBlockData<ExecutionData>,
 }
 
 impl Command {
@@ -178,44 +137,10 @@ impl Command {
         if let Some(duration) = self.wait_time {
             info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
         }
-        if self.wait_for_persistence {
-            info!(
-                target: "reth-bench",
-                "Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
-                self.persistence_threshold + 1,
-                self.persistence_threshold
-            );
-        }
         if self.reth_new_payload {
             info!("Using reth_newPayload and reth_forkchoiceUpdated endpoints");
         }
 
-        // Set up waiter based on configured options
-        // When both are set: wait at least wait_time, and also wait for persistence if needed
-        let mut waiter = match (self.wait_time, self.wait_for_persistence) {
-            (Some(duration), true) => {
-                let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_duration_and_subscription(
-                    duration,
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
-            (None, true) => {
-                let ws_url = derive_ws_rpc_url(self.ws_rpc_url.as_deref(), &self.engine_rpc_url)?;
-                let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
-                Some(PersistenceWaiter::with_subscription(
-                    sub,
-                    self.persistence_threshold,
-                    self.persistence_timeout,
-                ))
-            }
-            (None, false) => None,
-        };
-
         let mut metrics_scraper = MetricsScraper::maybe_new(self.metrics_url.clone());
 
         // Set up authenticated engine provider
@@ -245,74 +170,48 @@ impl Command {
             "Using initial parent block"
         );
 
-        // Load all payloads upfront to avoid I/O delays between phases
-        let gas_ramp_payloads = if let Some(ref gas_ramp_dir) = self.gas_ramp_dir {
-            let payloads = self.load_gas_ramp_payloads(gas_ramp_dir)?;
-            if payloads.is_empty() {
-                return Err(eyre::eyre!("No gas ramp payload files found in {:?}", gas_ramp_dir));
-            }
-            info!(target: "reth-bench", count = payloads.len(), "Loaded gas ramp payloads from disk");
-            payloads
-        } else {
-            Vec::new()
-        };
+        // Warn if deprecated --gas-ramp-dir is passed
+        if self.gas_ramp_dir.is_some() {
+            warn!(
+                target: "reth-bench",
+                "--gas-ramp-dir is deprecated and ignored."
+            );
+        }
 
+        // Load all payloads upfront to avoid I/O delays between phases
         let payloads = self.load_payloads()?;
         if payloads.is_empty() {
             return Err(eyre::eyre!("No payload files found in {:?}", self.payload_dir));
         }
         info!(target: "reth-bench", count = payloads.len(), "Loaded main payloads from disk");
 
+        // If any payload has env_switches but we're not using reth_newPayload, warn the user
+        if !self.reth_new_payload {
+            let has_env_switches =
+                payloads.iter().any(|p| !p.big_block_data.env_switches.is_empty());
+            if has_env_switches {
+                warn!(
+                    target: "reth-bench",
+                    "Payloads contain env_switches but --reth-new-payload is not set. \
+                     env_switches are only supported with reth_newPayload and will be ignored."
+                );
+            }
+        }
+
         let mut parent_hash = initial_parent_hash;
 
-        // Replay gas ramp payloads first
-        for (i, payload) in gas_ramp_payloads.iter().enumerate() {
-            info!(
-                target: "reth-bench",
-                gas_ramp_payload = i + 1,
-                total = gas_ramp_payloads.len(),
-                block_number = payload.block_number,
-                block_hash = %payload.file.block_hash,
-                "Executing gas ramp payload (newPayload + FCU)"
-            );
-
-            let _ = call_new_payload_with_reth(
-                &auth_provider,
-                payload.version,
-                payload.file.params.clone(),
-            )
-            .await?;
-
-            let fcu_state = ForkchoiceState {
-                head_block_hash: payload.file.block_hash,
-                safe_block_hash: parent_hash,
-                finalized_block_hash: parent_hash,
-            };
-            call_forkchoice_updated_with_reth(&auth_provider, payload.version, fcu_state).await?;
-
-            info!(target: "reth-bench", gas_ramp_payload = i + 1, "Gas ramp payload executed successfully");
-
-            parent_hash = payload.file.block_hash;
-        }
-
-        if !gas_ramp_payloads.is_empty() {
-            info!(target: "reth-bench", count = gas_ramp_payloads.len(), "All gas ramp payloads replayed");
-        }
-
         let mut results = Vec::new();
         let total_benchmark_duration = Instant::now();
 
         for (i, payload) in payloads.iter().enumerate() {
-            let envelope = &payload.envelope;
+            let execution_data = &payload.execution_data;
             let block_hash = payload.block_hash;
-            let execution_payload = &envelope.envelope_inner.execution_payload;
-            let inner_payload = &execution_payload.payload_inner.payload_inner;
+            let v1 = execution_data.payload.as_v1();
 
-            let gas_used = inner_payload.gas_used;
-            let gas_limit = inner_payload.gas_limit;
-            let block_number = inner_payload.block_number;
-            let transaction_count =
-                execution_payload.payload_inner.payload_inner.transactions.len() as u64;
+            let gas_used = v1.gas_used;
+            let gas_limit = v1.gas_limit;
+            let block_number = v1.block_number;
+            let transaction_count = v1.transactions.len() as u64;
 
             debug!(
                 target: "reth-bench",
@@ -333,27 +232,36 @@ impl Command {
             );
 
             let (version, params) = if self.reth_new_payload {
-                let reth_data = ExecutionData {
-                    payload: execution_payload.clone().into(),
-                    sidecar: ExecutionPayloadSidecar::v4(
-                        CancunPayloadFields {
-                            versioned_hashes: Vec::new(),
-                            parent_beacon_block_root: B256::ZERO,
-                        },
-                        PraguePayloadFields {
-                            requests: envelope.execution_requests.clone().into(),
-                        },
-                    ),
+                let big_block_data_param = if payload.big_block_data.env_switches.is_empty() &&
+                    payload.big_block_data.prior_block_hashes.is_empty()
+                {
+                    None
+                } else {
+                    Some(payload.big_block_data.clone())
                 };
-                (None, serde_json::to_value((RethNewPayloadInput::ExecutionData(reth_data),))?)
+                let wait_for_persistence = self
+                    .wait_for_persistence
+                    .unwrap_or(WaitForPersistence::Never)
+                    .rpc_value(block_number);
+                (
+                    None,
+                    serde_json::to_value((
+                        RethNewPayloadInput::ExecutionData(execution_data.clone()),
+                        wait_for_persistence,
+                        self.no_wait_for_caches.then_some(false),
+                        big_block_data_param,
+                    ))?,
+                )
             } else {
+                let requests =
+                    execution_data.sidecar.requests().cloned().unwrap_or_default().to_vec();
                 (
                     Some(EngineApiMessageVersion::V4),
                     serde_json::to_value((
-                        execution_payload.clone(),
+                        execution_data.payload.clone(),
                         Vec::<B256>::new(),
                         B256::ZERO,
-                        envelope.execution_requests.to_vec(),
+                        requests,
                     ))?,
                 )
             };
@@ -409,8 +317,8 @@ impl Command {
                 tracing::warn!(target: "reth-bench", %err, block_number, "Failed to scrape metrics");
             }
 
-            if let Some(w) = &mut waiter {
-                w.on_block(block_number).await?;
+            if let Some(wait_time) = self.wait_time {
+                tokio::time::sleep(wait_time).await;
             }
 
             let gas_row =
@@ -420,10 +328,6 @@ impl Command {
             parent_hash = block_hash;
         }
 
-        // Drop waiter - we don't need to wait for final blocks to persist
-        // since the benchmark goal is measuring Ggas/s of newPayload/FCU, not persistence.
-        drop(waiter);
-
         let (gas_output_results, combined_results): (Vec<TotalGasRow>, Vec<CombinedResult>) =
             results.into_iter().unzip();
 
@@ -452,28 +356,42 @@ impl Command {
     }
 
     /// Load and parse all payload files from the directory.
+    ///
+    /// Tries to load each file as a [`BigBlockPayload`] first (which includes `env_switches`),
+    /// falling back to [`ExecutionPayloadEnvelopeV4`] for backwards compatibility.
     fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
         let mut payloads = Vec::new();
 
-        // Read directory entries
+        // Read directory entries — match both legacy "payload_block_*.json" and new
+        // "big_block_*.json" formats
         let entries: Vec<_> = std::fs::read_dir(&self.payload_dir)
             .wrap_err_with(|| format!("Failed to read directory {:?}", self.payload_dir))?
             .filter_map(|e| e.ok())
             .filter(|e| {
+                let name = e.file_name();
+                let name_str = name.to_string_lossy();
                 e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
-                    e.file_name().to_string_lossy().starts_with("payload_block_")
+                    (name_str.starts_with("payload_block_") ||
+                        name_str.starts_with("big_block_"))
             })
             .collect();
 
-        // Parse filenames to get indices and sort
+        // Parse filenames to get indices and sort.
+        // Supports "payload_block_N.json" and "big_block_FROM_to_TO.json" naming.
         let mut indexed_paths: Vec<(u64, PathBuf)> = entries
             .into_iter()
             .filter_map(|e| {
                 let name = e.file_name();
                 let name_str = name.to_string_lossy();
-                // Extract index from "payload_NNN.json"
-                let index_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
-                let index: u64 = index_str.parse().ok()?;
+                let index = if let Some(rest) = name_str.strip_prefix("payload_block_") {
+                    rest.strip_suffix(".json")?.parse::<u64>().ok()?
+                } else if let Some(rest) = name_str.strip_prefix("big_block_") {
+                    // "big_block_FROM_to_TO.json" — use FROM as the index
+                    let rest = rest.strip_suffix(".json")?;
+                    rest.split("_to_").next()?.parse::<u64>().ok()?
+                } else {
+                    return None;
+                };
                 Some((index, e.path()))
             })
             .collect();
@@ -491,82 +409,42 @@ impl Command {
         for (index, path) in indexed_paths {
             let content = std::fs::read_to_string(&path)
                 .wrap_err_with(|| format!("Failed to read {:?}", path))?;
-            let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
-                .wrap_err_with(|| format!("Failed to parse {:?}", path))?;
 
-            let block_hash =
-                envelope.envelope_inner.execution_payload.payload_inner.payload_inner.block_hash;
+            // Try BigBlockPayload first, then fall back to legacy ExecutionPayloadEnvelopeV4
+            let (execution_data, big_block_data) =
+                if let Ok(big_block) = serde_json::from_str::<BigBlockPayload>(&content) {
+                    (big_block.execution_data, big_block.big_block_data)
+                } else {
+                    let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
+                        .wrap_err_with(|| format!("Failed to parse {:?}", path))?;
+                    let execution_data = ExecutionData {
+                        payload: envelope.envelope_inner.execution_payload.clone().into(),
+                        sidecar: ExecutionPayloadSidecar::v4(
+                            CancunPayloadFields {
+                                versioned_hashes: Vec::new(),
+                                parent_beacon_block_root: B256::ZERO,
+                            },
+                            PraguePayloadFields {
+                                requests: envelope.execution_requests.clone().into(),
+                            },
+                        ),
+                    };
+                    (execution_data, BigBlockData::default())
+                };
+
+            let block_hash = execution_data.payload.as_v1().block_hash;
 
             debug!(
                 target: "reth-bench",
                 index = index,
                 block_hash = %block_hash,
+                env_switches = big_block_data.env_switches.len(),
+                prior_block_hashes = big_block_data.prior_block_hashes.len(),
                 path = %path.display(),
                 "Loaded payload"
             );
 
-            payloads.push(LoadedPayload { index, envelope, block_hash });
-        }
-
-        Ok(payloads)
-    }
-
-    /// Load and parse gas ramp payload files from a directory.
-    fn load_gas_ramp_payloads(&self, dir: &PathBuf) -> eyre::Result<Vec<GasRampPayload>> {
-        let mut payloads = Vec::new();
-
-        let entries: Vec<_> = std::fs::read_dir(dir)
-            .wrap_err_with(|| format!("Failed to read directory {:?}", dir))?
-            .filter_map(|e| e.ok())
-            .filter(|e| {
-                e.path().extension().and_then(|s| s.to_str()) == Some("json") &&
-                    e.file_name().to_string_lossy().starts_with("payload_block_")
-            })
-            .collect();
-
-        // Parse filenames to get block numbers and sort
-        let mut indexed_paths: Vec<(u64, PathBuf)> = entries
-            .into_iter()
-            .filter_map(|e| {
-                let name = e.file_name();
-                let name_str = name.to_string_lossy();
-                // Extract block number from "payload_block_NNN.json"
-                let block_str = name_str.strip_prefix("payload_block_")?.strip_suffix(".json")?;
-                let block_number: u64 = block_str.parse().ok()?;
-                Some((block_number, e.path()))
-            })
-            .collect();
-
-        indexed_paths.sort_by_key(|(num, _)| *num);
-
-        for (block_number, path) in indexed_paths {
-            let content = std::fs::read_to_string(&path)
-                .wrap_err_with(|| format!("Failed to read {:?}", path))?;
-            let file: GasRampPayloadFile = serde_json::from_str(&content)
-                .wrap_err_with(|| format!("Failed to parse {:?}", path))?;
-
-            let version = if let Some(version) = file.version {
-                match version {
-                    1 => EngineApiMessageVersion::V1,
-                    2 => EngineApiMessageVersion::V2,
-                    3 => EngineApiMessageVersion::V3,
-                    4 => EngineApiMessageVersion::V4,
-                    5 => EngineApiMessageVersion::V5,
-                    v => return Err(eyre::eyre!("Invalid version {} in {:?}", v, path)),
-                }
-                .into()
-            } else {
-                None
-            };
-
-            info!(
-                block_number,
-                block_hash = %file.block_hash,
-                path = %path.display(),
-                "Loaded gas ramp payload"
-            );
-
-            payloads.push(GasRampPayload { block_number, version, file });
+            payloads.push(LoadedPayload { index, execution_data, block_hash, big_block_data });
         }
 
         Ok(payloads)

bin/reth-bench/src/main.rs

@@ -14,6 +14,9 @@
 #[global_allocator]
 static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
 
+#[cfg(all(feature = "jemalloc", unix))]
+use reth_cli_util::allocator::tikv_jemalloc_sys as _;
+
 pub mod authenticated_transport;
 pub mod bench;
 pub mod bench_mode;

bin/reth-bench/src/valid_payload.rs

@@ -12,6 +12,7 @@ use alloy_rpc_types_engine::{
 use alloy_transport::TransportResult;
 use op_alloy_rpc_types_engine::OpExecutionPayloadV4;
 use reth_node_api::EngineApiMessageVersion;
+use reth_node_core::args::WaitForPersistence;
 use reth_rpc_api::RethNewPayloadInput;
 use serde::Deserialize;
 use std::time::Duration;
@@ -167,16 +168,28 @@ where
 /// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
 ///
 /// Returns `(version, versioned_params, execution_data)`.
+///
+/// `wait_for_persistence` controls how `wait_for_persistence` is passed to
+/// `reth_newPayload` on a per-block basis.
 pub(crate) fn block_to_new_payload(
     block: AnyRpcBlock,
     is_optimism: bool,
     rlp: Option<Bytes>,
     reth_new_payload: bool,
+    wait_for_persistence: WaitForPersistence,
+    no_wait_for_caches: bool,
 ) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
+    let block_number = block.header.number;
+    let wait_for_persistence = wait_for_persistence.rpc_value(block_number);
+
     if let Some(rlp) = rlp {
         return Ok((
             None,
-            serde_json::to_value((RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),))?,
+            serde_json::to_value((
+                RethNewPayloadInput::<ExecutionData>::BlockRlp(rlp),
+                wait_for_persistence,
+                no_wait_for_caches.then_some(false),
+            ))?,
         ));
     }
     let block = block
@@ -194,7 +207,14 @@ pub(crate) fn block_to_new_payload(
         payload_to_new_payload(payload, sidecar, is_optimism, block.withdrawals_root, None)?;
 
     if reth_new_payload {
-        Ok((None, serde_json::to_value((RethNewPayloadInput::ExecutionData(execution_data),))?))
+        Ok((
+            None,
+            serde_json::to_value((
+                RethNewPayloadInput::ExecutionData(execution_data),
+                wait_for_persistence,
+                no_wait_for_caches.then_some(false),
+            ))?,
+        ))
     } else {
         Ok((Some(version), params))
     }

bin/reth/Cargo.toml

@@ -89,7 +89,6 @@ default = [
     "keccak-cache-global",
     "asm-keccak",
     "min-debug-logs",
-    "rocksdb",
 ]
 
 otlp = [
@@ -125,9 +124,10 @@ jemalloc = [
     "reth-node-core/jemalloc",
     "reth-node-metrics/jemalloc",
     "reth-ethereum-cli/jemalloc",
+    "reth-provider/jemalloc",
 ]
 jemalloc-prof = [
-    "reth-cli-util/jemalloc",
+    "jemalloc",
     "reth-cli-util/jemalloc-prof",
     "reth-ethereum-cli/jemalloc-prof",
     "reth-node-metrics/jemalloc-prof",
@@ -136,12 +136,6 @@ jemalloc-symbols = [
     "jemalloc-prof",
     "reth-ethereum-cli/jemalloc-symbols",
 ]
-jemalloc-unprefixed = [
-    "reth-cli-util/jemalloc-unprefixed",
-    "reth-node-core/jemalloc",
-    "reth-node-metrics/jemalloc",
-    "reth-ethereum-cli/jemalloc",
-]
 tracy-allocator = [
     "reth-cli-util/tracy-allocator",
     "reth-ethereum-cli/tracy-allocator",
@@ -191,8 +185,6 @@ min-trace-logs = [
 ]
 
 trie-debug = ["reth-node-builder/trie-debug", "reth-node-core/trie-debug"]
-rocksdb = ["reth-ethereum-cli/rocksdb", "reth-node-core/rocksdb"]
-edge = ["rocksdb"]
 
 [[bin]]
 name = "reth"

bin/reth/src/lib.rs

@@ -22,7 +22,6 @@
 //!   and leak detection functionality. See [jemalloc's opt.prof](https://jemalloc.net/jemalloc.3.html#opt.prof)
 //!   documentation for usage details. This is **not recommended on Windows**.
 //! - `jemalloc-symbols`: Enables jemalloc symbols for profiling. Includes `jemalloc-prof`.
-//! - `jemalloc-unprefixed`: Uses unprefixed jemalloc symbols.
 //! - `tracy-allocator`: Enables [Tracy](https://github.com/wolfpld/tracy) profiler allocator
 //!   integration for memory profiling.
 //! - `snmalloc`: Uses [snmalloc](https://github.com/microsoft/snmalloc) as the global allocator.

bin/reth/src/main.rs

@@ -3,8 +3,12 @@
 #[global_allocator]
 static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
 
+// Required for "override_allocator_on_supported_platforms".
+#[cfg(all(feature = "jemalloc", unix))]
+use reth_cli_util::allocator::tikv_jemalloc_sys as _;
+
 #[cfg(all(feature = "jemalloc-prof", unix))]
-#[unsafe(export_name = "_rjem_malloc_conf")]
+#[unsafe(export_name = "malloc_conf")]
 static MALLOC_CONF: &[u8] = b"prof:true,prof_active:true,lg_prof_sample:19\0";
 
 use clap::Parser;

crates/chain-state/src/in_memory.rs

@@ -992,7 +992,7 @@ impl<N: NodePrimitives<SignedTx: SignedTransaction>> NewCanonicalChain<N> {
     ///
     /// Returns the new tip for [`Self::Reorg`] and [`Self::Commit`] variants which commit at least
     /// 1 new block.
-    pub fn tip(&self) -> &SealedBlock<N::Block> {
+    pub fn tip(&self) -> &RecoveredBlock<N::Block> {
         match self {
             Self::Commit { new } | Self::Reorg { new, .. } => {
                 new.last().expect("non empty blocks").recovered_block()

crates/chainspec/src/lib.rs

@@ -36,15 +36,6 @@ pub use spec::{
     DepositContract, ForkBaseFeeParams, DEV, HOLESKY, HOODI, MAINNET, SEPOLIA,
 };
 
-use reth_primitives_traits::sync::OnceLock;
-
-/// Simple utility to create a thread-safe sync cell with a value set.
-pub fn once_cell_set<T>(value: T) -> OnceLock<T> {
-    let once = OnceLock::new();
-    let _ = once.set(value);
-    once
-}
-
 #[cfg(test)]
 mod tests {
     use super::*;

crates/cli/commands/Cargo.toml

@@ -84,7 +84,7 @@ tracing.workspace = true
 backon.workspace = true
 secp256k1 = { workspace = true, features = ["global-context", "std", "recovery"] }
 tokio-stream.workspace = true
-reqwest.workspace = true
+reqwest = { workspace = true, features = ["blocking"] }
 url.workspace = true
 metrics.workspace = true
 blake3.workspace = true
@@ -110,7 +110,6 @@ reth-provider = { workspace = true, features = ["test-utils"] }
 tempfile.workspace = true
 
 [features]
-default = []
 arbitrary = [
     "dep:proptest",
     "dep:arbitrary",
@@ -135,6 +134,3 @@ arbitrary = [
     "reth-primitives-traits/arbitrary",
     "reth-ethereum-primitives/arbitrary",
 ]
-
-rocksdb = ["reth-db-common/rocksdb", "reth-stages/rocksdb", "reth-provider/rocksdb", "reth-prune/rocksdb"]
-edge = ["rocksdb"]

crates/cli/commands/src/common.rs

@@ -75,10 +75,15 @@ pub struct EnvironmentArgs<C: ChainSpecParser> {
 impl<C: ChainSpecParser> EnvironmentArgs<C> {
     /// Returns the storage settings for new database initialization.
     ///
-    /// Always returns [`StorageSettings::v2()`] — v2 is the default for all new
-    /// databases. Existing databases use the settings persisted in their metadata.
+    /// Determined by the `--storage.v2` flag (defaults to `true`).
+    /// Existing databases retain whatever settings are persisted in their
+    /// metadata (checked during genesis init).
     pub fn storage_settings(&self) -> StorageSettings {
-        StorageSettings::v2()
+        if self.storage.v2 {
+            StorageSettings::v2()
+        } else {
+            StorageSettings::v1()
+        }
     }
 
     /// Initializes environment according to [`AccessRights`] and returns an instance of
@@ -187,7 +192,6 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
     where
         C: ChainSpecParser<ChainSpec = N::ChainSpec>,
     {
-        let prune_modes = config.prune.segments.clone();
         let factory = ProviderFactory::<NodeTypesWithDBAdapter<N, DatabaseEnv>>::new(
             db,
             self.chain.clone(),
@@ -195,7 +199,8 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
             rocksdb_provider,
             runtime,
         )?
-        .with_prune_modes(prune_modes.clone());
+        .with_prune_modes(config.prune.segments.clone())
+        .with_minimum_pruning_distance(config.prune.minimum_pruning_distance);
 
         // Check for consistency between database and static files.
         if !access.is_read_only_inconsistent() &&
@@ -229,10 +234,13 @@ impl<C: ChainSpecParser> EnvironmentArgs<C> {
                     NoopBodiesDownloader::default(),
                     NoopEvmConfig::<N::Evm>::default(),
                     config.stages.clone(),
-                    prune_modes.clone(),
+                    config.prune.segments.clone(),
                     None,
                 ))
-                .build(factory.clone(), StaticFileProducer::new(factory.clone(), prune_modes));
+                .build(
+                    factory.clone(),
+                    StaticFileProducer::new(factory.clone(), config.prune.segments.clone()),
+                );
 
             // Move all applicable data from database to static files.
             pipeline.move_to_static_files()?;

crates/cli/commands/src/db/checksum/mod.rs

@@ -21,7 +21,6 @@ use std::{
 };
 use tracing::{info, warn};
 
-#[cfg(all(unix, feature = "rocksdb"))]
 mod rocksdb;
 
 /// Interval for logging progress during checksum computation.
@@ -73,7 +72,6 @@ enum Subcommand {
         limit: Option<usize>,
     },
     /// Calculates the checksum of a RocksDB table
-    #[cfg(all(unix, feature = "rocksdb"))]
     Rocksdb {
         /// The RocksDB table
         #[arg(value_enum)]
@@ -100,7 +98,6 @@ impl Command {
             Subcommand::StaticFile { segment, start_block, end_block, limit } => {
                 checksum_static_file(tool, segment, start_block, end_block, limit)?;
             }
-            #[cfg(all(unix, feature = "rocksdb"))]
             Subcommand::Rocksdb { table, limit } => {
                 rocksdb::checksum_rocksdb(tool, table, limit)?;
             }

crates/cli/commands/src/db/checksum/rocksdb.rs

@@ -8,8 +8,9 @@ use reth_db::{tables, DatabaseEnv};
 use reth_db_api::table::Table;
 use reth_db_common::DbTool;
 use reth_node_builder::NodeTypesWithDBAdapter;
+use reth_primitives_traits::FastInstant as Instant;
 use reth_provider::RocksDBProviderFactory;
-use std::{hash::Hasher, time::Instant};
+use std::hash::Hasher;
 use tracing::info;
 
 /// RocksDB tables that can be checksummed.

crates/cli/commands/src/db/mod.rs

@@ -22,6 +22,7 @@ mod settings;
 mod stage_checkpoints;
 mod state;
 mod static_file_header;
+mod static_files;
 mod stats;
 /// DB List TUI
 mod tui;
@@ -63,6 +64,8 @@ pub enum Subcommands {
     RepairTrie(repair_trie::Command),
     /// Reads and displays the static file segment header
     StaticFileHeader(static_file_header::Command),
+    /// Static file operations (split, etc.)
+    StaticFiles(static_files::Command),
     /// Lists current and local database versions
     Version,
     /// Returns the full database path
@@ -188,6 +191,11 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
                     command.execute(&tool)?;
                 });
             }
+            Subcommands::StaticFiles(command) => {
+                db_exec!(self.env, tool, N, AccessRights::RO, {
+                    command.execute(&tool)?;
+                });
+            }
             Subcommands::Version => {
                 let local_db_version = match get_db_version(&db_path) {
                     Ok(version) => Some(version),

crates/cli/commands/src/db/static_files/mod.rs

@@ -0,0 +1,31 @@
+//! Static file related CLI commands
+
+mod split;
+
+pub use split::SplitCommand;
+
+use clap::{Parser, Subcommand};
+use reth_db_common::DbTool;
+use reth_provider::providers::ProviderNodeTypes;
+
+/// Static files subcommands
+#[derive(Debug, Parser)]
+pub struct Command {
+    #[command(subcommand)]
+    command: Subcommands,
+}
+
+#[derive(Debug, Subcommand)]
+enum Subcommands {
+    /// Split static files into new files with different blocks-per-file setting
+    Split(SplitCommand),
+}
+
+impl Command {
+    /// Execute the static files command
+    pub fn execute<N: ProviderNodeTypes>(self, tool: &DbTool<N>) -> eyre::Result<()> {
+        match self.command {
+            Subcommands::Split(cmd) => cmd.execute(tool),
+        }
+    }
+}

crates/cli/commands/src/db/static_files/split.rs

@@ -0,0 +1,878 @@
+use clap::Parser;
+use reth_codecs::Compact;
+use reth_db::{
+    cursor::DbCursorRO,
+    static_file::{
+        AccountChangesetMask, BlockHashMask, HeaderMask, ReceiptMask, StorageChangesetMask,
+        TotalDifficultyMask, TransactionMask, TransactionSenderMask,
+    },
+    tables,
+    transaction::DbTx,
+};
+use reth_db_api::models::{CompactU256, StoredBlockBodyIndices};
+use reth_db_common::DbTool;
+use reth_primitives_traits::NodePrimitives;
+use reth_provider::{
+    providers::{ProviderNodeTypes, StaticFileProvider},
+    DBProvider, StaticFileProviderBuilder, StaticFileProviderFactory, StaticFileWriter,
+};
+use reth_static_file_types::StaticFileSegment;
+use std::{collections::HashMap, path::PathBuf};
+use tracing::info;
+
+/// Split static files into new files with different blocks-per-file setting
+#[derive(Debug, Parser)]
+pub struct SplitCommand {
+    /// Source static files directory.
+    /// If not specified, uses the datadir's static_files directory.
+    #[arg(long, value_name = "PATH")]
+    static_files_dir: Option<PathBuf>,
+
+    /// Output directory for the new static files.
+    /// Required unless --in-place is specified.
+    #[arg(long, value_name = "PATH", required_unless_present = "in_place")]
+    output_dir: Option<PathBuf>,
+
+    /// Number of blocks per output file
+    #[arg(long, value_name = "NUM")]
+    blocks_per_file: u64,
+
+    /// Segments to split (default: all)
+    #[arg(long, value_delimiter = ',')]
+    segments: Option<Vec<StaticFileSegment>>,
+
+    /// Start block number (default: 0)
+    #[arg(long)]
+    from_block: Option<u64>,
+
+    /// End block number (default: highest available)
+    #[arg(long)]
+    to_block: Option<u64>,
+
+    /// Print what would be done without writing
+    #[arg(long)]
+    dry_run: bool,
+
+    /// Split in-place: write to temp dir, verify, then atomically swap.
+    /// Original files are preserved in static_files.bak
+    #[arg(long, conflicts_with = "output_dir")]
+    in_place: bool,
+
+    /// Skip verification step when using --in-place
+    #[arg(long, requires = "in_place")]
+    skip_verify: bool,
+}
+
+impl SplitCommand {
+    /// Execute the split command
+    pub fn execute<N: ProviderNodeTypes>(self, tool: &DbTool<N>) -> eyre::Result<()>
+    where
+        N::Primitives: NodePrimitives<BlockHeader: Compact, SignedTx: Compact, Receipt: Compact>,
+    {
+        let segments = self.segments.clone().unwrap_or_else(|| StaticFileSegment::iter().collect());
+
+        // Use custom static files dir if provided, otherwise use datadir's static files
+        let (source_provider, source_dir) =
+            if let Some(ref static_files_dir) = self.static_files_dir {
+                let provider = StaticFileProviderBuilder::read_only(static_files_dir)
+                    .build::<N::Primitives>()?;
+                let dir = static_files_dir.clone();
+                (provider, dir)
+            } else {
+                let provider = tool.provider_factory.static_file_provider();
+                let dir = provider.directory().to_path_buf();
+                (provider, dir)
+            };
+
+        // Determine output directory
+        let (output_dir, is_in_place) = if self.in_place {
+            let temp_dir = source_dir.with_file_name("static_files.tmp");
+            (temp_dir, true)
+        } else {
+            (self.output_dir.clone().expect("output_dir required when not in_place"), false)
+        };
+
+        info!(
+            target: "reth::cli",
+            output_dir = %output_dir.display(),
+            blocks_per_file = self.blocks_per_file,
+            ?segments,
+            from_block = ?self.from_block,
+            to_block = ?self.to_block,
+            dry_run = self.dry_run,
+            in_place = is_in_place,
+            "Splitting static files"
+        );
+
+        if self.dry_run {
+            println!("Dry run mode - no files will be written");
+            if is_in_place {
+                println!("In-place mode:");
+                println!("  1. Write to: {}", output_dir.display());
+                println!("  2. Verify output integrity");
+                println!("  3. Rename {} -> {}.bak", source_dir.display(), source_dir.display());
+                println!("  4. Rename {} -> {}", output_dir.display(), source_dir.display());
+            }
+            for segment in &segments {
+                let min_block = source_provider.get_lowest_range_start(*segment);
+                let max_block = source_provider.get_highest_static_file_block(*segment);
+                if let (Some(min_block), Some(max_block)) = (min_block, max_block) {
+                    let from_block = self.from_block.unwrap_or(min_block).max(min_block);
+                    let to_block = self.to_block.unwrap_or(max_block).min(max_block);
+                    let num_blocks = to_block.saturating_sub(from_block) + 1;
+                    let num_files = num_blocks.div_ceil(self.blocks_per_file);
+                    println!(
+                        "  {segment}: blocks {from_block}..={to_block} ({num_blocks} blocks) -> {num_files} files"
+                    );
+                } else {
+                    println!("  {segment}: no data available");
+                }
+            }
+            return Ok(());
+        }
+
+        // Clean up output directory if it exists
+        // For in-place mode: remove previous incomplete temp directory
+        // For regular mode: ensure we start fresh to avoid block number mismatches
+        if output_dir.exists() {
+            info!(target: "reth::cli", output_dir = %output_dir.display(), "Removing existing output directory");
+            reth_fs_util::remove_dir_all(&output_dir)?;
+        }
+
+        reth_fs_util::create_dir_all(&output_dir)?;
+
+        // Calculate segment ranges first to determine the global starting block
+        let mut segment_ranges = Vec::new();
+        for &segment in &segments {
+            let Some(min_block) = source_provider.get_lowest_range_start(segment) else {
+                continue;
+            };
+            let Some(max_block) = source_provider.get_highest_static_file_block(segment) else {
+                continue;
+            };
+            let from_block = self.from_block.unwrap_or(min_block).max(min_block);
+            let to_block = self.to_block.unwrap_or(max_block).min(max_block);
+            if from_block <= to_block {
+                segment_ranges.push((segment, from_block, to_block));
+            }
+        }
+
+        // Pre-load block body indices for segments that need them (transactions, receipts,
+        // transaction senders). This avoids holding a long-lived DB read transaction open and
+        // is much faster than seeking per-block since the entire table is small.
+        let needs_indices = segment_ranges.iter().any(|(seg, _, _)| {
+            matches!(
+                seg,
+                StaticFileSegment::Transactions |
+                    StaticFileSegment::Receipts |
+                    StaticFileSegment::TransactionSenders
+            )
+        });
+        let block_body_indices = if needs_indices {
+            let global_from = segment_ranges
+                .iter()
+                .filter(|(seg, _, _)| {
+                    matches!(
+                        seg,
+                        StaticFileSegment::Transactions |
+                            StaticFileSegment::Receipts |
+                            StaticFileSegment::TransactionSenders
+                    )
+                })
+                .map(|(_, from, _)| *from)
+                .min()
+                .unwrap();
+            let global_to = segment_ranges
+                .iter()
+                .filter(|(seg, _, _)| {
+                    matches!(
+                        seg,
+                        StaticFileSegment::Transactions |
+                            StaticFileSegment::Receipts |
+                            StaticFileSegment::TransactionSenders
+                    )
+                })
+                .map(|(_, _, to)| *to)
+                .max()
+                .unwrap();
+
+            info!(target: "reth::cli", from_block = global_from, to_block = global_to, "Loading block body indices");
+            Self::load_block_body_indices(tool, global_from, global_to)?
+        } else {
+            HashMap::new()
+        };
+
+        for (segment, from_block, to_block) in segment_ranges {
+            info!(target: "reth::cli", ?segment, from_block, to_block, "Processing segment");
+
+            // Build output provider per-segment with genesis_block_number set to this segment's
+            // starting block. This prevents the writer from trying to load non-existent previous
+            // files when segments have different starting blocks (e.g., pruned transactions).
+            let output_provider = StaticFileProviderBuilder::read_write(&output_dir)
+                .with_blocks_per_file(self.blocks_per_file)
+                .with_genesis_block_number(from_block)
+                .build::<N::Primitives>()?;
+
+            match segment {
+                StaticFileSegment::Headers => {
+                    self.split_headers::<N>(
+                        &source_provider,
+                        &output_provider,
+                        from_block,
+                        to_block,
+                    )?;
+                }
+                StaticFileSegment::Transactions => {
+                    self.split_transactions::<N>(
+                        &block_body_indices,
+                        &source_provider,
+                        &output_provider,
+                        from_block,
+                        to_block,
+                    )?;
+                }
+                StaticFileSegment::Receipts => {
+                    self.split_receipts::<N>(
+                        &block_body_indices,
+                        &source_provider,
+                        &output_provider,
+                        from_block,
+                        to_block,
+                    )?;
+                }
+                StaticFileSegment::TransactionSenders => {
+                    self.split_transaction_senders::<N>(
+                        &block_body_indices,
+                        &source_provider,
+                        &output_provider,
+                        from_block,
+                        to_block,
+                    )?;
+                }
+                StaticFileSegment::AccountChangeSets => {
+                    self.split_account_changesets::<N>(
+                        &source_provider,
+                        &output_provider,
+                        from_block,
+                        to_block,
+                    )?;
+                }
+                StaticFileSegment::StorageChangeSets => {
+                    self.split_storage_changesets::<N>(
+                        &source_provider,
+                        &output_provider,
+                        from_block,
+                        to_block,
+                    )?;
+                }
+            }
+
+            info!(target: "reth::cli", ?segment, "Segment complete");
+
+            // Drop the output provider to release file handles before processing next segment
+            drop(output_provider);
+        }
+
+        // In-place mode: verify and swap directories
+        if is_in_place {
+            // Verification step
+            if !self.skip_verify {
+                info!(target: "reth::cli", "Verifying output integrity");
+                self.verify_output::<N>(&output_dir, &segments)?;
+            }
+
+            // Atomic swap
+            let backup_dir = source_dir.with_file_name("static_files.bak");
+
+            // Remove old backup if exists
+            if backup_dir.exists() {
+                info!(target: "reth::cli", backup_dir = %backup_dir.display(), "Removing old backup");
+                reth_fs_util::remove_dir_all(&backup_dir)?;
+            }
+
+            // Drop source provider to release file handles
+            drop(source_provider);
+
+            // Rename: source -> backup
+            info!(target: "reth::cli",
+                from = %source_dir.display(),
+                to = %backup_dir.display(),
+                "Moving original to backup"
+            );
+            reth_fs_util::rename(&source_dir, &backup_dir)?;
+
+            // Rename: temp -> source
+            info!(target: "reth::cli",
+                from = %output_dir.display(),
+                to = %source_dir.display(),
+                "Moving new files into place"
+            );
+            reth_fs_util::rename(&output_dir, &source_dir)?;
+
+            info!(target: "reth::cli",
+                backup = %backup_dir.display(),
+                "In-place split complete. Original files preserved in backup directory"
+            );
+        }
+
+        info!(target: "reth::cli", "Static file split complete");
+        Ok(())
+    }
+
+    /// Verify the output static files have valid data
+    fn verify_output<N: ProviderNodeTypes>(
+        &self,
+        output_dir: &PathBuf,
+        segments: &[StaticFileSegment],
+    ) -> eyre::Result<()> {
+        let provider = StaticFileProviderBuilder::read_only(output_dir).build::<N::Primitives>()?;
+
+        for &segment in segments {
+            let Some(lowest) = provider.get_lowest_range_start(segment) else {
+                return Err(eyre::eyre!("Verification failed: no data for segment {segment}"));
+            };
+            let Some(highest) = provider.get_highest_static_file_block(segment) else {
+                return Err(eyre::eyre!("Verification failed: no data for segment {segment}"));
+            };
+
+            // Verify we can read the first and last blocks
+            provider.get_segment_provider(segment, lowest)?;
+            provider.get_segment_provider(segment, highest)?;
+
+            info!(target: "reth::cli", ?segment, from_block = lowest, to_block = highest, "Verified");
+        }
+
+        Ok(())
+    }
+
+    fn split_headers<N: ProviderNodeTypes>(
+        &self,
+        source: &StaticFileProvider<N::Primitives>,
+        output: &StaticFileProvider<N::Primitives>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<()>
+    where
+        <N::Primitives as NodePrimitives>::BlockHeader: Compact,
+    {
+        let mut writer = output.get_writer(from_block, StaticFileSegment::Headers)?;
+
+        for block in from_block..=to_block {
+            let jar = source.get_segment_provider(StaticFileSegment::Headers, block)?;
+            let mut cursor = jar.cursor()?;
+
+            let header: <N::Primitives as NodePrimitives>::BlockHeader = cursor
+                .get_one::<HeaderMask<_>>(block.into())?
+                .ok_or_else(|| eyre::eyre!("Missing header for block {block}"))?;
+
+            let td: CompactU256 = cursor
+                .get_one::<TotalDifficultyMask>(block.into())?
+                .ok_or_else(|| eyre::eyre!("Missing TD for block {block}"))?;
+
+            let hash = cursor
+                .get_one::<BlockHashMask>(block.into())?
+                .ok_or_else(|| eyre::eyre!("Missing hash for block {block}"))?;
+
+            writer.append_header_with_td(&header, td.into(), &hash)?;
+
+            if block % 100_000 == 0 {
+                info!(target: "reth::cli", block, to_block, "Headers progress");
+            }
+        }
+
+        writer.commit()?;
+        Ok(())
+    }
+
+    fn load_block_body_indices<N: ProviderNodeTypes>(
+        tool: &DbTool<N>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<HashMap<u64, StoredBlockBodyIndices>> {
+        let provider = tool.provider_factory.provider()?.disable_long_read_transaction_safety();
+        let tx = provider.tx_ref();
+        let mut cursor = tx.cursor_read::<tables::BlockBodyIndices>()?;
+        let mut indices = HashMap::with_capacity((to_block - from_block + 1) as usize);
+        for entry in cursor.walk_range(from_block..=to_block)? {
+            let (block, body_indices) = entry?;
+            indices.insert(block, body_indices);
+        }
+        info!(target: "reth::cli", count = indices.len(), "Loaded block body indices");
+        Ok(indices)
+    }
+
+    fn split_transactions<N: ProviderNodeTypes>(
+        &self,
+        block_body_indices: &HashMap<u64, StoredBlockBodyIndices>,
+        source: &StaticFileProvider<N::Primitives>,
+        output: &StaticFileProvider<N::Primitives>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<()>
+    where
+        <N::Primitives as NodePrimitives>::SignedTx: Compact,
+    {
+        let mut writer = output.get_writer(from_block, StaticFileSegment::Transactions)?;
+        let mut block = from_block;
+        let mut block_incremented = false;
+
+        while block <= to_block {
+            if !block_incremented {
+                writer.increment_block(block)?;
+            }
+            block_incremented = false;
+
+            // Skip blocks with no transactions until we find one that needs a jar
+            let Some(indices) =
+                block_body_indices.get(&block).filter(|i| i.tx_count > 0)
+            else {
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Transactions progress");
+                }
+                block += 1;
+                continue;
+            };
+
+            // Open jar + cursor, reuse for all subsequent blocks within this jar's range
+            let jar =
+                source.get_segment_provider(StaticFileSegment::Transactions, indices.first_tx_num)?;
+            let jar_tx_end =
+                jar.user_header().tx_range().map(|r| r.end()).unwrap_or(u64::MAX);
+            let mut cursor = jar.cursor()?;
+
+            loop {
+                if let Some(indices) = block_body_indices.get(&block) {
+                    for tx_num in indices.first_tx_num..indices.first_tx_num + indices.tx_count {
+                        let transaction: <N::Primitives as NodePrimitives>::SignedTx = cursor
+                            .get_one::<TransactionMask<_>>(tx_num.into())?
+                            .ok_or_else(|| eyre::eyre!("Missing transaction {tx_num}"))?;
+                        writer.append_transaction(tx_num, &transaction)?;
+                    }
+                }
+
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Transactions progress");
+                }
+                block += 1;
+                if block > to_block {
+                    break;
+                }
+
+                writer.increment_block(block)?;
+                block_incremented = true;
+
+                // Check if next block's txs need a different jar
+                if let Some(next_indices) = block_body_indices.get(&block) &&
+                    next_indices.tx_count > 0 && next_indices.first_tx_num > jar_tx_end
+                {
+                    break;
+                }
+            }
+        }
+
+        writer.commit()?;
+        Ok(())
+    }
+
+    fn split_receipts<N: ProviderNodeTypes>(
+        &self,
+        block_body_indices: &HashMap<u64, StoredBlockBodyIndices>,
+        source: &StaticFileProvider<N::Primitives>,
+        output: &StaticFileProvider<N::Primitives>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<()>
+    where
+        <N::Primitives as NodePrimitives>::Receipt: Compact,
+    {
+        let mut writer = output.get_writer(from_block, StaticFileSegment::Receipts)?;
+        let mut block = from_block;
+        let mut block_incremented = false;
+
+        while block <= to_block {
+            if !block_incremented {
+                writer.increment_block(block)?;
+            }
+            block_incremented = false;
+
+            let Some(indices) =
+                block_body_indices.get(&block).filter(|i| i.tx_count > 0)
+            else {
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Receipts progress");
+                }
+                block += 1;
+                continue;
+            };
+
+            let jar =
+                source.get_segment_provider(StaticFileSegment::Receipts, indices.first_tx_num)?;
+            let jar_tx_end =
+                jar.user_header().tx_range().map(|r| r.end()).unwrap_or(u64::MAX);
+            let mut cursor = jar.cursor()?;
+
+            loop {
+                if let Some(indices) = block_body_indices.get(&block) {
+                    for tx_num in indices.first_tx_num..indices.first_tx_num + indices.tx_count {
+                        let receipt: <N::Primitives as NodePrimitives>::Receipt = cursor
+                            .get_one::<ReceiptMask<_>>(tx_num.into())?
+                            .ok_or_else(|| eyre::eyre!("Missing receipt {tx_num}"))?;
+                        writer.append_receipt(tx_num, &receipt)?;
+                    }
+                }
+
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Receipts progress");
+                }
+                block += 1;
+                if block > to_block {
+                    break;
+                }
+
+                writer.increment_block(block)?;
+                block_incremented = true;
+
+                if let Some(next_indices) = block_body_indices.get(&block) &&
+                    next_indices.tx_count > 0 && next_indices.first_tx_num > jar_tx_end
+                {
+                    break;
+                }
+            }
+        }
+
+        writer.commit()?;
+        Ok(())
+    }
+
+    fn split_transaction_senders<N: ProviderNodeTypes>(
+        &self,
+        block_body_indices: &HashMap<u64, StoredBlockBodyIndices>,
+        source: &StaticFileProvider<N::Primitives>,
+        output: &StaticFileProvider<N::Primitives>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<()> {
+        let mut writer = output.get_writer(from_block, StaticFileSegment::TransactionSenders)?;
+        let mut block = from_block;
+        let mut block_incremented = false;
+
+        while block <= to_block {
+            if !block_incremented {
+                writer.increment_block(block)?;
+            }
+            block_incremented = false;
+
+            let Some(indices) =
+                block_body_indices.get(&block).filter(|i| i.tx_count > 0)
+            else {
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Transaction senders progress");
+                }
+                block += 1;
+                continue;
+            };
+
+            let jar = source
+                .get_segment_provider(StaticFileSegment::TransactionSenders, indices.first_tx_num)?;
+            let jar_tx_end =
+                jar.user_header().tx_range().map(|r| r.end()).unwrap_or(u64::MAX);
+            let mut cursor = jar.cursor()?;
+
+            loop {
+                if let Some(indices) = block_body_indices.get(&block) {
+                    for tx_num in indices.first_tx_num..indices.first_tx_num + indices.tx_count {
+                        let sender = cursor
+                            .get_one::<TransactionSenderMask>(tx_num.into())?
+                            .ok_or_else(|| eyre::eyre!("Missing sender {tx_num}"))?;
+                        writer.append_transaction_sender(tx_num, &sender)?;
+                    }
+                }
+
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Transaction senders progress");
+                }
+                block += 1;
+                if block > to_block {
+                    break;
+                }
+
+                writer.increment_block(block)?;
+                block_incremented = true;
+
+                if let Some(next_indices) = block_body_indices.get(&block) &&
+                    next_indices.tx_count > 0 && next_indices.first_tx_num > jar_tx_end
+                {
+                    break;
+                }
+            }
+        }
+
+        writer.commit()?;
+        Ok(())
+    }
+
+    fn split_account_changesets<N: ProviderNodeTypes>(
+        &self,
+        source: &StaticFileProvider<N::Primitives>,
+        output: &StaticFileProvider<N::Primitives>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<()> {
+        let mut writer = output.get_writer(from_block, StaticFileSegment::AccountChangeSets)?;
+        let mut block = from_block;
+
+        while block <= to_block {
+            // Open jar + cursor, reuse for all blocks within this jar's range
+            let jar =
+                source.get_segment_provider(StaticFileSegment::AccountChangeSets, block)?;
+            let jar_block_end = jar
+                .user_header()
+                .block_range()
+                .map(|r| r.end())
+                .unwrap_or(u64::MAX);
+            let mut cursor = jar.cursor()?;
+
+            loop {
+                let mut changes = Vec::new();
+                if let Some(offset) = jar.read_changeset_offset(block)? {
+                    for i in offset.changeset_range() {
+                        if let Some(change) =
+                            cursor.get_one::<AccountChangesetMask>(i.into())?
+                        {
+                            changes.push(change);
+                        }
+                    }
+                }
+
+                writer.append_account_changeset(changes, block)?;
+
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Account changesets progress");
+                }
+                block += 1;
+                if block > to_block || block > jar_block_end {
+                    break;
+                }
+            }
+        }
+
+        writer.commit()?;
+        Ok(())
+    }
+
+    fn split_storage_changesets<N: ProviderNodeTypes>(
+        &self,
+        source: &StaticFileProvider<N::Primitives>,
+        output: &StaticFileProvider<N::Primitives>,
+        from_block: u64,
+        to_block: u64,
+    ) -> eyre::Result<()> {
+        let mut writer = output.get_writer(from_block, StaticFileSegment::StorageChangeSets)?;
+        let mut block = from_block;
+
+        while block <= to_block {
+            let jar =
+                source.get_segment_provider(StaticFileSegment::StorageChangeSets, block)?;
+            let jar_block_end = jar
+                .user_header()
+                .block_range()
+                .map(|r| r.end())
+                .unwrap_or(u64::MAX);
+            let mut cursor = jar.cursor()?;
+
+            loop {
+                let mut changes = Vec::new();
+                if let Some(offset) = jar.read_changeset_offset(block)? {
+                    for i in offset.changeset_range() {
+                        if let Some(change) =
+                            cursor.get_one::<StorageChangesetMask>(i.into())?
+                        {
+                            changes.push(change);
+                        }
+                    }
+                }
+
+                writer.append_storage_changeset(changes, block)?;
+
+                if block.is_multiple_of(100_000) {
+                    info!(target: "reth::cli", block, to_block, "Storage changesets progress");
+                }
+                block += 1;
+                if block > to_block || block > jar_block_end {
+                    break;
+                }
+            }
+        }
+
+        writer.commit()?;
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+    use clap::Parser;
+
+    #[derive(Parser)]
+    struct TestCli {
+        #[command(subcommand)]
+        command: TestCommand,
+    }
+
+    #[derive(clap::Subcommand)]
+    enum TestCommand {
+        Split(SplitCommand),
+    }
+
+    #[test]
+    fn parse_split_command_minimal() {
+        let args = TestCli::try_parse_from([
+            "test",
+            "split",
+            "--output-dir",
+            "/tmp/output",
+            "--blocks-per-file",
+            "100000",
+        ])
+        .unwrap();
+
+        match args.command {
+            TestCommand::Split(cmd) => {
+                assert_eq!(cmd.output_dir, Some(PathBuf::from("/tmp/output")));
+                assert_eq!(cmd.blocks_per_file, 100000);
+                assert!(cmd.segments.is_none());
+                assert!(cmd.from_block.is_none());
+                assert!(cmd.to_block.is_none());
+                assert!(!cmd.dry_run);
+                assert!(!cmd.in_place);
+            }
+        }
+    }
+
+    #[test]
+    fn parse_split_command_full() {
+        let args = TestCli::try_parse_from([
+            "test",
+            "split",
+            "--output-dir",
+            "/tmp/output",
+            "--blocks-per-file",
+            "50000",
+            "--segments",
+            "headers,receipts",
+            "--from-block",
+            "1000",
+            "--to-block",
+            "500000",
+            "--dry-run",
+        ])
+        .unwrap();
+
+        match args.command {
+            TestCommand::Split(cmd) => {
+                assert_eq!(cmd.output_dir, Some(PathBuf::from("/tmp/output")));
+                assert_eq!(cmd.blocks_per_file, 50000);
+                assert_eq!(
+                    cmd.segments,
+                    Some(vec![StaticFileSegment::Headers, StaticFileSegment::Receipts])
+                );
+                assert_eq!(cmd.from_block, Some(1000));
+                assert_eq!(cmd.to_block, Some(500000));
+                assert!(cmd.dry_run);
+                assert!(!cmd.in_place);
+            }
+        }
+    }
+
+    #[test]
+    fn parse_split_command_in_place() {
+        let args =
+            TestCli::try_parse_from(["test", "split", "--in-place", "--blocks-per-file", "100000"])
+                .unwrap();
+
+        match args.command {
+            TestCommand::Split(cmd) => {
+                assert!(cmd.output_dir.is_none());
+                assert_eq!(cmd.blocks_per_file, 100000);
+                assert!(cmd.in_place);
+                assert!(!cmd.skip_verify);
+            }
+        }
+    }
+
+    #[test]
+    fn parse_split_command_in_place_skip_verify() {
+        let args = TestCli::try_parse_from([
+            "test",
+            "split",
+            "--in-place",
+            "--skip-verify",
+            "--blocks-per-file",
+            "100000",
+        ])
+        .unwrap();
+
+        match args.command {
+            TestCommand::Split(cmd) => {
+                assert!(cmd.in_place);
+                assert!(cmd.skip_verify);
+            }
+        }
+    }
+
+    #[test]
+    fn parse_split_command_output_dir_conflicts_with_in_place() {
+        let result = TestCli::try_parse_from([
+            "test",
+            "split",
+            "--output-dir",
+            "/tmp/out",
+            "--in-place",
+            "--blocks-per-file",
+            "100000",
+        ]);
+        assert!(result.is_err());
+    }
+
+    #[test]
+    fn parse_split_command_skip_verify_requires_in_place() {
+        // --skip-verify without --in-place should fail
+        let result = TestCli::try_parse_from([
+            "test",
+            "split",
+            "--skip-verify",
+            "--blocks-per-file",
+            "100000",
+        ]);
+        assert!(result.is_err(), "--skip-verify should require --in-place");
+    }
+
+    #[test]
+    fn parse_split_command_all_segments() {
+        let args = TestCli::try_parse_from([
+            "test",
+            "split",
+            "--output-dir",
+            "/tmp/out",
+            "--blocks-per-file",
+            "10",
+            "--segments",
+            "headers,transactions,receipts,transaction-senders,account-change-sets,storage-change-sets",
+        ])
+        .unwrap();
+
+        match args.command {
+            TestCommand::Split(cmd) => {
+                let segments = cmd.segments.unwrap();
+                assert_eq!(segments.len(), 6);
+                assert!(segments.contains(&StaticFileSegment::Headers));
+                assert!(segments.contains(&StaticFileSegment::Transactions));
+                assert!(segments.contains(&StaticFileSegment::Receipts));
+                assert!(segments.contains(&StaticFileSegment::TransactionSenders));
+                assert!(segments.contains(&StaticFileSegment::AccountChangeSets));
+                assert!(segments.contains(&StaticFileSegment::StorageChangeSets));
+            }
+        }
+    }
+}

crates/cli/commands/src/download/config_gen.rs

@@ -199,7 +199,7 @@ pub(crate) fn config_for_selections(
         }
 
         return Config {
-            prune: PruneConfig { block_interval: PruneConfig::default().block_interval, segments },
+            prune: PruneConfig { segments, ..Default::default() },
             static_files,
             ..Default::default()
         };

crates/cli/commands/src/download/manifest_cmd.rs

@@ -3,9 +3,10 @@ use clap::Parser;
 use eyre::{Result, WrapErr};
 use reth_db::{mdbx::DatabaseArguments, open_db_read_only, tables, Database};
 use reth_db_api::transaction::DbTx;
+use reth_primitives_traits::FastInstant as Instant;
 use reth_stages_types::StageId;
 use reth_static_file_types::DEFAULT_BLOCKS_PER_STATIC_FILE;
-use std::{path::PathBuf, time::Instant};
+use std::path::PathBuf;
 use tracing::{info, warn};
 
 /// Generate modular chunk archives and a snapshot manifest from a source datadir.

crates/cli/commands/src/download/mod.rs

@@ -84,6 +84,10 @@ pub struct DownloadDefaults {
     ///
     /// Falls back to [`default_base_url`](Self::default_base_url) when `None`.
     pub default_chain_aware_base_url: Option<Cow<'static, str>>,
+    /// URL for the snapshot discovery API that lists available snapshots.
+    ///
+    /// Defaults to `https://snapshots.reth.rs/api/snapshots`.
+    pub snapshot_api_url: Cow<'static, str>,
     /// Optional custom long help text that overrides the generated help
     pub long_help: Option<String>,
 }
@@ -108,6 +112,7 @@ impl DownloadDefaults {
             ],
             default_base_url: Cow::Borrowed(RETH_SNAPSHOTS_BASE_URL),
             default_chain_aware_base_url: None,
+            snapshot_api_url: Cow::Borrowed(RETH_SNAPSHOTS_API_URL),
             long_help: None,
         }
     }
@@ -121,10 +126,11 @@ impl DownloadDefaults {
             return custom_help.clone();
         }
 
-        let mut help = String::from(
+        let mut help = format!(
             "Specify a snapshot URL or let the command propose a default one.\n\n\
-             Browse available snapshots at https://snapshots.reth.rs\n\
+             Browse available snapshots at {}\n\
              or use --list-snapshots to see them from the CLI.\n\nAvailable snapshot sources:\n",
+            self.snapshot_api_url.trim_end_matches("/api/snapshots"),
         );
 
         for source in &self.available_snapshots {
@@ -169,6 +175,12 @@ impl DownloadDefaults {
         self
     }
 
+    /// Set the snapshot discovery API URL.
+    pub fn with_snapshot_api_url(mut self, url: impl Into<Cow<'static, str>>) -> Self {
+        self.snapshot_api_url = url.into();
+        self
+    }
+
     /// Builder: Set custom long help text, overriding the generated help
     pub fn with_long_help(mut self, help: impl Into<String>) -> Self {
         self.long_help = Some(help.into());
@@ -191,7 +203,7 @@ pub struct DownloadCommand<C: ChainSpecParser> {
     /// Custom URL to download a single snapshot archive (legacy mode).
     ///
     /// When provided, downloads and extracts a single archive without component selection.
-    /// Browse available snapshots at <https://snapshots.reth.rs> or use --list-snapshots.
+    /// Browse available snapshots with --list-snapshots.
     #[arg(long, short, long_help = DownloadDefaults::get_global().long_help())]
     url: Option<String>,
 
@@ -261,7 +273,7 @@ pub struct DownloadCommand<C: ChainSpecParser> {
     #[arg(long, default_value_t = MAX_CONCURRENT_DOWNLOADS)]
     download_concurrency: usize,
 
-    /// List available snapshots from snapshots.reth.rs and exit.
+    /// List available snapshots and exit.
     ///
     /// Queries the snapshots API and prints all available snapshots for the selected chain,
     /// including block number, size, and manifest URL.
@@ -1328,7 +1340,17 @@ fn streaming_download_and_extract(
         let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
             Ok(r) => r,
             Err(e) => {
-                last_error = Some(e.into());
+                let err = eyre::Error::from(e);
+                if attempt < MAX_DOWNLOAD_RETRIES {
+                    warn!(target: "reth::cli",
+                        url = %url,
+                        attempt,
+                        max = MAX_DOWNLOAD_RETRIES,
+                        err = %err,
+                        "Streaming request failed, retrying"
+                    );
+                }
+                last_error = Some(err);
                 if attempt < MAX_DOWNLOAD_RETRIES {
                     std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
                 }
@@ -1355,6 +1377,15 @@ fn streaming_download_and_extract(
         match result {
             Ok(()) => return Ok(()),
             Err(e) => {
+                if attempt < MAX_DOWNLOAD_RETRIES {
+                    warn!(target: "reth::cli",
+                        url = %url,
+                        attempt,
+                        max = MAX_DOWNLOAD_RETRIES,
+                        err = %e,
+                        "Streaming extraction failed, retrying"
+                    );
+                }
                 last_error = Some(e);
                 if attempt < MAX_DOWNLOAD_RETRIES {
                     std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
@@ -1520,6 +1551,7 @@ fn blocking_process_modular_archive(
     }
 
     let format = CompressionFormat::from_url(&archive.file_name)?;
+    let mut last_error: Option<eyre::Error> = None;
     for attempt in 1..=MAX_DOWNLOAD_RETRIES {
         cleanup_output_files(target_dir, &archive.output_files);
 
@@ -1527,13 +1559,31 @@ fn blocking_process_modular_archive(
             let cache_dir = cache_dir.ok_or_else(|| eyre::eyre!("Missing cache directory"))?;
             let archive_path = cache_dir.join(&archive.file_name);
             let part_path = cache_dir.join(format!("{}.part", archive.file_name));
-            let (downloaded_path, _downloaded_size) =
-                resumable_download(&archive.url, cache_dir, shared.as_ref(), cancel_token.clone())?;
-            let file = fs::open(&downloaded_path)?;
-            extract_archive_raw(file, format, target_dir)?;
+            let result =
+                resumable_download(&archive.url, cache_dir, shared.as_ref(), cancel_token.clone())
+                    .and_then(|(downloaded_path, _)| {
+                        let file = fs::open(&downloaded_path)?;
+                        extract_archive_raw(file, format, target_dir)
+                    });
             let _ = fs::remove_file(&archive_path);
             let _ = fs::remove_file(&part_path);
+
+            if let Err(e) = result {
+                warn!(target: "reth::cli",
+                    file = %archive.file_name,
+                    component = %planned.component,
+                    attempt,
+                    err = %e,
+                    "Download or extraction failed, retrying"
+                );
+                last_error = Some(e);
+                if attempt < MAX_DOWNLOAD_RETRIES {
+                    std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
+                }
+                continue;
+            }
         } else {
+            // streaming_download_and_extract already has its own internal retry loop
             streaming_download_and_extract(
                 &archive.url,
                 format,
@@ -1553,6 +1603,13 @@ fn blocking_process_modular_archive(
         warn!(target: "reth::cli", file = %archive.file_name, component = %planned.component, attempt, "Extracted files failed integrity checks, retrying");
     }
 
+    if let Some(e) = last_error {
+        return Err(e.wrap_err(format!(
+            "Failed after {} attempts for {}",
+            MAX_DOWNLOAD_RETRIES, archive.file_name
+        )));
+    }
+
     eyre::bail!(
         "Failed integrity validation after {} attempts for {}",
         MAX_DOWNLOAD_RETRIES,
@@ -1608,10 +1665,11 @@ fn file_blake3_hex(path: &Path) -> Result<String> {
 
 /// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
 ///
-/// Queries `snapshots.reth.rs/api/snapshots` and returns the manifest URL for the most
+/// Queries the configured snapshot API and returns the manifest URL for the most
 /// recent modular snapshot matching the requested chain.
 async fn discover_manifest_url(chain_id: u64) -> Result<String> {
-    let api_url = RETH_SNAPSHOTS_API_URL;
+    let defaults = DownloadDefaults::get_global();
+    let api_url = &*defaults.snapshot_api_url;
 
     info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
 
@@ -1624,8 +1682,9 @@ async fn discover_manifest_url(chain_id: u64) -> Result<String> {
                  {chain_id} at {api_url}\n\n\
                  You can provide a manifest URL directly with --manifest-url, or\n\
                  use a direct snapshot URL with -u from:\n\
-                 \t- https://snapshots.reth.rs\n\n\
-                 Use --list to see all available snapshots."
+                 \t- {}\n\n\
+                 Use --list to see all available snapshots.",
+                api_url.trim_end_matches("/api/snapshots"),
             )
         })?;
 
@@ -1656,7 +1715,7 @@ where
     }
 }
 
-/// An entry from the `snapshots.reth.rs/api/snapshots` listing.
+/// An entry from the snapshot discovery API listing.
 #[derive(serde::Deserialize)]
 #[serde(rename_all = "camelCase")]
 struct SnapshotApiEntry {
@@ -1681,7 +1740,7 @@ impl SnapshotApiEntry {
 
 /// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
 async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
-    let api_url = RETH_SNAPSHOTS_API_URL;
+    let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
 
     let entries: Vec<SnapshotApiEntry> = Client::new()
         .get(api_url)
@@ -1699,7 +1758,11 @@ async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntr
 fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
     let modular: Vec<_> = entries.iter().filter(|e| e.is_modular()).collect();
 
-    println!("Available snapshots for chain {chain_id} (https://snapshots.reth.rs):\n");
+    let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
+    println!(
+        "Available snapshots for chain {chain_id} ({}):\n",
+        api_url.trim_end_matches("/api/snapshots"),
+    );
     println!("{:<12}  {:>10}  {:<10}  {:>10}  MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
     println!("{}", "-".repeat(100));
 
@@ -1738,14 +1801,18 @@ async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
                     .await
                     .and_then(|r| r.error_for_status())
                     .wrap_err_with(|| {
+                        let sources = DownloadDefaults::get_global()
+                            .available_snapshots
+                            .iter()
+                            .map(|s| format!("\t- {s}"))
+                            .collect::<Vec<_>>()
+                            .join("\n");
                         format!(
                             "Failed to fetch snapshot manifest from {source}\n\n\
                              The manifest endpoint may not be available for this snapshot source.\n\
                              You can use a direct snapshot URL instead:\n\n\
                              \treth download -u <snapshot-url>\n\n\
-                             Available snapshot sources:\n\
-                             \t- https://snapshots.reth.rs\n\
-                             \t- https://publicnode.com/snapshots"
+                             Available snapshot sources:\n{sources}"
                         )
                     })?;
                 Ok(response.json().await?)

crates/cli/commands/src/import_core.rs

@@ -19,11 +19,12 @@ use reth_node_api::BlockTy;
 use reth_node_events::node::NodeEvent;
 use reth_provider::{
     providers::ProviderNodeTypes, BlockNumReader, HeaderProvider, ProviderError, ProviderFactory,
-    StageCheckpointReader,
+    RocksDBProviderFactory, StageCheckpointReader,
 };
 use reth_prune::PruneModes;
 use reth_stages::{prelude::*, ControlFlow, Pipeline, StageId, StageSet};
 use reth_static_file::StaticFileProducer;
+use reth_storage_api::StorageSettingsCache;
 use std::{path::Path, sync::Arc};
 use tokio::sync::watch;
 use tracing::{debug, error, info, warn};
@@ -108,7 +109,11 @@ where
 
     let provider = provider_factory.provider()?;
     let init_blocks = provider.tx_ref().entries::<tables::HeaderNumbers>()?;
-    let init_txns = provider.tx_ref().entries::<tables::TransactionHashNumbers>()?;
+    let init_txns = if provider_factory.cached_storage_settings().storage_v2 {
+        provider_factory.rocksdb_provider().iter::<tables::TransactionHashNumbers>()?.count()
+    } else {
+        provider.tx_ref().entries::<tables::TransactionHashNumbers>()?
+    };
     drop(provider);
 
     let mut total_decoded_blocks = 0;
@@ -215,8 +220,12 @@ where
 
     let provider = provider_factory.provider()?;
     let total_imported_blocks = provider.tx_ref().entries::<tables::HeaderNumbers>()? - init_blocks;
-    let total_imported_txns =
-        provider.tx_ref().entries::<tables::TransactionHashNumbers>()? - init_txns;
+    let current_txns = if provider_factory.cached_storage_settings().storage_v2 {
+        provider_factory.rocksdb_provider().iter::<tables::TransactionHashNumbers>()?.count()
+    } else {
+        provider.tx_ref().entries::<tables::TransactionHashNumbers>()?
+    };
+    let total_imported_txns = current_txns - init_txns;
 
     let result = ImportResult {
         total_decoded_blocks,

crates/cli/commands/src/node.rs

@@ -178,6 +178,8 @@ where
             ext,
         } = self;
 
+        engine.validate()?;
+
         // set up node config
         let mut node_config = NodeConfig {
             datadir,

crates/cli/commands/src/p2p/mod.rs

@@ -193,11 +193,15 @@ impl<C: ChainSpecParser> DownloadArgs<C> {
         let default_secret_key_path = data_dir.p2p_secret();
         let p2p_secret_key = self.network.secret_key(default_secret_key_path)?;
         let rlpx_socket = (self.network.addr, self.network.port).into();
-        let boot_nodes = self.chain.bootnodes().unwrap_or_default();
+        let boot_nodes = self
+            .network
+            .resolved_bootnodes()
+            .unwrap_or_else(|| self.chain.bootnodes().unwrap_or_default());
 
         let net =
             NetworkConfigBuilder::<N::NetworkPrimitives>::new(p2p_secret_key, Runtime::test())
                 .peer_config(config.peers_config_with_basic_nodes_from_file(None))
+                .sessions_config(config.sessions)
                 .external_ip_resolver(self.network.nat.clone())
                 .network_id(self.network.network_id)
                 .boot_nodes(boot_nodes.clone())

crates/cli/commands/src/prune.rs

@@ -12,7 +12,6 @@ use reth_node_metrics::{
     server::{MetricServer, MetricServerConfig},
     version::VersionInfo,
 };
-#[cfg(all(unix, feature = "rocksdb"))]
 use reth_provider::RocksDBProviderFactory;
 use reth_prune::PrunerBuilder;
 use reth_static_file::StaticFileProducer;
@@ -122,7 +121,6 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> PruneComma
         }
 
         // Flush and compact RocksDB to reclaim disk space after pruning
-        #[cfg(all(unix, feature = "rocksdb"))]
         {
             info!(target: "reth::cli", "Flushing and compacting RocksDB...");
             provider_factory.rocksdb_provider().flush_and_compact()?;

crates/cli/commands/src/stage/dump/execution.rs

@@ -6,7 +6,7 @@ use reth_db_api::{
 };
 use reth_db_common::DbTool;
 use reth_evm::ConfigureEvm;
-use reth_node_builder::NodeTypesWithDB;
+use reth_node_api::HeaderTy;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_provider::{
     providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
@@ -58,7 +58,7 @@ where
 }
 
 /// Imports all the tables that can be copied over a range.
-fn import_tables_with_range<N: NodeTypesWithDB>(
+fn import_tables_with_range<N: ProviderNodeTypes>(
     output_db: &DatabaseEnv,
     db_tool: &DbTool<N>,
     from: u64,
@@ -74,7 +74,7 @@ fn import_tables_with_range<N: NodeTypesWithDB>(
         )
     })??;
     output_db.update(|tx| {
-        tx.import_table_with_range::<tables::Headers, _>(
+        tx.import_table_with_range::<tables::Headers<HeaderTy<N>>, _>(
             &db_tool.provider_factory.db_ref().tx()?,
             Some(from),
             to,

crates/cli/commands/src/stage/dump/merkle.rs

@@ -10,6 +10,7 @@ use reth_db_api::{database::Database, models::BlockNumberAddress, table::TableIm
 use reth_db_common::DbTool;
 use reth_evm::ConfigureEvm;
 use reth_exex::ExExManagerHandle;
+use reth_node_api::HeaderTy;
 use reth_node_core::dirs::{ChainPath, DataDirPath};
 use reth_provider::{
     providers::{ProviderNodeTypes, RocksDBProvider, StaticFileProvider},
@@ -41,7 +42,7 @@ where
     let (output_db, tip_block_number) = setup(from, to, &output_datadir.db(), db_tool)?;
 
     output_db.update(|tx| {
-        tx.import_table_with_range::<tables::Headers, _>(
+        tx.import_table_with_range::<tables::Headers<HeaderTy<N>>, _>(
             &db_tool.provider_factory.db_ref().tx()?,
             Some(from),
             to,

crates/cli/commands/src/stage/run.rs

@@ -28,6 +28,7 @@ use reth_node_metrics::{
     server::{MetricServer, MetricServerConfig},
     version::VersionInfo,
 };
+use reth_primitives_traits::FastInstant as Instant;
 use reth_provider::{
     ChainSpecProvider, DBProvider, DatabaseProviderFactory, StageCheckpointReader,
     StageCheckpointWriter,
@@ -40,7 +41,7 @@ use reth_stages::{
     },
     ExecInput, ExecOutput, ExecutionStageThresholds, Stage, StageExt, UnwindInput, UnwindOutput,
 };
-use std::{any::Any, net::SocketAddr, sync::Arc, time::Instant};
+use std::{any::Any, net::SocketAddr, sync::Arc};
 use tokio::sync::watch;
 use tracing::*;
 

crates/cli/util/Cargo.toml

@@ -33,19 +33,21 @@ reth-tracing = { workspace = true, optional = true }
 rand.workspace = true
 
 [target.'cfg(unix)'.dependencies]
+tikv-jemalloc-sys = { workspace = true, optional = true }
 tikv-jemallocator = { workspace = true, optional = true }
 snmalloc-rs = { workspace = true, optional = true }
 libc = "0.2"
 
 [features]
-jemalloc = ["dep:tikv-jemallocator"]
+jemalloc = [
+    "dep:tikv-jemallocator",
+    "dep:tikv-jemalloc-sys",
+    "tikv-jemallocator?/override_allocator_on_supported_platforms",
+]
 
 # Enables jemalloc profiling features
 jemalloc-prof = ["jemalloc", "tikv-jemallocator?/profiling"]
 
-# Enables unprefixed malloc (reproducible builds support)
-jemalloc-unprefixed = ["jemalloc", "tikv-jemallocator?/unprefixed_malloc_on_supported_platforms"]
-
 # Wraps the selected allocator in the tracy profiling allocator
 tracy-allocator = ["dep:tracy-client", "dep:reth-tracing"]