ci(bench): big blocks in CI benchmarks (#22802)

Co-authored-by: Georgios Konstantopoulos <me@gakonst.com>
Co-authored-by: Amp <amp@ampcode.com>

.github/scripts/bench-reth-run.sh

@@ -6,6 +6,7 @@
 # Usage: bench-reth-run.sh <label> <binary> <output-dir>
 #
 # Required env: SCHELK_MOUNT, BENCH_RPC_URL, BENCH_BLOCKS, BENCH_WARMUP_BLOCKS
+# Optional env: BENCH_BIG_BLOCKS (true/false), BENCH_WORK_DIR (for big blocks path)
 set -euo pipefail
 
 LABEL="$1"
@@ -73,6 +74,8 @@ if [ "${BENCH_CORES:-0}" -gt 0 ] && [ "$BENCH_CORES" -lt "$MAX_RETH" ]; then
 fi
 RETH_CPUS="1-${MAX_RETH}"
 
+BIG_BLOCKS="${BENCH_BIG_BLOCKS:-false}"
+
 RETH_ARGS=(
   node
   --datadir "$DATADIR"
@@ -87,6 +90,11 @@ RETH_ARGS=(
   --no-persist-peers
 )
 
+# Big blocks mode requires the testing API and skip-invalid-transactions
+if [ "$BIG_BLOCKS" = "true" ]; then
+  RETH_ARGS+=(--http.api eth,net,web3,reth,testing --testing.skip-invalid-transactions)
+fi
+
 if [ "${BENCH_SAMPLY:-false}" = "true" ]; then
   RETH_ARGS+=(--log.samply)
   SAMPLY="$(which samply)"
@@ -124,21 +132,35 @@ done
 # files are not root-owned (avoids EACCES on next checkout).
 BENCH_NICE="sudo nice -n -20 sudo -u $(id -un)"
 
-# Warmup
-$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
-  --rpc-url "$BENCH_RPC_URL" \
-  --engine-rpc-url http://127.0.0.1:8551 \
-  --jwt-secret "$DATADIR/jwt.hex" \
-  --advance "${BENCH_WARMUP_BLOCKS:-50}" \
-  --reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
+if [ "$BIG_BLOCKS" = "true" ]; then
+  # Big blocks mode: replay pre-generated payloads with gas ramp
+  BIG_BLOCKS_DIR="${BENCH_WORK_DIR}/big-blocks"
+  echo "Running big blocks benchmark (replay-payloads)..."
+  $BENCH_NICE "$RETH_BENCH" replay-payloads \
+    --reth-new-payload \
+    --gas-ramp-dir "$BIG_BLOCKS_DIR/gas-ramp-dir" \
+    --payload-dir "$BIG_BLOCKS_DIR/payloads" \
+    --engine-rpc-url http://127.0.0.1:8551 \
+    --jwt-secret "$DATADIR/jwt.hex" \
+    --output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
+else
+  # Standard mode: warmup + new-payload-fcu
+  # Warmup
+  $BENCH_NICE "$RETH_BENCH" new-payload-fcu \
+    --rpc-url "$BENCH_RPC_URL" \
+    --engine-rpc-url http://127.0.0.1:8551 \
+    --jwt-secret "$DATADIR/jwt.hex" \
+    --advance "${BENCH_WARMUP_BLOCKS:-50}" \
+    --reth-new-payload 2>&1 | sed -u "s/^/[bench] /"
 
-# Benchmark
-$BENCH_NICE "$RETH_BENCH" new-payload-fcu \
-  --rpc-url "$BENCH_RPC_URL" \
-  --engine-rpc-url http://127.0.0.1:8551 \
-  --jwt-secret "$DATADIR/jwt.hex" \
-  --advance "$BENCH_BLOCKS" \
-  --reth-new-payload \
-  --output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
+  # Benchmark
+  $BENCH_NICE "$RETH_BENCH" new-payload-fcu \
+    --rpc-url "$BENCH_RPC_URL" \
+    --engine-rpc-url http://127.0.0.1:8551 \
+    --jwt-secret "$DATADIR/jwt.hex" \
+    --advance "$BENCH_BLOCKS" \
+    --reth-new-payload \
+    --output "$OUTPUT_DIR" 2>&1 | sed -u "s/^/[bench] /"
+fi
 
 # cleanup runs via trap

.github/scripts/bench-reth-summary.py

@@ -350,6 +350,7 @@ def generate_comparison_table(
     baseline_name: str,
     feature_name: str,
     feature_sha: str,
+    big_blocks: bool = False,
 ) -> str:
     """Generate a markdown comparison table between baseline and feature."""
     n = paired["blocks"]
@@ -390,7 +391,7 @@ def generate_comparison_table(
         f"| Mgas/s | {fmt_mgas(run1['mean_mgas_s'])} | {fmt_mgas(run2['mean_mgas_s'])} | {change_str(gas_pct, mgas_ci_pct, lower_is_better=False)} |",
         f"| Wall Clock | {fmt_s(run1['wall_clock_s'])} | {fmt_s(run2['wall_clock_s'])} | {change_str(wall_pct, wall_ci_pct, lower_is_better=True)} |",
         "",
-        f"*{n} blocks*",
+        f"*{n} {'big blocks' if big_blocks else 'blocks'}*",
     ]
     return "\n".join(lines)
 
@@ -466,6 +467,7 @@ def main():
     parser.add_argument("--feature-name", "--branch-name", default=None, help="Feature branch name")
     parser.add_argument("--feature-ref", "--branch-sha", "--feature-sha", default=None, help="Feature commit SHA")
     parser.add_argument("--behind-baseline", "--behind-main", type=int, default=0, help="Commits behind baseline")
+    parser.add_argument("--big-blocks", action="store_true", default=False, help="Big blocks mode")
     args = parser.parse_args()
 
     if len(args.baseline_csv) != len(args.feature_csv):
@@ -514,6 +516,7 @@ def main():
         baseline_name=baseline_name,
         feature_name=feature_name,
         feature_sha=feature_sha,
+        big_blocks=args.big_blocks,
     )
     print(f"Generated comparison ({paired_stats['n']} paired blocks, "
           f"mean diff {paired_stats['mean_diff_ms']:+.3f}ms ± {paired_stats['ci_ms']:.3f}ms)")