Compare commits

..

37 Commits
t4 ... v2.1.0

Author SHA1 Message Date
DaniPopes
d58c6e3d07 chore(docs): normalize Grafana dashboard JSON formatting and tags (#23266) 2026-04-20 13:42:53 +00:00
Matthias Seitz
d577814eb1 fix(engine): align Amsterdam endpoint validation (#23625) 2026-04-20 13:34:46 +00:00
Emma Jamieson-Hoare
8b46f1a6d0 chore: release 2.1.0 (#23641)
Co-authored-by: Amp <amp@ampcode.com>
2026-04-20 13:27:51 +00:00
Brian Picciano
c527c2e7d6 fix(engine): revert #23541 and #23578 (#23646)
Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-04-20 13:23:24 +00:00
Derek Cofausper
14570f325a perf(txpool): replace BTreeMap with imbl::OrdMap in BestTransactions (#23621)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-04-20 12:56:04 +00:00
Emma Jamieson-Hoare
41fe41f2f2 perf(re-execute): relax executor reset thresholds (#23617)
Co-authored-by: Amp <amp@ampcode.com>
2026-04-20 09:39:06 +00:00
Tim
27bfddeada feat: add fetch-grafana-dashboard workflow (#23585) 2026-04-20 08:02:06 +00:00
github-actions[bot]
981a7ef99b chore(deps): weekly cargo update (#23628)
Co-authored-by: github-merge-queue <118344674+github-merge-queue@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-04-19 08:38:10 +00:00
Matthias Seitz
8c826a5cd0 fix: address nightly clippy warnings (#23630) 2026-04-19 10:13:27 +02:00
Derek Cofausper
6465997ea1 refactor(tasks): make WorkerPool lazy by default (#23627)
Co-authored-by: DaniPopes <57450786+DaniPopes@users.noreply.github.com>
2026-04-18 18:59:13 +00:00
Derek Cofausper
03a308da63 feat(cli): add reth db migrate-v2 for v1→v2 storage migration (#23422)
Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com>
Co-authored-by: klkvr <klkvrr@gmail.com>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
2026-04-18 18:12:28 +00:00
Soubhik Singha Mahapatra
af84b982c3 chore: add slotnum to payload (#23626) 2026-04-18 15:51:44 +00:00
Ishika Choudhury
77c3e86ec6 chore: added gas limit to BlockOrPayload (#23624) 2026-04-18 14:28:07 +00:00
Arsenii Kulikov
98ebc3454f fix: don't cache stateful precompiles (#23619) 2026-04-17 18:42:47 +00:00
Derek Cofausper
c8979d0a1d fix(txpool,rpc): skip tx gas limit cap enforcement when EIP-8037 is active (#23612)
Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com>
Co-authored-by: klkvr <klkvrr@gmail.com>
2026-04-17 14:35:34 +00:00
Alexey Shekhirin
742a7e7a18 ci: use reth 2.0 banner image in release draft (#23404) 2026-04-17 14:31:41 +00:00
Derek Cofausper
99bf7a17c0 refactor(rpc): accept BlockId in block_access_list_raw (#23615)
Co-authored-by: Matthias Seitz <19890894+mattsse@users.noreply.github.com>
2026-04-17 14:10:56 +00:00
Ishika Choudhury
24436ca9f9 chore(BAL): added changes for slotnum (#23605)
Co-authored-by: klkvr <klkvrr@gmail.com>
2026-04-17 13:17:52 +00:00
Derek Cofausper
c26ec53d7d fix(bench): use previous snapshot to avoid block fetch failures (#23608)
Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com>
2026-04-17 11:22:14 +00:00
Dan Cline
3a136fc8c3 fix(db): use sync=true for rocksdb WriteOptions (#23603) 2026-04-17 11:17:02 +00:00
Soubhik Singha Mahapatra
d215d16a7d chore(BAL): add eth bal rpc methods to EngineEth (#23609) 2026-04-17 10:39:50 +00:00
Soubhik Singha Mahapatra
b36fff0ab8 feat(BAL): use new engine-api methods in bench (#23517)
Co-authored-by: Ishika Choudhury <117741714+Rimeeeeee@users.noreply.github.com>
Co-authored-by: Matthias Seitz <matthias.seitz@outlook.de>
2026-04-17 10:36:30 +00:00
Derek Cofausper
e4d4ba30cb refactor(provider): simplify get_overlay cache miss tracking (#23584)
Co-authored-by: Sergei Shulepov <2205845+pepyakin@users.noreply.github.com>
2026-04-17 08:15:44 +00:00
John Chase
7c219fa955 fix(cli): open stage dump environment read-write (#23602) 2026-04-17 08:02:35 +00:00
Derek Cofausper
0ac36468c6 feat(cli): add RocksDB support to reth db get (#23032)
Co-authored-by: Arsenii Kulikov <62447812+klkvr@users.noreply.github.com>
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: Arsenii Kulikov <klkvrr@gmail.com>
2026-04-16 20:26:04 +00:00
Derek Cofausper
93b2201c76 fix(engine): include backpressure in newPayload prometheus latency (#23578)
Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
2026-04-16 18:26:00 +00:00
Dan Cline
9990670990 fix(cli): error on non-mainnet when no download url provided (#23570)
Co-authored-by: Derek Cofausper <256792747+decofe@users.noreply.github.com>
2026-04-16 18:07:26 +00:00
Derek Cofausper
1b69c9bb42 fix(ci): use proper Slack mention for AI bot in bench failure alerts (#23591)
Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com>
2026-04-16 17:29:14 +00:00
Derek Cofausper
c2e649fc90 perf: parallel segmented snapshot downloads (#23028)
Co-authored-by: Dan Cline <6798349+Rjected@users.noreply.github.com>
Co-authored-by: Alexey Shekhirin <github@shekhirin.com>
Co-authored-by: Emma Jamieson-Hoare <21029500+emmajam@users.noreply.github.com>
Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com>
2026-04-16 16:57:22 +00:00
Alexey Shekhirin
cff41bb9c2 feat(miner): add --dev.payload-wait-time to LocalMiner (#23598) 2026-04-16 16:04:56 +00:00
Brian Picciano
0a9af7907f fix(ci): clean up bench cpu dma latency helper (#23594)
Co-authored-by: Brian Picciano <933154+mediocregopher@users.noreply.github.com>
Co-authored-by: Amp <amp@ampcode.com>
2026-04-16 14:02:29 +00:00
figtracer
815d8407ce chore(examples): add custom auth HTTP middleware example (#23586) 2026-04-16 12:53:01 +00:00
cui
6cf6378e36 perf(eth-wire-types): encode DisconnectReason without heap allocation (#23479) 2026-04-16 10:31:52 +00:00
figtracer
39f078e40f feat(rpc): expose auth HTTP transport middleware (#23579) 2026-04-16 10:14:29 +00:00
dependabot[bot]
37a23ae169 chore(deps): bump actions/upload-pages-artifact from 4 to 5 (#23572)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-04-16 10:00:02 +00:00
dependabot[bot]
8da8f3e4bc chore(deps): bump actions/github-script from 8 to 9 (#23571)
Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
2026-04-16 09:59:49 +00:00
Derek Cofausper
f97947b5a5 bench: bump defaults to 200 warmup, 500 blocks (#23580)
Co-authored-by: Alexey Shekhirin <5773434+shekhirin@users.noreply.github.com>
2026-04-16 08:49:45 +00:00
96 changed files with 16497 additions and 11675 deletions

View File

@@ -24,7 +24,7 @@ MC="mc"
BUCKET="minio/reth-snapshots"
# Allow overriding the snapshot name (e.g. for big-blocks mode where the
# big-blocks manifest specifies which base snapshot to use).
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable}"
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable-previous}"
MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
DATADIR_NAME="datadir"
HASH_MODE_SUFFIX=""

View File

@@ -0,0 +1,244 @@
#!/usr/bin/env python3
"""
Fetch a Grafana dashboard and convert it to the portable import format.
Fetches the dashboard via API, replaces internal datasource/variable references
with template variables, and adds __inputs/__requires/__elements so the JSON is
importable on any Grafana instance.
Usage:
export FETCH_GRAFANA_DASHBOARD_URL=https://<NAMESPACE>.grafana.net
export FETCH_GRAFANA_DASHBOARD_TOKEN=glsa_...
python3 .github/scripts/fetch-grafana-dashboard.py <dashboard-uid> > output.json
"""
import json
import os
import sys
import urllib.request
PANEL_TYPE_NAMES = {
"bargauge": "Bar gauge",
"gauge": "Gauge",
"heatmap": "Heatmap",
"piechart": "Pie chart",
"stat": "Stat",
"table": "Table",
"timeseries": "Time series",
"barchart": "Bar chart",
"text": "Text",
"dashlist": "Dashboard list",
"logs": "Logs",
"nodeGraph": "Node Graph",
"histogram": "Histogram",
"candlestick": "Candlestick",
"state-timeline": "State timeline",
"status-history": "Status history",
"geomap": "Geomap",
"canvas": "Canvas",
"news": "News",
"xychart": "XY Chart",
"trend": "Trend",
"datagrid": "Datagrid",
"flamegraph": "Flame Graph",
"traces": "Traces",
}
def fetch_json(base_url: str, token: str, path: str) -> dict:
url = f"{base_url}{path}"
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
with urllib.request.urlopen(req) as resp:
return json.loads(resp.read())
def fetch_dashboard(base_url: str, token: str, uid: str) -> dict:
return fetch_json(base_url, token, f"/api/dashboards/uid/{uid}")
def fetch_grafana_version(base_url: str) -> str:
req = urllib.request.Request(f"{base_url}/api/health")
with urllib.request.urlopen(req) as resp:
data = json.loads(resp.read())
# version string like "13.0.0-23940615780.patch2" -> take just the semver part
version = data.get("version", "")
# strip build metadata after the first hyphen if it looks like a pre-release
parts = version.split("-")
return parts[0] if parts else version
def collect_panel_types(panels: list) -> set[str]:
types = set()
for panel in panels:
ptype = panel.get("type", "")
if ptype and ptype != "row":
types.add(ptype)
# nested panels inside collapsed rows
for sub in panel.get("panels", []):
sub_type = sub.get("type", "")
if sub_type and sub_type != "row":
types.add(sub_type)
return types
def has_expression_datasource(dashboard: dict) -> bool:
return "__expr__" in json.dumps(dashboard)
def make_exportable(dashboard: dict, grafana_version: str = "") -> dict:
dash = json.loads(json.dumps(dashboard)) # deep copy
# --- Strip internal fields ---
dash.pop("id", None)
# --- Rewrite links: point to the public repo instead of internal ---
dash["links"] = [
{
"asDropdown": False,
"icon": "external link",
"includeVars": False,
"keepTime": False,
"tags": [],
"targetBlank": True,
"title": "Source (GitHub)",
"tooltip": "View source file in repository",
"type": "link",
"url": "https://github.com/paradigmxyz/reth/tree/main/etc/grafana/dashboards",
}
]
# --- Datasource: victoriametrics -> prometheus ---
dash_str = json.dumps(dash)
dash_str = dash_str.replace("victoriametrics-metrics-datasource", "prometheus")
dash = json.loads(dash_str)
# --- Templating: instance_label constant -> ${VAR_INSTANCE_LABEL} ---
# Also strip default-value fields the API returns that are not needed for import
STRIP_VAR_DEFAULTS = {"allowCustomValue", "regexApplyTo"}
for var in dash.get("templating", {}).get("list", []):
if var.get("name") == "instance_label" and var.get("type") == "constant":
var["query"] = "${VAR_INSTANCE_LABEL}"
var["current"] = {
"value": "${VAR_INSTANCE_LABEL}",
"text": "${VAR_INSTANCE_LABEL}",
"selected": False,
}
var["options"] = [
{
"value": "${VAR_INSTANCE_LABEL}",
"text": "${VAR_INSTANCE_LABEL}",
"selected": False,
}
]
# Clear current values for query/datasource vars (not meaningful for import)
elif var.get("type") in ("query", "datasource"):
var["current"] = {}
# Remove noisy default fields
for field in STRIP_VAR_DEFAULTS:
var.pop(field, None)
# Strip falsy defaults on query/datasource vars (API returns them, export omits them)
if var.get("type") in ("query", "datasource"):
for field in ("hide", "multi", "skipUrlSync"):
if not var.get(field):
var.pop(field, None)
# --- Build __inputs ---
inputs = [
{
"name": "DS_PROMETHEUS",
"label": "Prometheus",
"description": "",
"type": "datasource",
"pluginId": "prometheus",
"pluginName": "Prometheus",
},
]
if has_expression_datasource(dash):
inputs.append(
{
"name": "DS_EXPRESSION",
"label": "Expression",
"description": "",
"type": "datasource",
"pluginId": "__expr__",
}
)
inputs.append(
{
"name": "VAR_INSTANCE_LABEL",
"type": "constant",
"label": "Instance Label",
"value": "job",
"description": "",
}
)
# --- Build __requires ---
requires = []
if has_expression_datasource(dash):
requires.append({"type": "datasource", "id": "__expr__", "version": "1.0.0"})
panel_types = collect_panel_types(dash.get("panels", []))
for pt in sorted(panel_types):
requires.append(
{
"type": "panel",
"id": pt,
"name": PANEL_TYPE_NAMES.get(pt, pt),
"version": "",
}
)
requires.append(
{"type": "grafana", "id": "grafana", "name": "Grafana", "version": grafana_version}
)
requires.append(
{
"type": "datasource",
"id": "prometheus",
"name": "Prometheus",
"version": "1.0.0",
}
)
# --- Assemble output (with __inputs/__requires/__elements first) ---
output = {
"__inputs": inputs,
"__elements": {},
"__requires": requires,
}
output.update(dash)
return output
def main():
if len(sys.argv) < 2:
print(f"Usage: {sys.argv[0]} <dashboard-uid>", file=sys.stderr)
sys.exit(1)
uid = sys.argv[1]
base_url = os.environ.get("FETCH_GRAFANA_DASHBOARD_URL", "").rstrip("/")
token = os.environ.get("FETCH_GRAFANA_DASHBOARD_TOKEN", "")
if not base_url or not token:
print(
"Error: FETCH_GRAFANA_DASHBOARD_URL and FETCH_GRAFANA_DASHBOARD_TOKEN env vars required",
file=sys.stderr,
)
sys.exit(1)
resp = fetch_dashboard(base_url, token, uid)
dashboard = resp["dashboard"]
grafana_version = fetch_grafana_version(base_url)
exported = make_exportable(dashboard, grafana_version)
print(json.dumps(exported, indent=2))
if __name__ == "__main__":
main()

View File

@@ -112,7 +112,7 @@ jobs:
- name: Alert on long-running hourly
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
uses: actions/github-script@v8
uses: actions/github-script@v9
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -154,7 +154,7 @@ jobs:
- name: Alert on stale nightly
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
uses: actions/github-script@v8
uses: actions/github-script@v9
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -278,7 +278,7 @@ jobs:
- name: Resolve job URL
id: job-url
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
@@ -680,7 +680,7 @@ jobs:
- name: Write job summary
if: success()
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
const fs = require('fs');
@@ -759,7 +759,7 @@ jobs:
- name: Send Slack notification (success)
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
uses: actions/github-script@v8
uses: actions/github-script@v9
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -914,7 +914,7 @@ jobs:
- name: Send Slack notification (failure)
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
uses: actions/github-script@v8
uses: actions/github-script@v9
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -948,7 +948,7 @@ jobs:
},
{
type: 'section',
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n@ai investigate this` },
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n<@U0AAA8F0JEM> investigate this` },
},
{
type: 'actions',

View File

@@ -14,7 +14,7 @@ on:
blocks:
description: "Number of blocks to benchmark"
required: false
default: "200"
default: "500"
type: string
big_blocks:
description: "Use big blocks mode (pre-generated merged payloads with reth-bb)"
@@ -34,7 +34,7 @@ on:
warmup:
description: "Number of warmup blocks"
required: false
default: "100"
default: "200"
type: string
baseline:
description: "Baseline git ref (default: merge-base)"
@@ -133,7 +133,7 @@ jobs:
steps:
- name: Check org membership
if: github.event_name == 'issue_comment'
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.GITHUB_TOKEN }}
script: |
@@ -152,7 +152,7 @@ jobs:
- name: Parse arguments
id: args
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -164,9 +164,9 @@ jobs:
if (context.eventName === 'workflow_dispatch') {
actor = '${{ github.actor }}';
blocks = '${{ github.event.inputs.blocks }}' || '200';
warmup = '${{ github.event.inputs.warmup }}' || '100';
if (warmup !== '100') explicitWarmup = true;
blocks = '${{ github.event.inputs.blocks }}' || '500';
warmup = '${{ github.event.inputs.warmup }}' || '200';
if (warmup !== '200') explicitWarmup = true;
baseline = '${{ github.event.inputs.baseline }}';
feature = '${{ github.event.inputs.feature }}';
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
@@ -205,7 +205,7 @@ jobs:
const enumArgs = new Map([['bal', validBalModes], ['slack', validSlackModes]]);
const durationArgs = new Set(['wait-time']);
const stringArgs = new Set(['baseline-args', 'feature-args']);
const defaults = { blocks: '200', warmup: '100', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const defaults = { blocks: '500', warmup: '200', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
const unknown = [];
const invalid = [];
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
@@ -359,7 +359,7 @@ jobs:
- name: Acknowledge request
id: ack
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -445,7 +445,7 @@ jobs:
- name: Poll queue position
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -560,7 +560,7 @@ jobs:
- name: Resolve checkout ref
id: checkout-ref
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
if (!process.env.BENCH_PR) {
@@ -586,7 +586,7 @@ jobs:
- name: Resolve job URL and update status
if: env.BENCH_COMMENT_ID
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -702,7 +702,7 @@ jobs:
# Build binaries
- name: Resolve PR head branch
id: pr-info
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
if (process.env.BENCH_PR) {
@@ -720,7 +720,7 @@ jobs:
- name: Resolve refs
id: refs
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
const { execSync } = require('child_process');
@@ -815,7 +815,7 @@ jobs:
- name: Update status (snapshot needed)
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -910,8 +910,11 @@ jobs:
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
done || true
# Replace any stale PM QoS holders left behind by earlier benchmark jobs.
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
# Prevent deep C-states (avoids wake-up latency jitter)
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
sudo bash -c 'exec 3<>/dev/cpu_dma_latency; printf "\0\0\0\0" >&3; exec -a bench-cpu-dma-latency sleep infinity' &
echo "BENCH_CPU_DMA_LATENCY_PID=$!" >> "$GITHUB_ENV"
# Move all IRQs to core 0 (housekeeping core)
for irq in /proc/irq/*/smp_affinity_list; do
echo 0 | sudo tee "$irq" 2>/dev/null || true
@@ -999,7 +1002,7 @@ jobs:
- name: Update status (running benchmarks)
if: success() && env.BENCH_COMMENT_ID
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1261,7 +1264,7 @@ jobs:
- name: Compare & comment
if: success() && env.BENCH_COMMENT_ID
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1341,7 +1344,7 @@ jobs:
- name: Write job summary
if: success()
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
const jobSummary = require('./.github/scripts/bench-job-summary.js');
@@ -1355,7 +1358,7 @@ jobs:
- name: Send Slack notification (success)
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
uses: actions/github-script@v8
uses: actions/github-script@v9
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -1366,7 +1369,7 @@ jobs:
- name: Update status (failed)
if: failure() && env.BENCH_COMMENT_ID
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1400,7 +1403,7 @@ jobs:
- name: Send Slack notification (failure)
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
uses: actions/github-script@v8
uses: actions/github-script@v9
env:
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
@@ -1422,7 +1425,7 @@ jobs:
- name: Update status (cancelled)
if: cancelled() && env.BENCH_COMMENT_ID
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
github-token: ${{ secrets.DEREK_PAT }}
script: |
@@ -1445,5 +1448,9 @@ jobs:
done
# Restore amd-pstate to active (EPP) mode with powersave governor
echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
if [ -n "${BENCH_CPU_DMA_LATENCY_PID:-}" ]; then
sudo kill "$BENCH_CPU_DMA_LATENCY_PID" 2>/dev/null || true
fi
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
sudo cpupower frequency-set -g powersave 2>/dev/null || true
sudo systemctl start irqbalance cron atd 2>/dev/null || true

View File

@@ -50,7 +50,7 @@ jobs:
uses: actions/configure-pages@v6
- name: Upload artifact
uses: actions/upload-pages-artifact@v4
uses: actions/upload-pages-artifact@v5
with:
path: "./docs/vocs/docs/dist"

View File

@@ -0,0 +1,62 @@
name: Fetch Grafana Dashboard
on:
workflow_dispatch:
inputs:
dashboard_uid:
description: "Grafana dashboard UID to export"
required: true
default: "2k8BXz24x"
target_path:
description: "Target file path in the repo (e.g. etc/grafana/dashboards/overview.json)"
required: true
default: "etc/grafana/dashboards/overview.json"
jobs:
fetch:
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v6
- uses: actions/setup-python@v5
with:
python-version: "3.12"
- name: Fetch dashboard from Grafana
env:
FETCH_GRAFANA_DASHBOARD_URL: ${{ secrets.FETCH_GRAFANA_DASHBOARD_URL }}
FETCH_GRAFANA_DASHBOARD_TOKEN: ${{ secrets.FETCH_GRAFANA_DASHBOARD_TOKEN }}
run: |
python3 .github/scripts/fetch-grafana-dashboard.py "${{ inputs.dashboard_uid }}" \
> "${{ inputs.target_path }}"
- name: Check for changes
id: diff
run: |
if git diff --quiet "${{ inputs.target_path }}"; then
echo "changed=false" >> "$GITHUB_OUTPUT"
echo "No changes detected."
else
echo "changed=true" >> "$GITHUB_OUTPUT"
fi
- name: Create pull request
if: steps.diff.outputs.changed == 'true'
env:
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
run: |
TARGET="${{ inputs.target_path }}"
FILENAME="$(basename "$TARGET")"
BRANCH="chore/sync-grafana-${FILENAME%.*}-$(date +%Y%m%d-%H%M%S)"
git config user.name "github-actions[bot]"
git config user.email "github-actions[bot]@users.noreply.github.com"
git checkout -b "$BRANCH"
git add "$TARGET"
git commit -m "chore: update Grafana dashboard ${FILENAME}"
git push origin "$BRANCH"
gh pr create \
--title "chore: update Grafana dashboard ${FILENAME}" \
--body "Automated export from Grafana (dashboard UID: \`${{ inputs.dashboard_uid }}\`, target: \`${TARGET}\`)."

View File

@@ -11,11 +11,22 @@ jobs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
- name: Check for ${DS_PROMETHEUS} in overview.json
- name: Validate dashboard format
run: |
if grep -Fn '${DS_PROMETHEUS}' etc/grafana/dashboards/overview.json; then
echo "Error: overview.json contains '\${DS_PROMETHEUS}' placeholder"
echo "Please replace it with '\${datasource}'"
exit 1
fi
echo "✓ overview.json does not contain '\${DS_PROMETHEUS}' placeholder"
python3 -c "
import json, sys
with open('etc/grafana/dashboards/overview.json') as f:
d = json.load(f)
errors = []
if '__inputs' not in d:
errors.append('missing __inputs')
if '__requires' not in d:
errors.append('missing __requires')
if d.get('id') is not None:
errors.append('contains internal id field — use export-dashboard.py')
if errors:
for e in errors:
print(f'Error: {e}', file=sys.stderr)
sys.exit(1)
print('✓ overview.json is a valid exported dashboard')
"

View File

@@ -16,7 +16,7 @@ jobs:
fetch-depth: 0
- name: Label PRs
uses: actions/github-script@v8
uses: actions/github-script@v9
with:
script: |
const label_pr = require('./.github/scripts/label_pr.js')

View File

@@ -195,7 +195,7 @@ jobs:
fi
body=$(cat <<- "ENDBODY"
![image](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-prod.png)
![image](https://raw.githubusercontent.com/paradigmxyz/reth/main/assets/reth-2.png)
## Testing Checklist (DELETE ME)

794
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -1,5 +1,5 @@
[workspace.package]
version = "2.0.0"
version = "2.1.0"
edition = "2024"
rust-version = "1.93"
license = "MIT OR Apache-2.0"
@@ -129,6 +129,7 @@ members = [
"examples/custom-node-components/",
"examples/custom-payload-builder/",
"examples/custom-rlpx-subprotocol",
"examples/custom-auth-http-middleware",
"examples/custom-rpc-middleware",
"examples/db-access",
"examples/exex-subscription",
@@ -432,14 +433,14 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
reth-zstd-compressors = { version = "0.3.0", default-features = false }
# revm
revm = { version = "37.0.0", default-features = false }
revm = { version = "38.0.0", default-features = false }
revm-bytecode = { version = "10.0.0", default-features = false }
revm-database = { version = "13.0.0", default-features = false }
revm-state = { version = "11.0.0", default-features = false }
revm-primitives = { version = "23.0.0", default-features = false }
revm-interpreter = { version = "35.0.0", default-features = false }
revm-database-interface = { version = "11.0.0", default-features = false }
revm-inspectors = "0.38.0"
revm-inspectors = "0.39.0"
# eth
alloy-dyn-abi = "1.5.6"
@@ -449,7 +450,7 @@ alloy-sol-types = { version = "1.5.6", default-features = false }
alloy-chains = { version = "0.2.33", default-features = false }
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.32.0", default-features = false }
alloy-evm = { version = "0.33.0", default-features = false }
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.4", default-features = false }
@@ -506,6 +507,7 @@ eyre = "0.6"
fdlimit = "0.3.0"
fixed-map = { version = "0.9", default-features = false }
humantime = "2.1"
imbl = "7"
humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
@@ -698,19 +700,3 @@ vergen-git2 = "9.1.0"
# networking
ipnet = "2.11"
[patch.crates-io]
revm = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-database = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-state = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "c6f88bbe7186d863f4667dd43c42608eb7a8ba5c" }
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "ff0bbec9ccaa818155e25003a77f4d73d350bbd7" }
reth-codecs = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
reth-codecs-derive = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
reth-rpc-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
reth-zstd-compressors = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }

View File

@@ -238,6 +238,7 @@ where
withdrawals: prev_segment.ctx.withdrawals.clone(),
extra_data: prev_segment.ctx.extra_data.clone(),
tx_count_hint: prev_segment.ctx.tx_count_hint,
slot_number: prev_segment.ctx.slot_number,
};
// Clone the next segment's data before we consume inner.
@@ -252,6 +253,7 @@ where
withdrawals: new_segment.ctx.withdrawals.clone(),
extra_data: new_segment.ctx.extra_data.clone(),
tx_count_hint: new_segment.ctx.tx_count_hint,
slot_number: new_segment.ctx.slot_number,
};
plan.next_segment += 1;
@@ -364,6 +366,7 @@ where
withdrawals: seg0.ctx.withdrawals.clone(),
extra_data: seg0.ctx.extra_data.clone(),
tx_count_hint: seg0.ctx.tx_count_hint,
slot_number: seg0.ctx.slot_number,
};
let inner = self.inner_mut();
@@ -422,6 +425,7 @@ where
withdrawals: last_seg.ctx.withdrawals.clone(),
extra_data: last_seg.ctx.extra_data.clone(),
tx_count_hint: last_seg.ctx.tx_count_hint,
slot_number: last_seg.ctx.slot_number,
};
self.inner_mut().ctx = last_ctx;
}

View File

@@ -266,6 +266,7 @@ where
ommers: &[],
withdrawals: ctx.withdrawals.map(|w| std::borrow::Cow::Owned(w.into_owned())),
extra_data: ctx.extra_data,
slot_number: ctx.slot_number,
};
BigBlockSegment { start_tx, evm_env, ctx }
})

View File

@@ -176,6 +176,7 @@ impl BbAddOns {
BasicEngineApiBuilder::default(),
BasicEngineValidatorBuilder::default(),
Default::default(),
Default::default(),
)
}
}

View File

@@ -10,7 +10,7 @@ use alloy_eips::{
eip1559::BaseFeeParams,
eip7840::BlobParams,
eip7928::{AccountChanges, BlockAccessList, SlotChanges},
BlockNumberOrTag, Typed2718,
Typed2718,
};
use alloy_primitives::{Bloom, Bytes, B256};
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
@@ -32,6 +32,8 @@ use serde::{Deserialize, Serialize};
use std::{collections::HashMap, future::Future};
use tracing::{info, warn};
use crate::bench::helpers::fetch_block_access_list;
/// A single transaction with its gas used and raw encoded bytes.
#[derive(Debug, Clone)]
pub struct RawTransaction {
@@ -669,20 +671,6 @@ impl Command {
}
}
async fn fetch_block_access_list(
provider: &RootProvider<AnyNetwork>,
block_number: u64,
) -> eyre::Result<BlockAccessList> {
provider
.client()
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
.await
.map_err(Into::into)
.and_then(|block_access_list: Option<BlockAccessList>| {
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
})
}
fn merge_block_access_list(
merged: &mut BlockAccessList,
incoming: BlockAccessList,

View File

@@ -1,5 +1,7 @@
//! Common helpers for reth-bench commands.
use alloy_eips::{eip7928::BlockAccessList, BlockNumberOrTag};
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use eyre::Result;
use std::{
io::{BufReader, Read},
@@ -69,6 +71,21 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
}
}
/// Fetches the block access list for a given block number using the provided provider.
pub(crate) async fn fetch_block_access_list(
provider: &RootProvider<AnyNetwork>,
block_number: u64,
) -> eyre::Result<BlockAccessList> {
provider
.client()
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
.await
.map_err(Into::into)
.and_then(|block_access_list: Option<BlockAccessList>| {
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
})
}
#[cfg(test)]
mod tests {
use super::*;

View File

@@ -4,7 +4,7 @@
use crate::{
bench::{
context::BenchContext,
helpers::parse_duration,
helpers::{fetch_block_access_list, parse_duration},
metrics_scraper::MetricsScraper,
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
@@ -198,12 +198,19 @@ impl Command {
finalized_block_hash: finalized,
};
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
} else {
None
};
let (version, params) = block_to_new_payload(
block,
rlp,
use_reth_namespace,
wait_for_persistence,
no_wait_for_caches,
bal,
)?;
let start = Instant::now();
let server_timings =

View File

@@ -3,6 +3,7 @@
use crate::{
bench::{
context::BenchContext,
helpers::fetch_block_access_list,
metrics_scraper::MetricsScraper,
output::{
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
@@ -69,7 +70,9 @@ impl Command {
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
let block_provider_clone = block_provider.clone();
tokio::task::spawn(async move {
let block_provider = block_provider_clone;
while benchmark_mode.contains(next_block) {
let block_res = block_provider
.get_block_by_number(next_block.into())
@@ -123,12 +126,19 @@ impl Command {
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
} else {
None
};
let (version, params) = block_to_new_payload(
block,
rlp,
use_reth_namespace,
wait_for_persistence,
no_wait_for_caches,
bal,
)?;
let start = Instant::now();

View File

@@ -18,7 +18,7 @@ use alloy_primitives::B256;
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV4,
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV6,
ExecutionPayloadSidecar, ExecutionPayloadV4, ForkchoiceState, JwtSecret, PraguePayloadFields,
};
use clap::Parser;
@@ -315,7 +315,7 @@ impl Command {
let requests =
execution_data.sidecar.requests().cloned().unwrap_or_default().to_vec();
(
Some(EngineApiMessageVersion::V4),
Some(EngineApiMessageVersion::V6),
serde_json::to_value((
execution_data.payload.clone(),
Vec::<B256>::new(),
@@ -423,7 +423,7 @@ impl Command {
/// Load and parse all payload files from the directory.
///
/// Tries to load each file as a [`BigBlockPayload`] first (which includes `env_switches`),
/// falling back to [`ExecutionPayloadEnvelopeV4`] for backwards compatibility.
/// falling back to [`ExecutionPayloadEnvelopeV6`] for backwards compatibility.
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
let mut payloads = Vec::new();
@@ -450,12 +450,11 @@ impl Command {
let name_str = name.to_string_lossy();
let index = if let Some(rest) = name_str.strip_prefix("payload_block_") {
rest.strip_suffix(".json")?.parse::<u64>().ok()?
} else if let Some(rest) = name_str.strip_prefix("big_block_") {
} else {
let rest = name_str.strip_prefix("big_block_")?;
// "big_block_FROM_to_TO.json" — use FROM as the index
let rest = rest.strip_suffix(".json")?;
rest.split("_to_").next()?.parse::<u64>().ok()?
} else {
return None;
};
Some((index, e.path()))
})
@@ -481,10 +480,10 @@ impl Command {
{
(big_block.execution_data, big_block.big_block_data, big_block.block_access_list)
} else {
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
let envelope: ExecutionPayloadEnvelopeV6 = serde_json::from_str(&content)
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
let execution_data = ExecutionData {
payload: envelope.envelope_inner.execution_payload.clone().into(),
payload: envelope.execution_payload.clone().into(),
sidecar: ExecutionPayloadSidecar::v4(
CancunPayloadFields {
versioned_hashes: Vec::new(),

View File

@@ -1,6 +1,8 @@
use alloy_eips::eip4895::Withdrawal;
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
use alloy_rpc_types_engine::{
ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4,
};
/// Configuration for invalidating payload fields
#[derive(Debug, Default)]
@@ -21,6 +23,7 @@ pub(super) struct InvalidationConfig {
pub(super) block_hash: Option<B256>,
pub(super) blob_gas_used: Option<u64>,
pub(super) excess_blob_gas: Option<u64>,
pub(super) slot_number: Option<u64>,
// Auto-invalidation flags
pub(super) invalidate_parent_hash: bool,
@@ -35,6 +38,8 @@ pub(super) struct InvalidationConfig {
pub(super) invalidate_withdrawals: bool,
pub(super) invalidate_blob_gas_used: bool,
pub(super) invalidate_excess_blob_gas: bool,
pub(super) invalidate_block_access_list: bool,
pub(super) invalidate_slot_number: bool,
}
impl InvalidationConfig {
@@ -216,4 +221,30 @@ impl InvalidationConfig {
changes
}
/// Applies invalidations to a V4 payload, returns list of what was changed.
pub(super) fn apply_to_payload_v4(&self, payload: &mut ExecutionPayloadV4) -> Vec<String> {
let mut changes = self.apply_to_payload_v3(&mut payload.payload_inner);
// Explicit override for slot_number
if let Some(slot_number) = self.slot_number {
payload.slot_number = slot_number;
changes.push(format!("slot_number = {slot_number}"));
}
// Handle block access list invalidation (V4+)
if self.invalidate_block_access_list {
let fake_bal = Bytes::from_static(&[0x01, 0x02, 0x03]);
payload.block_access_list = fake_bal.clone();
changes.push(format!("block_access_list = {fake_bal} (auto-invalidated)"));
}
// Handle slot number invalidation (V4+)
if self.invalidate_slot_number {
payload.slot_number = u64::MAX;
changes.push("slot_number = MAX (auto-invalidated)".to_string());
}
changes
}
}

View File

@@ -1,12 +1,18 @@
//! Command for sending invalid payloads to test Engine API rejection.
mod invalidation;
use alloy_rpc_client::ClientBuilder;
use invalidation::InvalidationConfig;
use crate::bench::helpers::fetch_block_access_list;
use super::helpers::{load_jwt_secret, read_input};
use alloy_consensus::TxEnvelope;
use alloy_primitives::{Address, B256};
use alloy_provider::network::AnyRpcBlock;
use alloy_primitives::{Address, Bytes, B256};
use alloy_provider::{
network::{AnyNetwork, AnyRpcBlock},
RootProvider,
};
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
@@ -105,6 +111,9 @@ pub struct Command {
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
requests_hash: Option<B256>,
/// Override the slot number with a specific value.
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
slot_number: Option<u64>,
// ==================== Auto-Invalidation Flags ====================
/// Invalidate the parent hash by setting it to a random value.
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
@@ -158,6 +167,14 @@ pub struct Command {
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_requests_hash: bool,
/// Invalidate the block access list by setting it to a random value (EIP-7928).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_block_access_list: bool,
/// Invalidate the slot number by setting it to an random value.(EIP-7843).
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
invalidate_slot_number: bool,
// ==================== Meta Flags ====================
/// Skip block hash recalculation after modifications.
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
@@ -199,6 +216,7 @@ impl Command {
block_hash: self.block_hash,
blob_gas_used: self.blob_gas_used,
excess_blob_gas: self.excess_blob_gas,
slot_number: self.slot_number,
invalidate_parent_hash: self.invalidate_parent_hash,
invalidate_state_root: self.invalidate_state_root,
invalidate_receipts_root: self.invalidate_receipts_root,
@@ -211,6 +229,8 @@ impl Command {
invalidate_withdrawals: self.invalidate_withdrawals,
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
invalidate_block_access_list: self.invalidate_block_access_list,
invalidate_slot_number: self.invalidate_slot_number,
}
}
@@ -234,15 +254,21 @@ impl Command {
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
let use_v4 = block.header.requests_hash.is_some();
let use_v5 = block.header.block_access_list_hash.is_some();
let requests_hash = self.requests_hash.or(block.header.requests_hash);
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
let mut execution_payload = if use_v5 {
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
} else {
ExecutionPayload::from_block_slow(&block).0
};
let changes = match &mut execution_payload {
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
ExecutionPayload::V4(p) => config.apply_to_payload_v4(p),
};
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
@@ -312,7 +338,13 @@ impl Command {
match self.mode {
Mode::Execute => {
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
let method = if use_v5 {
"engine_newPayloadV5"
} else if use_v4 {
"engine_newPayloadV4"
} else {
"engine_newPayloadV3"
};
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
@@ -353,4 +385,17 @@ impl Command {
Ok(())
}
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
let rpc_url = self
.rpc_url
.as_deref()
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
let client = ClientBuilder::default()
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let provider = RootProvider::<AnyNetwork>::new(client);
let bal = fetch_block_access_list(&provider, block_number).await?;
Ok(alloy_rlp::encode(bal).into())
}
}

View File

@@ -1,6 +1,11 @@
use super::helpers::{load_jwt_secret, read_input};
use super::helpers::{fetch_block_access_list, load_jwt_secret, read_input};
use alloy_consensus::TxEnvelope;
use alloy_provider::network::AnyRpcBlock;
use alloy_primitives::Bytes;
use alloy_provider::{
network::{AnyNetwork, AnyRpcBlock},
RootProvider,
};
use alloy_rpc_client::ClientBuilder;
use alloy_rpc_types_engine::ExecutionPayload;
use clap::Parser;
use eyre::{OptionExt, Result};
@@ -69,6 +74,9 @@ impl Command {
})?
.into_consensus();
let use_v4 = block.header.requests_hash.is_some();
let use_v5 = block.header.block_access_list_hash.is_some();
// Extract parent beacon block root
let parent_beacon_block_root = block.header.parent_beacon_block_root;
@@ -76,10 +84,14 @@ impl Command {
let blob_versioned_hashes =
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
// Convert to execution payload
let execution_payload = ExecutionPayload::from_block_slow(&block).0;
let use_v4 = block.header.requests_hash.is_some();
// V5 payloads must carry the full RLP-encoded block access list, not just the hash stored
// in the header.
let execution_payload = if use_v5 {
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
} else {
ExecutionPayload::from_block_slow(&block).0
};
// Create JSON request data
let json_request = if use_v4 {
@@ -102,7 +114,13 @@ impl Command {
Mode::Execute => {
// Create cast command
let mut command = std::process::Command::new("cast");
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
let method = if use_v5 {
"engine_newPayloadV5"
} else if use_v4 {
"engine_newPayloadV4"
} else {
"engine_newPayloadV3"
};
command.arg("rpc").arg(method).arg("--raw");
if let Some(rpc_url) = self.rpc_url {
command.arg("--rpc-url").arg(rpc_url);
@@ -146,4 +164,17 @@ impl Command {
Ok(())
}
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
let rpc_url = self
.rpc_url
.as_deref()
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
let client = ClientBuilder::default()
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
.http(rpc_url.parse()?);
let provider = RootProvider::<AnyNetwork>::new(client);
let bal = fetch_block_access_list(&provider, block_number).await?;
Ok(alloy_rlp::encode(bal).into())
}
}

View File

@@ -3,6 +3,7 @@
//! before sending additional calls.
use alloy_consensus::TxEnvelope;
use alloy_eips::eip7928::BlockAccessList;
use alloy_primitives::Bytes;
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
use alloy_rpc_types_engine::{
@@ -43,6 +44,14 @@ pub trait EngineApiValidWaitExt<N>: Send + Sync {
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated>;
/// Calls `engine_forkChoiceUpdatedV4` with the given [`ForkchoiceState`] and optional
/// [`PayloadAttributes`], and waits until the response is VALID.
async fn fork_choice_updated_v4_wait(
&self,
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated>;
}
#[async_trait::async_trait]
@@ -162,6 +171,40 @@ where
Ok(status)
}
async fn fork_choice_updated_v4_wait(
&self,
fork_choice_state: ForkchoiceState,
payload_attributes: Option<PayloadAttributes>,
) -> TransportResult<ForkchoiceUpdated> {
debug!(
target: "reth-bench",
method = "engine_forkchoiceUpdatedV3",
?fork_choice_state,
?payload_attributes,
"Sending forkchoiceUpdated"
);
let mut status =
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
while !status.is_valid() {
if status.is_invalid() {
error!(
target: "reth-bench",
?status,
?fork_choice_state,
?payload_attributes,
"Invalid forkchoiceUpdatedV4 message",
);
panic!("Invalid forkchoiceUpdatedV4: {status:?}");
}
status =
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
}
Ok(status)
}
}
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
@@ -176,6 +219,7 @@ pub(crate) fn block_to_new_payload(
reth_new_payload: bool,
wait_for_persistence: WaitForPersistence,
no_wait_for_caches: bool,
bal: Option<BlockAccessList>,
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
let block_number = block.header.number;
let wait_for_persistence = wait_for_persistence.rpc_value(block_number);
@@ -198,7 +242,11 @@ pub(crate) fn block_to_new_payload(
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
})?
.into_consensus();
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
let block_access_list = alloy_rlp::encode(bal.unwrap_or_default());
let (payload, sidecar) =
ExecutionPayload::from_block_slow_with_bal(&block, block_access_list.into());
let (version, params, execution_data) = payload_to_new_payload(payload, sidecar, None)?;
if reth_new_payload {
@@ -227,6 +275,22 @@ pub(crate) fn payload_to_new_payload(
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
let (version, params) = match payload {
ExecutionPayload::V4(payload) => {
let cancun = sidecar
.cancun()
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
let version = target_version.unwrap_or(EngineApiMessageVersion::V6);
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
(
version,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
requests,
))?,
)
}
ExecutionPayload::V3(payload) => {
let cancun = sidecar
.cancun()
@@ -266,22 +330,6 @@ pub(crate) fn payload_to_new_payload(
ExecutionPayload::V1(payload) => {
(EngineApiMessageVersion::V1, serde_json::to_value((payload,))?)
}
ExecutionPayload::V4(payload) => {
let cancun = sidecar
.cancun()
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
(
version,
serde_json::to_value((
payload,
cancun.versioned_hashes.clone(),
cancun.parent_beacon_block_root,
requests,
))?,
)
}
};
Ok((version, params, execution_data))
@@ -386,10 +434,10 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
) -> TransportResult<ForkchoiceUpdated> {
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
match message_version {
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 |
EngineApiMessageVersion::V6 => {
provider.fork_choice_updated_v4_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
}
EngineApiMessageVersion::V2 => {

View File

@@ -1,4 +1,4 @@
use alloy_primitives::{hex, BlockHash};
use alloy_primitives::{hex, Address, BlockHash, B256};
use clap::Parser;
use reth_db::{
static_file::{
@@ -10,6 +10,7 @@ use reth_db::{
use reth_db_api::{
cursor::{DbCursorRO, DbDupCursorRO},
database::Database,
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
table::{Compress, Decompress, DupSort, Table},
tables,
transaction::DbTx,
@@ -19,7 +20,10 @@ use reth_db_common::DbTool;
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
use reth_node_builder::NodeTypesWithDB;
use reth_primitives_traits::ValueWithSubKey;
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
use reth_provider::{
providers::ProviderNodeTypes, ChangeSetReader, RocksDBProviderFactory,
StaticFileProviderFactory,
};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StorageChangeSetReader;
use tracing::error;
@@ -73,6 +77,55 @@ enum Subcommand {
#[arg(long)]
raw: bool,
},
/// Gets the content of a RocksDB table for the given key
///
/// For history tables (accounts-history, storages-history), you can pass a plain address
/// instead of a full JSON ShardedKey. Use --block to query a specific block number
/// (seeks to the shard containing that block), or --all-shards to list all shards for
/// the address.
///
/// Examples:
/// reth db get rocksdb accounts-history 0xdBBE3D8c2d2b22A2611c5A94A9a12C2fCD49Eb29
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --block 1000000
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --all-shards
/// reth db get rocksdb storages-history 0xdBBE...Eb29 --storage-key 0x0000...0003
Rocksdb {
/// The RocksDB table
#[arg(value_enum)]
table: RocksDbTable,
/// The key to get content for. For history tables, this can be a plain address.
#[arg(value_parser = maybe_json_value_parser)]
key: String,
/// Target block number for history tables. Seeks to the shard containing this block.
/// Defaults to the latest shard if not specified.
#[arg(long)]
block: Option<u64>,
/// Storage key for storages-history table lookups.
#[arg(long)]
storage_key: Option<String>,
/// List all shards for the given key (history tables only).
#[arg(long)]
all_shards: bool,
/// Output bytes instead of human-readable decoded value
#[arg(long)]
raw: bool,
},
}
/// RocksDB tables that can be queried.
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
pub enum RocksDbTable {
/// Transaction hash to transaction number mapping
TransactionHashNumbers,
/// Account history indices
AccountsHistory,
/// Storage history indices
StoragesHistory,
}
impl Command {
@@ -82,6 +135,9 @@ impl Command {
Subcommand::Mdbx { table, key, subkey, end_key, end_subkey, raw } => {
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
}
Subcommand::Rocksdb { table, key, block, storage_key, all_shards, raw } => {
get_rocksdb(tool, table, &key, block, storage_key.as_deref(), all_shards, raw)?;
}
Subcommand::StaticFile { segment, key, subkey, raw } => {
if let StaticFileSegment::StorageChangeSets = segment {
let storage_key =
@@ -246,6 +302,208 @@ impl Command {
}
}
/// Gets a value from a RocksDB table by key.
fn get_rocksdb<N: ProviderNodeTypes>(
tool: &DbTool<N>,
table: RocksDbTable,
key: &str,
block: Option<u64>,
storage_key: Option<&str>,
all_shards: bool,
raw: bool,
) -> eyre::Result<()> {
let rocksdb = tool.provider_factory.rocksdb_provider();
match table {
RocksDbTable::TransactionHashNumbers => {
if block.is_some() || all_shards || storage_key.is_some() {
return Err(eyre::eyre!(
"--block, --all-shards, and --storage-key are only supported for history tables"
));
}
get_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, key, raw)
}
RocksDbTable::AccountsHistory => {
if storage_key.is_some() {
return Err(eyre::eyre!("--storage-key is only supported for storages-history"));
}
get_rocksdb_account_history(&rocksdb, key, block, all_shards, raw)
}
RocksDbTable::StoragesHistory => {
get_rocksdb_storage_history(&rocksdb, key, storage_key, block, all_shards, raw)
}
}
}
/// Try to parse a key string as a plain address, falling back to JSON `ShardedKey` parsing.
fn parse_address(key: &str) -> eyre::Result<Address> {
// Strip surrounding quotes that `maybe_json_value_parser` may have added
let stripped = key.trim_matches('"');
stripped.parse::<Address>().map_err(|e| eyre::eyre!("failed to parse address: {e}"))
}
/// Gets account history from RocksDB with ergonomic key parsing.
///
/// Accepts a plain address and uses seek to find the relevant shard.
fn get_rocksdb_account_history(
rocksdb: &reth_provider::providers::RocksDBProvider,
key: &str,
block: Option<u64>,
all_shards: bool,
raw: bool,
) -> eyre::Result<()> {
// Try parsing as a plain address first, fall back to full JSON ShardedKey
match parse_address(key) {
Ok(address) => {
let block_number = block.unwrap_or(u64::MAX);
let seek_key = ShardedKey::new(address, block_number);
if all_shards {
// Iterate all shards: seek from (address, 0) until address changes
let start = ShardedKey::new(address, 0);
let iter = rocksdb.iter_from::<tables::AccountsHistory>(start)?;
for result in iter {
let (k, v) = result?;
if k.key != address {
break;
}
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"highest_block_number": k.highest_block_number,
"value": v,
}))?
);
}
} else {
// Seek to the first shard with highest_block_number >= target
let mut iter = rocksdb.iter_from::<tables::AccountsHistory>(seek_key)?;
match iter.next() {
Some(Ok((k, v))) if k.key == address => {
if raw {
let raw_val = rocksdb.get_raw::<tables::AccountsHistory>(k)?;
if let Some(bytes) = raw_val {
println!("{}", hex::encode_prefixed(&bytes));
}
} else {
println!("{}", serde_json::to_string_pretty(&v)?);
}
}
_ => {
error!(target: "reth::cli", "No content for the given table key.");
}
}
}
Ok(())
}
Err(_) => {
// Fall back to full JSON key parsing (e.g.
// `{"key":"0x...","highest_block_number":...}`)
if all_shards || block.is_some() {
return Err(eyre::eyre!(
"--block and --all-shards require a plain address, not a JSON key"
));
}
get_rocksdb_table::<tables::AccountsHistory>(rocksdb, key, raw)
}
}
}
/// Gets storage history from RocksDB with ergonomic key parsing.
///
/// Accepts a plain address + optional `--storage-key` and uses seek.
fn get_rocksdb_storage_history(
rocksdb: &reth_provider::providers::RocksDBProvider,
key: &str,
storage_key: Option<&str>,
block: Option<u64>,
all_shards: bool,
raw: bool,
) -> eyre::Result<()> {
match parse_address(key) {
Ok(address) => {
let storage_key = storage_key
.map(|s| s.trim_matches('"').parse::<B256>())
.transpose()
.map_err(|e| eyre::eyre!("failed to parse storage key: {e}"))?
.unwrap_or_default();
let block_number = block.unwrap_or(u64::MAX);
let seek_key = StorageShardedKey::new(address, storage_key, block_number);
if all_shards {
let start = StorageShardedKey::new(address, storage_key, 0);
let iter = rocksdb.iter_from::<tables::StoragesHistory>(start)?;
for result in iter {
let (k, v) = result?;
if k.address != address || k.sharded_key.key != storage_key {
break;
}
println!(
"{}",
serde_json::to_string_pretty(&serde_json::json!({
"highest_block_number": k.sharded_key.highest_block_number,
"value": v,
}))?
);
}
} else {
let mut iter = rocksdb.iter_from::<tables::StoragesHistory>(seek_key)?;
match iter.next() {
Some(Ok((k, v)))
if k.address == address && k.sharded_key.key == storage_key =>
{
if raw {
let raw_val = rocksdb.get_raw::<tables::StoragesHistory>(k)?;
if let Some(bytes) = raw_val {
println!("{}", hex::encode_prefixed(&bytes));
}
} else {
println!("{}", serde_json::to_string_pretty(&v)?);
}
}
_ => {
error!(target: "reth::cli", "No content for the given table key.");
}
}
}
Ok(())
}
Err(_) => {
if all_shards || block.is_some() || storage_key.is_some() {
return Err(eyre::eyre!(
"--block, --all-shards, and --storage-key require a plain address, not a JSON key"
));
}
get_rocksdb_table::<tables::StoragesHistory>(rocksdb, key, raw)
}
}
}
/// Gets a value from a specific RocksDB table by exact key and prints it.
fn get_rocksdb_table<T: Table>(
rocksdb: &reth_provider::providers::RocksDBProvider,
key_str: &str,
raw: bool,
) -> eyre::Result<()> {
let key = table_key::<T>(key_str)?;
if raw {
let content = rocksdb.get_raw::<T>(key)?;
match content {
Some(bytes) => println!("{}", hex::encode_prefixed(&bytes)),
None => error!(target: "reth::cli", "No content for the given table key."),
}
} else {
let content = rocksdb.get::<T>(key)?;
match content {
Some(value) => println!("{}", serde_json::to_string_pretty(&value)?),
None => error!(target: "reth::cli", "No content for the given table key."),
}
}
Ok(())
}
/// Get an instance of key for given table
pub(crate) fn table_key<T: Table>(key: &str) -> Result<T::Key, eyre::Error> {
serde_json::from_str(key).map_err(|e| eyre::eyre!(e))

View File

@@ -0,0 +1,361 @@
//! `reth db migrate-v2` command for migrating v1 storage layout to v2.
//!
//! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to
//! static files, clears recomputable tables (senders, indices, trie, plain
//! state), compacts MDBX, then runs the pipeline to rebuild them.
use crate::common::CliNodeTypes;
use clap::Parser;
use reth_db::{
mdbx::{self, ffi},
models::StorageBeforeTx,
DatabaseEnv,
};
use reth_db_api::{
cursor::DbCursorRO,
database::Database,
table::Table,
tables,
transaction::{DbTx, DbTxMut},
};
use reth_node_builder::NodeTypesWithDBAdapter;
use reth_provider::{
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider,
MetadataWriter, ProviderFactory, PruneCheckpointReader, StageCheckpointWriter,
StaticFileProviderFactory, StaticFileWriter, StorageSettings,
};
use reth_prune_types::PruneSegment;
use reth_stages_types::{StageCheckpoint, StageId};
use reth_static_file_types::StaticFileSegment;
use reth_storage_api::StageCheckpointReader;
use tracing::info;
/// `reth db migrate-v2` command
#[derive(Debug, Parser)]
pub struct Command;
impl Command {
/// Execute the full v1 → v2 migration:
///
/// 1. Migrate changesets + receipts to static files
/// 2. Flip `StorageSettings` to v2
/// 3. Clear recomputable MDBX tables + reset stage checkpoints
/// 4. Compact MDBX
pub async fn execute<N: CliNodeTypes>(
self,
provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
) -> eyre::Result<()>
where
N::Primitives: reth_primitives_traits::NodePrimitives<
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
>,
{
// === Phase 0: Preflight ===
info!(target: "reth::cli", "Starting v1 → v2 storage migration");
let provider = provider_factory.provider()?;
let current_settings = provider.storage_settings()?;
if current_settings.is_some_and(|s| s.is_v2()) {
info!(target: "reth::cli", "Storage is already v2, nothing to do");
return Ok(());
}
let tip =
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number).unwrap_or(0);
info!(target: "reth::cli", tip, "Chain tip block number");
let sf_provider = provider_factory.static_file_provider();
for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets]
{
if sf_provider.get_highest_static_file_block(segment).is_some() {
eyre::bail!(
"Static file segment {segment:?} already contains data. \
Cannot migrate — target must be empty."
);
}
}
drop(provider);
// === Phase 1: Migrate changesets → static files ===
Self::migrate_account_changesets(&provider_factory, tip)?;
Self::migrate_storage_changesets(&provider_factory, tip)?;
// === Phase 2: Migrate receipts → static files ===
Self::migrate_receipts::<NodeTypesWithDBAdapter<N, DatabaseEnv>>(&provider_factory, tip)?;
// === Phase 3: Flip metadata to v2 ===
info!(target: "reth::cli", "Writing StorageSettings v2 metadata");
{
let provider_rw = provider_factory.database_provider_rw()?;
provider_rw.write_storage_settings(StorageSettings::v2())?;
provider_rw.commit()?;
}
info!(target: "reth::cli", "Storage settings updated to v2");
// === Phase 4: Clear recomputable tables ===
Self::clear_recomputable_tables(&provider_factory)?;
// === Phase 5: Compact MDBX (before pipeline, so it runs on a smaller DB) ===
let db_path = provider_factory.db_ref().path();
Self::compact_mdbx(provider_factory.db_ref())?;
// Drop to release DB handle for swap
drop(provider_factory);
let compact_path = db_path.with_file_name("db_compact");
Self::swap_compacted_db(&db_path, &compact_path)?;
// === Phase 6: Reopen DB and run pipeline ===
// The caller will reopen the environment and run the pipeline.
// We return here — the pipeline step is handled in mod.rs after
// reopening the database with the compacted copy.
info!(target: "reth::cli", "Migration complete. You should now restart the node and let it run the pipeline to rebuild the remaining data.");
Ok(())
}
fn migrate_account_changesets<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
tip: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Migrating AccountChangeSets → static files");
let provider = factory.provider()?.disable_long_read_transaction_safety();
let sf_provider = factory.static_file_provider();
let mut cursor = provider.tx_ref().cursor_read::<tables::AccountChangeSets>()?;
let first_block = provider
.get_prune_checkpoint(PruneSegment::AccountHistory)?
.and_then(|cp| cp.block_number)
.map_or(0, |b| b + 1);
let mut writer =
sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?;
let mut count = 0u64;
let mut walker = cursor.walk(Some(first_block))?.peekable();
for block in first_block..=tip {
let mut entries = Vec::new();
while let Some(Ok((block_number, _))) = walker.peek() {
if *block_number != block {
break;
}
let (_, entry) = walker.next().expect("peeked")?;
entries.push(entry);
}
count += entries.len() as u64;
writer.append_account_changeset(entries, block)?;
}
writer.commit()?;
info!(target: "reth::cli", count, "AccountChangeSets migrated");
Ok(())
}
fn migrate_storage_changesets<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
tip: u64,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Migrating StorageChangeSets → static files");
let provider = factory.provider()?.disable_long_read_transaction_safety();
let sf_provider = factory.static_file_provider();
let mut cursor = provider.tx_ref().cursor_read::<tables::StorageChangeSets>()?;
let first_block = provider
.get_prune_checkpoint(PruneSegment::StorageHistory)?
.and_then(|cp| cp.block_number)
.map_or(0, |b| b + 1);
let mut writer =
sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?;
let mut count = 0u64;
let mut walker = cursor.walk(Some(Default::default()))?.peekable();
for block in first_block..=tip {
let mut entries = Vec::new();
while let Some(Ok((key, _))) = walker.peek() {
if key.block_number() != block {
break;
}
let (key, entry) = walker.next().expect("peeked")?;
entries.push(StorageBeforeTx {
address: key.address(),
key: entry.key,
value: entry.value,
});
}
count += entries.len() as u64;
writer.append_storage_changeset(entries, block)?;
}
writer.commit()?;
info!(target: "reth::cli", count, "StorageChangeSets migrated");
Ok(())
}
fn migrate_receipts<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
tip: u64,
) -> eyre::Result<()>
where
N::Primitives: reth_primitives_traits::NodePrimitives<
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
>,
{
let provider = factory.provider()?;
if !provider.prune_modes_ref().receipts_log_filter.is_empty() {
info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX");
return Ok(());
}
drop(provider);
let sf_provider = factory.static_file_provider();
let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts);
if existing.is_some_and(|b| b >= tip) {
info!(target: "reth::cli", "Receipts already in static files, skipping");
return Ok(());
}
info!(target: "reth::cli", "Migrating Receipts → static files");
let provider = factory.provider()?.disable_long_read_transaction_safety();
let prune_start = provider
.get_prune_checkpoint(PruneSegment::Receipts)?
.and_then(|cp| cp.block_number)
.map_or(0, |b| b + 1);
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
let block_range = first_block..=tip;
let segment = reth_static_file::segments::Receipts;
reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?;
sf_provider.commit()?;
info!(target: "reth::cli", "Receipts migrated");
Ok(())
}
/// Clears tables that can be recomputed by the pipeline and resets their
/// stage checkpoints.
fn clear_recomputable_tables<N: ProviderNodeTypes>(
factory: &ProviderFactory<N>,
) -> eyre::Result<()> {
info!(target: "reth::cli", "Clearing recomputable MDBX tables");
let db = factory.db_ref();
macro_rules! clear_table {
($table:ty) => {{
let tx = db.tx_mut()?;
tx.clear::<$table>()?;
tx.commit()?;
info!(target: "reth::cli", table = <$table as Table>::NAME, "Cleared");
}};
}
// Migrated changeset tables (now in static files)
clear_table!(tables::AccountChangeSets);
clear_table!(tables::StorageChangeSets);
// Senders — rebuilt by SenderRecovery
clear_table!(tables::TransactionSenders);
// Indices — rebuilt by TransactionLookup / IndexAccountHistory / IndexStorageHistory
clear_table!(tables::TransactionHashNumbers);
clear_table!(tables::AccountsHistory);
clear_table!(tables::StoragesHistory);
// Plain state — superseded by hashed state in v2
clear_table!(tables::PlainAccountState);
clear_table!(tables::PlainStorageState);
// Trie — rebuilt by MerkleExecute
clear_table!(tables::AccountsTrie);
clear_table!(tables::StoragesTrie);
// Reset stage checkpoints so the pipeline rebuilds everything
info!(target: "reth::cli", "Resetting stage checkpoints");
let provider_rw = factory.database_provider_rw()?;
for stage in [
StageId::SenderRecovery,
StageId::TransactionLookup,
StageId::IndexAccountHistory,
StageId::IndexStorageHistory,
StageId::MerkleExecute,
StageId::MerkleUnwind,
] {
provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(0))?;
info!(target: "reth::cli", %stage, "Checkpoint reset to 0");
}
provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?;
provider_rw.commit()?;
info!(target: "reth::cli", "Recomputable tables cleared");
Ok(())
}
/// Creates a compacted copy of the MDBX database.
fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
let db_path = db.path();
let compact_path = db_path.with_file_name("db_compact");
reth_fs_util::create_dir_all(&compact_path)?;
info!(target: "reth::cli", ?db_path, ?compact_path, "Compacting MDBX database");
let compact_dest = compact_path.join("mdbx.dat");
let dest_cstr = std::ffi::CString::new(
compact_dest.to_str().ok_or_else(|| eyre::eyre!("compact path must be valid UTF-8"))?,
)?;
let flags = ffi::MDBX_CP_COMPACT | ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
});
if rc != 0 {
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
});
}
info!(target: "reth::cli", "MDBX compaction complete");
Ok(())
}
/// Swaps the original MDBX database with a compacted copy.
fn swap_compacted_db(
db_path: &std::path::Path,
compact_path: &std::path::Path,
) -> eyre::Result<()> {
let backup_path = db_path.with_file_name("db_pre_compact");
info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database");
std::fs::rename(db_path, &backup_path)?;
if let Err(e) = std::fs::rename(compact_path, db_path) {
let _ = std::fs::rename(&backup_path, db_path);
return Err(e.into());
}
std::fs::remove_dir_all(&backup_path)?;
info!(target: "reth::cli", "Database compaction swap complete");
Ok(())
}
}

View File

@@ -16,6 +16,7 @@ mod copy;
mod diff;
mod get;
mod list;
mod migrate_v2;
mod prune_checkpoints;
mod repair_trie;
mod settings;
@@ -77,6 +78,9 @@ pub enum Subcommands {
AccountStorage(account_storage::Command),
/// Gets account state and storage at a specific block
State(state::Command),
/// Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
#[command(name = "migrate-v2")]
MigrateV2(migrate_v2::Command),
}
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
@@ -231,6 +235,13 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
command.execute(&tool)?;
});
}
Subcommands::MigrateV2(command) => {
let Environment { provider_factory, .. } =
self.env.init::<N>(AccessRights::RW, ctx.task_executor.clone())?;
// Migrate changesets+receipts, clear tables, compact MDBX
command.execute::<N>(provider_factory).await?;
}
}
Ok(())

View File

@@ -0,0 +1,368 @@
use super::{
extract::{extract_archive_raw, streaming_download_and_extract, CompressionFormat},
fetch::ArchiveFetcher,
manifest::SnapshotArchive,
planning::{PlannedArchive, PlannedDownloads},
progress::{
spawn_progress_display, ArchiveDownloadProgress, ArchiveExtractionProgress,
ArchiveVerificationProgress, DownloadRequestLimiter, SharedProgress,
},
session::{ArchiveProcessContext, DownloadSession},
verify::OutputVerifier,
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
};
use eyre::Result;
use futures::stream::{self, StreamExt};
use reth_cli_util::cancellation::CancellationToken;
use reth_fs_util as fs;
use std::{
path::Path,
sync::{atomic::Ordering, Arc},
time::Duration,
};
use tokio::task;
use tracing::{debug, info, warn};
const DOWNLOAD_CACHE_DIR: &str = ".download-cache";
/// Runs all planned modular archive downloads for one command invocation.
pub(crate) async fn run_modular_downloads(
planned_downloads: PlannedDownloads,
target_dir: &Path,
download_concurrency: usize,
cancel_token: CancellationToken,
) -> Result<()> {
let download_cache_dir = target_dir.join(DOWNLOAD_CACHE_DIR);
fs::create_dir_all(&download_cache_dir)?;
let shared = SharedProgress::new(
planned_downloads.total_download_size,
planned_downloads.total_output_size,
planned_downloads.total_archives() as u64,
cancel_token.clone(),
);
let session = DownloadSession::new(
Some(Arc::clone(&shared)),
Some(DownloadRequestLimiter::new(download_concurrency)),
cancel_token,
);
let ctx =
ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session);
ModularDownloadJob::new(ctx, download_concurrency).run(planned_downloads).await
}
/// Schedules modular archive work for one run of `reth download`.
struct ModularDownloadJob {
/// Shared paths and session state for each archive in this job.
ctx: ArchiveProcessContext,
/// Maximum number of archives processed at once.
archive_concurrency: usize,
}
impl ModularDownloadJob {
/// Creates the modular download job for one command run.
const fn new(ctx: ArchiveProcessContext, archive_concurrency: usize) -> Self {
Self { ctx, archive_concurrency }
}
/// Runs all planned archives and waits for the shared progress task to finish.
async fn run(self, planned_downloads: PlannedDownloads) -> Result<()> {
let shared = Arc::clone(
self.ctx.session().progress().expect("modular downloads always use shared progress"),
);
let progress_handle = spawn_progress_display(Arc::clone(&shared));
let ctx = self.ctx.clone();
let results: Vec<Result<()>> = stream::iter(planned_downloads.archives)
.map(move |archive| {
let ctx = ctx.clone();
async move { Self::process_archive(ctx, archive).await }
})
.buffer_unordered(self.archive_concurrency)
.collect()
.await;
shared.done.store(true, Ordering::Relaxed);
let _ = progress_handle.await;
for result in results {
result?;
}
Ok(())
}
/// Runs one archive on the blocking pool so fetch and extraction stay off the async executor.
async fn process_archive(ctx: ArchiveProcessContext, archive: PlannedArchive) -> Result<()> {
task::spawn_blocking(move || ArchiveProcessor::new(archive, ctx).run()).await??;
Ok(())
}
}
/// Explicit retry states for one modular archive.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ArchiveAttemptState {
/// Start or restart one full archive attempt.
RunAttempt,
/// Check whether the extracted outputs verify.
VerifyOutputs,
/// Wait and decide whether another full attempt should run.
RetryAttempt,
/// Finish successfully.
Complete,
/// Stop with an error after retries are exhausted.
Fail,
}
/// Processes one modular archive from reuse check through extraction and verification.
struct ArchiveProcessor {
/// The concrete archive and component being processed.
archive: PlannedArchive,
/// Shared paths and session state for this archive attempt.
ctx: ArchiveProcessContext,
}
impl ArchiveProcessor {
/// Creates a processor for one archive and the shared download context.
fn new(archive: PlannedArchive, ctx: ArchiveProcessContext) -> Self {
Self { archive, ctx }
}
/// Runs the archive retry state machine until outputs are verified or retries are exhausted.
fn run(self) -> Result<()> {
let archive = self.archive();
if self.try_reuse_outputs()? {
info!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, "Skipping already verified plain files");
return Ok(());
}
let mode = ArchiveMode::new(&self.ctx)?;
let format = CompressionFormat::from_url(&archive.file_name)?;
let mut attempt = 1;
let mut last_error: Option<eyre::Error> = None;
let mut state = ArchiveAttemptState::RunAttempt;
loop {
match state {
ArchiveAttemptState::RunAttempt => {
self.cleanup_outputs();
if attempt > 1 {
info!(target: "reth::cli",
file = %archive.file_name,
component = %self.archive.component,
attempt,
max = MAX_DOWNLOAD_RETRIES,
"Retrying archive from scratch"
);
}
match self.run_attempt(mode, format) {
Ok(()) => state = ArchiveAttemptState::VerifyOutputs,
Err(error) if mode.retries_fetch_errors() => {
warn!(target: "reth::cli",
file = %archive.file_name,
component = %self.archive.component,
attempt,
err = %format_args!("{error:#}"),
"Archive attempt failed, retrying from scratch"
);
last_error = Some(error);
state = ArchiveAttemptState::RetryAttempt;
}
Err(error) => return Err(error),
}
}
ArchiveAttemptState::VerifyOutputs => {
if self.verify_outputs_with_progress()? {
state = ArchiveAttemptState::Complete;
} else {
warn!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, attempt, "Archive extracted, but output verification failed, retrying");
state = ArchiveAttemptState::RetryAttempt;
}
}
ArchiveAttemptState::RetryAttempt => {
if attempt >= MAX_DOWNLOAD_RETRIES {
state = ArchiveAttemptState::Fail;
} else {
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
attempt += 1;
state = ArchiveAttemptState::RunAttempt;
}
}
ArchiveAttemptState::Complete => return Ok(()),
ArchiveAttemptState::Fail => {
if let Some(error) = last_error {
return Err(error.wrap_err(format!(
"Failed after {} attempts for {}",
MAX_DOWNLOAD_RETRIES, archive.file_name
)));
}
eyre::bail!(
"Failed integrity validation after {} attempts for {}",
MAX_DOWNLOAD_RETRIES,
archive.file_name
)
}
}
}
}
/// Returns the concrete archive being fetched or verified.
fn archive(&self) -> &SnapshotArchive {
&self.archive.archive
}
/// Returns the verifier for this archive's output files.
fn output_verifier(&self) -> OutputVerifier<'_> {
OutputVerifier::new(self.ctx.target_dir())
}
/// Returns `true` if this archive can be reused from existing verified outputs.
/// Returns `false` if a fresh archive attempt is still needed.
fn try_reuse_outputs(&self) -> Result<bool> {
if self.verify_outputs()? {
self.mark_complete();
return Ok(true);
}
Ok(false)
}
/// Removes any partial outputs before a fresh archive attempt.
fn cleanup_outputs(&self) {
self.output_verifier().cleanup(&self.archive().output_files);
}
/// Returns `true` if all declared plain outputs verify.
/// Returns `false` if any output is missing or does not match.
fn verify_outputs(&self) -> Result<bool> {
self.output_verifier().verify(&self.archive().output_files)
}
/// Records archive completion in shared progress once outputs verify.
fn mark_complete(&self) {
self.ctx.session().record_reused_archive(self.archive().size, self.archive().output_size());
}
/// Executes one archive attempt according to the selected cache-vs-stream mode.
fn run_attempt(&self, mode: ArchiveMode, format: CompressionFormat) -> Result<()> {
mode.execute(self, format)
}
/// Downloads the archive into the cache, then extracts from the cached file.
fn run_cached_attempt(&self, format: CompressionFormat) -> Result<()> {
let cache_dir =
self.ctx.cache_dir().ok_or_else(|| eyre::eyre!("Missing download cache directory"))?;
let fetcher =
ArchiveFetcher::new(self.archive().url.clone(), cache_dir, self.ctx.session().clone());
if self.archive.ty == super::manifest::SnapshotComponentType::State {
debug!(target: "reth::cli", url = %self.archive().url, "Downloading state snapshot archive");
}
let download_result = {
let mut download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
let result = fetcher.download(Some(&mut download_progress));
if let Ok(ref downloaded) = result &&
download_progress.has_tracked_bytes()
{
download_progress.complete(downloaded.size);
}
result
};
let downloaded = match download_result {
Ok(downloaded) => downloaded,
Err(error) => {
fetcher.cleanup_downloaded_files();
return Err(error);
}
};
info!(target: "reth::cli",
file = %self.archive().file_name,
component = %self.archive.component,
size = %super::progress::DownloadProgress::format_size(downloaded.size),
"Archive download complete"
);
let extract_result = self.extract_cached_archive(&downloaded.path, format);
fetcher.cleanup_downloaded_files();
extract_result
}
/// Streams the archive directly into extraction without keeping a cached copy.
fn run_streaming_attempt(&self, format: CompressionFormat) -> Result<()> {
let _download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
streaming_download_and_extract(
&self.archive().url,
format,
self.ctx.target_dir(),
self.ctx.session(),
)
}
/// Extracts a cached archive file while updating shared extraction activity.
fn extract_cached_archive(&self, archive_path: &Path, format: CompressionFormat) -> Result<()> {
let mut extraction_progress = ArchiveExtractionProgress::new(self.ctx.session().progress());
let file = fs::open(archive_path)?;
let result = extract_archive_raw(
file,
format,
self.ctx.target_dir(),
Some(&mut extraction_progress),
);
extraction_progress.finish();
result
}
/// Returns `true` if all declared plain outputs verify while updating shared verification
/// progress.
fn verify_outputs_with_progress(&self) -> Result<bool> {
let mut verification_progress =
ArchiveVerificationProgress::new(self.ctx.session().progress());
let verified = self
.output_verifier()
.verify_with_progress(&self.archive().output_files, Some(&mut verification_progress))?;
if verified {
verification_progress.complete(self.archive().output_size());
}
Ok(verified)
}
}
/// Chooses whether an archive attempt uses the cache or streams directly.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum ArchiveMode {
/// Download the archive to the cache, then extract it.
Cached,
/// Stream the archive directly into extraction.
Streaming,
}
impl ArchiveMode {
/// Picks the archive mode from the process context.
fn new(ctx: &ArchiveProcessContext) -> Result<Self> {
if ctx.cache_dir().is_some() {
ctx.session().require_request_limiter()?;
return Ok(Self::Cached)
}
Ok(Self::Streaming)
}
/// Returns `true` when fetch failures should retry the whole archive attempt.
const fn retries_fetch_errors(&self) -> bool {
matches!(self, Self::Cached)
}
/// Runs the selected archive mode for a single attempt.
fn execute(&self, processor: &ArchiveProcessor, format: CompressionFormat) -> Result<()> {
match self {
Self::Cached => processor.run_cached_attempt(format),
Self::Streaming => processor.run_streaming_attempt(format),
}
}
}

View File

@@ -270,6 +270,7 @@ pub(crate) fn describe_prune_config(config: &Config) -> Vec<String> {
.collect()
}
/// Formats one prune mode for the generated config summary.
fn format_mode(mode: &PruneMode) -> String {
match mode {
PruneMode::Full => "\"full\"".to_string(),

View File

@@ -0,0 +1,490 @@
use super::{
fetch::{ArchiveFetcher, DownloadedArchive},
progress::{
ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress,
DownloadRequestLimiter, ProgressReader, SharedProgress, SharedProgressReader,
},
session::DownloadSession,
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
};
use eyre::{Result, WrapErr};
use lz4::Decoder;
use reqwest::blocking::Client as BlockingClient;
use reth_cli_util::cancellation::CancellationToken;
use reth_fs_util as fs;
use std::{
io::Read,
path::{Component, Path, PathBuf},
sync::{
atomic::{AtomicBool, Ordering},
Arc,
},
thread,
time::{Duration, Instant},
};
use tar::Archive;
use tokio::task;
use tracing::{info, warn};
use url::Url;
use zstd::stream::read::Decoder as ZstdDecoder;
const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
const STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE: u64 = 64 * 1024 * 1024;
const EXTRACTION_PROGRESS_POLL_INTERVAL: Duration = Duration::from_millis(100);
/// Supported compression formats for snapshots
#[derive(Debug, Clone, Copy)]
pub(crate) enum CompressionFormat {
/// LZ4-compressed tar archive.
Lz4,
/// Zstandard-compressed tar archive.
Zstd,
}
impl CompressionFormat {
/// Detect compression format from file extension
pub(crate) fn from_url(url: &str) -> Result<Self> {
let path =
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
if path.ends_with(EXTENSION_TAR_LZ4) {
Ok(Self::Lz4)
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
Ok(Self::Zstd)
} else {
Err(eyre::eyre!(
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
path
))
}
}
}
/// Extracts a compressed tar archive to the target directory with progress tracking.
fn extract_archive<R: Read>(
reader: R,
total_size: u64,
format: CompressionFormat,
target_dir: &Path,
cancel_token: CancellationToken,
) -> Result<()> {
let progress_reader = ProgressReader::new(reader, total_size, cancel_token);
match format {
CompressionFormat::Lz4 => {
let decoder = Decoder::new(progress_reader)?;
Archive::new(decoder).unpack(target_dir)?;
}
CompressionFormat::Zstd => {
let decoder = ZstdDecoder::new(progress_reader)?;
Archive::new(decoder).unpack(target_dir)?;
}
}
println!();
Ok(())
}
/// Extracts a compressed tar archive without progress tracking.
pub(crate) fn extract_archive_raw<R: Read>(
reader: R,
format: CompressionFormat,
target_dir: &Path,
progress: Option<&mut ArchiveExtractionProgress>,
) -> Result<()> {
match format {
CompressionFormat::Lz4 => {
unpack_archive(Archive::new(Decoder::new(reader)?), target_dir, progress)?;
}
CompressionFormat::Zstd => {
unpack_archive(Archive::new(ZstdDecoder::new(reader)?), target_dir, progress)?;
}
}
Ok(())
}
fn unpack_archive<R: Read>(
mut archive: Archive<R>,
target_dir: &Path,
mut progress: Option<&mut ArchiveExtractionProgress>,
) -> Result<()> {
let entries = archive.entries().wrap_err_with(|| {
format!("failed to read archive entries for `{}`", target_dir.display())
})?;
for entry in entries {
let mut entry = entry.wrap_err_with(|| {
format!("failed to read archive entry for `{}`", target_dir.display())
})?;
extract_entry_with_progress(&mut entry, target_dir, progress.as_deref_mut())?;
}
Ok(())
}
fn extract_entry_with_progress<R: Read>(
entry: &mut tar::Entry<'_, R>,
target_dir: &Path,
progress: Option<&mut ArchiveExtractionProgress>,
) -> Result<()> {
let size = entry.header().entry_size().unwrap_or(0);
let entry_type = entry.header().entry_type();
if !entry_type.is_file() || size == 0 {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
return Ok(())
}
if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
if let Some(progress) = progress {
progress.record_extracted(size);
}
return Ok(())
}
let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
return Ok(())
};
let Some(entry_path) = entry_destination_path(entry, target_dir)? else {
entry.unpack_in(target_dir).wrap_err_with(|| {
format!("failed to extract archive into `{}`", target_dir.display())
})?;
return Ok(())
};
let stop = Arc::new(AtomicBool::new(false));
let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop));
let unpack_result = entry
.unpack_in(target_dir)
.wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display()));
stop.store(true, Ordering::Relaxed);
let monitor_result = monitor.join();
unpack_result?;
monitor_result.map_err(|_| eyre::eyre!("extraction progress monitor panicked"))?;
Ok(())
}
fn entry_destination_path<R: Read>(
entry: &tar::Entry<'_, R>,
target_dir: &Path,
) -> Result<Option<PathBuf>> {
let mut file_dst = target_dir.to_path_buf();
let path = entry.path().wrap_err("invalid path in archive entry")?;
for part in path.components() {
match part {
Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
Component::ParentDir => return Ok(None),
Component::Normal(part) => file_dst.push(part),
}
}
if file_dst == target_dir {
return Ok(None)
}
Ok(Some(file_dst))
}
fn spawn_extraction_progress_monitor(
entry_path: PathBuf,
progress: ArchiveExtractionProgressHandle,
stop: Arc<AtomicBool>,
) -> thread::JoinHandle<()> {
thread::spawn(move || {
let mut extracted = 0_u64;
loop {
record_extracted_file_bytes(&entry_path, &progress, &mut extracted);
if stop.load(Ordering::Relaxed) {
break;
}
thread::sleep(EXTRACTION_PROGRESS_POLL_INTERVAL);
}
})
}
fn record_extracted_file_bytes(
entry_path: &Path,
progress: &ArchiveExtractionProgressHandle,
extracted: &mut u64,
) {
let Ok(meta) = fs::metadata(entry_path) else { return };
let len = meta.len();
if len > *extracted {
progress.record_extracted(len - *extracted);
*extracted = len;
}
}
/// Extracts a snapshot from a local file.
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
let file = std::fs::File::open(path)?;
let total_size = file.metadata()?.len();
info!(target: "reth::cli",
file = %path.display(),
size = %DownloadProgress::format_size(total_size),
"Extracting local archive"
);
let start = Instant::now();
extract_archive(file, total_size, format, target_dir, CancellationToken::new())?;
info!(target: "reth::cli",
file = %path.display(),
elapsed = %DownloadProgress::format_duration(start.elapsed()),
"Local extraction complete"
);
Ok(())
}
/// Streams a remote archive directly into the extractor without writing to disk.
///
/// On failure, retries from scratch up to [`MAX_DOWNLOAD_RETRIES`] times.
pub(crate) fn streaming_download_and_extract(
url: &str,
format: CompressionFormat,
target_dir: &Path,
session: &DownloadSession,
) -> Result<()> {
let shared = session.progress();
let quiet = session.progress().is_some();
let mut last_error: Option<eyre::Error> = None;
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
if attempt > 1 {
info!(target: "reth::cli",
url = %url,
attempt,
max = MAX_DOWNLOAD_RETRIES,
"Retrying streaming download from scratch"
);
}
let client = BlockingClient::builder().connect_timeout(Duration::from_secs(30)).build()?;
let _request_permit = session
.request_limiter()
.map(|limiter| limiter.acquire(session.progress(), session.cancel_token()))
.transpose()?;
let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
Ok(r) => r,
Err(error) => {
let err = eyre::Error::from(error);
if attempt < MAX_DOWNLOAD_RETRIES {
warn!(target: "reth::cli",
url = %url,
attempt,
max = MAX_DOWNLOAD_RETRIES,
err = %err,
"Streaming request failed, retrying"
);
}
last_error = Some(err);
if attempt < MAX_DOWNLOAD_RETRIES {
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
continue;
}
};
if !quiet && let Some(size) = response.content_length() {
info!(target: "reth::cli",
url = %url,
size = %DownloadProgress::format_size(size),
"Streaming archive"
);
}
let result = if let Some(progress) = shared {
let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) };
extract_archive_raw(reader, format, target_dir, None)
} else {
let total_size = response.content_length().unwrap_or(0);
extract_archive(
response,
total_size,
format,
target_dir,
session.cancel_token().clone(),
)
};
match result {
Ok(()) => return Ok(()),
Err(error) => {
if attempt < MAX_DOWNLOAD_RETRIES {
warn!(target: "reth::cli",
url = %url,
attempt,
max = MAX_DOWNLOAD_RETRIES,
err = %error,
"Streaming extraction failed, retrying"
);
}
last_error = Some(error);
if attempt < MAX_DOWNLOAD_RETRIES {
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
}
}
}
}
Err(last_error.unwrap_or_else(|| {
eyre::eyre!("Streaming download failed after {MAX_DOWNLOAD_RETRIES} attempts")
}))
}
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
fn download_and_extract(
url: &str,
format: CompressionFormat,
target_dir: &Path,
session: DownloadSession,
) -> Result<()> {
let quiet = session.progress().is_some();
let fetcher = ArchiveFetcher::new(url.to_string(), target_dir, session.clone());
let DownloadedArchive { path: downloaded_path, size: total_size } = fetcher.download(None)?;
let file_name =
downloaded_path.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
if !quiet {
info!(target: "reth::cli",
file = %file_name,
size = %DownloadProgress::format_size(total_size),
"Extracting archive"
);
}
let file = fs::open(&downloaded_path)?;
if quiet {
extract_archive_raw(file, format, target_dir, None)?;
} else {
extract_archive(file, total_size, format, target_dir, session.cancel_token().clone())?;
info!(target: "reth::cli",
file = %file_name,
"Extraction complete"
);
}
fetcher.cleanup_downloaded_files();
session.record_archive_output_complete(total_size);
Ok(())
}
/// Downloads and extracts a snapshot, blocking until finished.
///
/// Supports `file://` URLs for local files and HTTP(S) URLs for remote downloads.
/// When `resumable` is true, downloads to a `.part` file first with HTTP Range resume
/// support. Otherwise streams directly into the extractor.
fn blocking_download_and_extract(
url: &str,
target_dir: &Path,
shared: Option<Arc<SharedProgress>>,
resumable: bool,
request_limiter: Option<Arc<DownloadRequestLimiter>>,
cancel_token: CancellationToken,
) -> Result<()> {
let format = CompressionFormat::from_url(url)?;
if let Ok(parsed_url) = Url::parse(url) &&
parsed_url.scheme() == "file"
{
let session = DownloadSession::new(shared, request_limiter, cancel_token);
let file_path = parsed_url
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
let result = extract_from_file(&file_path, format, target_dir);
if result.is_ok() {
session.record_archive_output_complete(file_path.metadata()?.len());
}
result
} else if let Some(request_limiter) = request_limiter {
download_and_extract(
url,
format,
target_dir,
DownloadSession::new(shared, Some(request_limiter), cancel_token),
)
} else if resumable {
let session =
DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token);
download_and_extract(url, format, target_dir, session)
} else {
let session = DownloadSession::new(shared, None, cancel_token);
let result = streaming_download_and_extract(url, format, target_dir, &session);
if result.is_ok() {
session.record_archive_output_complete(0);
}
result
}
}
/// Downloads and extracts a snapshot archive asynchronously.
///
/// When `shared` is provided, download progress is reported to the shared
/// counter for aggregated display. Otherwise uses a local progress bar.
/// When `resumable` is true, uses two-phase download with `.part` files.
pub(crate) async fn stream_and_extract(
url: &str,
target_dir: &Path,
shared: Option<Arc<SharedProgress>>,
resumable: bool,
request_limiter: Option<Arc<DownloadRequestLimiter>>,
cancel_token: CancellationToken,
) -> Result<()> {
let target_dir = target_dir.to_path_buf();
let url = url.to_string();
task::spawn_blocking(move || {
blocking_download_and_extract(
&url,
&target_dir,
shared,
resumable,
request_limiter,
cancel_token,
)
})
.await??;
Ok(())
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_compression_format_detection() {
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
Ok(CompressionFormat::Lz4)
));
assert!(matches!(
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
Ok(CompressionFormat::Zstd)
));
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -10,6 +10,10 @@ use std::{
};
use tracing::info;
fn is_zero(value: &u64) -> bool {
*value == 0
}
/// A snapshot manifest describes available components for a snapshot at a given block height.
///
/// Each component is either a single archive (state) or a set of chunked archives (static file
@@ -62,6 +66,12 @@ pub struct SingleArchive {
pub file: String,
/// Compressed archive size in bytes.
pub size: u64,
/// Total extracted plain-output size in bytes.
///
/// Older manifests may omit this, in which case downloaders should derive it from
/// `output_files`.
#[serde(default, skip_serializing_if = "is_zero")]
pub decompressed_size: u64,
/// Optional BLAKE3 checksum of the compressed archive.
#[serde(default, skip_serializing_if = "Option::is_none")]
pub blake3: Option<String>,
@@ -83,6 +93,12 @@ pub struct ChunkedArchive {
/// Computed during manifest generation. Older manifests may omit this.
#[serde(default)]
pub chunk_sizes: Vec<u64>,
/// Extracted plain-output size of each chunk in bytes, ordered from first to last.
///
/// Older manifests may omit this, in which case downloaders should derive it from
/// `chunk_output_files`.
#[serde(default, skip_serializing_if = "Vec::is_empty")]
pub chunk_decompressed_sizes: Vec<u64>,
/// Expected extracted plain files per chunk, ordered from first to last.
///
/// This is the authoritative integrity source for the modular download path.
@@ -101,9 +117,9 @@ pub struct OutputFileChecksum {
pub blake3: String,
}
/// A single archive with concrete URL and optional integrity metadata.
/// A concrete snapshot archive with its download and verification metadata.
#[derive(Debug, Clone, PartialEq, Eq)]
pub struct ArchiveDescriptor {
pub struct SnapshotArchive {
pub url: String,
pub file_name: String,
pub size: u64,
@@ -111,6 +127,13 @@ pub struct ArchiveDescriptor {
pub output_files: Vec<OutputFileChecksum>,
}
impl SnapshotArchive {
/// Returns the total extracted plain-output size for this archive.
pub fn output_size(&self) -> u64 {
self.output_files.iter().map(|file| file.size).sum()
}
}
/// How much of a component to download.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
pub enum ComponentSelection {
@@ -315,19 +338,19 @@ impl SnapshotManifest {
}
}
/// Returns concrete archive descriptors for a component, optionally limited to distance.
pub fn archive_descriptors_for_distance(
/// Returns concrete snapshot archives for a component, optionally limited to distance.
pub fn snapshot_archives_for_distance(
&self,
ty: SnapshotComponentType,
distance: Option<u64>,
) -> Vec<ArchiveDescriptor> {
) -> Vec<SnapshotArchive> {
let Some(component) = self.component(ty) else {
return vec![];
};
match component {
ComponentManifest::Single(single) => {
vec![ArchiveDescriptor {
vec![SnapshotArchive {
url: format!("{}/{}", self.base_url_or_empty(), single.file),
file_name: single.file.clone(),
size: single.size,
@@ -357,7 +380,7 @@ impl SnapshotManifest {
let output_files =
chunked.chunk_output_files.get(i as usize).cloned().unwrap_or_default();
ArchiveDescriptor {
SnapshotArchive {
url: format!("{}/{}", self.base_url_or_empty(), file_name),
file_name,
size,
@@ -398,6 +421,36 @@ impl SnapshotManifest {
}
}
/// Returns the exact extracted plain-output size for a component given a distance selection.
pub fn output_size_for_distance(
&self,
ty: SnapshotComponentType,
distance: Option<u64>,
) -> u64 {
let Some(component) = self.component(ty) else {
return 0;
};
match component {
ComponentManifest::Single(single) => single.output_size(),
ComponentManifest::Chunked(chunked) => {
let num_chunks = chunked.num_chunks();
let start_chunk = match distance {
Some(dist) => {
let needed = dist.min(chunked.total_blocks);
let needed_chunks = needed.div_ceil(chunked.blocks_per_file);
num_chunks.saturating_sub(needed_chunks)
}
None => 0,
};
(start_chunk..num_chunks)
.map(|index| chunked.chunk_output_size(index as usize))
.sum()
}
}
}
/// Returns the number of chunks that would be downloaded for a given distance.
pub fn chunks_for_distance(&self, ty: SnapshotComponentType, distance: Option<u64>) -> u64 {
let Some(ComponentManifest::Chunked(chunked)) = self.component(ty) else {
@@ -421,6 +474,14 @@ impl ComponentManifest {
Self::Chunked(c) => c.chunk_sizes.iter().sum(),
}
}
/// Returns the total extracted plain-output size for this component.
pub fn total_output_size(&self) -> u64 {
match self {
Self::Single(single) => single.output_size(),
Self::Chunked(chunked) => chunked.total_output_size(),
}
}
}
impl ChunkedArchive {
@@ -428,6 +489,39 @@ impl ChunkedArchive {
pub fn num_chunks(&self) -> u64 {
self.total_blocks.div_ceil(self.blocks_per_file)
}
/// Returns the extracted plain-output size for one chunk.
pub fn chunk_output_size(&self, index: usize) -> u64 {
self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| {
self.chunk_output_files
.get(index)
.map(|files| files.iter().map(|file| file.size).sum())
.unwrap_or(0)
})
}
/// Returns the total extracted plain-output size across all chunks.
pub fn total_output_size(&self) -> u64 {
if !self.chunk_decompressed_sizes.is_empty() {
self.chunk_decompressed_sizes.iter().sum()
} else {
self.chunk_output_files
.iter()
.map(|files| files.iter().map(|file| file.size).sum::<u64>())
.sum()
}
}
}
impl SingleArchive {
/// Returns the total extracted plain-output size for this archive.
pub fn output_size(&self) -> u64 {
if self.decompressed_size != 0 {
self.decompressed_size
} else {
self.output_files.iter().map(|file| file.size).sum()
}
}
}
/// Fetch a snapshot manifest from a URL.
@@ -516,6 +610,10 @@ pub fn generate_manifest(
blocks_per_file,
total_blocks: block,
chunk_sizes,
chunk_decompressed_sizes: chunk_output_files
.iter()
.map(|files| files.iter().map(|file| file.size).sum())
.collect(),
chunk_output_files,
}),
);
@@ -532,6 +630,7 @@ pub fn generate_manifest(
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: state_size,
decompressed_size: state_output_files.iter().map(|file| file.size).sum(),
blake3: None,
output_files: state_output_files,
}),
@@ -546,6 +645,7 @@ pub fn generate_manifest(
ComponentManifest::Single(SingleArchive {
file: "rocksdb_indices.tar.zst".to_string(),
size: rocksdb_size,
decompressed_size: rocksdb_output_files.iter().map(|file| file.size).sum(),
blake3: None,
output_files: rocksdb_output_files,
}),
@@ -814,6 +914,7 @@ mod tests {
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 100,
decompressed_size: 0,
blake3: None,
output_files: vec![],
}),
@@ -824,6 +925,7 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 1_500_000,
chunk_sizes: vec![80_000, 100_000, 120_000],
chunk_decompressed_sizes: vec![],
chunk_output_files: vec![vec![], vec![], vec![]],
}),
);
@@ -833,6 +935,7 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 1_500_000,
chunk_sizes: vec![40_000, 50_000, 60_000],
chunk_decompressed_sizes: vec![],
chunk_output_files: vec![vec![], vec![], vec![]],
}),
);
@@ -883,6 +986,7 @@ mod tests {
ComponentManifest::Single(SingleArchive {
file: "rocksdb_indices.tar.zst".to_string(),
size: 777,
decompressed_size: 0,
blake3: None,
output_files: vec![],
}),
@@ -955,6 +1059,7 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 24_396_822,
chunk_sizes: vec![100; 49], // 49 chunks
chunk_decompressed_sizes: vec![],
chunk_output_files: vec![vec![]; 49],
}),
);
@@ -997,6 +1102,68 @@ mod tests {
assert_eq!(m.size_for_distance(SnapshotComponentType::Receipts, None), 0);
}
#[test]
fn output_size_for_distance_uses_manifest_or_output_files() {
let m = test_manifest();
assert_eq!(m.output_size_for_distance(SnapshotComponentType::Transactions, None), 0);
let mut components = BTreeMap::new();
components.insert(
"state".to_string(),
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 100,
decompressed_size: 1_000,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "db/mdbx.dat".to_string(),
size: 1_000,
blake3: "h0".to_string(),
}],
}),
);
components.insert(
"transactions".to_string(),
ComponentManifest::Chunked(ChunkedArchive {
blocks_per_file: 500_000,
total_blocks: 1_000_000,
chunk_sizes: vec![80_000, 120_000],
chunk_decompressed_sizes: vec![111, 222],
chunk_output_files: vec![
vec![OutputFileChecksum {
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
size: 111,
blake3: "h0".to_string(),
}],
vec![OutputFileChecksum {
path: "static_files/static_file_transactions_500000_999999.bin".to_string(),
size: 222,
blake3: "h1".to_string(),
}],
],
}),
);
let manifest = SnapshotManifest {
block: 1_000_000,
chain_id: 1,
storage_version: 2,
timestamp: 0,
base_url: Some("https://example.com".to_string()),
reth_version: None,
components,
};
assert_eq!(manifest.output_size_for_distance(SnapshotComponentType::State, None), 1_000);
assert_eq!(
manifest.output_size_for_distance(SnapshotComponentType::Transactions, None),
333
);
assert_eq!(
manifest.output_size_for_distance(SnapshotComponentType::Transactions, Some(500_000)),
222
);
}
#[test]
fn archive_descriptors_include_checksum_metadata() {
let mut components = BTreeMap::new();
@@ -1005,6 +1172,7 @@ mod tests {
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 100,
decompressed_size: 1_000,
blake3: Some("abc123".to_string()),
output_files: vec![OutputFileChecksum {
path: "db/mdbx.dat".to_string(),
@@ -1019,6 +1187,7 @@ mod tests {
blocks_per_file: 500_000,
total_blocks: 1_000_000,
chunk_sizes: vec![80_000, 120_000],
chunk_decompressed_sizes: vec![111, 222],
chunk_output_files: vec![
vec![OutputFileChecksum {
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
@@ -1044,13 +1213,13 @@ mod tests {
components,
};
let state = m.archive_descriptors_for_distance(SnapshotComponentType::State, None);
let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None);
assert_eq!(state.len(), 1);
assert_eq!(state[0].file_name, "state.tar.zst");
assert_eq!(state[0].blake3.as_deref(), Some("abc123"));
assert_eq!(state[0].output_files.len(), 1);
let tx = m.archive_descriptors_for_distance(SnapshotComponentType::Transactions, None);
let tx = m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, None);
assert_eq!(tx.len(), 2);
assert_eq!(tx[0].blake3, None);
assert_eq!(tx[1].blake3, None);
@@ -1073,6 +1242,7 @@ mod tests {
panic!("state should be a single archive")
};
assert_eq!(state.file, "state.tar.zst");
assert!(state.decompressed_size > 0);
assert!(!state.output_files.is_empty());
assert_eq!(state.output_files[0].path, "db/mdbx.dat");
assert!(output.path().join("state.tar.zst").exists());
@@ -1097,6 +1267,7 @@ mod tests {
panic!("rocksdb indices should be a single archive")
};
assert_eq!(rocksdb.file, "rocksdb_indices.tar.zst");
assert!(rocksdb.decompressed_size > 0);
assert!(!rocksdb.output_files.is_empty());
assert_eq!(rocksdb.output_files[0].path, "rocksdb/CURRENT");
assert!(output.path().join("rocksdb_indices.tar.zst").exists());

View File

@@ -45,6 +45,7 @@ pub struct SnapshotManifestCommand {
}
impl SnapshotManifestCommand {
/// Packages snapshot archives and writes the manifest file.
pub fn execute(self) -> Result<()> {
let block = match self.block {
Some(block) => block,
@@ -88,6 +89,7 @@ impl SnapshotManifestCommand {
}
}
/// Infers the snapshot block from the source datadir.
fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
if let Ok(block) = infer_snapshot_block_from_db(source_datadir) {
return Ok(block);
@@ -102,6 +104,7 @@ fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
Ok(block)
}
/// Reads the snapshot block from the source database Finish stage checkpoint.
fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64> {
let candidates = [source_datadir.join("db"), source_datadir.to_path_buf()];
@@ -126,6 +129,7 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64>
)
}
/// Infers the snapshot block from the highest header static-file range.
fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result<u64> {
let max_end = header_ranges(source_datadir)?
.into_iter()
@@ -135,6 +139,7 @@ fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result
Ok(max_end)
}
/// Infers the static-file block span from header file ranges.
fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
let mut inferred = None;
for (start, end) in header_ranges(source_datadir)? {
@@ -161,6 +166,7 @@ fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
})
}
/// Collects header static-file ranges from the source datadir.
fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
let static_files_dir = source_datadir.join("static_files");
let static_files_dir =
@@ -183,6 +189,7 @@ fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
Ok(ranges)
}
/// Parses the block range from a header static-file name.
fn parse_headers_range(file_name: &str) -> Option<(u64, u64)> {
let remainder = file_name.strip_prefix("static_file_headers_")?;
let (start, end_with_suffix) = remainder.split_once('_')?;

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,322 @@
use super::{manifest::*, verify::OutputVerifier};
use eyre::Result;
use std::{collections::BTreeMap, path::Path};
use tracing::info;
/// One archive selected from the manifest, along with its component name.
#[derive(Debug, Clone)]
pub(crate) struct PlannedArchive {
/// Snapshot component type this archive belongs to.
pub(crate) ty: SnapshotComponentType,
/// User-facing component name used in logs.
pub(crate) component: String,
/// Concrete snapshot archive metadata resolved from the manifest.
pub(crate) archive: SnapshotArchive,
}
/// The archive list for a modular snapshot download.
#[derive(Debug)]
pub(crate) struct PlannedDownloads {
/// Concrete archives that still need reuse checks or processing.
pub(crate) archives: Vec<PlannedArchive>,
/// Total compressed download size of all planned archives.
pub(crate) total_download_size: u64,
/// Total extracted plain-output size of all planned archives.
pub(crate) total_output_size: u64,
}
impl PlannedDownloads {
/// Returns the number of concrete archives queued for this snapshot selection.
pub(crate) const fn total_archives(&self) -> usize {
self.archives.len()
}
}
/// Returns the sort priority used to schedule archives.
pub(crate) const fn archive_priority_rank(ty: SnapshotComponentType) -> u8 {
match ty {
SnapshotComponentType::State => 0,
SnapshotComponentType::RocksdbIndices => 1,
_ => 2,
}
}
/// Startup summary showing how much of the selected work can be reused.
#[derive(Debug, Default, Clone, Copy)]
pub(crate) struct DownloadStartupSummary {
/// Archives whose declared outputs already verify on disk.
pub(crate) reusable: usize,
/// Archives that still need to be downloaded or retried.
pub(crate) needs_download: usize,
}
/// Checks selected archives against existing output files before work begins.
pub(crate) fn summarize_download_startup(
all_downloads: &[PlannedArchive],
target_dir: &Path,
) -> Result<DownloadStartupSummary> {
let mut summary = DownloadStartupSummary::default();
let verifier = OutputVerifier::new(target_dir);
for planned in all_downloads {
if verifier.verify(&planned.archive.output_files)? {
summary.reusable += 1;
} else {
summary.needs_download += 1;
}
}
Ok(summary)
}
/// Converts a selection into the manifest distance form used for archive lookup.
fn selection_archive_distance(
selection: &ComponentSelection,
snapshot_block: u64,
) -> Option<Option<u64>> {
match selection {
ComponentSelection::All => Some(None),
ComponentSelection::Distance(distance) => Some(Some(*distance)),
ComponentSelection::Since(block) => Some(Some(snapshot_block.saturating_sub(*block) + 1)),
ComponentSelection::None => None,
}
}
/// Sorts planned archives into a stable processing order.
fn sort_planned_archives(all_downloads: &mut [PlannedArchive]) {
all_downloads.sort_by(|a, b| {
archive_priority_rank(a.ty)
.cmp(&archive_priority_rank(b.ty))
.then_with(|| a.component.cmp(&b.component))
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
});
}
/// Expands component selections into the archives that need to be processed.
pub(crate) fn collect_planned_archives(
manifest: &SnapshotManifest,
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
) -> Result<PlannedDownloads> {
let mut archives = Vec::new();
let mut total_download_size = 0;
let mut total_output_size = 0;
for (ty, selection) in selections {
let Some(distance) = selection_archive_distance(selection, manifest.block) else {
continue;
};
total_download_size += manifest.size_for_distance(*ty, distance);
total_output_size += manifest.output_size_for_distance(*ty, distance);
let snapshot_archives = manifest.snapshot_archives_for_distance(*ty, distance);
let component = ty.display_name().to_string();
if !snapshot_archives.is_empty() {
info!(target: "reth::cli",
component = %component,
archives = snapshot_archives.len(),
selection = %selection,
"Queued component for download"
);
}
for archive in snapshot_archives {
if archive.output_files.is_empty() {
eyre::bail!(
"Invalid modular manifest: {} is missing plain output checksum metadata",
archive.file_name
);
}
archives.push(PlannedArchive { ty: *ty, component: component.clone(), archive });
}
}
sort_planned_archives(&mut archives);
Ok(PlannedDownloads { archives, total_download_size, total_output_size })
}
#[cfg(test)]
mod tests {
use super::*;
use tempfile::tempdir;
#[test]
fn summarize_download_startup_counts_reusable_and_needs_download() {
let dir = tempdir().unwrap();
let target_dir = dir.path();
let ok_file = target_dir.join("ok.bin");
std::fs::write(&ok_file, vec![1_u8; 4]).unwrap();
let ok_hash = blake3::hash(&[1_u8; 4]).to_hex().to_string();
let planned = vec![
PlannedArchive {
ty: SnapshotComponentType::State,
component: "State".to_string(),
archive: SnapshotArchive {
url: "https://example.com/ok.tar.zst".to_string(),
file_name: "ok.tar.zst".to_string(),
size: 10,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "ok.bin".to_string(),
size: 4,
blake3: ok_hash,
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::Headers,
component: "Headers".to_string(),
archive: SnapshotArchive {
url: "https://example.com/missing.tar.zst".to_string(),
file_name: "missing.tar.zst".to_string(),
size: 10,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "missing.bin".to_string(),
size: 1,
blake3: "deadbeef".to_string(),
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::Transactions,
component: "Transactions".to_string(),
archive: SnapshotArchive {
url: "https://example.com/bad-size.tar.zst".to_string(),
file_name: "bad-size.tar.zst".to_string(),
size: 10,
blake3: None,
output_files: vec![],
},
},
];
let summary = summarize_download_startup(&planned, target_dir).unwrap();
assert_eq!(summary.reusable, 1);
assert_eq!(summary.needs_download, 2);
}
#[test]
fn archive_priority_prefers_state_then_rocksdb() {
let mut planned = [
PlannedArchive {
ty: SnapshotComponentType::Transactions,
component: "Transactions".to_string(),
archive: SnapshotArchive {
url: "u3".to_string(),
file_name: "t.tar.zst".to_string(),
size: 1,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "a".to_string(),
size: 1,
blake3: "x".to_string(),
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::RocksdbIndices,
component: "RocksDB Indices".to_string(),
archive: SnapshotArchive {
url: "u2".to_string(),
file_name: "rocksdb_indices.tar.zst".to_string(),
size: 1,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "b".to_string(),
size: 1,
blake3: "y".to_string(),
}],
},
},
PlannedArchive {
ty: SnapshotComponentType::State,
component: "State (mdbx)".to_string(),
archive: SnapshotArchive {
url: "u1".to_string(),
file_name: "state.tar.zst".to_string(),
size: 1,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "c".to_string(),
size: 1,
blake3: "z".to_string(),
}],
},
},
];
planned.sort_by(|a, b| {
archive_priority_rank(a.ty)
.cmp(&archive_priority_rank(b.ty))
.then_with(|| a.component.cmp(&b.component))
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
});
assert_eq!(planned[0].ty, SnapshotComponentType::State);
assert_eq!(planned[1].ty, SnapshotComponentType::RocksdbIndices);
assert_eq!(planned[2].ty, SnapshotComponentType::Transactions);
}
#[test]
fn collect_planned_archives_tracks_download_and_output_totals() {
let mut components = BTreeMap::new();
components.insert(
"state".to_string(),
ComponentManifest::Single(SingleArchive {
file: "state.tar.zst".to_string(),
size: 10,
decompressed_size: 100,
blake3: None,
output_files: vec![OutputFileChecksum {
path: "db/mdbx.dat".to_string(),
size: 100,
blake3: "h0".to_string(),
}],
}),
);
components.insert(
"transactions".to_string(),
ComponentManifest::Chunked(ChunkedArchive {
blocks_per_file: 500_000,
total_blocks: 1_000_000,
chunk_sizes: vec![20, 30],
chunk_decompressed_sizes: vec![200, 300],
chunk_output_files: vec![
vec![OutputFileChecksum {
path: "static_files/tx-0".to_string(),
size: 200,
blake3: "h1".to_string(),
}],
vec![OutputFileChecksum {
path: "static_files/tx-1".to_string(),
size: 300,
blake3: "h2".to_string(),
}],
],
}),
);
let manifest = SnapshotManifest {
block: 1_000_000,
chain_id: 1,
storage_version: 2,
timestamp: 0,
base_url: Some("https://example.com".to_string()),
reth_version: None,
components,
};
let selections = BTreeMap::from([
(SnapshotComponentType::State, ComponentSelection::All),
(SnapshotComponentType::Transactions, ComponentSelection::Distance(500_000)),
]);
let planned = collect_planned_archives(&manifest, &selections).unwrap();
assert_eq!(planned.total_download_size, 40);
assert_eq!(planned.total_output_size, 400);
assert_eq!(planned.archives.len(), 2);
}
}

View File

@@ -0,0 +1,844 @@
use eyre::Result;
use reth_cli_util::cancellation::CancellationToken;
use std::{
io::{self, Read, Write},
sync::{
atomic::{AtomicBool, AtomicU64, Ordering},
Arc, Condvar, Mutex,
},
time::{Duration, Instant},
};
use tracing::info;
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
/// Tracks download progress and throttles display updates to every 100ms.
pub(crate) struct DownloadProgress {
/// Bytes copied so far for this single download.
pub(crate) downloaded: u64,
/// Total bytes expected for this single download.
total_size: u64,
/// Time when the progress line was last printed.
last_displayed: Instant,
/// Time when this progress tracker started.
started_at: Instant,
}
impl DownloadProgress {
/// Creates new progress tracker with given total size
pub(crate) fn new(total_size: u64) -> Self {
let now = Instant::now();
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
}
/// Converts bytes to human readable format (B, KB, MB, GB)
pub(crate) fn format_size(size: u64) -> String {
let mut size = size as f64;
let mut unit_index = 0;
while size >= 1024.0 && unit_index < BYTE_UNITS.len() - 1 {
size /= 1024.0;
unit_index += 1;
}
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
}
/// Format duration as human readable string
pub(crate) fn format_duration(duration: Duration) -> String {
let secs = duration.as_secs();
if secs < 60 {
format!("{secs}s")
} else if secs < 3600 {
format!("{}m {}s", secs / 60, secs % 60)
} else {
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
}
}
/// Updates progress bar (for single-archive legacy downloads)
pub(crate) fn update(&mut self, chunk_size: u64) -> Result<()> {
self.downloaded += chunk_size;
if self.last_displayed.elapsed() >= Duration::from_millis(100) {
let formatted_downloaded = Self::format_size(self.downloaded);
let formatted_total = Self::format_size(self.total_size);
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
let elapsed = self.started_at.elapsed();
let eta = if self.downloaded > 0 {
let remaining = self.total_size.saturating_sub(self.downloaded);
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
if speed > 0.0 {
Duration::from_secs_f64(remaining as f64 / speed)
} else {
Duration::ZERO
}
} else {
Duration::ZERO
};
let eta_str = Self::format_duration(eta);
print!(
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
);
io::stdout().flush()?;
self.last_displayed = Instant::now();
}
Ok(())
}
}
#[derive(Debug, Clone, Copy)]
struct PhaseStart {
started_at: Instant,
baseline_bytes: u64,
}
/// Shared progress counters for parallel downloads.
pub(crate) struct SharedProgress {
/// Raw HTTP bytes fetched during this session, including retries.
pub(crate) session_fetched_bytes: AtomicU64,
/// Compressed bytes from archives that have fully downloaded.
pub(crate) completed_download_bytes: AtomicU64,
/// Compressed bytes written for currently active archive download attempts.
pub(crate) active_download_bytes: AtomicU64,
/// Total compressed bytes expected across all planned archives.
pub(crate) total_download_bytes: u64,
/// Plain-output bytes from archives that have fully verified.
pub(crate) completed_output_bytes: AtomicU64,
/// Plain-output bytes unpacked by currently active extractions.
pub(crate) active_extracted_output_bytes: AtomicU64,
/// Plain-output bytes hashed by currently active verifications.
pub(crate) active_verified_output_bytes: AtomicU64,
/// Total plain-output bytes expected across all planned archives.
pub(crate) total_output_bytes: u64,
/// Total number of planned archives.
pub(crate) total_archives: u64,
/// Time when the modular download job started.
pub(crate) started_at: Instant,
/// Time and baseline when the current extraction phase started.
extraction_phase: Mutex<Option<PhaseStart>>,
/// Time and baseline when the current verification phase started.
verification_phase: Mutex<Option<PhaseStart>>,
/// Number of archives that have fully finished.
pub(crate) archives_done: AtomicU64,
/// Number of archives currently in the fetch phase.
pub(crate) active_downloads: AtomicU64,
/// Number of in-flight HTTP requests.
pub(crate) active_download_requests: AtomicU64,
/// Number of archives currently extracting.
pub(crate) active_extractions: AtomicU64,
/// Number of archives currently verifying extracted outputs.
pub(crate) active_verifications: AtomicU64,
/// Signals the background progress task to exit.
pub(crate) done: AtomicBool,
/// Cancellation token shared by the whole command.
cancel_token: CancellationToken,
}
impl SharedProgress {
/// Creates the shared progress state for a modular download job.
pub(crate) fn new(
total_download_bytes: u64,
total_output_bytes: u64,
total_archives: u64,
cancel_token: CancellationToken,
) -> Arc<Self> {
Arc::new(Self {
session_fetched_bytes: AtomicU64::new(0),
completed_download_bytes: AtomicU64::new(0),
active_download_bytes: AtomicU64::new(0),
total_download_bytes,
completed_output_bytes: AtomicU64::new(0),
active_extracted_output_bytes: AtomicU64::new(0),
active_verified_output_bytes: AtomicU64::new(0),
total_output_bytes,
total_archives,
started_at: Instant::now(),
extraction_phase: Mutex::new(None),
verification_phase: Mutex::new(None),
archives_done: AtomicU64::new(0),
active_downloads: AtomicU64::new(0),
active_download_requests: AtomicU64::new(0),
active_extractions: AtomicU64::new(0),
active_verifications: AtomicU64::new(0),
done: AtomicBool::new(false),
cancel_token,
})
}
/// Returns whether the whole command has been cancelled.
pub(crate) fn is_cancelled(&self) -> bool {
self.cancel_token.is_cancelled()
}
/// Adds raw session traffic bytes without affecting logical progress.
pub(crate) fn record_session_fetched_bytes(&self, bytes: u64) {
self.session_fetched_bytes.fetch_add(bytes, Ordering::Relaxed);
}
pub(crate) fn add_active_download_bytes(&self, bytes: u64) {
self.active_download_bytes.fetch_add(bytes, Ordering::Relaxed);
}
pub(crate) fn sub_active_download_bytes(&self, bytes: u64) {
sub_bytes(&self.active_download_bytes, bytes);
}
fn add_active_extracted_output_bytes(&self, bytes: u64) {
self.active_extracted_output_bytes.fetch_add(bytes, Ordering::Relaxed);
}
fn sub_active_extracted_output_bytes(&self, bytes: u64) {
sub_bytes(&self.active_extracted_output_bytes, bytes);
}
fn add_active_verified_output_bytes(&self, bytes: u64) {
self.active_verified_output_bytes.fetch_add(bytes, Ordering::Relaxed);
}
fn sub_active_verified_output_bytes(&self, bytes: u64) {
sub_bytes(&self.active_verified_output_bytes, bytes);
}
/// Records an archive whose outputs were already present locally.
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
self.completed_download_bytes.fetch_add(download_bytes, Ordering::Relaxed);
self.completed_output_bytes.fetch_add(output_bytes, Ordering::Relaxed);
self.archives_done.fetch_add(1, Ordering::Relaxed);
}
/// Records an archive whose compressed download completed successfully.
pub(crate) fn record_archive_download_complete(&self, bytes: u64) {
self.completed_download_bytes.fetch_add(bytes, Ordering::Relaxed);
}
/// Records an archive whose extracted outputs have fully verified.
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
self.completed_output_bytes.fetch_add(bytes, Ordering::Relaxed);
self.archives_done.fetch_add(1, Ordering::Relaxed);
}
/// Returns logical compressed download progress.
pub(crate) fn logical_downloaded_bytes(&self) -> u64 {
(self.completed_download_bytes.load(Ordering::Relaxed) +
self.active_download_bytes.load(Ordering::Relaxed))
.min(self.total_download_bytes)
}
/// Returns verified plain-output bytes.
pub(crate) fn verified_output_bytes(&self) -> u64 {
self.completed_output_bytes.load(Ordering::Relaxed).min(self.total_output_bytes)
}
/// Returns plain-output bytes currently represented by extraction progress.
pub(crate) fn extracting_output_bytes(&self) -> u64 {
(self.completed_output_bytes.load(Ordering::Relaxed) +
self.active_extracted_output_bytes.load(Ordering::Relaxed))
.min(self.total_output_bytes)
}
/// Returns plain-output bytes currently represented by verification progress.
pub(crate) fn verifying_output_bytes(&self) -> u64 {
(self.completed_output_bytes.load(Ordering::Relaxed) +
self.active_verified_output_bytes.load(Ordering::Relaxed))
.min(self.total_output_bytes)
}
fn restart_phase(slot: &Mutex<Option<PhaseStart>>, baseline_bytes: u64) {
*slot.lock().unwrap() = Some(PhaseStart { started_at: Instant::now(), baseline_bytes });
}
fn phase_eta(
slot: &Mutex<Option<PhaseStart>>,
current_bytes: u64,
total_bytes: u64,
) -> Option<Duration> {
let phase = *slot.lock().unwrap();
let phase = phase?;
let done = current_bytes.saturating_sub(phase.baseline_bytes);
let total = total_bytes.saturating_sub(phase.baseline_bytes);
eta_from_progress(phase.started_at.elapsed(), done, total)
}
fn extraction_eta(&self, current_bytes: u64) -> Option<Duration> {
Self::phase_eta(&self.extraction_phase, current_bytes, self.total_output_bytes)
}
fn verification_eta(&self, current_bytes: u64) -> Option<Duration> {
Self::phase_eta(&self.verification_phase, current_bytes, self.total_output_bytes)
}
/// Marks one archive as actively downloading.
pub(crate) fn download_started(&self) {
self.active_downloads.fetch_add(1, Ordering::Relaxed);
}
/// Marks one archive download as finished.
pub(crate) fn download_finished(&self) {
sub_bytes(&self.active_downloads, 1);
}
/// Marks one HTTP request as in flight.
pub(crate) fn request_started(&self) {
self.active_download_requests.fetch_add(1, Ordering::Relaxed);
}
/// Marks one HTTP request as finished.
pub(crate) fn request_finished(&self) {
sub_bytes(&self.active_download_requests, 1);
}
/// Marks one archive as actively extracting.
pub(crate) fn extraction_started(&self) {
if self.active_extractions.fetch_add(1, Ordering::Relaxed) == 0 {
Self::restart_phase(
&self.extraction_phase,
self.completed_output_bytes.load(Ordering::Relaxed),
);
}
}
/// Marks one archive extraction as finished.
pub(crate) fn extraction_finished(&self) {
sub_bytes(&self.active_extractions, 1);
}
/// Marks one archive as actively verifying outputs.
pub(crate) fn verification_started(&self) {
if self.active_verifications.fetch_add(1, Ordering::Relaxed) == 0 {
Self::restart_phase(
&self.verification_phase,
self.completed_output_bytes.load(Ordering::Relaxed),
);
}
}
/// Marks one archive verification as finished.
pub(crate) fn verification_finished(&self) {
sub_bytes(&self.active_verifications, 1);
}
}
fn sub_bytes(counter: &AtomicU64, bytes: u64) {
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
Some(current.saturating_sub(bytes))
});
}
fn eta_from_progress(elapsed: Duration, done: u64, total: u64) -> Option<Duration> {
if done == 0 || done >= total {
return None;
}
let secs = elapsed.as_secs_f64();
if secs <= 0.0 {
return None;
}
let speed = done as f64 / secs;
if speed <= 0.0 {
return None;
}
Some(Duration::from_secs_f64((total - done) as f64 / speed))
}
fn format_percent(done: u64, total: u64) -> String {
if total == 0 {
return "100.0%".to_string();
}
format!("{:.1}%", (done as f64 / total as f64) * 100.0)
}
fn format_eta(eta: Option<Duration>) -> String {
eta.map(DownloadProgress::format_duration).unwrap_or_else(|| "unknown".to_string())
}
/// Global request limit for the blocking downloader.
///
/// This uses `Mutex + Condvar` because the segmented path runs blocking reqwest
/// clients on OS threads.
pub(crate) struct DownloadRequestLimiter {
/// Maximum number of in-flight HTTP requests.
limit: usize,
/// Current number of acquired request slots.
active: Mutex<usize>,
/// Wakes blocked threads when a slot is released.
notify: Condvar,
}
impl DownloadRequestLimiter {
/// Creates the shared request limiter.
pub(crate) fn new(limit: usize) -> Arc<Self> {
Arc::new(Self { limit: limit.max(1), active: Mutex::new(0), notify: Condvar::new() })
}
/// Returns the configured request limit.
pub(crate) fn max_concurrency(&self) -> usize {
self.limit
}
pub(crate) fn acquire<'a>(
&'a self,
progress: Option<&'a Arc<SharedProgress>>,
cancel_token: &CancellationToken,
) -> Result<DownloadRequestPermit<'a>> {
let mut active = self.active.lock().unwrap();
loop {
if cancel_token.is_cancelled() {
return Err(eyre::eyre!("Download cancelled"));
}
if *active < self.limit {
*active += 1;
if let Some(progress) = progress {
progress.request_started();
}
return Ok(DownloadRequestPermit { limiter: self, progress });
}
// Wake periodically so cancellation can interrupt waiters even if
// no request finishes.
let (next_active, _) =
self.notify.wait_timeout(active, Duration::from_millis(100)).unwrap();
active = next_active;
}
}
}
/// RAII permit for one in-flight HTTP request.
///
/// Dropping the permit releases a slot in the shared request limit and updates
/// the live progress counters.
pub(crate) struct DownloadRequestPermit<'a> {
/// Limiter that owns the request slot.
limiter: &'a DownloadRequestLimiter,
/// Shared progress counters updated when the permit drops.
progress: Option<&'a Arc<SharedProgress>>,
}
impl Drop for DownloadRequestPermit<'_> {
/// Releases the request slot and updates shared progress counters.
fn drop(&mut self) {
let mut active = self.limiter.active.lock().unwrap();
*active = active.saturating_sub(1);
drop(active);
self.limiter.notify.notify_one();
if let Some(progress) = self.progress {
progress.request_finished();
}
}
}
/// Tracks one active archive download attempt.
pub(crate) struct ArchiveDownloadProgress<'a> {
progress: Option<&'a Arc<SharedProgress>>,
downloaded: u64,
completed: bool,
}
impl<'a> ArchiveDownloadProgress<'a> {
/// Starts tracking one archive download attempt.
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
if let Some(progress) = progress {
progress.download_started();
}
Self { progress, downloaded: 0, completed: false }
}
/// Adds logical compressed bytes written by this attempt.
pub(crate) fn record_downloaded(&mut self, bytes: u64) {
self.downloaded += bytes;
if let Some(progress) = self.progress {
progress.add_active_download_bytes(bytes);
}
}
/// Returns whether this tracker has recorded any logical bytes itself.
pub(crate) fn has_tracked_bytes(&self) -> bool {
self.downloaded > 0
}
/// Moves this archive from active download bytes into completed download bytes.
pub(crate) fn complete(&mut self, total_bytes: u64) {
if self.completed {
return;
}
if let Some(progress) = self.progress {
progress.sub_active_download_bytes(self.downloaded);
progress.record_archive_download_complete(total_bytes);
}
self.downloaded = 0;
self.completed = true;
}
}
impl Drop for ArchiveDownloadProgress<'_> {
fn drop(&mut self) {
if let Some(progress) = self.progress {
progress.sub_active_download_bytes(self.downloaded);
progress.download_finished();
}
}
}
/// Tracks one active archive extraction attempt.
pub(crate) struct ArchiveExtractionProgress {
progress: Option<Arc<SharedProgress>>,
extracted: Arc<AtomicU64>,
finished: bool,
}
/// Cloneable handle for reporting extracted bytes from background monitoring.
#[derive(Clone)]
pub(crate) struct ArchiveExtractionProgressHandle {
progress: Arc<SharedProgress>,
extracted: Arc<AtomicU64>,
}
impl ArchiveExtractionProgress {
/// Starts tracking one archive extraction attempt.
pub(crate) fn new(progress: Option<&Arc<SharedProgress>>) -> Self {
if let Some(progress) = progress {
progress.extraction_started();
}
Self {
progress: progress.cloned(),
extracted: Arc::new(AtomicU64::new(0)),
finished: false,
}
}
/// Returns a cloneable handle that can report extraction progress from another thread.
pub(crate) fn handle(&self) -> Option<ArchiveExtractionProgressHandle> {
Some(ArchiveExtractionProgressHandle {
progress: Arc::clone(self.progress.as_ref()?),
extracted: Arc::clone(&self.extracted),
})
}
/// Adds plain-output bytes extracted by this attempt.
pub(crate) fn record_extracted(&mut self, bytes: u64) {
if let Some(handle) = self.handle() {
handle.record_extracted(bytes);
}
}
/// Ends extraction tracking before verification begins.
pub(crate) fn finish(&mut self) {
if self.finished {
return;
}
if let Some(progress) = &self.progress {
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
}
self.finished = true;
}
}
impl Drop for ArchiveExtractionProgress {
fn drop(&mut self) {
if let Some(progress) = &self.progress {
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
progress.extraction_finished();
}
}
}
impl ArchiveExtractionProgressHandle {
/// Adds plain-output bytes extracted by this attempt.
pub(crate) fn record_extracted(&self, bytes: u64) {
self.extracted.fetch_add(bytes, Ordering::Relaxed);
self.progress.add_active_extracted_output_bytes(bytes);
}
}
/// Tracks one active archive verification attempt.
pub(crate) struct ArchiveVerificationProgress<'a> {
progress: Option<&'a Arc<SharedProgress>>,
verified: u64,
completed: bool,
}
impl<'a> ArchiveVerificationProgress<'a> {
/// Starts tracking one archive verification attempt.
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
if let Some(progress) = progress {
progress.verification_started();
}
Self { progress, verified: 0, completed: false }
}
/// Adds plain-output bytes hashed by this verification attempt.
pub(crate) fn record_verified(&mut self, bytes: u64) {
self.verified += bytes;
if let Some(progress) = self.progress {
progress.add_active_verified_output_bytes(bytes);
}
}
/// Moves this archive from active verification bytes into completed output bytes.
pub(crate) fn complete(&mut self, total_bytes: u64) {
if self.completed {
return;
}
if let Some(progress) = self.progress {
progress.sub_active_verified_output_bytes(self.verified);
progress.record_archive_output_complete(total_bytes);
}
self.verified = 0;
self.completed = true;
}
}
impl Drop for ArchiveVerificationProgress<'_> {
fn drop(&mut self) {
if let Some(progress) = self.progress {
progress.sub_active_verified_output_bytes(self.verified);
progress.verification_finished();
}
}
}
/// Adapter to track progress while reading (used for extraction in legacy path)
pub(crate) struct ProgressReader<R> {
/// Wrapped reader that provides archive bytes.
reader: R,
/// Per-download progress tracker for legacy paths.
progress: DownloadProgress,
/// Cancellation token checked between reads.
cancel_token: CancellationToken,
}
impl<R: Read> ProgressReader<R> {
/// Wraps a reader with per-download progress tracking.
pub(crate) fn new(reader: R, total_size: u64, cancel_token: CancellationToken) -> Self {
Self { reader, progress: DownloadProgress::new(total_size), cancel_token }
}
}
impl<R: Read> Read for ProgressReader<R> {
/// Reads bytes, checks cancellation, and updates the local progress bar.
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.cancel_token.is_cancelled() {
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
}
let bytes = self.reader.read(buf)?;
if bytes > 0 &&
let Err(error) = self.progress.update(bytes as u64)
{
return Err(io::Error::other(error));
}
Ok(bytes)
}
}
/// Wrapper that bumps a shared atomic counter while writing data.
/// Used for parallel downloads where a single display task shows aggregated progress.
pub(crate) struct SharedProgressWriter<'a, W> {
/// Wrapped writer receiving downloaded bytes.
pub(crate) inner: W,
/// Shared counters updated as bytes are written.
pub(crate) progress: Arc<SharedProgress>,
/// Optional callback for logical bytes written by the current archive attempt.
pub(crate) on_written: Option<&'a mut dyn FnMut(u64)>,
}
impl<W: Write> Write for SharedProgressWriter<'_, W> {
/// Writes bytes and records them in shared progress.
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
if self.progress.is_cancelled() {
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
}
let n = self.inner.write(buf)?;
self.progress.record_session_fetched_bytes(n as u64);
if let Some(on_written) = self.on_written.as_deref_mut() {
on_written(n as u64);
}
Ok(n)
}
/// Flushes the wrapped writer.
fn flush(&mut self) -> io::Result<()> {
self.inner.flush()
}
}
/// Wrapper that bumps a shared atomic counter while reading data.
/// Used for streaming downloads where a single display task shows aggregated progress.
pub(crate) struct SharedProgressReader<R> {
/// Wrapped reader producing streamed bytes.
pub(crate) inner: R,
/// Shared counters updated as bytes are read.
pub(crate) progress: Arc<SharedProgress>,
}
impl<R: Read> Read for SharedProgressReader<R> {
/// Reads bytes and records them in shared progress.
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
if self.progress.is_cancelled() {
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
}
let n = self.inner.read(buf)?;
self.progress.record_session_fetched_bytes(n as u64);
Ok(n)
}
}
/// Spawns a background task that prints aggregated download progress.
/// Returns a handle; drop it (or call `.abort()`) to stop.
pub(crate) fn spawn_progress_display(progress: Arc<SharedProgress>) -> tokio::task::JoinHandle<()> {
tokio::spawn(async move {
let mut interval = tokio::time::interval(Duration::from_secs(3));
interval.tick().await;
loop {
interval.tick().await;
if progress.done.load(Ordering::Relaxed) {
break;
}
let download_total = progress.total_download_bytes;
let output_total = progress.total_output_bytes;
if download_total == 0 && output_total == 0 {
continue;
}
let done = progress.archives_done.load(Ordering::Relaxed);
let all = progress.total_archives;
let active_downloads = progress.active_downloads.load(Ordering::Relaxed);
let active_requests = progress.active_download_requests.load(Ordering::Relaxed);
let active_extractions = progress.active_extractions.load(Ordering::Relaxed);
let active_verifications = progress.active_verifications.load(Ordering::Relaxed);
let downloaded = progress.logical_downloaded_bytes();
let extracted = progress.extracting_output_bytes();
let verified = progress.verifying_output_bytes();
let elapsed = DownloadProgress::format_duration(progress.started_at.elapsed());
let download_total_display = DownloadProgress::format_size(download_total);
let output_total_display = DownloadProgress::format_size(output_total);
let downloaded_display = DownloadProgress::format_size(downloaded);
let extracted_display = DownloadProgress::format_size(extracted);
let active_download_phase = active_downloads > 0 || active_requests > 0;
if active_download_phase {
info!(target: "reth::cli",
archives = format_args!("{done}/{all}"),
progress = %format_percent(downloaded, download_total),
elapsed = %elapsed,
eta = %format_eta(eta_from_progress(progress.started_at.elapsed(), downloaded, download_total)),
bytes = format_args!("{downloaded_display}/{download_total_display}"),
"Downloading snapshot archives"
);
} else if active_extractions > 0 {
info!(target: "reth::cli",
archives = format_args!("{done}/{all}"),
progress = %format_percent(extracted, output_total),
elapsed = %elapsed,
eta = %format_eta(progress.extraction_eta(extracted)),
bytes = format_args!("{extracted_display}/{output_total_display}"),
"Extracting snapshot archives"
);
} else if active_verifications > 0 {
info!(target: "reth::cli",
archives = format_args!("{done}/{all}"),
progress = %format_percent(verified, output_total),
elapsed = %elapsed,
eta = %format_eta(progress.verification_eta(verified)),
bytes = format_args!("{}/{output_total_display}", DownloadProgress::format_size(verified)),
"Verifying snapshot archives"
);
} else {
continue;
}
}
let completed = progress.verified_output_bytes();
let completed_display = DownloadProgress::format_size(completed);
let output_total = DownloadProgress::format_size(progress.total_output_bytes);
info!(target: "reth::cli",
archives = format_args!("{}/{}", progress.total_archives, progress.total_archives),
progress = "100.0%",
elapsed = %DownloadProgress::format_duration(progress.started_at.elapsed()),
eta = "0s",
bytes = format_args!("{completed_display}/{output_total}"),
"Snapshot archive processing complete"
);
})
}
#[cfg(test)]
mod tests {
use super::*;
use std::sync::atomic::Ordering;
#[test]
fn shared_progress_separates_session_fetch_from_logical_progress() {
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
progress.record_session_fetched_bytes(10);
progress.record_session_fetched_bytes(10);
progress.record_archive_download_complete(10);
progress.record_archive_output_complete(20);
assert_eq!(progress.session_fetched_bytes.load(Ordering::Relaxed), 20);
assert_eq!(progress.logical_downloaded_bytes(), 10);
assert_eq!(progress.verified_output_bytes(), 20);
assert_eq!(progress.archives_done.load(Ordering::Relaxed), 1);
}
#[test]
fn archive_download_progress_rolls_back_unfinished_attempts() {
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
{
let mut download = ArchiveDownloadProgress::new(Some(&progress));
download.record_downloaded(4);
assert_eq!(progress.logical_downloaded_bytes(), 4);
}
assert_eq!(progress.logical_downloaded_bytes(), 0);
assert_eq!(progress.active_downloads.load(Ordering::Relaxed), 0);
}
#[test]
fn extraction_phase_baseline_restarts_after_idle() {
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
progress.extraction_started();
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.completed_output_bytes.store(25, Ordering::Relaxed);
progress.extraction_started();
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.extraction_finished();
progress.extraction_finished();
progress.extraction_started();
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 25);
}
#[test]
fn verification_phase_baseline_restarts_after_idle() {
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
progress.verification_started();
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.completed_output_bytes.store(40, Ordering::Relaxed);
progress.verification_started();
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
progress.verification_finished();
progress.verification_finished();
progress.verification_started();
assert_eq!(
progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes,
40
);
}
}

View File

@@ -0,0 +1,100 @@
use super::progress::{DownloadRequestLimiter, SharedProgress};
use eyre::Result;
use reth_cli_util::cancellation::CancellationToken;
use std::{
path::{Path, PathBuf},
sync::Arc,
};
/// Shared state for one run of `reth download`.
#[derive(Clone)]
pub(crate) struct DownloadSession {
/// Shared progress counters for this command, when enabled.
progress: Option<Arc<SharedProgress>>,
/// Shared limit for concurrent HTTP requests, when enabled.
request_limiter: Option<Arc<DownloadRequestLimiter>>,
/// Cancellation token shared by the whole command.
cancel_token: CancellationToken,
}
impl DownloadSession {
/// Stores the shared progress, request limiter, and cancellation token.
pub(crate) fn new(
progress: Option<Arc<SharedProgress>>,
request_limiter: Option<Arc<DownloadRequestLimiter>>,
cancel_token: CancellationToken,
) -> Self {
Self { progress, request_limiter, cancel_token }
}
/// Returns the shared progress tracker, if this flow uses one.
pub(crate) fn progress(&self) -> Option<&Arc<SharedProgress>> {
self.progress.as_ref()
}
/// Returns the shared HTTP request limiter, if this flow uses one.
pub(crate) fn request_limiter(&self) -> Option<&Arc<DownloadRequestLimiter>> {
self.request_limiter.as_ref()
}
/// Returns the request limiter or errors if the caller needs one.
pub(crate) fn require_request_limiter(&self) -> Result<&Arc<DownloadRequestLimiter>> {
self.request_limiter().ok_or_else(|| eyre::eyre!("Missing download request limiter"))
}
/// Returns the cancellation token for this command.
pub(crate) fn cancel_token(&self) -> &CancellationToken {
&self.cancel_token
}
/// Records one archive whose outputs were already reusable on disk.
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
if let Some(progress) = self.progress() {
progress.record_reused_archive(download_bytes, output_bytes);
}
}
/// Records one archive whose extracted outputs fully verified.
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
if let Some(progress) = self.progress() {
progress.record_archive_output_complete(bytes);
}
}
}
/// Paths used while processing one archive, plus the shared download session.
#[derive(Clone)]
pub(crate) struct ArchiveProcessContext {
/// Directory where extracted output files are written.
target_dir: PathBuf,
/// Directory used for cached archive downloads, when enabled.
cache_dir: Option<PathBuf>,
/// Shared command-scoped download state.
session: DownloadSession,
}
impl ArchiveProcessContext {
/// Creates the context used while processing modular archives.
pub(crate) fn new(
target_dir: PathBuf,
cache_dir: Option<PathBuf>,
session: DownloadSession,
) -> Self {
Self { target_dir, cache_dir, session }
}
/// Returns the directory where extracted outputs should be written.
pub(crate) fn target_dir(&self) -> &Path {
&self.target_dir
}
/// Returns the cache directory for two-phase downloads, if enabled.
pub(crate) fn cache_dir(&self) -> Option<&Path> {
self.cache_dir.as_deref()
}
/// Returns the shared download session.
pub(crate) fn session(&self) -> &DownloadSession {
&self.session
}
}

View File

@@ -0,0 +1,230 @@
use super::{manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults};
use eyre::{Result, WrapErr};
use reqwest::Client;
use reth_fs_util as fs;
use std::path::{Path, PathBuf};
use tracing::info;
use url::Url;
/// An entry from the snapshot discovery API listing.
#[derive(serde::Deserialize)]
#[serde(rename_all = "camelCase")]
pub(crate) struct SnapshotApiEntry {
#[serde(deserialize_with = "deserialize_string_or_u64")]
chain_id: u64,
#[serde(deserialize_with = "deserialize_string_or_u64")]
block: u64,
#[serde(default)]
date: Option<String>,
#[serde(default)]
profile: Option<String>,
metadata_url: String,
#[serde(default)]
size: u64,
}
impl SnapshotApiEntry {
/// Returns whether this discovery entry points to a modular manifest.
fn is_modular(&self) -> bool {
self.metadata_url.ends_with("manifest.json")
}
}
/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
///
/// Queries the configured snapshot API and returns the manifest URL for the most
/// recent modular snapshot matching the requested chain.
pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
let defaults = DownloadDefaults::get_global();
let api_url = &*defaults.snapshot_api_url;
info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
let entries = fetch_snapshot_api_entries(chain_id).await?;
let entry =
entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
eyre::eyre!(
"No modular snapshot manifest found for chain \
{chain_id} at {api_url}\n\n\
You can provide a manifest URL directly with --manifest-url, or\n\
use a direct snapshot URL with -u from:\n\
\t- {}\n\n\
Use --list to see all available snapshots.",
api_url.trim_end_matches("/api/snapshots"),
)
})?;
info!(target: "reth::cli",
block = entry.block,
url = %entry.metadata_url,
"Found latest snapshot manifest"
);
Ok(entry.metadata_url.clone())
}
/// Deserializes a JSON value that may be either a number or a string-encoded number.
fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
where
D: serde::Deserializer<'de>,
{
use serde::Deserialize;
let value = serde_json::Value::deserialize(deserializer)?;
match &value {
serde_json::Value::Number(n) => {
n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
}
serde_json::Value::String(s) => {
s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
}
_ => Err(serde::de::Error::custom("expected number or string")),
}
}
/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
let entries: Vec<SnapshotApiEntry> = Client::new()
.get(api_url)
.send()
.await
.and_then(|r| r.error_for_status())
.wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
.json()
.await?;
Ok(entries.into_iter().filter(|entry| entry.chain_id == chain_id).collect())
}
/// Prints a formatted table of available modular snapshots.
pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
println!(
"Available snapshots for chain {chain_id} ({}):\n",
api_url.trim_end_matches("/api/snapshots"),
);
println!("{:<12} {:>10} {:<10} {:>10} MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
println!("{}", "-".repeat(100));
for entry in &modular {
let date = entry.date.as_deref().unwrap_or("-");
let profile = entry.profile.as_deref().unwrap_or("-");
let size = if entry.size > 0 {
DownloadProgress::format_size(entry.size)
} else {
"-".to_string()
};
println!(
"{date:<12} {:>10} {profile:<10} {size:>10} {}",
entry.block, entry.metadata_url
);
}
if modular.is_empty() {
println!(" (no modular snapshots found)");
}
println!(
"\nTo download a specific snapshot, copy its manifest URL and run:\n \
reth download --manifest-url <URL>"
);
}
/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path.
pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
if let Ok(parsed) = Url::parse(source) {
return match parsed.scheme() {
"http" | "https" => {
let response = Client::new()
.get(source)
.send()
.await
.and_then(|r| r.error_for_status())
.wrap_err_with(|| {
let sources = DownloadDefaults::get_global()
.available_snapshots
.iter()
.map(|snapshot| format!("\t- {snapshot}"))
.collect::<Vec<_>>()
.join("\n");
format!(
"Failed to fetch snapshot manifest from {source}\n\n\
The manifest endpoint may not be available for this snapshot source.\n\
You can use a direct snapshot URL instead:\n\n\
\treth download -u <snapshot-url>\n\n\
Available snapshot sources:\n{sources}"
)
})?;
Ok(response.json().await?)
}
"file" => {
let path = parsed
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
let content = fs::read_to_string(path)?;
Ok(serde_json::from_str(&content)?)
}
_ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
};
}
let content = fs::read_to_string(source)?;
Ok(serde_json::from_str(&content)?)
}
/// Resolves the base URL used to join relative archive paths in a manifest.
pub(crate) fn resolve_manifest_base_url(
manifest: &SnapshotManifest,
source: &str,
) -> Result<String> {
if let Some(base_url) = manifest.base_url.as_deref() &&
!base_url.is_empty()
{
return Ok(base_url.trim_end_matches('/').to_string());
}
if let Ok(mut url) = Url::parse(source) {
if url.scheme() == "file" {
let mut path = url
.to_file_path()
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
path.pop();
let mut base = Url::from_directory_path(path)
.map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
.to_string();
if base.ends_with('/') {
base.pop();
}
return Ok(base);
}
{
let mut segments = url
.path_segments_mut()
.map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
segments.pop_if_empty();
segments.pop();
}
return Ok(url.as_str().trim_end_matches('/').to_string());
}
let path = Path::new(source);
let manifest_dir = if path.is_absolute() {
path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
} else {
let joined = std::env::current_dir()?.join(path);
joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
};
let mut base = Url::from_directory_path(&manifest_dir)
.map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
.to_string();
if base.ends_with('/') {
base.pop();
}
Ok(base)
}

View File

@@ -0,0 +1,84 @@
use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress};
use blake3::Hasher;
use eyre::Result;
use reth_fs_util as fs;
use std::{io::Read, path::Path};
/// Verifies and cleans up extracted output files in one target directory.
pub(crate) struct OutputVerifier<'a> {
/// Directory containing the output files declared by the manifest.
target_dir: &'a Path,
}
impl<'a> OutputVerifier<'a> {
/// Creates a verifier for one extraction target directory.
pub(crate) const fn new(target_dir: &'a Path) -> Self {
Self { target_dir }
}
/// Returns `true` only when every declared output file exists and matches size and BLAKE3.
/// Returns `false` if any file is missing, mismatched, or no outputs were declared.
pub(crate) fn verify(&self, output_files: &[OutputFileChecksum]) -> Result<bool> {
self.verify_with_progress(output_files, None)
}
/// Returns `true` only when every declared output file exists and matches size and BLAKE3,
/// updating the optional verification progress as file bytes are hashed.
pub(crate) fn verify_with_progress(
&self,
output_files: &[OutputFileChecksum],
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
) -> Result<bool> {
if output_files.is_empty() {
return Ok(false);
}
for expected in output_files {
let output_path = self.target_dir.join(&expected.path);
let meta = match fs::metadata(&output_path) {
Ok(meta) => meta,
Err(_) => return Ok(false),
};
if meta.len() != expected.size {
return Ok(false);
}
let actual = Self::file_blake3_hex(&output_path, progress.as_deref_mut())?;
if !actual.eq_ignore_ascii_case(&expected.blake3) {
return Ok(false);
}
}
Ok(true)
}
/// Removes any declared output files so a fresh archive attempt can restart cleanly.
pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) {
for output in output_files {
let _ = fs::remove_file(self.target_dir.join(&output.path));
}
}
/// Computes the hex-encoded BLAKE3 checksum for one plain output file.
fn file_blake3_hex(
path: &Path,
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
) -> Result<String> {
let mut file = fs::open(path)?;
let mut hasher = Hasher::new();
let mut buf = [0_u8; 64 * 1024];
loop {
let n = file.read(&mut buf)?;
if n == 0 {
break;
}
hasher.update(&buf[..n]);
if let Some(progress) = progress.as_deref_mut() {
progress.record_verified(n as u64);
}
}
Ok(hasher.finalize().to_hex().to_string())
}
}

View File

@@ -138,7 +138,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let cancellation = cancellation.clone();
let next_block = Arc::clone(&next_block);
tasks.spawn_blocking(move || {
let executor_lifetime = Duration::from_secs(120);
let executor_lifetime = Duration::from_secs(600);
loop {
if cancellation.is_cancelled() {
@@ -245,7 +245,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
let _ = stats_tx.send(block.gas_used());
// Reset DB once in a while to avoid OOM or read tx timeouts
if executor.size_hint() > 1_000_000 ||
if executor.size_hint() > 5_000_000 ||
executor_created.elapsed() > executor_lifetime
{
executor =

View File

@@ -29,7 +29,10 @@ use execution::dump_execution_stage;
mod merkle;
use merkle::dump_merkle_stage;
/// `reth dump-stage` command
/// `reth dump-stage` command.
///
/// Note: mutates the source datadir (unwinds hashing/merkle/execution before copying tables).
/// Stop the node and back up the datadir first.
#[derive(Debug, Parser)]
pub struct Command<C: ChainSpecParser> {
#[command(flatten)]
@@ -100,8 +103,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
Comp: CliNodeComponents<N>,
F: FnOnce(Arc<C::ChainSpec>) -> Comp,
{
// `unwind_and_copy` opens a RW provider on the source datadir, so open RW here.
let Environment { provider_factory, .. } =
self.env.init::<N>(AccessRights::RO, runtime.clone())?;
self.env.init::<N>(AccessRights::RW, runtime.clone())?;
let tool = DbTool::new(provider_factory)?;
let components = components(tool.chain());
let evm_config = components.evm_config().clone();

View File

@@ -141,6 +141,11 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
last_header: SealedHeaderFor<<T::BuiltPayload as BuiltPayload>::Primitives>,
/// Stores latest mined blocks.
last_block_hashes: VecDeque<B256>,
/// Optional sleep duration between initiating payload building and resolving.
///
/// When set, the miner sleeps after `fork_choice_updated` before calling
/// `resolve_kind`, giving the payload job time for multiple rebuild attempts.
payload_wait_time: Option<Duration>,
}
impl<T, B, Pool> LocalMiner<T, B, Pool>
@@ -170,9 +175,16 @@ where
payload_builder,
last_block_hashes: VecDeque::from([last_header.hash()]),
last_header,
payload_wait_time: None,
}
}
/// Sets the payload wait time, if any.
pub const fn with_payload_wait_time_opt(mut self, wait_time: Option<Duration>) -> Self {
self.payload_wait_time = wait_time;
self
}
/// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads.
pub async fn run(mut self) {
let mut fcu_interval = tokio::time::interval(Duration::from_secs(1));
@@ -238,6 +250,10 @@ where
let payload_id = res.payload_id.ok_or_eyre("No payload id")?;
if let Some(wait_time) = self.payload_wait_time {
tokio::time::sleep(wait_time).await;
}
let Some(Ok(payload)) =
self.payload_builder.resolve_kind(payload_id, PayloadKind::WaitForPending).await
else {

View File

@@ -57,7 +57,7 @@ where
.chain_spec
.is_cancun_active_at_timestamp(timestamp)
.then(B256::random),
slot_number: None,
slot_number: self.chain_spec.is_amsterdam_active_at_timestamp(timestamp).then_some(0),
}
}
}

View File

@@ -1693,6 +1693,7 @@ where
let gas_used = payload.gas_used();
let num_hash = payload.num_hash();
let mut output = self.on_new_payload(payload);
let latency = start.elapsed();
self.metrics.engine.new_payload.update_response_metrics(
start,
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
@@ -1700,12 +1701,6 @@ where
gas_used,
);
// Latency measures time from enqueue to completion, excluding
// only the explicit persistence wait. This means backpressure
// (time spent queued due to the engine being busy) is included,
// reflecting real-world engine responsiveness.
let latency = enqueued_at.elapsed() - explicit_persistence_wait;
let maybe_event =
output.as_mut().ok().and_then(|out| out.event.take());

View File

@@ -2142,4 +2142,15 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
Self::Block(block) => block.gas_used(),
}
}
/// Returns the gas limit used by the block.
pub fn gas_limit(&self) -> u64
where
T::ExecutionData: ExecutionPayload,
{
match self {
Self::Payload(payload) => payload.gas_limit(),
Self::Block(block) => block.gas_limit(),
}
}
}

View File

@@ -10,6 +10,7 @@ use reth_primitives_traits::dashmap::DashMap;
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
use revm_primitives::Address;
use std::{hash::Hash, sync::Arc};
use tracing::error;
/// Default max cache size for [`PrecompileCache`]
const MAX_CACHE_SIZE: u32 = 10_000;
@@ -87,8 +88,14 @@ impl<S> CacheEntry<S> {
self.output.gas_used
}
fn to_precompile_result(&self) -> PrecompileResult {
Ok(self.output.clone())
/// Converts the cache entry to a precompile result. Accepts state gas reservoir as input.
///
/// All cached precompiles are not expected to access/created state and thus reservoir is always
/// kept as is.
fn to_precompile_result(&self, reservoir: u64) -> PrecompileResult {
let mut output = self.output.clone();
output.reservoir = reservoir;
Ok(output)
}
}
@@ -175,22 +182,34 @@ where
input.gas >= entry.gas_used()
{
self.increment_by_one_precompile_cache_hits();
return entry.to_precompile_result()
return entry.to_precompile_result(input.reservoir);
}
let calldata = input.data;
let reservoir = input.reservoir;
let result = self.precompile.call(input);
match &result {
// Only successful outputs are cacheable. Non-success statuses and errors must execute
// again instead of poisoning the cache for subsequent calls.
Ok(output) if output.is_success() => {
let size = self.cache.insert(
Bytes::copy_from_slice(calldata),
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
);
self.set_precompile_cache_size_metric(size as f64);
self.increment_by_one_precompile_cache_misses();
// Sanity-check precompile output to ensure that it does not affect state gas in any
// way.
//
// This does not fully protect us from caching stateful precompiles but might make
// it obvious when the node is misconfigured.
if output.reservoir != reservoir {
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile decremented reservoir, skipping cache insertion");
} else if output.state_gas_used != 0 {
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile used state gas, skipping cache insertion");
} else {
let size = self.cache.insert(
Bytes::copy_from_slice(calldata),
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
);
self.set_precompile_cache_size_metric(size as f64);
self.increment_by_one_precompile_cache_misses();
}
}
_ => {
self.increment_by_one_precompile_errors();
@@ -246,6 +265,7 @@ mod tests {
gas_used: 0,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: Bytes::default(),
})
})
@@ -259,6 +279,7 @@ mod tests {
gas_used: 50,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
};
@@ -292,6 +313,7 @@ mod tests {
gas_used: 5000,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
})
}
@@ -308,6 +330,7 @@ mod tests {
gas_used: 7000,
state_gas_used: 0,
reservoir: 0,
gas_refunded: 0,
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
})
}

View File

@@ -195,6 +195,7 @@ where
ommers: &block.body().ommers,
withdrawals: block.body().withdrawals.as_ref().map(|w| Cow::Borrowed(w.as_slice())),
extra_data: block.header().extra_data.clone(),
slot_number: block.header().slot_number,
})
}
@@ -210,6 +211,7 @@ where
ommers: &[],
withdrawals: attributes.withdrawals.map(|w| Cow::Owned(w.into_inner())),
extra_data: attributes.extra_data,
slot_number: attributes.slot_number,
})
}
}
@@ -274,7 +276,7 @@ where
gas_limit: payload.payload.gas_limit(),
basefee: payload.payload.saturated_base_fee_per_gas(),
blob_excess_gas_and_price,
slot_num: 0,
slot_num: payload.payload.as_v4().map(|v4| v4.slot_number).unwrap_or_default(),
};
Ok(EvmEnv { cfg_env, block_env })
@@ -291,6 +293,7 @@ where
ommers: &[],
withdrawals: payload.payload.withdrawals().map(|w| Cow::Borrowed(w.as_slice())),
extra_data: payload.payload.as_v1().extra_data.clone(),
slot_number: payload.payload.as_v4().map(|v4| v4.slot_number),
})
}

View File

@@ -37,7 +37,6 @@ reth-rpc-eth-types.workspace = true
reth-engine-local.workspace = true
reth-engine-primitives = { workspace = true, features = ["std"] }
reth-payload-primitives.workspace = true
# ethereum
alloy-eips.workspace = true
alloy-network.workspace = true

View File

@@ -26,9 +26,10 @@ use reth_node_builder::{
},
node::{FullNodeTypes, NodeTypes},
rpc::{
BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn,
EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity, PayloadValidatorBuilder,
RethRpcAddOns, RpcAddOns, RpcHandle,
BasicEngineApiBuilder, BasicEngineValidatorBuilder, Either, EngineApiBuilder,
EngineValidatorAddOn, EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity,
PayloadValidatorBuilder, RethAuthHttpMiddleware, RethRpcAddOns, RethRpcMiddleware,
RpcAddOns, RpcHandle, Stack,
},
BuilderContext, DebugNode, Node, NodeAdapter,
};
@@ -39,7 +40,7 @@ use reth_rpc::{
TestingApi, ValidationApi,
};
use reth_rpc_api::servers::{BlockSubmissionValidationApiServer, TestingApiServer};
use reth_rpc_builder::{config::RethRpcServerConfig, middleware::RethRpcMiddleware};
use reth_rpc_builder::config::RethRpcServerConfig;
use reth_rpc_eth_api::{
helpers::{
config::{EthConfigApiServer, EthConfigHandler},
@@ -165,17 +166,21 @@ pub struct EthereumAddOns<
EB = BasicEngineApiBuilder<PVB>,
EVB = BasicEngineValidatorBuilder<PVB>,
RpcMiddleware = Identity,
AuthHttpMiddleware = Identity,
> {
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>,
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents,
EthB: EthApiBuilder<N>,
{
/// Creates a new instance from the inner `RpcAddOns`.
pub const fn new(inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>) -> Self {
pub const fn new(
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
) -> Self {
Self { inner }
}
}
@@ -199,11 +204,13 @@ where
BasicEngineApiBuilder::default(),
BasicEngineValidatorBuilder::default(),
Default::default(),
Identity::new(),
))
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents,
EthB: EthApiBuilder<N>,
@@ -212,7 +219,7 @@ where
pub fn with_engine_api<T>(
self,
engine_api_builder: T,
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware>
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware, AuthHttpMiddleware>
where
T: Send,
{
@@ -224,7 +231,7 @@ where
pub fn with_payload_validator<V, T>(
self,
payload_validator_builder: T,
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware> {
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware, AuthHttpMiddleware> {
let Self { inner } = self;
EthereumAddOns::new(inner.with_payload_validator(payload_validator_builder))
}
@@ -233,7 +240,7 @@ where
pub fn with_rpc_middleware<T>(
self,
rpc_middleware: T,
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T>
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T, AuthHttpMiddleware>
where
T: Send,
{
@@ -241,6 +248,45 @@ where
EthereumAddOns::new(inner.with_rpc_middleware(rpc_middleware))
}
/// Configures the HTTP transport middleware for the auth / Engine API server.
pub fn with_auth_http_middleware<T>(
self,
auth_http_middleware: T,
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, T>
where
T: Send,
{
let Self { inner } = self;
EthereumAddOns::new(inner.with_auth_http_middleware(auth_http_middleware))
}
/// Stacks an additional HTTP transport middleware layer for the auth / Engine API server.
pub fn layer_auth_http_middleware<T>(
self,
layer: T,
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, Stack<AuthHttpMiddleware, T>> {
let Self { inner } = self;
EthereumAddOns::new(inner.layer_auth_http_middleware(layer))
}
/// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server.
#[expect(clippy::type_complexity)]
pub fn option_layer_auth_http_middleware<T>(
self,
layer: Option<T>,
) -> EthereumAddOns<
N,
EthB,
PVB,
EB,
EVB,
RpcMiddleware,
Stack<AuthHttpMiddleware, Either<T, Identity>>,
> {
let Self { inner } = self;
EthereumAddOns::new(inner.option_layer_auth_http_middleware(layer))
}
/// Sets the tokio runtime for the RPC servers.
///
/// Caution: This runtime must not be created from within asynchronous context.
@@ -250,8 +296,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> NodeAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> NodeAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents<
Types: NodeTypes<
@@ -268,6 +314,7 @@ where
EthApiError: FromEvmError<N::Evm>,
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
RpcMiddleware: RethRpcMiddleware,
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
{
type Handle = RpcHandle<N, EthB::EthApi>;
@@ -323,8 +370,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> RethRpcAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> RethRpcAddOns<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents<
Types: NodeTypes<
@@ -341,6 +388,7 @@ where
EthApiError: FromEvmError<N::Evm>,
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
RpcMiddleware: RethRpcMiddleware,
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
{
type EthApi = EthB::EthApi;
@@ -349,8 +397,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EngineValidatorAddOn<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> EngineValidatorAddOn<N>
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents<
Types: NodeTypes<
@@ -367,6 +415,7 @@ where
EthApiError: FromEvmError<N::Evm>,
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
RpcMiddleware: Send,
AuthHttpMiddleware: Send,
{
type ValidatorBuilder = EVB;

View File

@@ -248,7 +248,7 @@ impl Discv5 {
discv5::Event::SocketUpdated(_) | discv5::Event::TalkRequest(_) |
// `Discovered` not unique discovered peers
discv5::Event::Discovered(_) => None,
discv5::Event::NodeInserted { replaced: _, .. } => {
discv5::Event::NodeInserted { .. } => {
// node has been inserted into kbuckets

View File

@@ -1,6 +1,5 @@
//! `RLPx` disconnect reason sent to/received from peer
use alloc::vec;
use alloy_primitives::bytes::{Buf, BufMut};
use alloy_rlp::{Decodable, Encodable, Header};
use derive_more::Display;
@@ -84,10 +83,10 @@ impl Encodable for DisconnectReason {
/// The [`Encodable`] implementation for [`DisconnectReason`] encodes the disconnect reason in
/// a single-element RLP list.
fn encode(&self, out: &mut dyn BufMut) {
vec![*self as u8].encode(out);
alloy_rlp::encode_list(&[*self as u8], out);
}
fn length(&self) -> usize {
vec![*self as u8].length()
alloy_rlp::list_length(&[*self as u8])
}
}

View File

@@ -307,6 +307,14 @@ where
let dev_mining_mode =
mining_mode.unwrap_or_else(|| handle.node.config.dev_mining_mode(pool));
let payload_wait_time = config.dev.payload_wait_time;
if let (Some(wait_time), Some(block_time)) = (payload_wait_time, config.dev.block_time)
{
eyre::ensure!(
wait_time <= block_time,
"--dev.payload-wait-time ({wait_time:?}) must be <= --dev.block-time ({block_time:?})"
);
}
handle.node.task_executor.spawn_critical_task("local engine", async move {
LocalMiner::new(
blockchain_db,
@@ -315,6 +323,7 @@ where
dev_mining_mode,
payload_builder_handle,
)
.with_payload_wait_time_opt(payload_wait_time)
.run()
.await
});

View File

@@ -1,9 +1,15 @@
//! Builder support for rpc components.
pub use jsonrpsee::server::middleware::rpc::{RpcService, RpcServiceBuilder};
pub use jsonrpsee::{
core::middleware::layer::Either,
server::middleware::rpc::{RpcService, RpcServiceBuilder},
};
use reth_engine_tree::tree::WaitForCaches;
pub use reth_engine_tree::tree::{BasicEngineValidator, EngineValidator};
pub use reth_rpc_builder::{middleware::RethRpcMiddleware, Identity, Stack};
pub use reth_rpc_builder::{
middleware::{RethAuthHttpMiddleware, RethRpcMiddleware},
Identity, Stack,
};
pub use reth_trie_db::ChangesetCache;
use crate::{
@@ -12,7 +18,7 @@ use crate::{
};
use alloy_rpc_types::engine::ClientVersionV1;
use alloy_rpc_types_engine::ExecutionData;
use jsonrpsee::{core::middleware::layer::Either, RpcModule};
use jsonrpsee::RpcModule;
use parking_lot::Mutex;
use reth_chain_state::CanonStateSubscriptions;
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks};
@@ -517,6 +523,7 @@ pub struct RpcAddOns<
EB = BasicEngineApiBuilder<PVB>,
EVB = BasicEngineValidatorBuilder<PVB>,
RpcMiddleware = Identity,
AuthHttpMiddleware = Identity,
> {
/// Additional RPC add-ons.
pub hooks: RpcHooks<Node, EthB::EthApi>,
@@ -533,12 +540,17 @@ pub struct RpcAddOns<
/// This middleware is applied to all RPC requests across all transports (HTTP, WS, IPC).
/// See [`RpcAddOns::with_rpc_middleware`] for more details.
rpc_middleware: RpcMiddleware,
/// Configurable HTTP transport middleware for the auth server.
///
/// This middleware is applied after JWT authentication and before JSON-RPC parsing on the
/// auth / Engine API server, giving access to the raw HTTP request.
auth_http_middleware: AuthHttpMiddleware,
/// Optional custom tokio runtime for the RPC server.
tokio_runtime: Option<tokio::runtime::Handle>,
}
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware> Debug
for RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware>
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> Debug
for RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
Node: FullNodeComponents,
EthB: EthApiBuilder<Node>,
@@ -558,7 +570,8 @@ where
}
}
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware> RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware>
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
Node: FullNodeComponents,
EthB: EthApiBuilder<Node>,
@@ -570,6 +583,7 @@ where
engine_api_builder: EB,
engine_validator_builder: EVB,
rpc_middleware: RpcMiddleware,
auth_http_middleware: AuthHttpMiddleware,
) -> Self {
Self {
hooks: RpcHooks::default(),
@@ -578,6 +592,7 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime: None,
}
}
@@ -586,13 +601,14 @@ where
pub fn with_engine_api<T>(
self,
engine_api_builder: T,
) -> RpcAddOns<Node, EthB, PVB, T, EVB, RpcMiddleware> {
) -> RpcAddOns<Node, EthB, PVB, T, EVB, RpcMiddleware, AuthHttpMiddleware> {
let Self {
hooks,
eth_api_builder,
payload_validator_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
..
} = self;
@@ -603,6 +619,7 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
@@ -611,13 +628,14 @@ where
pub fn with_payload_validator<T>(
self,
payload_validator_builder: T,
) -> RpcAddOns<Node, EthB, T, EB, EVB, RpcMiddleware> {
) -> RpcAddOns<Node, EthB, T, EB, EVB, RpcMiddleware, AuthHttpMiddleware> {
let Self {
hooks,
eth_api_builder,
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
..
} = self;
@@ -628,6 +646,7 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
@@ -636,13 +655,14 @@ where
pub fn with_engine_validator<T>(
self,
engine_validator_builder: T,
) -> RpcAddOns<Node, EthB, PVB, EB, T, RpcMiddleware> {
) -> RpcAddOns<Node, EthB, PVB, EB, T, RpcMiddleware, AuthHttpMiddleware> {
let Self {
hooks,
eth_api_builder,
payload_validator_builder,
engine_api_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
..
} = self;
@@ -653,6 +673,7 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
@@ -698,13 +719,14 @@ where
pub fn with_rpc_middleware<T>(
self,
rpc_middleware: T,
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, T> {
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, T, AuthHttpMiddleware> {
let Self {
hooks,
eth_api_builder,
payload_validator_builder,
engine_api_builder,
engine_validator_builder,
auth_http_middleware,
tokio_runtime,
..
} = self;
@@ -715,10 +737,87 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
/// Configures the HTTP transport middleware for the auth / Engine API server.
///
/// This middleware is applied after JWT authentication and before JSON-RPC parsing,
/// giving access to the raw HTTP request (headers, body, etc.).
pub fn with_auth_http_middleware<T>(
self,
auth_http_middleware: T,
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, T> {
let Self {
hooks,
eth_api_builder,
payload_validator_builder,
engine_api_builder,
engine_validator_builder,
rpc_middleware,
tokio_runtime,
..
} = self;
RpcAddOns {
hooks,
eth_api_builder,
payload_validator_builder,
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
/// Stacks an additional HTTP transport middleware layer for the auth / Engine API server.
pub fn layer_auth_http_middleware<T>(
self,
layer: T,
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, Stack<AuthHttpMiddleware, T>> {
let Self {
hooks,
eth_api_builder,
payload_validator_builder,
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
} = self;
let auth_http_middleware = Stack::new(auth_http_middleware, layer);
RpcAddOns {
hooks,
eth_api_builder,
payload_validator_builder,
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
/// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server.
#[expect(clippy::type_complexity)]
pub fn option_layer_auth_http_middleware<T>(
self,
layer: Option<T>,
) -> RpcAddOns<
Node,
EthB,
PVB,
EB,
EVB,
RpcMiddleware,
Stack<AuthHttpMiddleware, Either<T, Identity>>,
> {
let layer = layer.map(Either::Left).unwrap_or(Either::Right(Identity::new()));
self.layer_auth_http_middleware(layer)
}
/// Sets the tokio runtime for the RPC servers.
///
/// Caution: This runtime must not be created from within asynchronous context.
@@ -730,6 +829,7 @@ where
engine_validator_builder,
engine_api_builder,
rpc_middleware,
auth_http_middleware,
..
} = self;
Self {
@@ -739,6 +839,7 @@ where
engine_validator_builder,
engine_api_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
@@ -747,7 +848,7 @@ where
pub fn layer_rpc_middleware<T>(
self,
layer: T,
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, Stack<RpcMiddleware, T>> {
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, Stack<RpcMiddleware, T>, AuthHttpMiddleware> {
let Self {
hooks,
eth_api_builder,
@@ -755,6 +856,7 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
} = self;
let rpc_middleware = Stack::new(rpc_middleware, layer);
@@ -765,6 +867,7 @@ where
engine_api_builder,
engine_validator_builder,
rpc_middleware,
auth_http_middleware,
tokio_runtime,
}
}
@@ -774,7 +877,15 @@ where
pub fn option_layer_rpc_middleware<T>(
self,
layer: Option<T>,
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, Stack<RpcMiddleware, Either<T, Identity>>> {
) -> RpcAddOns<
Node,
EthB,
PVB,
EB,
EVB,
Stack<RpcMiddleware, Either<T, Identity>>,
AuthHttpMiddleware,
> {
let layer = layer.map(Either::Left).unwrap_or(Either::Right(Identity::new()));
self.layer_rpc_middleware(layer)
}
@@ -800,7 +911,8 @@ where
}
}
impl<Node, EthB, EV, EB, Engine> Default for RpcAddOns<Node, EthB, EV, EB, Engine, Identity>
impl<Node, EthB, EV, EB, Engine> Default
for RpcAddOns<Node, EthB, EV, EB, Engine, Identity, Identity>
where
Node: FullNodeComponents,
EthB: EthApiBuilder<Node>,
@@ -815,11 +927,13 @@ where
EB::default(),
Engine::default(),
Default::default(),
Identity::new(),
)
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents,
N::Provider: ChainSpecProvider<ChainSpec: EthereumHardforks>,
@@ -827,6 +941,7 @@ where
EB: EngineApiBuilder<N>,
EVB: EngineValidatorBuilder<N>,
RpcMiddleware: RethRpcMiddleware,
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
{
/// Launches only the regular RPC server (HTTP/WS/IPC), without the authenticated Engine API
/// server.
@@ -913,6 +1028,7 @@ where
F: FnOnce(RpcModuleContainer<'_, N, EthB::EthApi>) -> eyre::Result<()>,
{
let rpc_middleware = self.rpc_middleware.clone();
let auth_http_middleware = self.auth_http_middleware.clone();
let tokio_runtime = self.tokio_runtime.clone();
let setup_ctx = self.setup_rpc_components(ctx, ext).await?;
let RpcSetupContext {
@@ -933,6 +1049,8 @@ where
.set_rpc_middleware(rpc_middleware)
.with_tokio_runtime(tokio_runtime);
let auth_config = auth_config.with_http_middleware(auth_http_middleware);
let (rpc, auth) = if disable_auth {
// Only launch the RPC server, use a noop auth handle
let rpc = Self::launch_rpc_server_internal(server_config, &modules).await?;
@@ -942,7 +1060,7 @@ where
// launch servers concurrently
let (rpc, auth) = futures::future::try_join(
Self::launch_rpc_server_internal(server_config, &modules),
Self::launch_auth_server_internal(auth_module_clone, auth_config),
Self::launch_auth_server_internal(auth_config.start(auth_module_clone)),
)
.await?;
(rpc, auth)
@@ -1087,10 +1205,9 @@ where
/// Helper to launch the auth server
async fn launch_auth_server_internal(
auth_module: AuthRpcModule,
auth_config: reth_rpc_builder::auth::AuthServerConfig,
start_fut: impl Future<Output = Result<AuthServerHandle, reth_rpc_builder::error::RpcError>>,
) -> eyre::Result<AuthServerHandle> {
auth_module.start_server(auth_config)
start_fut
.await
.map_err(Into::into)
.inspect(|handle| {
@@ -1120,8 +1237,8 @@ where
}
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> NodeAddOns<N>
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> NodeAddOns<N>
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents,
<N as FullNodeTypes>::Provider: ChainSpecProvider<ChainSpec: EthereumHardforks>,
@@ -1130,6 +1247,7 @@ where
EB: EngineApiBuilder<N>,
EVB: EngineValidatorBuilder<N>,
RpcMiddleware: RethRpcMiddleware,
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
{
type Handle = RpcHandle<N, EthB::EthApi>;
@@ -1150,8 +1268,8 @@ pub trait RethRpcAddOns<N: FullNodeComponents>:
fn hooks_mut(&mut self) -> &mut RpcHooks<N, Self::EthApi>;
}
impl<N: FullNodeComponents, EthB, EV, EB, Engine, RpcMiddleware> RethRpcAddOns<N>
for RpcAddOns<N, EthB, EV, EB, Engine, RpcMiddleware>
impl<N: FullNodeComponents, EthB, EV, EB, Engine, RpcMiddleware, AuthHttpMiddleware>
RethRpcAddOns<N> for RpcAddOns<N, EthB, EV, EB, Engine, RpcMiddleware, AuthHttpMiddleware>
where
Self: NodeAddOns<N, Handle = RpcHandle<N, EthB::EthApi>>,
EthB: EthApiBuilder<N>,
@@ -1221,8 +1339,8 @@ pub trait EngineValidatorAddOn<Node: FullNodeComponents>: Send {
fn engine_validator_builder(&self) -> Self::ValidatorBuilder;
}
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EngineValidatorAddOn<N>
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> EngineValidatorAddOn<N>
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
where
N: FullNodeComponents,
EthB: EthApiBuilder<N>,
@@ -1230,6 +1348,7 @@ where
EB: EngineApiBuilder<N>,
EVB: EngineValidatorBuilder<N>,
RpcMiddleware: Send,
AuthHttpMiddleware: Send,
{
type ValidatorBuilder = EVB;

View File

@@ -42,6 +42,22 @@ pub struct DevArgs {
)]
pub block_time: Option<Duration>,
/// Time to wait after initiating payload building before resolving.
///
/// Introduces a sleep between `fork_choice_updated` and `resolve_kind` in the
/// local miner, giving the payload job time for multiple rebuild attempts with
/// new transactions from the pool.
///
/// Parses strings using [`humantime::parse_duration`]
/// --dev.payload-wait-time 450ms
#[arg(
long = "dev.payload-wait-time",
help_heading = "Dev testnet",
value_parser = parse_duration,
verbatim_doc_comment
)]
pub payload_wait_time: Option<Duration>,
/// Derive dev accounts from a fixed mnemonic instead of random ones.
#[arg(
long = "dev.mnemonic",
@@ -60,6 +76,7 @@ impl Default for DevArgs {
dev: false,
block_max_transactions: None,
block_time: None,
payload_wait_time: None,
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
}
}
@@ -86,6 +103,7 @@ mod tests {
dev: false,
block_max_transactions: None,
block_time: None,
payload_wait_time: None,
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
}
);
@@ -97,6 +115,7 @@ mod tests {
dev: true,
block_max_transactions: None,
block_time: None,
payload_wait_time: None,
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
}
);
@@ -108,6 +127,7 @@ mod tests {
dev: true,
block_max_transactions: None,
block_time: None,
payload_wait_time: None,
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
}
);
@@ -125,6 +145,7 @@ mod tests {
dev: true,
block_max_transactions: Some(2),
block_time: None,
payload_wait_time: None,
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
}
);
@@ -137,6 +158,7 @@ mod tests {
dev: true,
block_max_transactions: None,
block_time: Some(std::time::Duration::from_secs(1)),
payload_wait_time: None,
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
}
);

View File

@@ -126,9 +126,9 @@ pub enum VersionSpecificValidationError {
/// root after Cancun
#[error("no parent beacon block root post-cancun")]
NoParentBeaconBlockRootPostCancun,
/// Thrown if the pre-V6 `PayloadAttributes` or `ExecutionPayload` contains a block access list
#[error("block access list not before V6")]
BlockAccessListNotSupportedBeforeV6,
/// Thrown if the current engine method version does not support a block access list
#[error("block access list not supported in this engine API version")]
BlockAccessListNotSupported,
/// Thrown if `engine_newPayload` contains no block access list
/// after Amsterdam
#[error("no block access list post-Amsterdam")]
@@ -137,9 +137,9 @@ pub enum VersionSpecificValidationError {
/// before Amsterdam
#[error("block access list pre-Amsterdam")]
HasBlockAccessListPreAmsterdam,
/// Thrown if the pre-V6 `PayloadAttributes` or `ExecutionPayload` contains a slot number
#[error("slot number not before V6")]
SlotNumberNotSupportedBeforeV6,
/// Thrown if the current engine method version does not support a slot number
#[error("slot number not supported in this engine API version")]
SlotNumberNotSupported,
/// Thrown if `engine_newPayload` contains no slot number
/// after Amsterdam
#[error("no slot number post-Amsterdam")]

View File

@@ -62,8 +62,10 @@ pub trait PayloadTypes: Send + Sync + Unpin + core::fmt::Debug + Clone + 'static
/// * If V3, this ensures that the payload timestamp is within the Cancun timestamp.
/// * If V4, this ensures that the payload timestamp is within the Prague timestamp.
/// * If V5, this ensures that the payload timestamp is within the Osaka timestamp.
/// * If V6, this ensures that the payload timestamp is within the Amsterdam timestamp.
///
/// Additionally, it ensures that `engine_getPayloadV4` is not used for an Osaka payload.
/// Additionally, it ensures that `engine_getPayloadV4` is not used for an Osaka payload and that
/// staggered endpoint upgrades reject the next fork once a newer method version is required.
///
/// Otherwise, this will return [`EngineObjectValidationError::UnsupportedFork`].
pub fn validate_payload_timestamp(
@@ -151,12 +153,26 @@ pub fn validate_payload_timestamp(
return Err(EngineObjectValidationError::UnsupportedFork)
}
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
// Staggered endpoint upgrades must reject Amsterdam payloads until the Amsterdam-specific
// method version is used.
if is_amsterdam &&
matches!(
(version, kind),
(EngineApiMessageVersion::V3, MessageValidationKind::PayloadAttributes) |
(EngineApiMessageVersion::V4, MessageValidationKind::Payload) |
(EngineApiMessageVersion::V5, MessageValidationKind::GetPayload)
)
{
return Err(EngineObjectValidationError::UnsupportedFork)
}
// `engine_getPayloadV4` MUST reject payloads with a timestamp >= Osaka.
if version.is_v4() && kind == MessageValidationKind::GetPayload && is_osaka {
return Err(EngineObjectValidationError::UnsupportedFork)
}
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
if version.is_v6() && !is_amsterdam {
// From the Engine API spec:
// <https://github.com/ethereum/execution-apis/blob/15399c2e2f16a5f800bf3f285640357e2c245ad9/src/engine/osaka.md#specification>
@@ -183,16 +199,30 @@ pub fn validate_block_access_list_presence<T: EthereumHardforks>(
has_block_access_list: bool,
) -> Result<(), EngineObjectValidationError> {
let is_amsterdam_active = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
match version {
EngineApiMessageVersion::V1 |
EngineApiMessageVersion::V2 |
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 => {
EngineApiMessageVersion::V4 => {
if has_block_access_list {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::BlockAccessListNotSupportedBeforeV6))
.to_error(VersionSpecificValidationError::BlockAccessListNotSupported))
}
}
EngineApiMessageVersion::V5 => {
if message_validation_kind == MessageValidationKind::Payload {
if is_amsterdam_active && !has_block_access_list {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::NoBlockAccessListPostAmsterdam))
}
if !is_amsterdam_active && has_block_access_list {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::HasBlockAccessListPreAmsterdam))
}
} else if has_block_access_list {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::BlockAccessListNotSupported))
}
}
@@ -224,14 +254,42 @@ pub fn validate_slot_number_presence<T: EthereumHardforks>(
let is_amsterdam_active = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
match version {
EngineApiMessageVersion::V1 |
EngineApiMessageVersion::V2 |
EngineApiMessageVersion::V3 |
EngineApiMessageVersion::V4 |
EngineApiMessageVersion::V5 => {
EngineApiMessageVersion::V1 | EngineApiMessageVersion::V2 | EngineApiMessageVersion::V3 => {
if has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::SlotNumberNotSupportedBeforeV6))
.to_error(VersionSpecificValidationError::SlotNumberNotSupported))
}
}
EngineApiMessageVersion::V4 => {
if message_validation_kind == MessageValidationKind::PayloadAttributes {
if is_amsterdam_active && !has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::NoSlotNumberPostAmsterdam))
}
if !is_amsterdam_active && has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::HasSlotNumberPreAmsterdam))
}
} else if has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::SlotNumberNotSupported))
}
}
EngineApiMessageVersion::V5 => {
if message_validation_kind == MessageValidationKind::Payload {
if is_amsterdam_active && !has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::NoSlotNumberPostAmsterdam))
}
if !is_amsterdam_active && has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::HasSlotNumberPreAmsterdam))
}
} else if has_slot_number {
return Err(message_validation_kind
.to_error(VersionSpecificValidationError::SlotNumberNotSupported))
}
}
@@ -651,6 +709,75 @@ mod tests {
assert_matches!(res, Ok(()));
}
#[test]
fn validate_amsterdam_staggered_version_restrictions() {
let chain_spec = ChainSpecBuilder::mainnet().amsterdam_activated().build();
let res = validate_payload_timestamp(
&chain_spec,
EngineApiMessageVersion::V3,
0,
MessageValidationKind::PayloadAttributes,
);
assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork));
let res = validate_payload_timestamp(
&chain_spec,
EngineApiMessageVersion::V4,
0,
MessageValidationKind::Payload,
);
assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork));
let res = validate_payload_timestamp(
&chain_spec,
EngineApiMessageVersion::V5,
0,
MessageValidationKind::GetPayload,
);
assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork));
let res = validate_payload_timestamp(
&chain_spec,
EngineApiMessageVersion::V6,
0,
MessageValidationKind::GetPayload,
);
assert_matches!(res, Ok(()));
}
#[test]
fn validate_amsterdam_slot_and_bal_presence() {
let chain_spec = ChainSpecBuilder::mainnet().amsterdam_activated().build();
let res = validate_slot_number_presence(
&chain_spec,
EngineApiMessageVersion::V4,
MessageValidationKind::PayloadAttributes,
0,
true,
);
assert_matches!(res, Ok(()));
let res = validate_slot_number_presence(
&chain_spec,
EngineApiMessageVersion::V5,
MessageValidationKind::Payload,
0,
true,
);
assert_matches!(res, Ok(()));
let res = validate_block_access_list_presence(
&chain_spec,
EngineApiMessageVersion::V5,
MessageValidationKind::Payload,
0,
true,
);
assert_matches!(res, Ok(()));
}
#[test]
fn execution_requests_validation() {
assert_matches!(validate_execution_requests(&[]), Ok(()));

View File

@@ -21,6 +21,7 @@ use alloy_rpc_types_eth::{
use alloy_serde::JsonStorageKey;
use jsonrpsee::{core::RpcResult, proc_macros::rpc, RpcModule};
use reth_engine_primitives::EngineTypes;
use serde_json::Value;
/// Helper trait for the engine api server.
///
@@ -392,4 +393,19 @@ pub trait EngineEthApi<TxReq: RpcObject, B: RpcObject, R: RpcObject> {
keys: Vec<JsonStorageKey>,
block_number: Option<BlockId>,
) -> RpcResult<EIP1186AccountProofResponse>;
/// Returns the EIP-7928 block access list for a block by hash.
#[method(name = "getBlockAccessListByBlockHash")]
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Value>>;
/// Returns the EIP-7928 block access list for a block by number.
#[method(name = "getBlockAccessListByBlockNumber")]
async fn block_access_list_by_block_number(
&self,
number: BlockNumberOrTag,
) -> RpcResult<Option<Value>>;
/// Returns the EIP-7928 block access list bytes for a block by number.
#[method(name = "getBlockAccessListRaw")]
async fn block_access_list_raw(&self, block: BlockId) -> RpcResult<Option<Bytes>>;
}

View File

@@ -77,3 +77,4 @@ alloy-rpc-types-engine.workspace = true
serde_json.workspace = true
clap = { workspace = true, features = ["derive"] }
reqwest.workspace = true

View File

@@ -1,12 +1,12 @@
use crate::{
error::{RpcError, ServerKind},
middleware::RethRpcMiddleware,
middleware::{RethAuthHttpMiddleware, RethRpcMiddleware},
};
use http::header::AUTHORIZATION;
use jsonrpsee::{
core::{client::SubscriptionClientT, RegisterMethodError},
http_client::HeaderMap,
server::{AlreadyStoppedError, RpcModule},
server::{AlreadyStoppedError, RpcModule, ServerConfig, ServerConfigBuilder},
ws_client::RpcServiceBuilder,
Methods,
};
@@ -20,12 +20,11 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
use tower::layer::util::Identity;
pub use jsonrpsee::server::ServerBuilder;
use jsonrpsee::server::{ServerConfig, ServerConfigBuilder};
pub use reth_ipc::server::Builder as IpcServerBuilder;
/// Server configuration for the auth server.
#[derive(Debug)]
pub struct AuthServerConfig<RpcMiddleware = Identity> {
pub struct AuthServerConfig<RpcMiddleware = Identity, HttpMiddleware = Identity> {
/// Where the server should listen.
pub(crate) socket_addr: SocketAddr,
/// The secret for the auth layer of the server.
@@ -38,6 +37,8 @@ pub struct AuthServerConfig<RpcMiddleware = Identity> {
pub(crate) ipc_endpoint: Option<String>,
/// Configurable RPC middleware
pub(crate) rpc_middleware: RpcMiddleware,
/// Configurable HTTP transport middleware, applied after JWT authentication.
pub(crate) http_middleware: HttpMiddleware,
}
// === impl AuthServerConfig ===
@@ -48,15 +49,23 @@ impl AuthServerConfig {
AuthServerConfigBuilder::new(secret)
}
}
impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
impl<RpcMiddleware, HttpMiddleware> AuthServerConfig<RpcMiddleware, HttpMiddleware> {
/// Returns the address the server will listen on.
pub const fn address(&self) -> SocketAddr {
self.socket_addr
}
/// Configures the rpc middleware.
pub fn with_rpc_middleware<T>(self, rpc_middleware: T) -> AuthServerConfig<T> {
let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint, .. } = self;
pub fn with_rpc_middleware<T>(self, rpc_middleware: T) -> AuthServerConfig<T, HttpMiddleware> {
let Self {
socket_addr,
secret,
server_config,
ipc_server_config,
ipc_endpoint,
http_middleware,
..
} = self;
AuthServerConfig {
socket_addr,
secret,
@@ -64,13 +73,44 @@ impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
ipc_server_config,
ipc_endpoint,
rpc_middleware,
http_middleware,
}
}
/// Configures the HTTP transport middleware.
///
/// This middleware is applied after JWT authentication and before JSON-RPC parsing,
/// giving access to the raw HTTP request (headers, body, etc.).
pub fn with_http_middleware<T>(self, http_middleware: T) -> AuthServerConfig<RpcMiddleware, T> {
let Self {
socket_addr,
secret,
server_config,
ipc_server_config,
ipc_endpoint,
rpc_middleware,
..
} = self;
AuthServerConfig {
socket_addr,
secret,
server_config,
ipc_server_config,
ipc_endpoint,
rpc_middleware,
http_middleware,
}
}
/// Convenience function to start a server in one step.
///
/// The `HttpMiddleware` type parameter configures additional HTTP transport middleware
/// that runs after JWT authentication. When set to `Identity` (the default), only JWT
/// authentication is applied.
pub async fn start(self, module: AuthRpcModule) -> Result<AuthServerHandle, RpcError>
where
RpcMiddleware: RethRpcMiddleware,
HttpMiddleware: RethAuthHttpMiddleware<RpcMiddleware>,
{
let Self {
socket_addr,
@@ -79,11 +119,13 @@ impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
ipc_server_config,
ipc_endpoint,
rpc_middleware,
http_middleware,
} = self;
// Create auth middleware.
// Create auth middleware with JWT authentication in front of the user-provided
// transport middleware.
let middleware =
tower::ServiceBuilder::new().layer(AuthLayer::new(JwtAuthValidator::new(secret)));
tower::ServiceBuilder::new().layer(AuthHttpLayer::new(secret, http_middleware));
let rpc_middleware = RpcServiceBuilder::default().layer(rpc_middleware);
@@ -117,15 +159,47 @@ impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
}
}
/// A combined tower layer that applies JWT authentication before custom HTTP middleware.
///
/// This composes `AuthLayer<JwtAuthValidator>` around a user-provided `HttpMiddleware` into a
/// single `tower::Layer`. Requests first pass through JWT validation and only authenticated
/// requests are forwarded into the custom middleware.
struct AuthHttpLayer<HttpMiddleware> {
auth_layer: AuthLayer<JwtAuthValidator>,
http_middleware: HttpMiddleware,
}
impl<HttpMiddleware> AuthHttpLayer<HttpMiddleware> {
const fn new(secret: JwtSecret, http_middleware: HttpMiddleware) -> Self {
Self { auth_layer: AuthLayer::new(JwtAuthValidator::new(secret)), http_middleware }
}
}
impl<S, HttpMiddleware> tower::Layer<S> for AuthHttpLayer<HttpMiddleware>
where
HttpMiddleware: tower::Layer<S> + Clone,
AuthLayer<JwtAuthValidator>: tower::Layer<<HttpMiddleware as tower::Layer<S>>::Service>,
{
type Service = <AuthLayer<JwtAuthValidator> as tower::Layer<
<HttpMiddleware as tower::Layer<S>>::Service,
>>::Service;
fn layer(&self, inner: S) -> Self::Service {
let http_service = self.http_middleware.layer(inner);
self.auth_layer.layer(http_service)
}
}
/// Builder type for configuring an `AuthServerConfig`.
#[derive(Debug)]
pub struct AuthServerConfigBuilder<RpcMiddleware = Identity> {
pub struct AuthServerConfigBuilder<RpcMiddleware = Identity, HttpMiddleware = Identity> {
socket_addr: Option<SocketAddr>,
secret: JwtSecret,
server_config: Option<ServerConfigBuilder>,
ipc_server_config: Option<IpcServerBuilder<Identity, Identity>>,
ipc_endpoint: Option<String>,
rpc_middleware: RpcMiddleware,
http_middleware: HttpMiddleware,
}
// === impl AuthServerConfigBuilder ===
@@ -140,14 +214,26 @@ impl AuthServerConfigBuilder {
ipc_server_config: None,
ipc_endpoint: None,
rpc_middleware: Identity::new(),
http_middleware: Identity::new(),
}
}
}
impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
impl<RpcMiddleware, HttpMiddleware> AuthServerConfigBuilder<RpcMiddleware, HttpMiddleware> {
/// Configures the rpc middleware.
pub fn with_rpc_middleware<T>(self, rpc_middleware: T) -> AuthServerConfigBuilder<T> {
let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint, .. } = self;
pub fn with_rpc_middleware<T>(
self,
rpc_middleware: T,
) -> AuthServerConfigBuilder<T, HttpMiddleware> {
let Self {
socket_addr,
secret,
server_config,
ipc_server_config,
ipc_endpoint,
http_middleware,
..
} = self;
AuthServerConfigBuilder {
socket_addr,
secret,
@@ -155,6 +241,35 @@ impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
ipc_server_config,
ipc_endpoint,
rpc_middleware,
http_middleware,
}
}
/// Configures the HTTP transport middleware.
///
/// This middleware is applied after JWT authentication and before JSON-RPC parsing,
/// giving access to the raw HTTP request (headers, body, etc.).
pub fn with_http_middleware<T>(
self,
http_middleware: T,
) -> AuthServerConfigBuilder<RpcMiddleware, T> {
let Self {
socket_addr,
secret,
server_config,
ipc_server_config,
ipc_endpoint,
rpc_middleware,
..
} = self;
AuthServerConfigBuilder {
socket_addr,
secret,
server_config,
ipc_server_config,
ipc_endpoint,
rpc_middleware,
http_middleware,
}
}
@@ -200,7 +315,7 @@ impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
}
/// Build the `AuthServerConfig`.
pub fn build(self) -> AuthServerConfig<RpcMiddleware> {
pub fn build(self) -> AuthServerConfig<RpcMiddleware, HttpMiddleware> {
AuthServerConfig {
socket_addr: self.socket_addr.unwrap_or_else(|| {
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), constants::DEFAULT_AUTH_PORT)
@@ -233,6 +348,7 @@ impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
}),
ipc_endpoint: self.ipc_endpoint,
rpc_middleware: self.rpc_middleware,
http_middleware: self.http_middleware,
}
}
}
@@ -289,10 +405,14 @@ impl AuthRpcModule {
}
/// Convenience function for starting a server
pub async fn start_server(
pub async fn start_server<RpcMiddleware, HttpMiddleware>(
self,
config: AuthServerConfig,
) -> Result<AuthServerHandle, RpcError> {
config: AuthServerConfig<RpcMiddleware, HttpMiddleware>,
) -> Result<AuthServerHandle, RpcError>
where
RpcMiddleware: RethRpcMiddleware,
HttpMiddleware: RethAuthHttpMiddleware<RpcMiddleware>,
{
config.start(self).await
}
}
@@ -397,7 +517,7 @@ impl AuthServerHandle {
.build(ipc_endpoint)
.await
.expect("Failed to create ipc client"),
)
);
}
None
}

View File

@@ -1,5 +1,10 @@
use jsonrpsee::server::middleware::rpc::RpcService;
use tower::Layer;
use jsonrpsee::server::{
middleware::rpc::RpcService, HttpRequest, HttpResponse, TowerServiceNoHttp,
};
use tower::{
layer::util::{Identity, Stack},
Layer,
};
/// A Helper alias trait for the RPC middleware supported by the server.
pub trait RethRpcMiddleware:
@@ -35,3 +40,39 @@ impl<T> RethRpcMiddleware for T where
+ 'static
{
}
/// Inner HTTP transport service type for auth-server middleware.
pub type AuthHttpService<RM> = TowerServiceNoHttp<Stack<RM, Identity>>;
/// Helper alias trait for auth-server HTTP transport middleware layers.
pub trait RethAuthHttpMiddleware<RM>:
tower::Layer<
AuthHttpService<RM>,
Service: tower::Service<
HttpRequest,
Response = HttpResponse,
Error = tower::BoxError,
Future: Send,
> + Send
+ Clone,
> + Clone
+ Send
+ 'static
{
}
impl<T, RM> RethAuthHttpMiddleware<RM> for T where
T: tower::Layer<
AuthHttpService<RM>,
Service: tower::Service<
HttpRequest,
Response = HttpResponse,
Error = tower::BoxError,
Future: Send,
> + Send
+ Clone,
> + Clone
+ Send
+ 'static
{
}

View File

@@ -1,16 +1,90 @@
//! Auth server tests
use crate::utils::launch_auth;
use crate::utils::{launch_auth, launch_auth_with_config, test_address};
use alloy_primitives::U64;
use alloy_rpc_types_engine::{
ExecutionPayloadInputV2, ExecutionPayloadV1, ForkchoiceState, PayloadId,
};
use jsonrpsee::core::client::{ClientT, SubscriptionClientT};
use http::header::{AUTHORIZATION, CONTENT_TYPE};
use jsonrpsee::{
core::client::{ClientT, SubscriptionClientT},
server::{HttpRequest, HttpResponse},
};
use reth_ethereum_engine_primitives::EthEngineTypes;
use reth_ethereum_primitives::{Block, TransactionSigned};
use reth_primitives_traits::block::Block as _;
use reth_rpc_api::clients::EngineApiClient;
use reth_rpc_layer::JwtSecret;
use reth_rpc_builder::auth::AuthServerConfig;
use reth_rpc_layer::{secret_to_bearer_header, JwtSecret};
use std::{
sync::{
atomic::{AtomicUsize, Ordering},
Arc, Mutex,
},
task::{Context, Poll},
};
use tower::{Layer, Service};
#[derive(Clone, Default)]
struct CountingAuthHttpLayer {
count: Arc<AtomicUsize>,
content_types: Arc<Mutex<Vec<Option<String>>>>,
}
impl CountingAuthHttpLayer {
fn count(&self) -> usize {
self.count.load(Ordering::Relaxed)
}
fn seen_content_types(&self) -> Vec<Option<String>> {
self.content_types.lock().unwrap().clone()
}
}
impl<S> Layer<S> for CountingAuthHttpLayer {
type Service = CountingAuthHttpService<S>;
fn layer(&self, inner: S) -> Self::Service {
CountingAuthHttpService {
inner,
count: self.count.clone(),
content_types: self.content_types.clone(),
}
}
}
#[derive(Clone)]
struct CountingAuthHttpService<S> {
inner: S,
count: Arc<AtomicUsize>,
content_types: Arc<Mutex<Vec<Option<String>>>>,
}
impl<S> Service<HttpRequest> for CountingAuthHttpService<S>
where
S: Service<HttpRequest, Response = HttpResponse, Error = tower::BoxError> + Send + Clone,
S::Future: Send + 'static,
{
type Response = HttpResponse;
type Error = tower::BoxError;
type Future = S::Future;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, request: HttpRequest) -> Self::Future {
self.count.fetch_add(1, Ordering::Relaxed);
self.content_types.lock().unwrap().push(
request
.headers()
.get(CONTENT_TYPE)
.and_then(|value| value.to_str().ok())
.map(str::to_owned),
);
self.inner.call(request)
}
}
#[expect(unused_must_use)]
async fn test_basic_engine_calls<C>(client: &C)
@@ -58,3 +132,55 @@ async fn test_auth_endpoints_ws() {
let client = handle.ws_client().await;
test_basic_engine_calls(&client).await
}
#[tokio::test(flavor = "multi_thread")]
async fn test_auth_http_middleware_runs_only_after_jwt() {
reth_tracing::init_test_tracing();
let secret = JwtSecret::random();
let layer = CountingAuthHttpLayer::default();
let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build();
let handle = launch_auth_with_config(config.with_http_middleware(layer.clone())).await;
let response = reqwest::Client::new()
.post(handle.http_url())
.header(CONTENT_TYPE, "application/json")
.body(
serde_json::to_vec(&serde_json::json!({
"jsonrpc": "2.0",
"method": "engine_exchangeCapabilities",
"params": [[]],
"id": 1
}))
.unwrap(),
)
.send()
.await
.unwrap();
assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED);
assert_eq!(layer.count(), 0);
}
#[tokio::test(flavor = "multi_thread")]
async fn test_auth_http_middleware_sees_transport_headers_before_json_rpc_parsing() {
reth_tracing::init_test_tracing();
let secret = JwtSecret::random();
let layer = CountingAuthHttpLayer::default();
let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build();
let handle = launch_auth_with_config(config.with_http_middleware(layer.clone())).await;
let response = reqwest::Client::new()
.post(handle.http_url())
.header(AUTHORIZATION, secret_to_bearer_header(&secret))
.header(CONTENT_TYPE, "application/ssz")
.body("not-json")
.send()
.await
.unwrap();
assert!(response.status().is_success() || response.status().is_client_error());
assert_eq!(layer.count(), 1);
assert_eq!(layer.seen_content_types(), vec![Some("application/ssz".to_owned())]);
}

View File

@@ -14,6 +14,7 @@ use reth_payload_builder::test_utils::spawn_test_payload_service;
use reth_provider::test_utils::NoopProvider;
use reth_rpc_builder::{
auth::{AuthRpcModule, AuthServerConfig, AuthServerHandle},
middleware::{RethAuthHttpMiddleware, RethRpcMiddleware},
RpcModuleBuilder, RpcServerConfig, RpcServerHandle, TransportRpcModuleConfig,
};
use reth_rpc_engine_api::{capabilities::EngineCapabilities, EngineApi};
@@ -28,12 +29,23 @@ use tokio::sync::mpsc::unbounded_channel;
/// Localhost with port 0 so a free port is used.
pub const fn test_address() -> SocketAddr {
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0))
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))
}
/// Launches a new server for the auth module
pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle {
let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build();
launch_auth_with_config(config).await
}
/// Launches a new server for the auth module with the given config.
pub async fn launch_auth_with_config<RpcMiddleware, HttpMiddleware>(
config: AuthServerConfig<RpcMiddleware, HttpMiddleware>,
) -> AuthServerHandle
where
RpcMiddleware: RethRpcMiddleware,
HttpMiddleware: RethAuthHttpMiddleware<RpcMiddleware>,
{
let (tx, _rx) = unbounded_channel();
let beacon_engine_handle = ConsensusEngineHandle::<EthEngineTypes>::new(tx);
let client = ClientVersionV1 {

View File

@@ -479,7 +479,6 @@ impl<Network, Evm, Receipt, Header, Map, SimTx, RpcTx, TxEnv>
evm,
sim_tx_converter,
rpc_tx_converter,
tx_env_converter: _,
..
} = self;
RpcConverter {

View File

@@ -269,7 +269,7 @@ where
>::from_execution_payload(&payload);
self.inner
.validator
.validate_version_specific_fields(EngineApiMessageVersion::V6, payload_or_attrs)?;
.validate_version_specific_fields(EngineApiMessageVersion::V5, payload_or_attrs)?;
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
}
@@ -386,7 +386,7 @@ where
state: ForkchoiceState,
payload_attrs: Option<EngineT::PayloadAttributes>,
) -> EngineApiResult<ForkchoiceUpdated> {
self.validate_and_execute_forkchoice(EngineApiMessageVersion::V6, state, payload_attrs)
self.validate_and_execute_forkchoice(EngineApiMessageVersion::V4, state, payload_attrs)
.await
}
@@ -1438,9 +1438,10 @@ struct EngineApiInner<Provider, PayloadT: PayloadTypes, Pool, Validator, ChainSp
#[cfg(test)]
mod tests {
use super::*;
use alloy_primitives::{Address, B256};
use alloy_eips::eip7685::Requests;
use alloy_primitives::{Address, Bytes, B256};
use alloy_rpc_types_engine::{
ClientCode, ClientVersionV1, PayloadAttributes, PayloadStatusEnum,
ClientCode, ClientVersionV1, ExecutionPayloadV2, PayloadAttributes, PayloadStatusEnum,
};
use assert_matches::assert_matches;
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
@@ -1532,6 +1533,63 @@ mod tests {
assert_matches!(handle.from_api.recv().await, Some(BeaconEngineMessage::NewPayload { .. }));
}
#[tokio::test]
async fn new_payload_v5_accepts_amsterdam_payloads() {
let chain_spec = Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().build());
let provider = Arc::new(MockEthProvider::default());
let payload_store = spawn_test_payload_service::<EthEngineTypes>();
let (to_engine, mut engine_rx) = unbounded_channel();
let api = EngineApi::new(
provider,
chain_spec.clone(),
ConsensusEngineHandle::new(to_engine),
payload_store.into(),
NoopTransactionPool::default(),
Runtime::test(),
ClientVersionV1 {
code: ClientCode::RH,
name: "Reth".to_string(),
version: "v0.0.0-test".to_string(),
commit: "test".to_string(),
},
EngineCapabilities::default(),
EthereumEngineValidator::new(chain_spec),
false,
NoopNetwork::default(),
);
tokio::spawn(async move {
let payload_v1 = ExecutionPayloadV1::from_block_slow(&Block::default());
let payload = ExecutionPayloadV4 {
payload_inner: ExecutionPayloadV3 {
payload_inner: ExecutionPayloadV2 {
payload_inner: payload_v1,
withdrawals: Vec::new(),
},
blob_gas_used: 0,
excess_blob_gas: 0,
},
block_access_list: Bytes::from_static(b"bal"),
slot_number: 1,
};
let execution_data = ExecutionData {
payload: payload.into(),
sidecar: ExecutionPayloadSidecar::v4(
CancunPayloadFields {
versioned_hashes: Vec::new(),
parent_beacon_block_root: B256::ZERO,
},
PraguePayloadFields { requests: RequestsOrHash::Requests(Requests::default()) },
),
};
api.new_payload_v5(execution_data).await.unwrap();
});
assert_matches!(engine_rx.recv().await, Some(BeaconEngineMessage::NewPayload { .. }));
}
#[derive(Clone)]
struct TestNetworkInfo {
syncing: bool,

View File

@@ -121,12 +121,12 @@ impl From<EngineApiError> for jsonrpsee_types::error::ErrorObject<'static> {
VersionSpecificValidationError::WithdrawalsNotSupportedInV1 |
VersionSpecificValidationError::NoWithdrawalsPostShanghai |
VersionSpecificValidationError::HasWithdrawalsPreShanghai |
VersionSpecificValidationError::BlockAccessListNotSupportedBeforeV6 |
VersionSpecificValidationError::BlockAccessListNotSupported |
VersionSpecificValidationError::HasBlockAccessListPreAmsterdam |
VersionSpecificValidationError::NoBlockAccessListPostAmsterdam |
VersionSpecificValidationError::HasSlotNumberPreAmsterdam |
VersionSpecificValidationError::NoSlotNumberPostAmsterdam |
VersionSpecificValidationError::SlotNumberNotSupportedBeforeV6,
VersionSpecificValidationError::SlotNumberNotSupported,
),
) |
EngineApiError::UnexpectedRequestsHash => {

View File

@@ -419,7 +419,7 @@ pub trait EthApi<
/// Returns the EIP-7928 block access list bytes for a block by number.
#[method(name = "getBlockAccessListRaw")]
async fn block_access_list_raw(&self, number: BlockNumberOrTag) -> RpcResult<Option<Bytes>>;
async fn block_access_list_raw(&self, block: BlockId) -> RpcResult<Option<Bytes>>;
}
#[async_trait::async_trait]
@@ -942,10 +942,10 @@ where
Ok(Some(json))
}
/// Handler for: `eth_getBlockAccessListRaw`
async fn block_access_list_raw(&self, number: BlockNumberOrTag) -> RpcResult<Option<Bytes>> {
trace!(target: "rpc::eth", ?number, "Serving eth_getBlockAccessListRaw");
async fn block_access_list_raw(&self, block: BlockId) -> RpcResult<Option<Bytes>> {
trace!(target: "rpc::eth", ?block, "Serving eth_getBlockAccessListRaw");
let bal = self.get_block_access_list(number.into()).await?;
let bal = self.get_block_access_list(block).await?;
Ok(bal.map(|b: BlockAccessList| alloy_rlp::encode(b).into()))
}
}

View File

@@ -25,7 +25,7 @@ use reth_rpc_eth_types::{
use reth_rpc_server_types::constants::gas_oracle::{CALL_STIPEND_GAS, ESTIMATE_GAS_ERROR_RATIO};
use revm::{
context::Block,
context_interface::{result::ExecutionResult, Transaction},
context_interface::{result::ExecutionResult, Cfg, Transaction},
primitives::KECCAK_EMPTY,
};
use tracing::trace;
@@ -74,10 +74,15 @@ pub trait EstimateCall: Call {
let tx_request_gas_limit = request.as_ref().gas_limit();
let tx_request_gas_price = request.as_ref().gas_price();
// the gas limit of the corresponding block
let max_gas_limit = evm_env.cfg_env.tx_gas_limit_cap.map_or_else(
|| evm_env.block_env.gas_limit(),
|cap| cap.min(evm_env.block_env.gas_limit()),
);
let max_gas_limit = evm_env
.cfg_env
.tx_gas_limit_cap
// If EIP-8037 is enabled, the transaction gas limit cap is not applicable
.filter(|_| !evm_env.cfg_env.is_amsterdam_eip8037_enabled())
.map_or_else(
|| evm_env.block_env.gas_limit(),
|cap| cap.min(evm_env.block_env.gas_limit()),
);
// Determine the highest possible gas limit, considering both the request's specified limit
// and the block's limit.

View File

@@ -377,7 +377,7 @@ pub trait LoadPendingBlock:
}
}
let BlockBuilderOutcome { execution_result, block, hashed_state, trie_updates } =
let BlockBuilderOutcome { execution_result, block, hashed_state, trie_updates, .. } =
builder.finish(NoopProvider::default(), None).map_err(Self::Error::from_eth_err)?;
let execution_outcome =

View File

@@ -13,6 +13,7 @@ pub use reth_rpc_engine_api::EngineApi;
use reth_rpc_eth_api::{
EngineEthFilter, FullEthApiTypes, QueryLimits, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction,
};
use serde_json::Value;
use tracing_futures::Instrument;
macro_rules! engine_span {
@@ -145,4 +146,22 @@ where
) -> Result<EIP1186AccountProofResponse> {
self.eth.get_proof(address, keys, block_number).instrument(engine_span!()).await
}
/// Handler for `eth_getBlockAccessListByBlockHash`
async fn block_access_list_by_block_hash(&self, hash: B256) -> Result<Option<Value>> {
self.eth.block_access_list_by_block_hash(hash).instrument(engine_span!()).await
}
/// Handler for `eth_getBlockAccessListByBlockNumber`
async fn block_access_list_by_block_number(
&self,
block_number: BlockNumberOrTag,
) -> Result<Option<Value>> {
self.eth.block_access_list_by_block_number(block_number).instrument(engine_span!()).await
}
/// Handler for `getBlockAccessListRaw`
async fn block_access_list_raw(&self, block: BlockId) -> Result<Option<Bytes>> {
self.eth.block_access_list_raw(block).instrument(engine_span!()).await
}
}

View File

@@ -38,6 +38,13 @@ use std::{
};
use tracing::instrument;
/// Returns [`WriteOptions`] with WAL sync enabled for crash durability.
fn synced_write_options() -> WriteOptions {
let mut opts = WriteOptions::default();
opts.set_sync(true);
opts
}
/// Pending `RocksDB` batches type alias.
pub(crate) type PendingRocksDBBatches = Arc<Mutex<Vec<WriteBatchWithTransaction<true>>>>;
@@ -765,7 +772,7 @@ impl RocksDBProvider {
/// # Panics
/// Panics if the provider is in read-only mode.
pub fn tx(&self) -> RocksTx<'_> {
let write_options = WriteOptions::default();
let write_options = synced_write_options();
let txn_options = OptimisticTransactionOptions::default();
let inner = self.0.db_rw().transaction_opt(&write_options, &txn_options);
RocksTx { inner, provider: self }
@@ -845,6 +852,19 @@ impl RocksDBProvider {
})
}
/// Gets raw bytes from the specified table without decompressing.
pub fn get_raw<T: Table>(&self, key: T::Key) -> ProviderResult<Option<Vec<u8>>> {
let encoded = key.encode();
self.execute_with_operation_metric(RocksDBOperation::Get, T::NAME, |this| {
this.0.get_cf(this.get_cf_handle::<T>()?, encoded.as_ref()).map_err(|e| {
ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
message: e.to_string().into(),
code: -1,
}))
})
})
}
/// Puts upsert a value into the specified table with the given key.
///
/// # Panics
@@ -963,6 +983,18 @@ impl RocksDBProvider {
Ok(RocksDBIter { inner: iter, _marker: std::marker::PhantomData })
}
/// Creates an iterator starting from the given key (inclusive, seek forward).
///
/// Returns decoded `(Key, Value)` pairs starting from the first key >= `key`.
pub fn iter_from<T: Table>(&self, key: T::Key) -> ProviderResult<RocksDBIter<'_, T>> {
let cf = self.get_cf_handle::<T>()?;
let encoded_key = key.encode();
let iter = self
.0
.iterator_cf(cf, IteratorMode::From(encoded_key.as_ref(), rocksdb::Direction::Forward));
Ok(RocksDBIter { inner: iter, _marker: std::marker::PhantomData })
}
/// Returns statistics for all column families in the database.
///
/// Returns a vector of (`table_name`, `estimated_keys`, `estimated_size_bytes`) tuples.
@@ -1243,7 +1275,7 @@ impl RocksDBProvider {
/// Panics if the provider is in read-only mode.
#[instrument(level = "debug", target = "providers::rocksdb", skip_all, fields(batch_len = batch.len(), batch_size = batch.size_in_bytes()))]
pub fn commit_batch(&self, batch: WriteBatchWithTransaction<true>) -> ProviderResult<()> {
self.0.db_rw().write_opt(batch, &WriteOptions::default()).map_err(|e| {
self.0.db_rw().write_opt(batch, &synced_write_options()).map_err(|e| {
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
message: e.to_string().into(),
code: -1,
@@ -1726,14 +1758,12 @@ impl<'a> RocksDBBatch<'a> {
"Auto-committing RocksDB batch"
);
let old_batch = std::mem::take(&mut self.inner);
self.provider.0.db_rw().write_opt(old_batch, &WriteOptions::default()).map_err(
|e| {
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
message: e.to_string().into(),
code: -1,
}))
},
)?;
self.provider.0.db_rw().write_opt(old_batch, &synced_write_options()).map_err(|e| {
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
message: e.to_string().into(),
code: -1,
}))
})?;
}
Ok(())
}
@@ -1746,7 +1776,7 @@ impl<'a> RocksDBBatch<'a> {
/// Panics if the provider is in read-only mode.
#[instrument(level = "debug", target = "providers::rocksdb", skip_all, fields(batch_len = self.inner.len(), batch_size = self.inner.size_in_bytes()))]
pub fn commit(self) -> ProviderResult<()> {
self.provider.0.db_rw().write_opt(self.inner, &WriteOptions::default()).map_err(|e| {
self.provider.0.db_rw().write_opt(self.inner, &synced_write_options()).map_err(|e| {
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
message: e.to_string().into(),
code: -1,

View File

@@ -408,8 +408,7 @@ where
/// cached value then this calculates the [`Overlay`] and populates the cache.
#[instrument(level = "debug", target = "providers::state::overlay", skip_all)]
fn get_overlay(&self, provider: &F::Provider) -> ProviderResult<Overlay> {
// If we have no anchor block configured then we will never need to get trie reverts, just
// return the in-memory overlay (resolving lazy overlay if set).
// No anchor block — just resolve the in-memory overlay directly.
if self.block_hash.is_none() {
let (trie_updates, hashed_post_state) = self.resolve_overlays();
return Ok(Overlay { trie_updates, hashed_post_state })
@@ -417,28 +416,16 @@ where
let db_tip_block = self.get_db_tip_block_number(provider)?;
// If the overlay is present in the cache then return it directly.
if let Some(entry) = self.overlay_cache.get(&db_tip_block) {
return Ok(entry.value().clone());
}
// If the overlay is not present then we need to calculate a new one.
// DashMap's entry API handles the race condition internally.
let mut cache_miss = false;
let overlay = match self.overlay_cache.entry(db_tip_block) {
dashmap::Entry::Occupied(entry) => entry.get().clone(),
dashmap::Entry::Vacant(entry) => {
cache_miss = true;
self.metrics.overlay_cache_misses.increment(1);
let overlay = self.calculate_overlay(provider, db_tip_block)?;
entry.insert(overlay.clone());
overlay
}
};
if cache_miss {
self.metrics.overlay_cache_misses.increment(1);
}
Ok(overlay)
}
}

View File

@@ -8,7 +8,7 @@ use std::{
pin::Pin,
sync::{
atomic::{AtomicUsize, Ordering},
Arc,
Arc, OnceLock,
},
task::{ready, Context, Poll},
thread,
@@ -168,34 +168,50 @@ thread_local! {
///
/// The pool supports multiple init/clear cycles, allowing reuse of the same threads with
/// different state configurations.
///
/// The underlying rayon pool is created lazily on first access.
#[derive(Debug)]
pub struct WorkerPool {
pool: rayon::ThreadPool,
pool: OnceLock<rayon::ThreadPool>,
num_threads: usize,
thread_name_prefix: &'static str,
}
impl WorkerPool {
/// Creates a new `WorkerPool` with the given number of threads.
pub fn new(num_threads: usize) -> Result<Self, rayon::ThreadPoolBuildError> {
Self::from_builder(rayon::ThreadPoolBuilder::new().num_threads(num_threads))
/// Creates a new lazy `WorkerPool` with the given number of threads and a thread name prefix.
///
/// The underlying rayon pool is not created until the first method that requires it is called.
/// Thread names follow the pattern `"{prefix}-{index:02}"`.
pub const fn new(num_threads: usize, thread_name_prefix: &'static str) -> Self {
Self { pool: OnceLock::new(), num_threads, thread_name_prefix }
}
/// Creates a new `WorkerPool` from a [`rayon::ThreadPoolBuilder`].
///
/// Installs a panic handler that logs panics instead of aborting the process.
pub fn from_builder(
builder: rayon::ThreadPoolBuilder,
) -> Result<Self, rayon::ThreadPoolBuildError> {
Ok(Self { pool: build_pool_with_panic_handler(builder)? })
/// Returns a reference to the underlying rayon pool, creating it on first access.
fn pool(&self) -> &rayon::ThreadPool {
self.pool.get_or_init(|| {
let prefix = self.thread_name_prefix;
build_pool_with_panic_handler(
rayon::ThreadPoolBuilder::new()
.num_threads(self.num_threads)
.thread_name(move |i| format!("{prefix}-{i:02}")),
)
.unwrap_or_else(|err| panic!("failed to build {prefix} worker pool: {err}"))
})
}
/// Returns `true` if the underlying rayon pool has been initialized.
pub fn is_initialized(&self) -> bool {
self.pool.get().is_some()
}
/// Returns the total number of threads in the underlying rayon pool.
pub fn current_num_threads(&self) -> usize {
self.pool.current_num_threads()
self.pool().current_num_threads()
}
/// Initializes per-thread [`Worker`] state on every thread in the pool.
pub fn init<T: 'static>(&self, f: impl Fn(Option<&mut T>) -> T + Sync) {
self.broadcast(self.pool.current_num_threads(), |worker| {
self.broadcast(self.pool().current_num_threads(), |worker| {
worker.init::<T>(&f);
});
}
@@ -206,14 +222,14 @@ impl WorkerPool {
/// Use this to initialize or re-initialize per-thread state via [`Worker::init`].
/// Only `num_threads` threads execute the closure; the rest skip it.
pub fn broadcast(&self, num_threads: usize, f: impl Fn(&mut Worker) + Sync) {
if num_threads >= self.pool.current_num_threads() {
if num_threads >= self.pool().current_num_threads() {
// Fast path: run on every thread, no atomic coordination needed.
self.pool.broadcast(|_| {
self.pool().broadcast(|_| {
WORKER.with_borrow_mut(|worker| f(worker));
});
} else {
let remaining = AtomicUsize::new(num_threads);
self.pool.broadcast(|_| {
self.pool().broadcast(|_| {
// Atomically claim a slot; threads that can't decrement skip the closure.
let mut current = remaining.load(Ordering::Relaxed);
loop {
@@ -237,7 +253,7 @@ impl WorkerPool {
/// Clears the state on every thread in the pool.
pub fn clear(&self) {
self.pool.broadcast(|_| {
self.pool().broadcast(|_| {
WORKER.with_borrow_mut(Worker::clear);
});
}
@@ -248,7 +264,7 @@ impl WorkerPool {
/// Each thread can access its own [`Worker`] via the provided reference or through additional
/// [`WorkerPool::with_worker`] calls.
pub fn install<R: Send>(&self, f: impl FnOnce(&Worker) -> R + Send) -> R {
self.pool.install(|| WORKER.with_borrow(|worker| f(worker)))
self.pool().install(|| WORKER.with_borrow(|worker| f(worker)))
}
/// Runs a closure on the pool without worker state access.
@@ -256,19 +272,19 @@ impl WorkerPool {
/// Like [`install`](Self::install) but for closures that don't need per-thread [`Worker`]
/// state.
pub fn install_fn<R: Send>(&self, f: impl FnOnce() -> R + Send) -> R {
self.pool.install(f)
self.pool().install(f)
}
/// Spawns a closure on the pool.
pub fn spawn(&self, f: impl FnOnce() + Send + 'static) {
self.pool.spawn(f);
self.pool().spawn(f);
}
/// Executes `f` on this pool using [`rayon::in_place_scope`], which converts the calling
/// thread into a worker for the duration — tasks spawned inside the scope run on the pool
/// and the call blocks until all of them complete.
pub fn in_place_scope<'scope, R>(&self, f: impl FnOnce(&rayon::Scope<'scope>) -> R) -> R {
self.pool.in_place_scope(f)
self.pool().in_place_scope(f)
}
/// Access the current thread's [`Worker`] from within an [`install`](Self::install) closure.
@@ -398,7 +414,7 @@ mod tests {
#[test]
fn worker_pool_init_and_access() {
let pool = WorkerPool::new(2).unwrap();
let pool = WorkerPool::new(2, "test");
pool.broadcast(2, |worker| {
worker.init::<Vec<u8>>(|_| vec![1, 2, 3]);
@@ -415,7 +431,7 @@ mod tests {
#[test]
fn worker_pool_reinit_reuses_resources() {
let pool = WorkerPool::new(1).unwrap();
let pool = WorkerPool::new(1, "test");
pool.broadcast(1, |worker| {
worker.init::<Vec<u8>>(|existing| {
@@ -441,7 +457,7 @@ mod tests {
#[test]
fn worker_pool_clear_and_reinit() {
let pool = WorkerPool::new(1).unwrap();
let pool = WorkerPool::new(1, "test");
pool.broadcast(1, |worker| {
worker.init::<u64>(|_| 42);
@@ -464,7 +480,7 @@ mod tests {
fn worker_pool_par_iter_with_worker() {
use rayon::prelude::*;
let pool = WorkerPool::new(2).unwrap();
let pool = WorkerPool::new(2, "test");
pool.broadcast(2, |worker| {
worker.init::<u64>(|_| 10);

View File

@@ -16,8 +16,6 @@ use crate::{
};
use futures_util::{future::select, Future, FutureExt, TryFutureExt};
#[cfg(feature = "rayon")]
use std::sync::OnceLock;
#[cfg(feature = "rayon")]
use std::{num::NonZeroUsize, thread::available_parallelism};
use std::{
pin::pin,
@@ -237,34 +235,6 @@ pub enum RuntimeBuildError {
RayonBuild(#[from] rayon::ThreadPoolBuildError),
}
#[cfg(feature = "rayon")]
#[derive(Debug)]
struct LazyWorkerPool {
pool: OnceLock<WorkerPool>,
num_threads: usize,
thread_name_prefix: &'static str,
}
#[cfg(feature = "rayon")]
impl LazyWorkerPool {
const fn new(num_threads: usize, thread_name_prefix: &'static str) -> Self {
Self { pool: OnceLock::new(), num_threads, thread_name_prefix }
}
fn get(&self) -> &WorkerPool {
let num_threads = self.num_threads;
let thread_name_prefix = self.thread_name_prefix;
self.pool.get_or_init(|| {
WorkerPool::from_builder(
rayon::ThreadPoolBuilder::new()
.num_threads(num_threads)
.thread_name(move |i| format!("{thread_name_prefix}-{i:02}")),
)
.unwrap_or_else(|err| panic!("failed to build {thread_name_prefix} worker pool: {err}"))
})
}
}
// ── RuntimeInner ──────────────────────────────────────────────────────
struct RuntimeInner {
@@ -303,7 +273,7 @@ struct RuntimeInner {
prewarming_pool: WorkerPool,
/// BAL streaming pool (BAL hashed state streaming).
#[cfg(feature = "rayon")]
bal_streaming_pool: LazyWorkerPool,
bal_streaming_pool: WorkerPool,
/// Named single-thread worker map. Each unique name gets a dedicated OS thread
/// that is reused across all tasks submitted under that name.
worker_map: WorkerMap,
@@ -392,7 +362,7 @@ impl Runtime {
/// Get the BAL streaming pool.
#[cfg(feature = "rayon")]
pub fn bal_streaming_pool(&self) -> &WorkerPool {
self.0.bal_streaming_pool.get()
&self.0.bal_streaming_pool
}
}
@@ -837,30 +807,20 @@ impl RuntimeBuilder {
let proof_storage_worker_threads =
config.rayon.proof_storage_worker_threads.unwrap_or(default_threads * 2);
let proof_storage_worker_pool = WorkerPool::from_builder(
rayon::ThreadPoolBuilder::new()
.num_threads(proof_storage_worker_threads)
.thread_name(|i| format!("proof-strg-{i:02}")),
)?;
let proof_storage_worker_pool =
WorkerPool::new(proof_storage_worker_threads, "proof-strg");
let proof_account_worker_threads =
config.rayon.proof_account_worker_threads.unwrap_or(default_threads * 2);
let proof_account_worker_pool = WorkerPool::from_builder(
rayon::ThreadPoolBuilder::new()
.num_threads(proof_account_worker_threads)
.thread_name(|i| format!("proof-acct-{i:02}")),
)?;
let proof_account_worker_pool =
WorkerPool::new(proof_account_worker_threads, "proof-acct");
let prewarming_threads = config.rayon.prewarming_threads.unwrap_or(default_threads);
let prewarming_pool = WorkerPool::from_builder(
rayon::ThreadPoolBuilder::new()
.num_threads(prewarming_threads)
.thread_name(|i| format!("prewarm-{i:02}")),
)?;
let prewarming_pool = WorkerPool::new(prewarming_threads, "prewarm");
let bal_streaming_threads =
config.rayon.bal_streaming_threads.unwrap_or(default_threads);
let bal_streaming_pool = LazyWorkerPool::new(bal_streaming_threads, "bal-stream");
let bal_streaming_pool = WorkerPool::new(bal_streaming_threads, "bal-stream");
debug!(
default_threads,
@@ -871,7 +831,7 @@ impl RuntimeBuilder {
prewarming_threads,
bal_streaming_threads,
max_blocking_tasks = config.rayon.max_blocking_tasks,
"Initialized rayon thread pools and configured lazy BAL streaming pool"
"Configured lazy rayon worker pools"
);
(
@@ -962,12 +922,19 @@ mod tests {
#[cfg(feature = "rayon")]
#[test]
fn test_bal_streaming_pool_is_lazy() {
fn test_worker_pools_are_lazy() {
let runtime = Runtime::test();
assert!(runtime.0.bal_streaming_pool.pool.get().is_none());
// Worker pools are lazy — not initialized until first access.
assert!(!runtime.0.bal_streaming_pool.is_initialized());
assert!(!runtime.0.proof_storage_worker_pool.is_initialized());
// Accessing them triggers initialization and returns the configured thread count.
assert_eq!(runtime.bal_streaming_pool().current_num_threads(), 2);
assert!(runtime.0.bal_streaming_pool.pool.get().is_some());
assert!(runtime.0.bal_streaming_pool.is_initialized());
assert_eq!(runtime.proof_storage_worker_pool().current_num_threads(), 2);
assert_eq!(runtime.proof_account_worker_pool().current_num_threads(), 2);
assert_eq!(runtime.prewarming_pool().current_num_threads(), 2);
}
}

View File

@@ -55,6 +55,7 @@ serde = { workspace = true, features = ["derive", "rc"] }
serde_json.workspace = true
bitflags.workspace = true
auto_impl.workspace = true
imbl.workspace = true
smallvec.workspace = true
# testing
@@ -84,6 +85,7 @@ serde = [
"alloy-eips/serde",
"alloy-primitives/serde",
"bitflags/serde",
"imbl/serde",
"parking_lot/serde",
"rand?/serde",
"smallvec/serde",
@@ -112,6 +114,7 @@ test-utils = [
arbitrary = [
"proptest",
"proptest-arbitrary-interop",
"imbl/arbitrary",
"reth-chainspec/arbitrary",
"reth-eth-wire-types/arbitrary",
"alloy-consensus/arbitrary",
@@ -124,4 +127,5 @@ arbitrary = [
"reth-ethereum-primitives/arbitrary",
"revm-primitives/arbitrary",
"revm/arbitrary",
"imbl/arbitrary",
]

View File

@@ -276,6 +276,8 @@
#![cfg_attr(docsrs, feature(doc_cfg))]
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
pub use imbl::OrdMap;
pub use crate::{
batcher::{BatchTxProcessor, BatchTxRequest},
blobstore::{BlobStore, BlobStoreError},

View File

@@ -7,9 +7,10 @@ use crate::{
use alloy_consensus::Transaction;
use alloy_primitives::map::AddressSet;
use core::fmt;
use imbl::OrdMap;
use reth_primitives_traits::transaction::error::InvalidTransactionError;
use std::{
collections::{BTreeMap, BTreeSet, HashSet, VecDeque},
collections::{BTreeSet, HashSet, VecDeque},
sync::Arc,
};
use tokio::sync::broadcast::{error::TryRecvError, Receiver};
@@ -84,7 +85,7 @@ impl<T: TransactionOrdering> Iterator for BestTransactionsWithFees<T> {
pub struct BestTransactions<T: TransactionOrdering> {
/// Contains a copy of _all_ transactions of the pending pool at the point in time this
/// iterator was created.
pub(crate) all: BTreeMap<TransactionId, PendingTransaction<T>>,
pub(crate) all: OrdMap<TransactionId, PendingTransaction<T>>,
/// Transactions that can be executed right away: these have the expected nonce.
///
/// Once an `independent` transaction with the nonce `N` is returned, it unlocks `N+1`, which

View File

@@ -8,13 +8,9 @@ use crate::{
},
Priority, SubPoolLimit, TransactionOrdering, ValidPoolTransaction,
};
use imbl::OrdMap;
use rustc_hash::{FxHashMap, FxHashSet};
use std::{
cmp::Ordering,
collections::{hash_map::Entry, BTreeMap},
ops::Bound::Unbounded,
sync::Arc,
};
use std::{cmp::Ordering, collections::hash_map::Entry, ops::Bound::Unbounded, sync::Arc};
use tokio::sync::broadcast;
/// A pool of validated and gapless transactions that are ready to be executed on the current state
@@ -36,7 +32,7 @@ pub struct PendingPool<T: TransactionOrdering> {
/// This way we can determine when transactions were submitted to the pool.
submission_id: u64,
/// _All_ Transactions that are currently inside the pool grouped by their identifier.
by_id: BTreeMap<TransactionId, PendingTransaction<T>>,
by_id: OrdMap<TransactionId, PendingTransaction<T>>,
/// The highest nonce transactions for each sender - like the `independent` set, but the
/// highest instead of lowest nonce.
highest_nonces: FxHashMap<SenderId, PendingTransaction<T>>,
@@ -80,7 +76,7 @@ impl<T: TransactionOrdering> PendingPool<T> {
/// # Returns
///
/// Returns all transactions by id.
fn clear_transactions(&mut self) -> BTreeMap<TransactionId, PendingTransaction<T>> {
fn clear_transactions(&mut self) -> OrdMap<TransactionId, PendingTransaction<T>> {
self.independent_transactions.clear();
self.highest_nonces.clear();
self.size_of.reset();
@@ -525,7 +521,7 @@ impl<T: TransactionOrdering> PendingPool<T> {
}
/// All transactions grouped by id
pub const fn by_id(&self) -> &BTreeMap<TransactionId, PendingTransaction<T>> {
pub const fn by_id(&self) -> &OrdMap<TransactionId, PendingTransaction<T>> {
&self.by_id
}

View File

@@ -1801,13 +1801,9 @@ impl<Tx: PoolTransaction> NewSubpoolTransactionStream<Tx> {
&mut self,
) -> Result<NewTransactionEvent<Tx>, tokio::sync::mpsc::error::TryRecvError> {
loop {
match self.st.try_recv() {
Ok(event) => {
if event.subpool == self.subpool {
return Ok(event)
}
}
Err(e) => return Err(e),
let event = self.st.try_recv()?;
if event.subpool == self.subpool {
return Ok(event)
}
}
}

View File

@@ -876,9 +876,17 @@ where
self.fork_tracker
.max_initcode_size
.store(evm_env.cfg_env.max_initcode_size(), std::sync::atomic::Ordering::Relaxed);
// EIP-8037: When state gas is enabled, `tx.gas` can exceed the per-tx gas limit cap
// because the cap only applies to regular gas (state gas uses a reservoir).
// Store 0 to disable the txpool-level check.
let tx_gas_limit_cap = if evm_env.cfg_env.is_amsterdam_eip8037_enabled() {
0
} else {
evm_env.cfg_env.tx_gas_limit_cap()
};
self.fork_tracker
.tx_gas_limit_cap
.store(evm_env.cfg_env.tx_gas_limit_cap(), std::sync::atomic::Ordering::Relaxed);
.store(tx_gas_limit_cap, std::sync::atomic::Ordering::Relaxed);
}
fn max_gas_limit(&self) -> u64 {
@@ -1060,7 +1068,12 @@ impl<Client, Evm> EthTransactionValidatorBuilder<Client, Evm> {
// no custom transaction types by default
other_tx_types: U256::ZERO,
tx_gas_limit_cap: evm_env.cfg_env.tx_gas_limit_cap(),
// EIP-8037: When state gas is enabled, tx.gas can exceed the per-tx cap
tx_gas_limit_cap: if evm_env.cfg_env.is_amsterdam_eip8037_enabled() {
0
} else {
evm_env.cfg_env.tx_gas_limit_cap()
},
max_initcode_size: evm_env.cfg_env.max_initcode_size(),
// EIP-7594 sidecars are accepted by default (standard Ethereum behavior)

View File

@@ -61,6 +61,7 @@ allow = [
# https://github.com/rustls/webpki/blob/main/LICENSE ISC Style
"LicenseRef-rustls-webpki",
"CDLA-Permissive-2.0",
"MPL-2.0",
]
# Allow 1 or more licenses on a per-crate basis, so that particular licenses

View File

@@ -18,6 +18,7 @@
- [`reth db get`](./reth/db/get.mdx)
- [`reth db get mdbx`](./reth/db/get/mdbx.mdx)
- [`reth db get static-file`](./reth/db/get/static-file.mdx)
- [`reth db get rocksdb`](./reth/db/get/rocksdb.mdx)
- [`reth db drop`](./reth/db/drop.mdx)
- [`reth db clear`](./reth/db/clear.mdx)
- [`reth db clear mdbx`](./reth/db/clear/mdbx.mdx)
@@ -40,6 +41,7 @@
- [`reth db stage-checkpoints set`](./reth/db/stage-checkpoints/set.mdx)
- [`reth db account-storage`](./reth/db/account-storage.mdx)
- [`reth db state`](./reth/db/state.mdx)
- [`reth db migrate-v2`](./reth/db/migrate-v2.mdx)
- [`reth download`](./reth/download.mdx)
- [`reth snapshot-manifest`](./reth/snapshot-manifest.mdx)
- [`reth stage`](./reth/stage.mdx)

View File

@@ -26,6 +26,7 @@ Commands:
stage-checkpoints `reth db stage-checkpoints` subcommand
account-storage Gets storage size information for an account
state Gets account state and storage at a specific block
migrate-v2 Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
help Print this message or the help of the given subcommand(s)
Options:

View File

@@ -11,6 +11,7 @@ Usage: reth db get [OPTIONS] <COMMAND>
Commands:
mdbx Gets the content of a database table for the given key
static-file Gets the content of a static file segment for the given key
rocksdb Gets the content of a RocksDB table for the given key
help Print this message or the help of the given subcommand(s)
Options:

View File

@@ -0,0 +1,190 @@
# reth db get rocksdb
Gets the content of a RocksDB table for the given key
```bash
$ reth db get rocksdb --help
```
```txt
Usage: reth db get rocksdb [OPTIONS] <TABLE> <KEY>
Arguments:
<TABLE>
The RocksDB table
Possible values:
- transaction-hash-numbers: Transaction hash to transaction number mapping
- accounts-history: Account history indices
- storages-history: Storage history indices
<KEY>
The key to get content for. For history tables, this can be a plain address
Options:
--block <BLOCK>
Target block number for history tables. Seeks to the shard containing this block. Defaults to the latest shard if not specified
--storage-key <STORAGE_KEY>
Storage key for storages-history table lookups
--all-shards
List all shards for the given key (history tables only)
--raw
Output bytes instead of human-readable decoded value
-h, --help
Print help (see a summary with '-h')
Datadir:
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, holesky, hoodi, dev
[default: mainnet]
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
[default: terminal]
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ""]
--log.file.format <FORMAT>
The format to use for logs written to the log file
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
[default: terminal]
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.name <NAME>
The prefix name of the log files
[default: reth.log]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled.
Default: 5 for `node` command, 0 for non-node utility subcommands.
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
Possible values:
- always: Colors on
- auto: Auto-detect
- never: Colors off
[default: always]
--logs-otlp[=<URL>]
Enable `Opentelemetry` logs export to an OTLP endpoint.
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
Example: --logs-otlp=http://collector:4318/v1/logs
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
--logs-otlp.filter <FILTER>
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
Example: --logs-otlp.filter=info,reth=debug
Defaults to INFO if not specified.
[default: info]
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
Tracing:
--tracing-otlp[=<URL>]
Enable `Opentelemetry` tracing export to an OTLP endpoint.
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
Example: --tracing-otlp=http://collector:4318/v1/traces
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
--tracing-otlp-protocol <PROTOCOL>
OTLP transport protocol to use for exporting traces and logs.
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
Defaults to HTTP if not specified.
Possible values:
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
- grpc: gRPC transport, port 4317
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
[default: http]
--tracing-otlp.filter <FILTER>
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

View File

@@ -0,0 +1,166 @@
# reth db migrate-v2
Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
```bash
$ reth db migrate-v2 --help
```
```txt
Usage: reth db migrate-v2 [OPTIONS]
Options:
-h, --help
Print help (see a summary with '-h')
Datadir:
--chain <CHAIN_OR_PATH>
The chain this node is running.
Possible values are either a built-in chain or the path to a chain specification file.
Built-in chains:
mainnet, sepolia, holesky, hoodi, dev
[default: mainnet]
Logging:
--log.stdout.format <FORMAT>
The format to use for logs written to stdout
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
[default: terminal]
--log.stdout.filter <FILTER>
The filter to use for logs written to stdout
[default: ""]
--log.file.format <FORMAT>
The format to use for logs written to the log file
Possible values:
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
- terminal: Represents terminal-friendly formatting for logs
[default: terminal]
--log.file.filter <FILTER>
The filter to use for logs written to the log file
[default: debug]
--log.file.directory <PATH>
The path to put log files in
[default: <CACHE_DIR>/logs]
--log.file.name <NAME>
The prefix name of the log files
[default: reth.log]
--log.file.max-size <SIZE>
The maximum size (in MB) of one log file
[default: 200]
--log.file.max-files <COUNT>
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled.
Default: 5 for `node` command, 0 for non-node utility subcommands.
--log.journald
Write logs to journald
--log.journald.filter <FILTER>
The filter to use for logs written to journald
[default: error]
--color <COLOR>
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
Possible values:
- always: Colors on
- auto: Auto-detect
- never: Colors off
[default: always]
--logs-otlp[=<URL>]
Enable `Opentelemetry` logs export to an OTLP endpoint.
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
Example: --logs-otlp=http://collector:4318/v1/logs
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
--logs-otlp.filter <FILTER>
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
Example: --logs-otlp.filter=info,reth=debug
Defaults to INFO if not specified.
[default: info]
Display:
-v, --verbosity...
Set the minimum log level.
-v Errors
-vv Warnings
-vvv Info
-vvvv Debug
-vvvvv Traces (warning: very verbose!)
-q, --quiet
Silence all log output
Tracing:
--tracing-otlp[=<URL>]
Enable `Opentelemetry` tracing export to an OTLP endpoint.
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
Example: --tracing-otlp=http://collector:4318/v1/traces
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
--tracing-otlp-protocol <PROTOCOL>
OTLP transport protocol to use for exporting traces and logs.
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
Defaults to HTTP if not specified.
Possible values:
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
- grpc: gRPC transport, port 4317
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
[default: http]
--tracing-otlp.filter <FILTER>
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
Defaults to TRACE if not specified.
[default: debug]
--tracing-otlp.sample-ratio <RATIO>
Trace sampling ratio to control the percentage of traces to export.
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
Example: --tracing-otlp.sample-ratio=0.0.
[env: OTEL_TRACES_SAMPLER_ARG=]
```

View File

@@ -130,8 +130,9 @@ Storage:
- https://snapshots.reth.rs (default)
- https://publicnode.com/snapshots (full nodes & testnets)
If no URL is provided, the latest archive snapshot for the selected chain
will be proposed for download from https://snapshots-r2.reth.rs.
If no URL is provided, the latest archive snapshot will only be proposed
for Ethereum mainnet. For other chains, provide --manifest-url, --manifest-path,
or -u explicitly.
Local file:// URLs are also supported for extracting snapshots from disk.
@@ -202,7 +203,9 @@ Storage:
[possible values: true, false]
--download-concurrency <DOWNLOAD_CONCURRENCY>
Maximum number of concurrent modular archive workers
Maximum number of simultaneous HTTP downloads.
Applies across the entire snapshot download. Small files use one slot, while large files may use multiple slots by splitting into fixed-size pieces.
[default: 8]

View File

@@ -840,6 +840,16 @@ Dev testnet:
Parses strings using [`humantime::parse_duration`]
--dev.block-time 12s
--dev.payload-wait-time <PAYLOAD_WAIT_TIME>
Time to wait after initiating payload building before resolving.
Introduces a sleep between `fork_choice_updated` and `resolve_kind` in the
local miner, giving the payload job time for multiple rebuild attempts with
new transactions from the pool.
Parses strings using [`humantime::parse_duration`]
--dev.payload-wait-time 450ms
--dev.mnemonic <MNEMONIC>
Derive dev accounts from a fixed mnemonic instead of random ones.

View File

@@ -85,6 +85,10 @@ export const rethCliSidebar: SidebarItem = {
{
text: "reth db get static-file",
link: "/cli/reth/db/get/static-file"
},
{
text: "reth db get rocksdb",
link: "/cli/reth/db/get/rocksdb"
}
]
},
@@ -193,6 +197,10 @@ export const rethCliSidebar: SidebarItem = {
{
text: "reth db state",
link: "/cli/reth/db/state"
},
{
text: "reth db migrate-v2",
link: "/cli/reth/db/migrate-v2"
}
]
},

View File

@@ -22,7 +22,7 @@ export default defineConfig({
},
{ text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' },
{
text: 'v2.0.0',
text: 'v2.1.0',
items: [
{
text: 'Releases',

File diff suppressed because it is too large Load Diff

View File

@@ -1154,7 +1154,9 @@
],
"refresh": "30s",
"schemaVersion": 39,
"tags": [],
"tags": [
"reth"
],
"templating": {
"list": [
{
@@ -1191,4 +1193,4 @@
"uid": "fd2d69b5-ca32-45d0-946e-c00ddcd7052c",
"version": 1,
"weekStart": ""
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -1321,7 +1321,9 @@
],
"refresh": "",
"schemaVersion": 40,
"tags": [],
"tags": [
"reth"
],
"templating": {
"list": [
{
@@ -1406,4 +1408,4 @@
"uid": "df9ob3njh2dxcd",
"version": 4,
"weekStart": ""
}
}

File diff suppressed because it is too large Load Diff

View File

@@ -28,9 +28,11 @@ See examples in a [dedicated repository](https://github.com/paradigmxyz/reth-exe
## RPC
| Example | Description |
| ----------------------- | --------------------------------------------------------------------------- |
| [DB over RPC](./rpc-db) | Illustrates how to run a standalone RPC server over a Reth database instance |
| Example | Description |
| ------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- |
| [Custom auth HTTP middleware](./custom-auth-http-middleware) | Illustrates how to add HTTP transport middleware to the auth server for path-based request proxy |
| [Custom RPC middleware](./custom-rpc-middleware) | Illustrates how to add JSON-RPC layer middleware that alters RPC error responses |
| [DB over RPC](./rpc-db) | Illustrates how to run a standalone RPC server over a Reth database instance |
## Database

View File

@@ -0,0 +1,16 @@
[package]
name = "example-custom-auth-http-middleware"
version = "0.0.0"
publish = false
edition.workspace = true
license.workspace = true
[dependencies]
reth-ethereum = { workspace = true, features = ["node", "rpc", "cli"] }
clap = { workspace = true, features = ["derive"] }
http = { workspace = true }
jsonrpsee = { workspace = true, features = ["server"] }
serde_json.workspace = true
tower.workspace = true
tracing.workspace = true

View File

@@ -0,0 +1,142 @@
//! Example of how to create a node with custom auth HTTP middleware that proxies requests based on
//! the URL path.
//!
//! This middleware operates at the HTTP transport layer on the auth/Engine API server (port 8551),
//! applied **after** JWT authentication and **before** JSON-RPC parsing. It inspects the raw HTTP
//! request path and can proxy matching requests to an upstream server, while forwarding everything
//! else to the default Engine API handler.
//!
//! ## Architecture
//!
//! ```text
//! HTTP Request → JWT Auth → PathProxyMiddleware → JSON-RPC Parsing → Engine API Handler
//! │
//! └── if path matches "/proxy/*" → forward to upstream
//! ```
//!
//! Run with
//!
//! ```sh
//! cargo run -p example-custom-auth-http-middleware node \
//! --authrpc.jwtsecret /tmp/jwt.hex \
//! --dev --dev.block-time 12s
//! ```
//!
//! Then send an authenticated request to the proxy path:
//!
//! ```sh
//! # Generate a JWT token for the secret in /tmp/jwt.hex and send a proxied request:
//! curl -s -X POST http://localhost:8551/proxy/anything \
//! -H "Content-Type: application/json" \
//! -H "Authorization: Bearer <jwt-token>" \
//! -d '{"hello":"world"}'
//! ```
use clap::Parser;
use http::{header::CONTENT_TYPE, Response, StatusCode};
use jsonrpsee::server::{HttpBody, HttpRequest, HttpResponse};
use reth_ethereum::{
cli::{chainspec::EthereumChainSpecParser, interface::Cli},
node::{EthereumAddOns, EthereumNode},
};
use std::{
future::Future,
pin::Pin,
task::{Context, Poll},
};
use tower::{BoxError, Layer, Service};
fn main() {
Cli::<EthereumChainSpecParser>::parse()
.run(async move |builder, _| {
let handle = builder
.with_types::<EthereumNode>()
.with_components(EthereumNode::components())
.with_add_ons(
EthereumAddOns::default()
.with_auth_http_middleware(PathProxyLayer { proxy_prefix: "/proxy/" }),
)
.launch_with_debug_capabilities()
.await?;
handle.wait_for_node_exit().await
})
.unwrap();
}
/// A [`Layer`] that intercepts HTTP requests matching a path prefix and returns a custom
/// response instead of forwarding to the Engine API.
///
/// In a real-world scenario, you would forward matching requests to an upstream HTTP server
/// (e.g., via `hyper` or `reqwest`). This example returns a simple JSON response to demonstrate
/// the path-based routing pattern.
#[derive(Clone)]
struct PathProxyLayer {
proxy_prefix: &'static str,
}
impl<S> Layer<S> for PathProxyLayer {
type Service = PathProxyService<S>;
fn layer(&self, inner: S) -> Self::Service {
PathProxyService { inner, proxy_prefix: self.proxy_prefix }
}
}
/// The [`Service`] produced by [`PathProxyLayer`].
///
/// For requests whose path starts with the configured prefix, this service short-circuits the
/// normal Engine API pipeline and returns a custom HTTP response. All other requests are forwarded
/// unchanged to the inner service (the JSON-RPC handler).
#[derive(Clone)]
struct PathProxyService<S> {
inner: S,
proxy_prefix: &'static str,
}
impl<S> Service<HttpRequest> for PathProxyService<S>
where
S: Service<HttpRequest, Response = HttpResponse, Error = BoxError> + Send + Clone,
S::Future: Send + 'static,
{
type Response = HttpResponse;
type Error = BoxError;
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
self.inner.poll_ready(cx)
}
fn call(&mut self, req: HttpRequest) -> Self::Future {
let path = req.uri().path().to_owned();
let proxy_prefix = self.proxy_prefix;
if let Some(suffix) = path.strip_prefix(proxy_prefix) {
// The request path matches our proxy prefix — handle it here instead of forwarding
// to the Engine API JSON-RPC handler.
//
// In production, you might forward this to an upstream HTTP server:
// let upstream_url = format!("http://127.0.0.1:9000{}", path);
// let resp = reqwest::Client::new().post(&upstream_url).body(body).send().await?;
tracing::info!(path = %path, "intercepted request on proxy path");
let body = serde_json::json!({
"status": "proxied",
"matched_prefix": proxy_prefix,
"remaining_path": suffix,
});
let response = Response::builder()
.status(StatusCode::OK)
.header(CONTENT_TYPE, "application/json")
.body(HttpBody::from(body.to_string()))
.expect("valid response");
Box::pin(async move { Ok(response) })
} else {
// Not a proxy path — forward to the inner Engine API handler as usual.
let fut = self.inner.call(req);
Box::pin(fut)
}
}
}