mirror of
https://github.com/paradigmxyz/reth.git
synced 2026-04-30 03:01:58 -04:00
Compare commits
37 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
d58c6e3d07 | ||
|
|
d577814eb1 | ||
|
|
8b46f1a6d0 | ||
|
|
c527c2e7d6 | ||
|
|
14570f325a | ||
|
|
41fe41f2f2 | ||
|
|
27bfddeada | ||
|
|
981a7ef99b | ||
|
|
8c826a5cd0 | ||
|
|
6465997ea1 | ||
|
|
03a308da63 | ||
|
|
af84b982c3 | ||
|
|
77c3e86ec6 | ||
|
|
98ebc3454f | ||
|
|
c8979d0a1d | ||
|
|
742a7e7a18 | ||
|
|
99bf7a17c0 | ||
|
|
24436ca9f9 | ||
|
|
c26ec53d7d | ||
|
|
3a136fc8c3 | ||
|
|
d215d16a7d | ||
|
|
b36fff0ab8 | ||
|
|
e4d4ba30cb | ||
|
|
7c219fa955 | ||
|
|
0ac36468c6 | ||
|
|
93b2201c76 | ||
|
|
9990670990 | ||
|
|
1b69c9bb42 | ||
|
|
c2e649fc90 | ||
|
|
cff41bb9c2 | ||
|
|
0a9af7907f | ||
|
|
815d8407ce | ||
|
|
6cf6378e36 | ||
|
|
39f078e40f | ||
|
|
37a23ae169 | ||
|
|
8da8f3e4bc | ||
|
|
f97947b5a5 |
2
.github/scripts/bench-reth-snapshot.sh
vendored
2
.github/scripts/bench-reth-snapshot.sh
vendored
@@ -24,7 +24,7 @@ MC="mc"
|
||||
BUCKET="minio/reth-snapshots"
|
||||
# Allow overriding the snapshot name (e.g. for big-blocks mode where the
|
||||
# big-blocks manifest specifies which base snapshot to use).
|
||||
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable}"
|
||||
SNAPSHOT_NAME="${BENCH_SNAPSHOT_NAME:-reth-1-minimal-stable-previous}"
|
||||
MANIFEST_PATH="${SNAPSHOT_NAME}/manifest.json"
|
||||
DATADIR_NAME="datadir"
|
||||
HASH_MODE_SUFFIX=""
|
||||
|
||||
244
.github/scripts/fetch-grafana-dashboard.py
vendored
Normal file
244
.github/scripts/fetch-grafana-dashboard.py
vendored
Normal file
@@ -0,0 +1,244 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Fetch a Grafana dashboard and convert it to the portable import format.
|
||||
|
||||
Fetches the dashboard via API, replaces internal datasource/variable references
|
||||
with template variables, and adds __inputs/__requires/__elements so the JSON is
|
||||
importable on any Grafana instance.
|
||||
|
||||
Usage:
|
||||
export FETCH_GRAFANA_DASHBOARD_URL=https://<NAMESPACE>.grafana.net
|
||||
export FETCH_GRAFANA_DASHBOARD_TOKEN=glsa_...
|
||||
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py <dashboard-uid> > output.json
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import urllib.request
|
||||
|
||||
PANEL_TYPE_NAMES = {
|
||||
"bargauge": "Bar gauge",
|
||||
"gauge": "Gauge",
|
||||
"heatmap": "Heatmap",
|
||||
"piechart": "Pie chart",
|
||||
"stat": "Stat",
|
||||
"table": "Table",
|
||||
"timeseries": "Time series",
|
||||
"barchart": "Bar chart",
|
||||
"text": "Text",
|
||||
"dashlist": "Dashboard list",
|
||||
"logs": "Logs",
|
||||
"nodeGraph": "Node Graph",
|
||||
"histogram": "Histogram",
|
||||
"candlestick": "Candlestick",
|
||||
"state-timeline": "State timeline",
|
||||
"status-history": "Status history",
|
||||
"geomap": "Geomap",
|
||||
"canvas": "Canvas",
|
||||
"news": "News",
|
||||
"xychart": "XY Chart",
|
||||
"trend": "Trend",
|
||||
"datagrid": "Datagrid",
|
||||
"flamegraph": "Flame Graph",
|
||||
"traces": "Traces",
|
||||
}
|
||||
|
||||
|
||||
def fetch_json(base_url: str, token: str, path: str) -> dict:
|
||||
url = f"{base_url}{path}"
|
||||
req = urllib.request.Request(url, headers={"Authorization": f"Bearer {token}"})
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
return json.loads(resp.read())
|
||||
|
||||
|
||||
def fetch_dashboard(base_url: str, token: str, uid: str) -> dict:
|
||||
return fetch_json(base_url, token, f"/api/dashboards/uid/{uid}")
|
||||
|
||||
|
||||
def fetch_grafana_version(base_url: str) -> str:
|
||||
req = urllib.request.Request(f"{base_url}/api/health")
|
||||
with urllib.request.urlopen(req) as resp:
|
||||
data = json.loads(resp.read())
|
||||
# version string like "13.0.0-23940615780.patch2" -> take just the semver part
|
||||
version = data.get("version", "")
|
||||
# strip build metadata after the first hyphen if it looks like a pre-release
|
||||
parts = version.split("-")
|
||||
return parts[0] if parts else version
|
||||
|
||||
|
||||
def collect_panel_types(panels: list) -> set[str]:
|
||||
types = set()
|
||||
for panel in panels:
|
||||
ptype = panel.get("type", "")
|
||||
if ptype and ptype != "row":
|
||||
types.add(ptype)
|
||||
# nested panels inside collapsed rows
|
||||
for sub in panel.get("panels", []):
|
||||
sub_type = sub.get("type", "")
|
||||
if sub_type and sub_type != "row":
|
||||
types.add(sub_type)
|
||||
return types
|
||||
|
||||
|
||||
def has_expression_datasource(dashboard: dict) -> bool:
|
||||
return "__expr__" in json.dumps(dashboard)
|
||||
|
||||
|
||||
def make_exportable(dashboard: dict, grafana_version: str = "") -> dict:
|
||||
dash = json.loads(json.dumps(dashboard)) # deep copy
|
||||
|
||||
# --- Strip internal fields ---
|
||||
dash.pop("id", None)
|
||||
|
||||
# --- Rewrite links: point to the public repo instead of internal ---
|
||||
dash["links"] = [
|
||||
{
|
||||
"asDropdown": False,
|
||||
"icon": "external link",
|
||||
"includeVars": False,
|
||||
"keepTime": False,
|
||||
"tags": [],
|
||||
"targetBlank": True,
|
||||
"title": "Source (GitHub)",
|
||||
"tooltip": "View source file in repository",
|
||||
"type": "link",
|
||||
"url": "https://github.com/paradigmxyz/reth/tree/main/etc/grafana/dashboards",
|
||||
}
|
||||
]
|
||||
|
||||
# --- Datasource: victoriametrics -> prometheus ---
|
||||
dash_str = json.dumps(dash)
|
||||
dash_str = dash_str.replace("victoriametrics-metrics-datasource", "prometheus")
|
||||
dash = json.loads(dash_str)
|
||||
|
||||
# --- Templating: instance_label constant -> ${VAR_INSTANCE_LABEL} ---
|
||||
# Also strip default-value fields the API returns that are not needed for import
|
||||
STRIP_VAR_DEFAULTS = {"allowCustomValue", "regexApplyTo"}
|
||||
for var in dash.get("templating", {}).get("list", []):
|
||||
if var.get("name") == "instance_label" and var.get("type") == "constant":
|
||||
var["query"] = "${VAR_INSTANCE_LABEL}"
|
||||
var["current"] = {
|
||||
"value": "${VAR_INSTANCE_LABEL}",
|
||||
"text": "${VAR_INSTANCE_LABEL}",
|
||||
"selected": False,
|
||||
}
|
||||
var["options"] = [
|
||||
{
|
||||
"value": "${VAR_INSTANCE_LABEL}",
|
||||
"text": "${VAR_INSTANCE_LABEL}",
|
||||
"selected": False,
|
||||
}
|
||||
]
|
||||
# Clear current values for query/datasource vars (not meaningful for import)
|
||||
elif var.get("type") in ("query", "datasource"):
|
||||
var["current"] = {}
|
||||
# Remove noisy default fields
|
||||
for field in STRIP_VAR_DEFAULTS:
|
||||
var.pop(field, None)
|
||||
# Strip falsy defaults on query/datasource vars (API returns them, export omits them)
|
||||
if var.get("type") in ("query", "datasource"):
|
||||
for field in ("hide", "multi", "skipUrlSync"):
|
||||
if not var.get(field):
|
||||
var.pop(field, None)
|
||||
|
||||
# --- Build __inputs ---
|
||||
inputs = [
|
||||
{
|
||||
"name": "DS_PROMETHEUS",
|
||||
"label": "Prometheus",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "prometheus",
|
||||
"pluginName": "Prometheus",
|
||||
},
|
||||
]
|
||||
|
||||
if has_expression_datasource(dash):
|
||||
inputs.append(
|
||||
{
|
||||
"name": "DS_EXPRESSION",
|
||||
"label": "Expression",
|
||||
"description": "",
|
||||
"type": "datasource",
|
||||
"pluginId": "__expr__",
|
||||
}
|
||||
)
|
||||
|
||||
inputs.append(
|
||||
{
|
||||
"name": "VAR_INSTANCE_LABEL",
|
||||
"type": "constant",
|
||||
"label": "Instance Label",
|
||||
"value": "job",
|
||||
"description": "",
|
||||
}
|
||||
)
|
||||
|
||||
# --- Build __requires ---
|
||||
requires = []
|
||||
|
||||
if has_expression_datasource(dash):
|
||||
requires.append({"type": "datasource", "id": "__expr__", "version": "1.0.0"})
|
||||
|
||||
panel_types = collect_panel_types(dash.get("panels", []))
|
||||
for pt in sorted(panel_types):
|
||||
requires.append(
|
||||
{
|
||||
"type": "panel",
|
||||
"id": pt,
|
||||
"name": PANEL_TYPE_NAMES.get(pt, pt),
|
||||
"version": "",
|
||||
}
|
||||
)
|
||||
|
||||
requires.append(
|
||||
{"type": "grafana", "id": "grafana", "name": "Grafana", "version": grafana_version}
|
||||
)
|
||||
requires.append(
|
||||
{
|
||||
"type": "datasource",
|
||||
"id": "prometheus",
|
||||
"name": "Prometheus",
|
||||
"version": "1.0.0",
|
||||
}
|
||||
)
|
||||
|
||||
# --- Assemble output (with __inputs/__requires/__elements first) ---
|
||||
output = {
|
||||
"__inputs": inputs,
|
||||
"__elements": {},
|
||||
"__requires": requires,
|
||||
}
|
||||
output.update(dash)
|
||||
|
||||
return output
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(f"Usage: {sys.argv[0]} <dashboard-uid>", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
uid = sys.argv[1]
|
||||
base_url = os.environ.get("FETCH_GRAFANA_DASHBOARD_URL", "").rstrip("/")
|
||||
token = os.environ.get("FETCH_GRAFANA_DASHBOARD_TOKEN", "")
|
||||
|
||||
if not base_url or not token:
|
||||
print(
|
||||
"Error: FETCH_GRAFANA_DASHBOARD_URL and FETCH_GRAFANA_DASHBOARD_TOKEN env vars required",
|
||||
file=sys.stderr,
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
resp = fetch_dashboard(base_url, token, uid)
|
||||
dashboard = resp["dashboard"]
|
||||
|
||||
grafana_version = fetch_grafana_version(base_url)
|
||||
exported = make_exportable(dashboard, grafana_version)
|
||||
print(json.dumps(exported, indent=2))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
14
.github/workflows/bench-scheduled.yml
vendored
14
.github/workflows/bench-scheduled.yml
vendored
@@ -112,7 +112,7 @@ jobs:
|
||||
|
||||
- name: Alert on long-running hourly
|
||||
if: steps.mode.outputs.mode == 'hourly' && steps.refs.outputs.long-running == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -154,7 +154,7 @@ jobs:
|
||||
|
||||
- name: Alert on stale nightly
|
||||
if: steps.mode.outputs.mode == 'nightly' && steps.refs.outputs.is-stale == 'true' && !(github.event_name == 'workflow_dispatch' && inputs.slack == 'never')
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -278,7 +278,7 @@ jobs:
|
||||
|
||||
- name: Resolve job URL
|
||||
id: job-url
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const { data: jobs } = await github.rest.actions.listJobsForWorkflowRun({
|
||||
@@ -680,7 +680,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const fs = require('fs');
|
||||
@@ -759,7 +759,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -914,7 +914,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -948,7 +948,7 @@ jobs:
|
||||
},
|
||||
{
|
||||
type: 'section',
|
||||
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n@ai investigate this` },
|
||||
text: { type: 'mrkdwn', text: `*${modeLabel} regression* failed while *${failedStep}*\ncc <@U09FARE0B9Q> <@U09FAL2UMLJ>\n<@U0AAA8F0JEM> investigate this` },
|
||||
},
|
||||
{
|
||||
type: 'actions',
|
||||
|
||||
53
.github/workflows/bench.yml
vendored
53
.github/workflows/bench.yml
vendored
@@ -14,7 +14,7 @@ on:
|
||||
blocks:
|
||||
description: "Number of blocks to benchmark"
|
||||
required: false
|
||||
default: "200"
|
||||
default: "500"
|
||||
type: string
|
||||
big_blocks:
|
||||
description: "Use big blocks mode (pre-generated merged payloads with reth-bb)"
|
||||
@@ -34,7 +34,7 @@ on:
|
||||
warmup:
|
||||
description: "Number of warmup blocks"
|
||||
required: false
|
||||
default: "100"
|
||||
default: "200"
|
||||
type: string
|
||||
baseline:
|
||||
description: "Baseline git ref (default: merge-base)"
|
||||
@@ -133,7 +133,7 @@ jobs:
|
||||
steps:
|
||||
- name: Check org membership
|
||||
if: github.event_name == 'issue_comment'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.GITHUB_TOKEN }}
|
||||
script: |
|
||||
@@ -152,7 +152,7 @@ jobs:
|
||||
|
||||
- name: Parse arguments
|
||||
id: args
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -164,9 +164,9 @@ jobs:
|
||||
|
||||
if (context.eventName === 'workflow_dispatch') {
|
||||
actor = '${{ github.actor }}';
|
||||
blocks = '${{ github.event.inputs.blocks }}' || '200';
|
||||
warmup = '${{ github.event.inputs.warmup }}' || '100';
|
||||
if (warmup !== '100') explicitWarmup = true;
|
||||
blocks = '${{ github.event.inputs.blocks }}' || '500';
|
||||
warmup = '${{ github.event.inputs.warmup }}' || '200';
|
||||
if (warmup !== '200') explicitWarmup = true;
|
||||
baseline = '${{ github.event.inputs.baseline }}';
|
||||
feature = '${{ github.event.inputs.feature }}';
|
||||
samply = '${{ github.event.inputs.samply }}' === 'true' ? 'true' : 'false';
|
||||
@@ -205,7 +205,7 @@ jobs:
|
||||
const enumArgs = new Map([['bal', validBalModes], ['slack', validSlackModes]]);
|
||||
const durationArgs = new Set(['wait-time']);
|
||||
const stringArgs = new Set(['baseline-args', 'feature-args']);
|
||||
const defaults = { blocks: '200', warmup: '100', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const defaults = { blocks: '500', warmup: '200', baseline: '', feature: '', samply: 'false', slack: 'always', 'big-blocks': 'false', bal: 'false', cores: '0', abba: 'true', otlp: 'true', 'wait-time': '', 'baseline-args': '', 'feature-args': '' };
|
||||
const unknown = [];
|
||||
const invalid = [];
|
||||
const args = body.replace(/^(?:@decofe|derek) bench\s*/, '');
|
||||
@@ -359,7 +359,7 @@ jobs:
|
||||
|
||||
- name: Acknowledge request
|
||||
id: ack
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -445,7 +445,7 @@ jobs:
|
||||
|
||||
- name: Poll queue position
|
||||
if: steps.ack.outputs.comment-id && steps.ack.outputs.queue-position != '0'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -560,7 +560,7 @@ jobs:
|
||||
|
||||
- name: Resolve checkout ref
|
||||
id: checkout-ref
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
if (!process.env.BENCH_PR) {
|
||||
@@ -586,7 +586,7 @@ jobs:
|
||||
|
||||
- name: Resolve job URL and update status
|
||||
if: env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -702,7 +702,7 @@ jobs:
|
||||
# Build binaries
|
||||
- name: Resolve PR head branch
|
||||
id: pr-info
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
if (process.env.BENCH_PR) {
|
||||
@@ -720,7 +720,7 @@ jobs:
|
||||
|
||||
- name: Resolve refs
|
||||
id: refs
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const { execSync } = require('child_process');
|
||||
@@ -815,7 +815,7 @@ jobs:
|
||||
|
||||
- name: Update status (snapshot needed)
|
||||
if: env.BENCH_COMMENT_ID && steps.snapshot-check.outputs.needed == 'true'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -910,8 +910,11 @@ jobs:
|
||||
for p in /sys/kernel/mm/transparent_hugepage /sys/kernel/mm/transparent_hugepages; do
|
||||
[ -d "$p" ] && echo never | sudo tee "$p/enabled" && echo never | sudo tee "$p/defrag" && break
|
||||
done || true
|
||||
# Replace any stale PM QoS holders left behind by earlier benchmark jobs.
|
||||
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
|
||||
# Prevent deep C-states (avoids wake-up latency jitter)
|
||||
sudo sh -c 'exec 3<>/dev/cpu_dma_latency; echo -ne "\x00\x00\x00\x00" >&3; sleep infinity' &
|
||||
sudo bash -c 'exec 3<>/dev/cpu_dma_latency; printf "\0\0\0\0" >&3; exec -a bench-cpu-dma-latency sleep infinity' &
|
||||
echo "BENCH_CPU_DMA_LATENCY_PID=$!" >> "$GITHUB_ENV"
|
||||
# Move all IRQs to core 0 (housekeeping core)
|
||||
for irq in /proc/irq/*/smp_affinity_list; do
|
||||
echo 0 | sudo tee "$irq" 2>/dev/null || true
|
||||
@@ -999,7 +1002,7 @@ jobs:
|
||||
|
||||
- name: Update status (running benchmarks)
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1261,7 +1264,7 @@ jobs:
|
||||
|
||||
- name: Compare & comment
|
||||
if: success() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1341,7 +1344,7 @@ jobs:
|
||||
|
||||
- name: Write job summary
|
||||
if: success()
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const jobSummary = require('./.github/scripts/bench-job-summary.js');
|
||||
@@ -1355,7 +1358,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (success)
|
||||
if: success() && (env.BENCH_SLACK == 'always' || env.BENCH_SLACK == 'on-win')
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -1366,7 +1369,7 @@ jobs:
|
||||
|
||||
- name: Update status (failed)
|
||||
if: failure() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1400,7 +1403,7 @@ jobs:
|
||||
|
||||
- name: Send Slack notification (failure)
|
||||
if: failure() && env.BENCH_SLACK != 'never' && env.BENCH_SLACK != 'on-win'
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
env:
|
||||
SLACK_BENCH_BOT_TOKEN: ${{ secrets.SLACK_BENCH_BOT_TOKEN }}
|
||||
SLACK_BENCH_CHANNEL: ${{ secrets.SLACK_BENCH_CHANNEL }}
|
||||
@@ -1422,7 +1425,7 @@ jobs:
|
||||
|
||||
- name: Update status (cancelled)
|
||||
if: cancelled() && env.BENCH_COMMENT_ID
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
github-token: ${{ secrets.DEREK_PAT }}
|
||||
script: |
|
||||
@@ -1445,5 +1448,9 @@ jobs:
|
||||
done
|
||||
# Restore amd-pstate to active (EPP) mode with powersave governor
|
||||
echo active | sudo tee /sys/devices/system/cpu/amd_pstate/status 2>/dev/null || true
|
||||
if [ -n "${BENCH_CPU_DMA_LATENCY_PID:-}" ]; then
|
||||
sudo kill "$BENCH_CPU_DMA_LATENCY_PID" 2>/dev/null || true
|
||||
fi
|
||||
sudo pkill -f '^bench-cpu-dma-latency' 2>/dev/null || true
|
||||
sudo cpupower frequency-set -g powersave 2>/dev/null || true
|
||||
sudo systemctl start irqbalance cron atd 2>/dev/null || true
|
||||
|
||||
2
.github/workflows/book.yml
vendored
2
.github/workflows/book.yml
vendored
@@ -50,7 +50,7 @@ jobs:
|
||||
uses: actions/configure-pages@v6
|
||||
|
||||
- name: Upload artifact
|
||||
uses: actions/upload-pages-artifact@v4
|
||||
uses: actions/upload-pages-artifact@v5
|
||||
with:
|
||||
path: "./docs/vocs/docs/dist"
|
||||
|
||||
|
||||
62
.github/workflows/fetch-grafana-dashboard.yml
vendored
Normal file
62
.github/workflows/fetch-grafana-dashboard.yml
vendored
Normal file
@@ -0,0 +1,62 @@
|
||||
name: Fetch Grafana Dashboard
|
||||
|
||||
on:
|
||||
workflow_dispatch:
|
||||
inputs:
|
||||
dashboard_uid:
|
||||
description: "Grafana dashboard UID to export"
|
||||
required: true
|
||||
default: "2k8BXz24x"
|
||||
target_path:
|
||||
description: "Target file path in the repo (e.g. etc/grafana/dashboards/overview.json)"
|
||||
required: true
|
||||
default: "etc/grafana/dashboards/overview.json"
|
||||
|
||||
jobs:
|
||||
fetch:
|
||||
runs-on: ubuntu-latest
|
||||
permissions:
|
||||
contents: write
|
||||
pull-requests: write
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.12"
|
||||
|
||||
- name: Fetch dashboard from Grafana
|
||||
env:
|
||||
FETCH_GRAFANA_DASHBOARD_URL: ${{ secrets.FETCH_GRAFANA_DASHBOARD_URL }}
|
||||
FETCH_GRAFANA_DASHBOARD_TOKEN: ${{ secrets.FETCH_GRAFANA_DASHBOARD_TOKEN }}
|
||||
run: |
|
||||
python3 .github/scripts/fetch-grafana-dashboard.py "${{ inputs.dashboard_uid }}" \
|
||||
> "${{ inputs.target_path }}"
|
||||
|
||||
- name: Check for changes
|
||||
id: diff
|
||||
run: |
|
||||
if git diff --quiet "${{ inputs.target_path }}"; then
|
||||
echo "changed=false" >> "$GITHUB_OUTPUT"
|
||||
echo "No changes detected."
|
||||
else
|
||||
echo "changed=true" >> "$GITHUB_OUTPUT"
|
||||
fi
|
||||
|
||||
- name: Create pull request
|
||||
if: steps.diff.outputs.changed == 'true'
|
||||
env:
|
||||
GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
||||
run: |
|
||||
TARGET="${{ inputs.target_path }}"
|
||||
FILENAME="$(basename "$TARGET")"
|
||||
BRANCH="chore/sync-grafana-${FILENAME%.*}-$(date +%Y%m%d-%H%M%S)"
|
||||
git config user.name "github-actions[bot]"
|
||||
git config user.email "github-actions[bot]@users.noreply.github.com"
|
||||
git checkout -b "$BRANCH"
|
||||
git add "$TARGET"
|
||||
git commit -m "chore: update Grafana dashboard ${FILENAME}"
|
||||
git push origin "$BRANCH"
|
||||
gh pr create \
|
||||
--title "chore: update Grafana dashboard ${FILENAME}" \
|
||||
--body "Automated export from Grafana (dashboard UID: \`${{ inputs.dashboard_uid }}\`, target: \`${TARGET}\`)."
|
||||
25
.github/workflows/grafana.yml
vendored
25
.github/workflows/grafana.yml
vendored
@@ -11,11 +11,22 @@ jobs:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
- name: Check for ${DS_PROMETHEUS} in overview.json
|
||||
- name: Validate dashboard format
|
||||
run: |
|
||||
if grep -Fn '${DS_PROMETHEUS}' etc/grafana/dashboards/overview.json; then
|
||||
echo "Error: overview.json contains '\${DS_PROMETHEUS}' placeholder"
|
||||
echo "Please replace it with '\${datasource}'"
|
||||
exit 1
|
||||
fi
|
||||
echo "✓ overview.json does not contain '\${DS_PROMETHEUS}' placeholder"
|
||||
python3 -c "
|
||||
import json, sys
|
||||
with open('etc/grafana/dashboards/overview.json') as f:
|
||||
d = json.load(f)
|
||||
errors = []
|
||||
if '__inputs' not in d:
|
||||
errors.append('missing __inputs')
|
||||
if '__requires' not in d:
|
||||
errors.append('missing __requires')
|
||||
if d.get('id') is not None:
|
||||
errors.append('contains internal id field — use export-dashboard.py')
|
||||
if errors:
|
||||
for e in errors:
|
||||
print(f'Error: {e}', file=sys.stderr)
|
||||
sys.exit(1)
|
||||
print('✓ overview.json is a valid exported dashboard')
|
||||
"
|
||||
|
||||
2
.github/workflows/label-pr.yml
vendored
2
.github/workflows/label-pr.yml
vendored
@@ -16,7 +16,7 @@ jobs:
|
||||
fetch-depth: 0
|
||||
|
||||
- name: Label PRs
|
||||
uses: actions/github-script@v8
|
||||
uses: actions/github-script@v9
|
||||
with:
|
||||
script: |
|
||||
const label_pr = require('./.github/scripts/label_pr.js')
|
||||
|
||||
2
.github/workflows/release.yml
vendored
2
.github/workflows/release.yml
vendored
@@ -195,7 +195,7 @@ jobs:
|
||||
fi
|
||||
|
||||
body=$(cat <<- "ENDBODY"
|
||||

|
||||

|
||||
|
||||
## Testing Checklist (DELETE ME)
|
||||
|
||||
|
||||
794
Cargo.lock
generated
794
Cargo.lock
generated
File diff suppressed because it is too large
Load Diff
26
Cargo.toml
26
Cargo.toml
@@ -1,5 +1,5 @@
|
||||
[workspace.package]
|
||||
version = "2.0.0"
|
||||
version = "2.1.0"
|
||||
edition = "2024"
|
||||
rust-version = "1.93"
|
||||
license = "MIT OR Apache-2.0"
|
||||
@@ -129,6 +129,7 @@ members = [
|
||||
"examples/custom-node-components/",
|
||||
"examples/custom-payload-builder/",
|
||||
"examples/custom-rlpx-subprotocol",
|
||||
"examples/custom-auth-http-middleware",
|
||||
"examples/custom-rpc-middleware",
|
||||
"examples/db-access",
|
||||
"examples/exex-subscription",
|
||||
@@ -432,14 +433,14 @@ reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
|
||||
reth-zstd-compressors = { version = "0.3.0", default-features = false }
|
||||
|
||||
# revm
|
||||
revm = { version = "37.0.0", default-features = false }
|
||||
revm = { version = "38.0.0", default-features = false }
|
||||
revm-bytecode = { version = "10.0.0", default-features = false }
|
||||
revm-database = { version = "13.0.0", default-features = false }
|
||||
revm-state = { version = "11.0.0", default-features = false }
|
||||
revm-primitives = { version = "23.0.0", default-features = false }
|
||||
revm-interpreter = { version = "35.0.0", default-features = false }
|
||||
revm-database-interface = { version = "11.0.0", default-features = false }
|
||||
revm-inspectors = "0.38.0"
|
||||
revm-inspectors = "0.39.0"
|
||||
|
||||
# eth
|
||||
alloy-dyn-abi = "1.5.6"
|
||||
@@ -449,7 +450,7 @@ alloy-sol-types = { version = "1.5.6", default-features = false }
|
||||
alloy-chains = { version = "0.2.33", default-features = false }
|
||||
alloy-eip2124 = { version = "0.2.0", default-features = false }
|
||||
alloy-eip7928 = { version = "0.3.0", default-features = false }
|
||||
alloy-evm = { version = "0.32.0", default-features = false }
|
||||
alloy-evm = { version = "0.33.0", default-features = false }
|
||||
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
|
||||
alloy-trie = { version = "0.9.4", default-features = false }
|
||||
|
||||
@@ -506,6 +507,7 @@ eyre = "0.6"
|
||||
fdlimit = "0.3.0"
|
||||
fixed-map = { version = "0.9", default-features = false }
|
||||
humantime = "2.1"
|
||||
imbl = "7"
|
||||
humantime-serde = "1.1"
|
||||
itertools = { version = "0.14", default-features = false }
|
||||
linked_hash_set = "0.1"
|
||||
@@ -698,19 +700,3 @@ vergen-git2 = "9.1.0"
|
||||
|
||||
# networking
|
||||
ipnet = "2.11"
|
||||
|
||||
[patch.crates-io]
|
||||
revm = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-database = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-state = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "89ecb25dbe49e1c3a10d99529e42f027d0bd2386" }
|
||||
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "c6f88bbe7186d863f4667dd43c42608eb7a8ba5c" }
|
||||
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "ff0bbec9ccaa818155e25003a77f4d73d350bbd7" }
|
||||
reth-codecs = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
|
||||
reth-codecs-derive = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
|
||||
reth-primitives-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
|
||||
reth-rpc-traits = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
|
||||
reth-zstd-compressors = { git = "https://github.com/paradigmxyz/reth-core", rev = "6b12498871bc1b1d42c6dcf28968c271660de8c0" }
|
||||
|
||||
@@ -238,6 +238,7 @@ where
|
||||
withdrawals: prev_segment.ctx.withdrawals.clone(),
|
||||
extra_data: prev_segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: prev_segment.ctx.tx_count_hint,
|
||||
slot_number: prev_segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
// Clone the next segment's data before we consume inner.
|
||||
@@ -252,6 +253,7 @@ where
|
||||
withdrawals: new_segment.ctx.withdrawals.clone(),
|
||||
extra_data: new_segment.ctx.extra_data.clone(),
|
||||
tx_count_hint: new_segment.ctx.tx_count_hint,
|
||||
slot_number: new_segment.ctx.slot_number,
|
||||
};
|
||||
|
||||
plan.next_segment += 1;
|
||||
@@ -364,6 +366,7 @@ where
|
||||
withdrawals: seg0.ctx.withdrawals.clone(),
|
||||
extra_data: seg0.ctx.extra_data.clone(),
|
||||
tx_count_hint: seg0.ctx.tx_count_hint,
|
||||
slot_number: seg0.ctx.slot_number,
|
||||
};
|
||||
|
||||
let inner = self.inner_mut();
|
||||
@@ -422,6 +425,7 @@ where
|
||||
withdrawals: last_seg.ctx.withdrawals.clone(),
|
||||
extra_data: last_seg.ctx.extra_data.clone(),
|
||||
tx_count_hint: last_seg.ctx.tx_count_hint,
|
||||
slot_number: last_seg.ctx.slot_number,
|
||||
};
|
||||
self.inner_mut().ctx = last_ctx;
|
||||
}
|
||||
|
||||
@@ -266,6 +266,7 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: ctx.withdrawals.map(|w| std::borrow::Cow::Owned(w.into_owned())),
|
||||
extra_data: ctx.extra_data,
|
||||
slot_number: ctx.slot_number,
|
||||
};
|
||||
BigBlockSegment { start_tx, evm_env, ctx }
|
||||
})
|
||||
|
||||
@@ -176,6 +176,7 @@ impl BbAddOns {
|
||||
BasicEngineApiBuilder::default(),
|
||||
BasicEngineValidatorBuilder::default(),
|
||||
Default::default(),
|
||||
Default::default(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,7 +10,7 @@ use alloy_eips::{
|
||||
eip1559::BaseFeeParams,
|
||||
eip7840::BlobParams,
|
||||
eip7928::{AccountChanges, BlockAccessList, SlotChanges},
|
||||
BlockNumberOrTag, Typed2718,
|
||||
Typed2718,
|
||||
};
|
||||
use alloy_primitives::{Bloom, Bytes, B256};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
@@ -32,6 +32,8 @@ use serde::{Deserialize, Serialize};
|
||||
use std::{collections::HashMap, future::Future};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
|
||||
/// A single transaction with its gas used and raw encoded bytes.
|
||||
#[derive(Debug, Clone)]
|
||||
pub struct RawTransaction {
|
||||
@@ -669,20 +671,6 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
async fn fetch_block_access_list(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
) -> eyre::Result<BlockAccessList> {
|
||||
provider
|
||||
.client()
|
||||
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.and_then(|block_access_list: Option<BlockAccessList>| {
|
||||
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
|
||||
})
|
||||
}
|
||||
|
||||
fn merge_block_access_list(
|
||||
merged: &mut BlockAccessList,
|
||||
incoming: BlockAccessList,
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
//! Common helpers for reth-bench commands.
|
||||
|
||||
use alloy_eips::{eip7928::BlockAccessList, BlockNumberOrTag};
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use eyre::Result;
|
||||
use std::{
|
||||
io::{BufReader, Read},
|
||||
@@ -69,6 +71,21 @@ pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches the block access list for a given block number using the provided provider.
|
||||
pub(crate) async fn fetch_block_access_list(
|
||||
provider: &RootProvider<AnyNetwork>,
|
||||
block_number: u64,
|
||||
) -> eyre::Result<BlockAccessList> {
|
||||
provider
|
||||
.client()
|
||||
.request("eth_getBlockAccessListByBlockNumber", (BlockNumberOrTag::Number(block_number),))
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.and_then(|block_access_list: Option<BlockAccessList>| {
|
||||
block_access_list.ok_or_else(|| eyre::eyre!("BAL not found for block {block_number}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::parse_duration,
|
||||
helpers::{fetch_block_access_list, parse_duration},
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
|
||||
@@ -198,12 +198,19 @@ impl Command {
|
||||
finalized_block_hash: finalized,
|
||||
};
|
||||
|
||||
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
let start = Instant::now();
|
||||
let server_timings =
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
use crate::{
|
||||
bench::{
|
||||
context::BenchContext,
|
||||
helpers::fetch_block_access_list,
|
||||
metrics_scraper::MetricsScraper,
|
||||
output::{
|
||||
NewPayloadResult, TotalGasOutput, TotalGasRow, GAS_OUTPUT_SUFFIX,
|
||||
@@ -69,7 +70,9 @@ impl Command {
|
||||
let (error_sender, mut error_receiver) = tokio::sync::oneshot::channel();
|
||||
let (sender, mut receiver) = tokio::sync::mpsc::channel(buffer_size);
|
||||
|
||||
let block_provider_clone = block_provider.clone();
|
||||
tokio::task::spawn(async move {
|
||||
let block_provider = block_provider_clone;
|
||||
while benchmark_mode.contains(next_block) {
|
||||
let block_res = block_provider
|
||||
.get_block_by_number(next_block.into())
|
||||
@@ -123,12 +126,19 @@ impl Command {
|
||||
|
||||
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
|
||||
|
||||
let bal = if rlp.is_none() && block.header.block_access_list_hash.is_some() {
|
||||
Some(fetch_block_access_list(&block_provider, block.header.number).await?)
|
||||
} else {
|
||||
None
|
||||
};
|
||||
|
||||
let (version, params) = block_to_new_payload(
|
||||
block,
|
||||
rlp,
|
||||
use_reth_namespace,
|
||||
wait_for_persistence,
|
||||
no_wait_for_caches,
|
||||
bal,
|
||||
)?;
|
||||
|
||||
let start = Instant::now();
|
||||
|
||||
@@ -18,7 +18,7 @@ use alloy_primitives::B256;
|
||||
use alloy_provider::{network::AnyNetwork, Provider, RootProvider};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::{
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV4,
|
||||
CancunPayloadFields, ExecutionData, ExecutionPayload, ExecutionPayloadEnvelopeV6,
|
||||
ExecutionPayloadSidecar, ExecutionPayloadV4, ForkchoiceState, JwtSecret, PraguePayloadFields,
|
||||
};
|
||||
use clap::Parser;
|
||||
@@ -315,7 +315,7 @@ impl Command {
|
||||
let requests =
|
||||
execution_data.sidecar.requests().cloned().unwrap_or_default().to_vec();
|
||||
(
|
||||
Some(EngineApiMessageVersion::V4),
|
||||
Some(EngineApiMessageVersion::V6),
|
||||
serde_json::to_value((
|
||||
execution_data.payload.clone(),
|
||||
Vec::<B256>::new(),
|
||||
@@ -423,7 +423,7 @@ impl Command {
|
||||
/// Load and parse all payload files from the directory.
|
||||
///
|
||||
/// Tries to load each file as a [`BigBlockPayload`] first (which includes `env_switches`),
|
||||
/// falling back to [`ExecutionPayloadEnvelopeV4`] for backwards compatibility.
|
||||
/// falling back to [`ExecutionPayloadEnvelopeV6`] for backwards compatibility.
|
||||
fn load_payloads(&self) -> eyre::Result<Vec<LoadedPayload>> {
|
||||
let mut payloads = Vec::new();
|
||||
|
||||
@@ -450,12 +450,11 @@ impl Command {
|
||||
let name_str = name.to_string_lossy();
|
||||
let index = if let Some(rest) = name_str.strip_prefix("payload_block_") {
|
||||
rest.strip_suffix(".json")?.parse::<u64>().ok()?
|
||||
} else if let Some(rest) = name_str.strip_prefix("big_block_") {
|
||||
} else {
|
||||
let rest = name_str.strip_prefix("big_block_")?;
|
||||
// "big_block_FROM_to_TO.json" — use FROM as the index
|
||||
let rest = rest.strip_suffix(".json")?;
|
||||
rest.split("_to_").next()?.parse::<u64>().ok()?
|
||||
} else {
|
||||
return None;
|
||||
};
|
||||
Some((index, e.path()))
|
||||
})
|
||||
@@ -481,10 +480,10 @@ impl Command {
|
||||
{
|
||||
(big_block.execution_data, big_block.big_block_data, big_block.block_access_list)
|
||||
} else {
|
||||
let envelope: ExecutionPayloadEnvelopeV4 = serde_json::from_str(&content)
|
||||
let envelope: ExecutionPayloadEnvelopeV6 = serde_json::from_str(&content)
|
||||
.wrap_err_with(|| format!("Failed to parse {:?}", path))?;
|
||||
let execution_data = ExecutionData {
|
||||
payload: envelope.envelope_inner.execution_payload.clone().into(),
|
||||
payload: envelope.execution_payload.clone().into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
use alloy_eips::eip4895::Withdrawal;
|
||||
use alloy_primitives::{Address, Bloom, Bytes, B256, U256};
|
||||
use alloy_rpc_types_engine::{ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3};
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadV1, ExecutionPayloadV2, ExecutionPayloadV3, ExecutionPayloadV4,
|
||||
};
|
||||
|
||||
/// Configuration for invalidating payload fields
|
||||
#[derive(Debug, Default)]
|
||||
@@ -21,6 +23,7 @@ pub(super) struct InvalidationConfig {
|
||||
pub(super) block_hash: Option<B256>,
|
||||
pub(super) blob_gas_used: Option<u64>,
|
||||
pub(super) excess_blob_gas: Option<u64>,
|
||||
pub(super) slot_number: Option<u64>,
|
||||
|
||||
// Auto-invalidation flags
|
||||
pub(super) invalidate_parent_hash: bool,
|
||||
@@ -35,6 +38,8 @@ pub(super) struct InvalidationConfig {
|
||||
pub(super) invalidate_withdrawals: bool,
|
||||
pub(super) invalidate_blob_gas_used: bool,
|
||||
pub(super) invalidate_excess_blob_gas: bool,
|
||||
pub(super) invalidate_block_access_list: bool,
|
||||
pub(super) invalidate_slot_number: bool,
|
||||
}
|
||||
|
||||
impl InvalidationConfig {
|
||||
@@ -216,4 +221,30 @@ impl InvalidationConfig {
|
||||
|
||||
changes
|
||||
}
|
||||
|
||||
/// Applies invalidations to a V4 payload, returns list of what was changed.
|
||||
pub(super) fn apply_to_payload_v4(&self, payload: &mut ExecutionPayloadV4) -> Vec<String> {
|
||||
let mut changes = self.apply_to_payload_v3(&mut payload.payload_inner);
|
||||
|
||||
// Explicit override for slot_number
|
||||
if let Some(slot_number) = self.slot_number {
|
||||
payload.slot_number = slot_number;
|
||||
changes.push(format!("slot_number = {slot_number}"));
|
||||
}
|
||||
|
||||
// Handle block access list invalidation (V4+)
|
||||
if self.invalidate_block_access_list {
|
||||
let fake_bal = Bytes::from_static(&[0x01, 0x02, 0x03]);
|
||||
payload.block_access_list = fake_bal.clone();
|
||||
changes.push(format!("block_access_list = {fake_bal} (auto-invalidated)"));
|
||||
}
|
||||
|
||||
// Handle slot number invalidation (V4+)
|
||||
if self.invalidate_slot_number {
|
||||
payload.slot_number = u64::MAX;
|
||||
changes.push("slot_number = MAX (auto-invalidated)".to_string());
|
||||
}
|
||||
|
||||
changes
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,18 @@
|
||||
//! Command for sending invalid payloads to test Engine API rejection.
|
||||
|
||||
mod invalidation;
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use invalidation::InvalidationConfig;
|
||||
|
||||
use crate::bench::helpers::fetch_block_access_list;
|
||||
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use alloy_provider::{
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
RootProvider,
|
||||
};
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
@@ -105,6 +111,9 @@ pub struct Command {
|
||||
#[arg(long, value_name = "HASH", help_heading = "Explicit Value Overrides")]
|
||||
requests_hash: Option<B256>,
|
||||
|
||||
/// Override the slot number with a specific value.
|
||||
#[arg(long, value_name = "U64", help_heading = "Explicit Value Overrides")]
|
||||
slot_number: Option<u64>,
|
||||
// ==================== Auto-Invalidation Flags ====================
|
||||
/// Invalidate the parent hash by setting it to a random value.
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
@@ -158,6 +167,14 @@ pub struct Command {
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_requests_hash: bool,
|
||||
|
||||
/// Invalidate the block access list by setting it to a random value (EIP-7928).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_block_access_list: bool,
|
||||
|
||||
/// Invalidate the slot number by setting it to an random value.(EIP-7843).
|
||||
#[arg(long, default_value_t = false, help_heading = "Auto-Invalidation Flags")]
|
||||
invalidate_slot_number: bool,
|
||||
|
||||
// ==================== Meta Flags ====================
|
||||
/// Skip block hash recalculation after modifications.
|
||||
#[arg(long, default_value_t = false, help_heading = "Meta Flags")]
|
||||
@@ -199,6 +216,7 @@ impl Command {
|
||||
block_hash: self.block_hash,
|
||||
blob_gas_used: self.blob_gas_used,
|
||||
excess_blob_gas: self.excess_blob_gas,
|
||||
slot_number: self.slot_number,
|
||||
invalidate_parent_hash: self.invalidate_parent_hash,
|
||||
invalidate_state_root: self.invalidate_state_root,
|
||||
invalidate_receipts_root: self.invalidate_receipts_root,
|
||||
@@ -211,6 +229,8 @@ impl Command {
|
||||
invalidate_withdrawals: self.invalidate_withdrawals,
|
||||
invalidate_blob_gas_used: self.invalidate_blob_gas_used,
|
||||
invalidate_excess_blob_gas: self.invalidate_excess_blob_gas,
|
||||
invalidate_block_access_list: self.invalidate_block_access_list,
|
||||
invalidate_slot_number: self.invalidate_slot_number,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -234,15 +254,21 @@ impl Command {
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let use_v5 = block.header.block_access_list_hash.is_some();
|
||||
let requests_hash = self.requests_hash.or(block.header.requests_hash);
|
||||
|
||||
let mut execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
let mut execution_payload = if use_v5 {
|
||||
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
|
||||
} else {
|
||||
ExecutionPayload::from_block_slow(&block).0
|
||||
};
|
||||
|
||||
let changes = match &mut execution_payload {
|
||||
ExecutionPayload::V1(p) => config.apply_to_payload_v1(p),
|
||||
ExecutionPayload::V2(p) => config.apply_to_payload_v2(p),
|
||||
ExecutionPayload::V3(p) => config.apply_to_payload_v3(p),
|
||||
ExecutionPayload::V4(p) => config.apply_to_payload_v3(&mut p.payload_inner),
|
||||
ExecutionPayload::V4(p) => config.apply_to_payload_v4(p),
|
||||
};
|
||||
|
||||
let skip_recalc = self.skip_hash_recalc || config.should_skip_hash_recalc();
|
||||
@@ -312,7 +338,13 @@ impl Command {
|
||||
match self.mode {
|
||||
Mode::Execute => {
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
let method = if use_v5 {
|
||||
"engine_newPayloadV5"
|
||||
} else if use_v4 {
|
||||
"engine_newPayloadV4"
|
||||
} else {
|
||||
"engine_newPayloadV3"
|
||||
};
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
@@ -353,4 +385,17 @@ impl Command {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
|
||||
let rpc_url = self
|
||||
.rpc_url
|
||||
.as_deref()
|
||||
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
|
||||
let client = ClientBuilder::default()
|
||||
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
let bal = fetch_block_access_list(&provider, block_number).await?;
|
||||
Ok(alloy_rlp::encode(bal).into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
use super::helpers::{load_jwt_secret, read_input};
|
||||
use super::helpers::{fetch_block_access_list, load_jwt_secret, read_input};
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_provider::network::AnyRpcBlock;
|
||||
use alloy_primitives::Bytes;
|
||||
use alloy_provider::{
|
||||
network::{AnyNetwork, AnyRpcBlock},
|
||||
RootProvider,
|
||||
};
|
||||
use alloy_rpc_client::ClientBuilder;
|
||||
use alloy_rpc_types_engine::ExecutionPayload;
|
||||
use clap::Parser;
|
||||
use eyre::{OptionExt, Result};
|
||||
@@ -69,6 +74,9 @@ impl Command {
|
||||
})?
|
||||
.into_consensus();
|
||||
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
let use_v5 = block.header.block_access_list_hash.is_some();
|
||||
|
||||
// Extract parent beacon block root
|
||||
let parent_beacon_block_root = block.header.parent_beacon_block_root;
|
||||
|
||||
@@ -76,10 +84,14 @@ impl Command {
|
||||
let blob_versioned_hashes =
|
||||
block.body.blob_versioned_hashes_iter().copied().collect::<Vec<_>>();
|
||||
|
||||
// Convert to execution payload
|
||||
let execution_payload = ExecutionPayload::from_block_slow(&block).0;
|
||||
|
||||
let use_v4 = block.header.requests_hash.is_some();
|
||||
// V5 payloads must carry the full RLP-encoded block access list, not just the hash stored
|
||||
// in the header.
|
||||
let execution_payload = if use_v5 {
|
||||
let encoded_bal = self.fetch_encoded_block_access_list(block.header.number).await?;
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, encoded_bal).0
|
||||
} else {
|
||||
ExecutionPayload::from_block_slow(&block).0
|
||||
};
|
||||
|
||||
// Create JSON request data
|
||||
let json_request = if use_v4 {
|
||||
@@ -102,7 +114,13 @@ impl Command {
|
||||
Mode::Execute => {
|
||||
// Create cast command
|
||||
let mut command = std::process::Command::new("cast");
|
||||
let method = if use_v4 { "engine_newPayloadV4" } else { "engine_newPayloadV3" };
|
||||
let method = if use_v5 {
|
||||
"engine_newPayloadV5"
|
||||
} else if use_v4 {
|
||||
"engine_newPayloadV4"
|
||||
} else {
|
||||
"engine_newPayloadV3"
|
||||
};
|
||||
command.arg("rpc").arg(method).arg("--raw");
|
||||
if let Some(rpc_url) = self.rpc_url {
|
||||
command.arg("--rpc-url").arg(rpc_url);
|
||||
@@ -146,4 +164,17 @@ impl Command {
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
async fn fetch_encoded_block_access_list(&self, block_number: u64) -> Result<Bytes> {
|
||||
let rpc_url = self
|
||||
.rpc_url
|
||||
.as_deref()
|
||||
.ok_or_eyre("--rpc-url is required to fetch the block access list for V5 payloads")?;
|
||||
let client = ClientBuilder::default()
|
||||
.layer(alloy_transport::layers::RetryBackoffLayer::new(10, 800, u64::MAX))
|
||||
.http(rpc_url.parse()?);
|
||||
let provider = RootProvider::<AnyNetwork>::new(client);
|
||||
let bal = fetch_block_access_list(&provider, block_number).await?;
|
||||
Ok(alloy_rlp::encode(bal).into())
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
//! before sending additional calls.
|
||||
|
||||
use alloy_consensus::TxEnvelope;
|
||||
use alloy_eips::eip7928::BlockAccessList;
|
||||
use alloy_primitives::Bytes;
|
||||
use alloy_provider::{ext::EngineApi, network::AnyRpcBlock, Network, Provider};
|
||||
use alloy_rpc_types_engine::{
|
||||
@@ -43,6 +44,14 @@ pub trait EngineApiValidWaitExt<N>: Send + Sync {
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated>;
|
||||
|
||||
/// Calls `engine_forkChoiceUpdatedV4` with the given [`ForkchoiceState`] and optional
|
||||
/// [`PayloadAttributes`], and waits until the response is VALID.
|
||||
async fn fork_choice_updated_v4_wait(
|
||||
&self,
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -162,6 +171,40 @@ where
|
||||
|
||||
Ok(status)
|
||||
}
|
||||
|
||||
async fn fork_choice_updated_v4_wait(
|
||||
&self,
|
||||
fork_choice_state: ForkchoiceState,
|
||||
payload_attributes: Option<PayloadAttributes>,
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
debug!(
|
||||
target: "reth-bench",
|
||||
method = "engine_forkchoiceUpdatedV3",
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Sending forkchoiceUpdated"
|
||||
);
|
||||
|
||||
let mut status =
|
||||
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
|
||||
|
||||
while !status.is_valid() {
|
||||
if status.is_invalid() {
|
||||
error!(
|
||||
target: "reth-bench",
|
||||
?status,
|
||||
?fork_choice_state,
|
||||
?payload_attributes,
|
||||
"Invalid forkchoiceUpdatedV4 message",
|
||||
);
|
||||
panic!("Invalid forkchoiceUpdatedV4: {status:?}");
|
||||
}
|
||||
status =
|
||||
self.fork_choice_updated_v4(fork_choice_state, payload_attributes.clone()).await?;
|
||||
}
|
||||
|
||||
Ok(status)
|
||||
}
|
||||
}
|
||||
|
||||
/// Converts an RPC block into versioned engine API params and an [`ExecutionData`].
|
||||
@@ -176,6 +219,7 @@ pub(crate) fn block_to_new_payload(
|
||||
reth_new_payload: bool,
|
||||
wait_for_persistence: WaitForPersistence,
|
||||
no_wait_for_caches: bool,
|
||||
bal: Option<BlockAccessList>,
|
||||
) -> eyre::Result<(Option<EngineApiMessageVersion>, serde_json::Value)> {
|
||||
let block_number = block.header.number;
|
||||
let wait_for_persistence = wait_for_persistence.rpc_value(block_number);
|
||||
@@ -198,7 +242,11 @@ pub(crate) fn block_to_new_payload(
|
||||
tx.try_into().map_err(|_| eyre::eyre!("unsupported tx type"))
|
||||
})?
|
||||
.into_consensus();
|
||||
let (payload, sidecar) = ExecutionPayload::from_block_slow(&block);
|
||||
|
||||
let block_access_list = alloy_rlp::encode(bal.unwrap_or_default());
|
||||
|
||||
let (payload, sidecar) =
|
||||
ExecutionPayload::from_block_slow_with_bal(&block, block_access_list.into());
|
||||
let (version, params, execution_data) = payload_to_new_payload(payload, sidecar, None)?;
|
||||
|
||||
if reth_new_payload {
|
||||
@@ -227,6 +275,22 @@ pub(crate) fn payload_to_new_payload(
|
||||
let execution_data = ExecutionData { payload: payload.clone(), sidecar: sidecar.clone() };
|
||||
|
||||
let (version, params) = match payload {
|
||||
ExecutionPayload::V4(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
|
||||
let version = target_version.unwrap_or(EngineApiMessageVersion::V6);
|
||||
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
|
||||
(
|
||||
version,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
ExecutionPayload::V3(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
@@ -266,22 +330,6 @@ pub(crate) fn payload_to_new_payload(
|
||||
ExecutionPayload::V1(payload) => {
|
||||
(EngineApiMessageVersion::V1, serde_json::to_value((payload,))?)
|
||||
}
|
||||
ExecutionPayload::V4(payload) => {
|
||||
let cancun = sidecar
|
||||
.cancun()
|
||||
.ok_or_else(|| eyre::eyre!("missing cancun sidecar for V4 payload"))?;
|
||||
let version = target_version.unwrap_or(EngineApiMessageVersion::V4);
|
||||
let requests = sidecar.prague().map(|p| p.requests.clone()).unwrap_or_default();
|
||||
(
|
||||
version,
|
||||
serde_json::to_value((
|
||||
payload,
|
||||
cancun.versioned_hashes.clone(),
|
||||
cancun.parent_beacon_block_root,
|
||||
requests,
|
||||
))?,
|
||||
)
|
||||
}
|
||||
};
|
||||
|
||||
Ok((version, params, execution_data))
|
||||
@@ -386,10 +434,10 @@ pub(crate) async fn call_forkchoice_updated<N, P: EngineApiValidWaitExt<N>>(
|
||||
) -> TransportResult<ForkchoiceUpdated> {
|
||||
// FCU V3 is used for both Cancun and Prague (there is no FCU V4)
|
||||
match message_version {
|
||||
EngineApiMessageVersion::V3 |
|
||||
EngineApiMessageVersion::V4 |
|
||||
EngineApiMessageVersion::V5 |
|
||||
EngineApiMessageVersion::V6 => {
|
||||
provider.fork_choice_updated_v4_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
EngineApiMessageVersion::V3 | EngineApiMessageVersion::V4 | EngineApiMessageVersion::V5 => {
|
||||
provider.fork_choice_updated_v3_wait(forkchoice_state, payload_attributes).await
|
||||
}
|
||||
EngineApiMessageVersion::V2 => {
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
use alloy_primitives::{hex, BlockHash};
|
||||
use alloy_primitives::{hex, Address, BlockHash, B256};
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
static_file::{
|
||||
@@ -10,6 +10,7 @@ use reth_db::{
|
||||
use reth_db_api::{
|
||||
cursor::{DbCursorRO, DbDupCursorRO},
|
||||
database::Database,
|
||||
models::{storage_sharded_key::StorageShardedKey, ShardedKey},
|
||||
table::{Compress, Decompress, DupSort, Table},
|
||||
tables,
|
||||
transaction::DbTx,
|
||||
@@ -19,7 +20,10 @@ use reth_db_common::DbTool;
|
||||
use reth_node_api::{HeaderTy, ReceiptTy, TxTy};
|
||||
use reth_node_builder::NodeTypesWithDB;
|
||||
use reth_primitives_traits::ValueWithSubKey;
|
||||
use reth_provider::{providers::ProviderNodeTypes, ChangeSetReader, StaticFileProviderFactory};
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, ChangeSetReader, RocksDBProviderFactory,
|
||||
StaticFileProviderFactory,
|
||||
};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StorageChangeSetReader;
|
||||
use tracing::error;
|
||||
@@ -73,6 +77,55 @@ enum Subcommand {
|
||||
#[arg(long)]
|
||||
raw: bool,
|
||||
},
|
||||
/// Gets the content of a RocksDB table for the given key
|
||||
///
|
||||
/// For history tables (accounts-history, storages-history), you can pass a plain address
|
||||
/// instead of a full JSON ShardedKey. Use --block to query a specific block number
|
||||
/// (seeks to the shard containing that block), or --all-shards to list all shards for
|
||||
/// the address.
|
||||
///
|
||||
/// Examples:
|
||||
/// reth db get rocksdb accounts-history 0xdBBE3D8c2d2b22A2611c5A94A9a12C2fCD49Eb29
|
||||
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --block 1000000
|
||||
/// reth db get rocksdb accounts-history 0xdBBE...Eb29 --all-shards
|
||||
/// reth db get rocksdb storages-history 0xdBBE...Eb29 --storage-key 0x0000...0003
|
||||
Rocksdb {
|
||||
/// The RocksDB table
|
||||
#[arg(value_enum)]
|
||||
table: RocksDbTable,
|
||||
|
||||
/// The key to get content for. For history tables, this can be a plain address.
|
||||
#[arg(value_parser = maybe_json_value_parser)]
|
||||
key: String,
|
||||
|
||||
/// Target block number for history tables. Seeks to the shard containing this block.
|
||||
/// Defaults to the latest shard if not specified.
|
||||
#[arg(long)]
|
||||
block: Option<u64>,
|
||||
|
||||
/// Storage key for storages-history table lookups.
|
||||
#[arg(long)]
|
||||
storage_key: Option<String>,
|
||||
|
||||
/// List all shards for the given key (history tables only).
|
||||
#[arg(long)]
|
||||
all_shards: bool,
|
||||
|
||||
/// Output bytes instead of human-readable decoded value
|
||||
#[arg(long)]
|
||||
raw: bool,
|
||||
},
|
||||
}
|
||||
|
||||
/// RocksDB tables that can be queried.
|
||||
#[derive(Debug, Clone, Copy, clap::ValueEnum)]
|
||||
pub enum RocksDbTable {
|
||||
/// Transaction hash to transaction number mapping
|
||||
TransactionHashNumbers,
|
||||
/// Account history indices
|
||||
AccountsHistory,
|
||||
/// Storage history indices
|
||||
StoragesHistory,
|
||||
}
|
||||
|
||||
impl Command {
|
||||
@@ -82,6 +135,9 @@ impl Command {
|
||||
Subcommand::Mdbx { table, key, subkey, end_key, end_subkey, raw } => {
|
||||
table.view(&GetValueViewer { tool, key, subkey, end_key, end_subkey, raw })?
|
||||
}
|
||||
Subcommand::Rocksdb { table, key, block, storage_key, all_shards, raw } => {
|
||||
get_rocksdb(tool, table, &key, block, storage_key.as_deref(), all_shards, raw)?;
|
||||
}
|
||||
Subcommand::StaticFile { segment, key, subkey, raw } => {
|
||||
if let StaticFileSegment::StorageChangeSets = segment {
|
||||
let storage_key =
|
||||
@@ -246,6 +302,208 @@ impl Command {
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a value from a RocksDB table by key.
|
||||
fn get_rocksdb<N: ProviderNodeTypes>(
|
||||
tool: &DbTool<N>,
|
||||
table: RocksDbTable,
|
||||
key: &str,
|
||||
block: Option<u64>,
|
||||
storage_key: Option<&str>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
let rocksdb = tool.provider_factory.rocksdb_provider();
|
||||
|
||||
match table {
|
||||
RocksDbTable::TransactionHashNumbers => {
|
||||
if block.is_some() || all_shards || storage_key.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block, --all-shards, and --storage-key are only supported for history tables"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::TransactionHashNumbers>(&rocksdb, key, raw)
|
||||
}
|
||||
RocksDbTable::AccountsHistory => {
|
||||
if storage_key.is_some() {
|
||||
return Err(eyre::eyre!("--storage-key is only supported for storages-history"));
|
||||
}
|
||||
get_rocksdb_account_history(&rocksdb, key, block, all_shards, raw)
|
||||
}
|
||||
RocksDbTable::StoragesHistory => {
|
||||
get_rocksdb_storage_history(&rocksdb, key, storage_key, block, all_shards, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Try to parse a key string as a plain address, falling back to JSON `ShardedKey` parsing.
|
||||
fn parse_address(key: &str) -> eyre::Result<Address> {
|
||||
// Strip surrounding quotes that `maybe_json_value_parser` may have added
|
||||
let stripped = key.trim_matches('"');
|
||||
stripped.parse::<Address>().map_err(|e| eyre::eyre!("failed to parse address: {e}"))
|
||||
}
|
||||
|
||||
/// Gets account history from RocksDB with ergonomic key parsing.
|
||||
///
|
||||
/// Accepts a plain address and uses seek to find the relevant shard.
|
||||
fn get_rocksdb_account_history(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key: &str,
|
||||
block: Option<u64>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
// Try parsing as a plain address first, fall back to full JSON ShardedKey
|
||||
match parse_address(key) {
|
||||
Ok(address) => {
|
||||
let block_number = block.unwrap_or(u64::MAX);
|
||||
let seek_key = ShardedKey::new(address, block_number);
|
||||
|
||||
if all_shards {
|
||||
// Iterate all shards: seek from (address, 0) until address changes
|
||||
let start = ShardedKey::new(address, 0);
|
||||
let iter = rocksdb.iter_from::<tables::AccountsHistory>(start)?;
|
||||
for result in iter {
|
||||
let (k, v) = result?;
|
||||
if k.key != address {
|
||||
break;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"highest_block_number": k.highest_block_number,
|
||||
"value": v,
|
||||
}))?
|
||||
);
|
||||
}
|
||||
} else {
|
||||
// Seek to the first shard with highest_block_number >= target
|
||||
let mut iter = rocksdb.iter_from::<tables::AccountsHistory>(seek_key)?;
|
||||
match iter.next() {
|
||||
Some(Ok((k, v))) if k.key == address => {
|
||||
if raw {
|
||||
let raw_val = rocksdb.get_raw::<tables::AccountsHistory>(k)?;
|
||||
if let Some(bytes) = raw_val {
|
||||
println!("{}", hex::encode_prefixed(&bytes));
|
||||
}
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&v)?);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
// Fall back to full JSON key parsing (e.g.
|
||||
// `{"key":"0x...","highest_block_number":...}`)
|
||||
if all_shards || block.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block and --all-shards require a plain address, not a JSON key"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::AccountsHistory>(rocksdb, key, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets storage history from RocksDB with ergonomic key parsing.
|
||||
///
|
||||
/// Accepts a plain address + optional `--storage-key` and uses seek.
|
||||
fn get_rocksdb_storage_history(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key: &str,
|
||||
storage_key: Option<&str>,
|
||||
block: Option<u64>,
|
||||
all_shards: bool,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
match parse_address(key) {
|
||||
Ok(address) => {
|
||||
let storage_key = storage_key
|
||||
.map(|s| s.trim_matches('"').parse::<B256>())
|
||||
.transpose()
|
||||
.map_err(|e| eyre::eyre!("failed to parse storage key: {e}"))?
|
||||
.unwrap_or_default();
|
||||
let block_number = block.unwrap_or(u64::MAX);
|
||||
let seek_key = StorageShardedKey::new(address, storage_key, block_number);
|
||||
|
||||
if all_shards {
|
||||
let start = StorageShardedKey::new(address, storage_key, 0);
|
||||
let iter = rocksdb.iter_from::<tables::StoragesHistory>(start)?;
|
||||
for result in iter {
|
||||
let (k, v) = result?;
|
||||
if k.address != address || k.sharded_key.key != storage_key {
|
||||
break;
|
||||
}
|
||||
println!(
|
||||
"{}",
|
||||
serde_json::to_string_pretty(&serde_json::json!({
|
||||
"highest_block_number": k.sharded_key.highest_block_number,
|
||||
"value": v,
|
||||
}))?
|
||||
);
|
||||
}
|
||||
} else {
|
||||
let mut iter = rocksdb.iter_from::<tables::StoragesHistory>(seek_key)?;
|
||||
match iter.next() {
|
||||
Some(Ok((k, v)))
|
||||
if k.address == address && k.sharded_key.key == storage_key =>
|
||||
{
|
||||
if raw {
|
||||
let raw_val = rocksdb.get_raw::<tables::StoragesHistory>(k)?;
|
||||
if let Some(bytes) = raw_val {
|
||||
println!("{}", hex::encode_prefixed(&bytes));
|
||||
}
|
||||
} else {
|
||||
println!("{}", serde_json::to_string_pretty(&v)?);
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
error!(target: "reth::cli", "No content for the given table key.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
Err(_) => {
|
||||
if all_shards || block.is_some() || storage_key.is_some() {
|
||||
return Err(eyre::eyre!(
|
||||
"--block, --all-shards, and --storage-key require a plain address, not a JSON key"
|
||||
));
|
||||
}
|
||||
get_rocksdb_table::<tables::StoragesHistory>(rocksdb, key, raw)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Gets a value from a specific RocksDB table by exact key and prints it.
|
||||
fn get_rocksdb_table<T: Table>(
|
||||
rocksdb: &reth_provider::providers::RocksDBProvider,
|
||||
key_str: &str,
|
||||
raw: bool,
|
||||
) -> eyre::Result<()> {
|
||||
let key = table_key::<T>(key_str)?;
|
||||
|
||||
if raw {
|
||||
let content = rocksdb.get_raw::<T>(key)?;
|
||||
match content {
|
||||
Some(bytes) => println!("{}", hex::encode_prefixed(&bytes)),
|
||||
None => error!(target: "reth::cli", "No content for the given table key."),
|
||||
}
|
||||
} else {
|
||||
let content = rocksdb.get::<T>(key)?;
|
||||
match content {
|
||||
Some(value) => println!("{}", serde_json::to_string_pretty(&value)?),
|
||||
None => error!(target: "reth::cli", "No content for the given table key."),
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Get an instance of key for given table
|
||||
pub(crate) fn table_key<T: Table>(key: &str) -> Result<T::Key, eyre::Error> {
|
||||
serde_json::from_str(key).map_err(|e| eyre::eyre!(e))
|
||||
|
||||
361
crates/cli/commands/src/db/migrate_v2.rs
Normal file
361
crates/cli/commands/src/db/migrate_v2.rs
Normal file
@@ -0,0 +1,361 @@
|
||||
//! `reth db migrate-v2` command for migrating v1 storage layout to v2.
|
||||
//!
|
||||
//! Migrates data that cannot be recomputed (changesets + receipts) from MDBX to
|
||||
//! static files, clears recomputable tables (senders, indices, trie, plain
|
||||
//! state), compacts MDBX, then runs the pipeline to rebuild them.
|
||||
|
||||
use crate::common::CliNodeTypes;
|
||||
use clap::Parser;
|
||||
use reth_db::{
|
||||
mdbx::{self, ffi},
|
||||
models::StorageBeforeTx,
|
||||
DatabaseEnv,
|
||||
};
|
||||
use reth_db_api::{
|
||||
cursor::DbCursorRO,
|
||||
database::Database,
|
||||
table::Table,
|
||||
tables,
|
||||
transaction::{DbTx, DbTxMut},
|
||||
};
|
||||
use reth_node_builder::NodeTypesWithDBAdapter;
|
||||
use reth_provider::{
|
||||
providers::ProviderNodeTypes, DBProvider, DatabaseProviderFactory, MetadataProvider,
|
||||
MetadataWriter, ProviderFactory, PruneCheckpointReader, StageCheckpointWriter,
|
||||
StaticFileProviderFactory, StaticFileWriter, StorageSettings,
|
||||
};
|
||||
use reth_prune_types::PruneSegment;
|
||||
use reth_stages_types::{StageCheckpoint, StageId};
|
||||
use reth_static_file_types::StaticFileSegment;
|
||||
use reth_storage_api::StageCheckpointReader;
|
||||
use tracing::info;
|
||||
|
||||
/// `reth db migrate-v2` command
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command;
|
||||
|
||||
impl Command {
|
||||
/// Execute the full v1 → v2 migration:
|
||||
///
|
||||
/// 1. Migrate changesets + receipts to static files
|
||||
/// 2. Flip `StorageSettings` to v2
|
||||
/// 3. Clear recomputable MDBX tables + reset stage checkpoints
|
||||
/// 4. Compact MDBX
|
||||
pub async fn execute<N: CliNodeTypes>(
|
||||
self,
|
||||
provider_factory: ProviderFactory<NodeTypesWithDBAdapter<N, DatabaseEnv>>,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N::Primitives: reth_primitives_traits::NodePrimitives<
|
||||
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
|
||||
>,
|
||||
{
|
||||
// === Phase 0: Preflight ===
|
||||
info!(target: "reth::cli", "Starting v1 → v2 storage migration");
|
||||
|
||||
let provider = provider_factory.provider()?;
|
||||
let current_settings = provider.storage_settings()?;
|
||||
|
||||
if current_settings.is_some_and(|s| s.is_v2()) {
|
||||
info!(target: "reth::cli", "Storage is already v2, nothing to do");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let tip =
|
||||
provider.get_stage_checkpoint(StageId::Execution)?.map(|c| c.block_number).unwrap_or(0);
|
||||
|
||||
info!(target: "reth::cli", tip, "Chain tip block number");
|
||||
|
||||
let sf_provider = provider_factory.static_file_provider();
|
||||
|
||||
for segment in [StaticFileSegment::AccountChangeSets, StaticFileSegment::StorageChangeSets]
|
||||
{
|
||||
if sf_provider.get_highest_static_file_block(segment).is_some() {
|
||||
eyre::bail!(
|
||||
"Static file segment {segment:?} already contains data. \
|
||||
Cannot migrate — target must be empty."
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
drop(provider);
|
||||
|
||||
// === Phase 1: Migrate changesets → static files ===
|
||||
Self::migrate_account_changesets(&provider_factory, tip)?;
|
||||
Self::migrate_storage_changesets(&provider_factory, tip)?;
|
||||
|
||||
// === Phase 2: Migrate receipts → static files ===
|
||||
Self::migrate_receipts::<NodeTypesWithDBAdapter<N, DatabaseEnv>>(&provider_factory, tip)?;
|
||||
|
||||
// === Phase 3: Flip metadata to v2 ===
|
||||
info!(target: "reth::cli", "Writing StorageSettings v2 metadata");
|
||||
{
|
||||
let provider_rw = provider_factory.database_provider_rw()?;
|
||||
provider_rw.write_storage_settings(StorageSettings::v2())?;
|
||||
provider_rw.commit()?;
|
||||
}
|
||||
info!(target: "reth::cli", "Storage settings updated to v2");
|
||||
|
||||
// === Phase 4: Clear recomputable tables ===
|
||||
Self::clear_recomputable_tables(&provider_factory)?;
|
||||
|
||||
// === Phase 5: Compact MDBX (before pipeline, so it runs on a smaller DB) ===
|
||||
let db_path = provider_factory.db_ref().path();
|
||||
Self::compact_mdbx(provider_factory.db_ref())?;
|
||||
|
||||
// Drop to release DB handle for swap
|
||||
drop(provider_factory);
|
||||
|
||||
let compact_path = db_path.with_file_name("db_compact");
|
||||
Self::swap_compacted_db(&db_path, &compact_path)?;
|
||||
|
||||
// === Phase 6: Reopen DB and run pipeline ===
|
||||
// The caller will reopen the environment and run the pipeline.
|
||||
// We return here — the pipeline step is handled in mod.rs after
|
||||
// reopening the database with the compacted copy.
|
||||
info!(target: "reth::cli", "Migration complete. You should now restart the node and let it run the pipeline to rebuild the remaining data.");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_account_changesets<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Migrating AccountChangeSets → static files");
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let sf_provider = factory.static_file_provider();
|
||||
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::AccountChangeSets>()?;
|
||||
|
||||
let first_block = provider
|
||||
.get_prune_checkpoint(PruneSegment::AccountHistory)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::AccountChangeSets)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(first_block))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Some(Ok((block_number, _))) = walker.peek() {
|
||||
if *block_number != block {
|
||||
break;
|
||||
}
|
||||
let (_, entry) = walker.next().expect("peeked")?;
|
||||
entries.push(entry);
|
||||
}
|
||||
|
||||
count += entries.len() as u64;
|
||||
writer.append_account_changeset(entries, block)?;
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
|
||||
info!(target: "reth::cli", count, "AccountChangeSets migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_storage_changesets<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Migrating StorageChangeSets → static files");
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let sf_provider = factory.static_file_provider();
|
||||
|
||||
let mut cursor = provider.tx_ref().cursor_read::<tables::StorageChangeSets>()?;
|
||||
|
||||
let first_block = provider
|
||||
.get_prune_checkpoint(PruneSegment::StorageHistory)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
|
||||
let mut writer =
|
||||
sf_provider.get_writer(first_block, StaticFileSegment::StorageChangeSets)?;
|
||||
|
||||
let mut count = 0u64;
|
||||
let mut walker = cursor.walk(Some(Default::default()))?.peekable();
|
||||
|
||||
for block in first_block..=tip {
|
||||
let mut entries = Vec::new();
|
||||
|
||||
while let Some(Ok((key, _))) = walker.peek() {
|
||||
if key.block_number() != block {
|
||||
break;
|
||||
}
|
||||
let (key, entry) = walker.next().expect("peeked")?;
|
||||
entries.push(StorageBeforeTx {
|
||||
address: key.address(),
|
||||
key: entry.key,
|
||||
value: entry.value,
|
||||
});
|
||||
}
|
||||
|
||||
count += entries.len() as u64;
|
||||
writer.append_storage_changeset(entries, block)?;
|
||||
}
|
||||
|
||||
writer.commit()?;
|
||||
|
||||
info!(target: "reth::cli", count, "StorageChangeSets migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn migrate_receipts<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
tip: u64,
|
||||
) -> eyre::Result<()>
|
||||
where
|
||||
N::Primitives: reth_primitives_traits::NodePrimitives<
|
||||
Receipt: reth_db_api::table::Value + reth_codecs::Compact,
|
||||
>,
|
||||
{
|
||||
let provider = factory.provider()?;
|
||||
if !provider.prune_modes_ref().receipts_log_filter.is_empty() {
|
||||
info!(target: "reth::cli", "Receipt log filter pruning is enabled, keeping receipts in MDBX");
|
||||
return Ok(());
|
||||
}
|
||||
drop(provider);
|
||||
|
||||
let sf_provider = factory.static_file_provider();
|
||||
let existing = sf_provider.get_highest_static_file_block(StaticFileSegment::Receipts);
|
||||
|
||||
if existing.is_some_and(|b| b >= tip) {
|
||||
info!(target: "reth::cli", "Receipts already in static files, skipping");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "Migrating Receipts → static files");
|
||||
|
||||
let provider = factory.provider()?.disable_long_read_transaction_safety();
|
||||
let prune_start = provider
|
||||
.get_prune_checkpoint(PruneSegment::Receipts)?
|
||||
.and_then(|cp| cp.block_number)
|
||||
.map_or(0, |b| b + 1);
|
||||
let first_block = prune_start.max(existing.map_or(0, |b| b + 1));
|
||||
|
||||
let block_range = first_block..=tip;
|
||||
|
||||
let segment = reth_static_file::segments::Receipts;
|
||||
reth_static_file::segments::Segment::copy_to_static_files(&segment, provider, block_range)?;
|
||||
|
||||
sf_provider.commit()?;
|
||||
|
||||
info!(target: "reth::cli", "Receipts migrated");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Clears tables that can be recomputed by the pipeline and resets their
|
||||
/// stage checkpoints.
|
||||
fn clear_recomputable_tables<N: ProviderNodeTypes>(
|
||||
factory: &ProviderFactory<N>,
|
||||
) -> eyre::Result<()> {
|
||||
info!(target: "reth::cli", "Clearing recomputable MDBX tables");
|
||||
let db = factory.db_ref();
|
||||
|
||||
macro_rules! clear_table {
|
||||
($table:ty) => {{
|
||||
let tx = db.tx_mut()?;
|
||||
tx.clear::<$table>()?;
|
||||
tx.commit()?;
|
||||
info!(target: "reth::cli", table = <$table as Table>::NAME, "Cleared");
|
||||
}};
|
||||
}
|
||||
|
||||
// Migrated changeset tables (now in static files)
|
||||
clear_table!(tables::AccountChangeSets);
|
||||
clear_table!(tables::StorageChangeSets);
|
||||
|
||||
// Senders — rebuilt by SenderRecovery
|
||||
clear_table!(tables::TransactionSenders);
|
||||
|
||||
// Indices — rebuilt by TransactionLookup / IndexAccountHistory / IndexStorageHistory
|
||||
clear_table!(tables::TransactionHashNumbers);
|
||||
clear_table!(tables::AccountsHistory);
|
||||
clear_table!(tables::StoragesHistory);
|
||||
|
||||
// Plain state — superseded by hashed state in v2
|
||||
clear_table!(tables::PlainAccountState);
|
||||
clear_table!(tables::PlainStorageState);
|
||||
|
||||
// Trie — rebuilt by MerkleExecute
|
||||
clear_table!(tables::AccountsTrie);
|
||||
clear_table!(tables::StoragesTrie);
|
||||
|
||||
// Reset stage checkpoints so the pipeline rebuilds everything
|
||||
info!(target: "reth::cli", "Resetting stage checkpoints");
|
||||
let provider_rw = factory.database_provider_rw()?;
|
||||
for stage in [
|
||||
StageId::SenderRecovery,
|
||||
StageId::TransactionLookup,
|
||||
StageId::IndexAccountHistory,
|
||||
StageId::IndexStorageHistory,
|
||||
StageId::MerkleExecute,
|
||||
StageId::MerkleUnwind,
|
||||
] {
|
||||
provider_rw.save_stage_checkpoint(stage, StageCheckpoint::new(0))?;
|
||||
info!(target: "reth::cli", %stage, "Checkpoint reset to 0");
|
||||
}
|
||||
provider_rw.save_stage_checkpoint_progress(StageId::MerkleExecute, vec![])?;
|
||||
provider_rw.commit()?;
|
||||
|
||||
info!(target: "reth::cli", "Recomputable tables cleared");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Creates a compacted copy of the MDBX database.
|
||||
fn compact_mdbx(db: &mdbx::DatabaseEnv) -> eyre::Result<()> {
|
||||
let db_path = db.path();
|
||||
let compact_path = db_path.with_file_name("db_compact");
|
||||
|
||||
reth_fs_util::create_dir_all(&compact_path)?;
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?compact_path, "Compacting MDBX database");
|
||||
|
||||
let compact_dest = compact_path.join("mdbx.dat");
|
||||
let dest_cstr = std::ffi::CString::new(
|
||||
compact_dest.to_str().ok_or_else(|| eyre::eyre!("compact path must be valid UTF-8"))?,
|
||||
)?;
|
||||
|
||||
let flags = ffi::MDBX_CP_COMPACT | ffi::MDBX_CP_FORCE_DYNAMIC_SIZE;
|
||||
|
||||
let rc = db.with_raw_env_ptr(|env_ptr| unsafe {
|
||||
ffi::mdbx_env_copy(env_ptr, dest_cstr.as_ptr(), flags)
|
||||
});
|
||||
|
||||
if rc != 0 {
|
||||
eyre::bail!("mdbx_env_copy failed with error code {rc}: {}", unsafe {
|
||||
std::ffi::CStr::from_ptr(ffi::mdbx_strerror(rc)).to_string_lossy()
|
||||
});
|
||||
}
|
||||
|
||||
info!(target: "reth::cli", "MDBX compaction complete");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Swaps the original MDBX database with a compacted copy.
|
||||
fn swap_compacted_db(
|
||||
db_path: &std::path::Path,
|
||||
compact_path: &std::path::Path,
|
||||
) -> eyre::Result<()> {
|
||||
let backup_path = db_path.with_file_name("db_pre_compact");
|
||||
|
||||
info!(target: "reth::cli", ?db_path, ?compact_path, "Swapping compacted database");
|
||||
|
||||
std::fs::rename(db_path, &backup_path)?;
|
||||
|
||||
if let Err(e) = std::fs::rename(compact_path, db_path) {
|
||||
let _ = std::fs::rename(&backup_path, db_path);
|
||||
return Err(e.into());
|
||||
}
|
||||
|
||||
std::fs::remove_dir_all(&backup_path)?;
|
||||
|
||||
info!(target: "reth::cli", "Database compaction swap complete");
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
@@ -16,6 +16,7 @@ mod copy;
|
||||
mod diff;
|
||||
mod get;
|
||||
mod list;
|
||||
mod migrate_v2;
|
||||
mod prune_checkpoints;
|
||||
mod repair_trie;
|
||||
mod settings;
|
||||
@@ -77,6 +78,9 @@ pub enum Subcommands {
|
||||
AccountStorage(account_storage::Command),
|
||||
/// Gets account state and storage at a specific block
|
||||
State(state::Command),
|
||||
/// Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
|
||||
#[command(name = "migrate-v2")]
|
||||
MigrateV2(migrate_v2::Command),
|
||||
}
|
||||
|
||||
impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C> {
|
||||
@@ -231,6 +235,13 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
command.execute(&tool)?;
|
||||
});
|
||||
}
|
||||
Subcommands::MigrateV2(command) => {
|
||||
let Environment { provider_factory, .. } =
|
||||
self.env.init::<N>(AccessRights::RW, ctx.task_executor.clone())?;
|
||||
|
||||
// Migrate changesets+receipts, clear tables, compact MDBX
|
||||
command.execute::<N>(provider_factory).await?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
|
||||
368
crates/cli/commands/src/download/archive.rs
Normal file
368
crates/cli/commands/src/download/archive.rs
Normal file
@@ -0,0 +1,368 @@
|
||||
use super::{
|
||||
extract::{extract_archive_raw, streaming_download_and_extract, CompressionFormat},
|
||||
fetch::ArchiveFetcher,
|
||||
manifest::SnapshotArchive,
|
||||
planning::{PlannedArchive, PlannedDownloads},
|
||||
progress::{
|
||||
spawn_progress_display, ArchiveDownloadProgress, ArchiveExtractionProgress,
|
||||
ArchiveVerificationProgress, DownloadRequestLimiter, SharedProgress,
|
||||
},
|
||||
session::{ArchiveProcessContext, DownloadSession},
|
||||
verify::OutputVerifier,
|
||||
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
|
||||
};
|
||||
use eyre::Result;
|
||||
use futures::stream::{self, StreamExt};
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
path::Path,
|
||||
sync::{atomic::Ordering, Arc},
|
||||
time::Duration,
|
||||
};
|
||||
use tokio::task;
|
||||
use tracing::{debug, info, warn};
|
||||
|
||||
const DOWNLOAD_CACHE_DIR: &str = ".download-cache";
|
||||
|
||||
/// Runs all planned modular archive downloads for one command invocation.
|
||||
pub(crate) async fn run_modular_downloads(
|
||||
planned_downloads: PlannedDownloads,
|
||||
target_dir: &Path,
|
||||
download_concurrency: usize,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let download_cache_dir = target_dir.join(DOWNLOAD_CACHE_DIR);
|
||||
fs::create_dir_all(&download_cache_dir)?;
|
||||
|
||||
let shared = SharedProgress::new(
|
||||
planned_downloads.total_download_size,
|
||||
planned_downloads.total_output_size,
|
||||
planned_downloads.total_archives() as u64,
|
||||
cancel_token.clone(),
|
||||
);
|
||||
let session = DownloadSession::new(
|
||||
Some(Arc::clone(&shared)),
|
||||
Some(DownloadRequestLimiter::new(download_concurrency)),
|
||||
cancel_token,
|
||||
);
|
||||
let ctx =
|
||||
ArchiveProcessContext::new(target_dir.to_path_buf(), Some(download_cache_dir), session);
|
||||
|
||||
ModularDownloadJob::new(ctx, download_concurrency).run(planned_downloads).await
|
||||
}
|
||||
|
||||
/// Schedules modular archive work for one run of `reth download`.
|
||||
struct ModularDownloadJob {
|
||||
/// Shared paths and session state for each archive in this job.
|
||||
ctx: ArchiveProcessContext,
|
||||
/// Maximum number of archives processed at once.
|
||||
archive_concurrency: usize,
|
||||
}
|
||||
|
||||
impl ModularDownloadJob {
|
||||
/// Creates the modular download job for one command run.
|
||||
const fn new(ctx: ArchiveProcessContext, archive_concurrency: usize) -> Self {
|
||||
Self { ctx, archive_concurrency }
|
||||
}
|
||||
|
||||
/// Runs all planned archives and waits for the shared progress task to finish.
|
||||
async fn run(self, planned_downloads: PlannedDownloads) -> Result<()> {
|
||||
let shared = Arc::clone(
|
||||
self.ctx.session().progress().expect("modular downloads always use shared progress"),
|
||||
);
|
||||
let progress_handle = spawn_progress_display(Arc::clone(&shared));
|
||||
let ctx = self.ctx.clone();
|
||||
let results: Vec<Result<()>> = stream::iter(planned_downloads.archives)
|
||||
.map(move |archive| {
|
||||
let ctx = ctx.clone();
|
||||
async move { Self::process_archive(ctx, archive).await }
|
||||
})
|
||||
.buffer_unordered(self.archive_concurrency)
|
||||
.collect()
|
||||
.await;
|
||||
|
||||
shared.done.store(true, Ordering::Relaxed);
|
||||
let _ = progress_handle.await;
|
||||
|
||||
for result in results {
|
||||
result?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Runs one archive on the blocking pool so fetch and extraction stay off the async executor.
|
||||
async fn process_archive(ctx: ArchiveProcessContext, archive: PlannedArchive) -> Result<()> {
|
||||
task::spawn_blocking(move || ArchiveProcessor::new(archive, ctx).run()).await??;
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
/// Explicit retry states for one modular archive.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ArchiveAttemptState {
|
||||
/// Start or restart one full archive attempt.
|
||||
RunAttempt,
|
||||
/// Check whether the extracted outputs verify.
|
||||
VerifyOutputs,
|
||||
/// Wait and decide whether another full attempt should run.
|
||||
RetryAttempt,
|
||||
/// Finish successfully.
|
||||
Complete,
|
||||
/// Stop with an error after retries are exhausted.
|
||||
Fail,
|
||||
}
|
||||
|
||||
/// Processes one modular archive from reuse check through extraction and verification.
|
||||
struct ArchiveProcessor {
|
||||
/// The concrete archive and component being processed.
|
||||
archive: PlannedArchive,
|
||||
/// Shared paths and session state for this archive attempt.
|
||||
ctx: ArchiveProcessContext,
|
||||
}
|
||||
|
||||
impl ArchiveProcessor {
|
||||
/// Creates a processor for one archive and the shared download context.
|
||||
fn new(archive: PlannedArchive, ctx: ArchiveProcessContext) -> Self {
|
||||
Self { archive, ctx }
|
||||
}
|
||||
|
||||
/// Runs the archive retry state machine until outputs are verified or retries are exhausted.
|
||||
fn run(self) -> Result<()> {
|
||||
let archive = self.archive();
|
||||
if self.try_reuse_outputs()? {
|
||||
info!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, "Skipping already verified plain files");
|
||||
return Ok(());
|
||||
}
|
||||
|
||||
let mode = ArchiveMode::new(&self.ctx)?;
|
||||
let format = CompressionFormat::from_url(&archive.file_name)?;
|
||||
let mut attempt = 1;
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
let mut state = ArchiveAttemptState::RunAttempt;
|
||||
|
||||
loop {
|
||||
match state {
|
||||
ArchiveAttemptState::RunAttempt => {
|
||||
self.cleanup_outputs();
|
||||
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
file = %archive.file_name,
|
||||
component = %self.archive.component,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
"Retrying archive from scratch"
|
||||
);
|
||||
}
|
||||
|
||||
match self.run_attempt(mode, format) {
|
||||
Ok(()) => state = ArchiveAttemptState::VerifyOutputs,
|
||||
Err(error) if mode.retries_fetch_errors() => {
|
||||
warn!(target: "reth::cli",
|
||||
file = %archive.file_name,
|
||||
component = %self.archive.component,
|
||||
attempt,
|
||||
err = %format_args!("{error:#}"),
|
||||
"Archive attempt failed, retrying from scratch"
|
||||
);
|
||||
last_error = Some(error);
|
||||
state = ArchiveAttemptState::RetryAttempt;
|
||||
}
|
||||
Err(error) => return Err(error),
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::VerifyOutputs => {
|
||||
if self.verify_outputs_with_progress()? {
|
||||
state = ArchiveAttemptState::Complete;
|
||||
} else {
|
||||
warn!(target: "reth::cli", file = %archive.file_name, component = %self.archive.component, attempt, "Archive extracted, but output verification failed, retrying");
|
||||
state = ArchiveAttemptState::RetryAttempt;
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::RetryAttempt => {
|
||||
if attempt >= MAX_DOWNLOAD_RETRIES {
|
||||
state = ArchiveAttemptState::Fail;
|
||||
} else {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
attempt += 1;
|
||||
state = ArchiveAttemptState::RunAttempt;
|
||||
}
|
||||
}
|
||||
ArchiveAttemptState::Complete => return Ok(()),
|
||||
ArchiveAttemptState::Fail => {
|
||||
if let Some(error) = last_error {
|
||||
return Err(error.wrap_err(format!(
|
||||
"Failed after {} attempts for {}",
|
||||
MAX_DOWNLOAD_RETRIES, archive.file_name
|
||||
)));
|
||||
}
|
||||
|
||||
eyre::bail!(
|
||||
"Failed integrity validation after {} attempts for {}",
|
||||
MAX_DOWNLOAD_RETRIES,
|
||||
archive.file_name
|
||||
)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the concrete archive being fetched or verified.
|
||||
fn archive(&self) -> &SnapshotArchive {
|
||||
&self.archive.archive
|
||||
}
|
||||
|
||||
/// Returns the verifier for this archive's output files.
|
||||
fn output_verifier(&self) -> OutputVerifier<'_> {
|
||||
OutputVerifier::new(self.ctx.target_dir())
|
||||
}
|
||||
|
||||
/// Returns `true` if this archive can be reused from existing verified outputs.
|
||||
/// Returns `false` if a fresh archive attempt is still needed.
|
||||
fn try_reuse_outputs(&self) -> Result<bool> {
|
||||
if self.verify_outputs()? {
|
||||
self.mark_complete();
|
||||
return Ok(true);
|
||||
}
|
||||
|
||||
Ok(false)
|
||||
}
|
||||
|
||||
/// Removes any partial outputs before a fresh archive attempt.
|
||||
fn cleanup_outputs(&self) {
|
||||
self.output_verifier().cleanup(&self.archive().output_files);
|
||||
}
|
||||
|
||||
/// Returns `true` if all declared plain outputs verify.
|
||||
/// Returns `false` if any output is missing or does not match.
|
||||
fn verify_outputs(&self) -> Result<bool> {
|
||||
self.output_verifier().verify(&self.archive().output_files)
|
||||
}
|
||||
|
||||
/// Records archive completion in shared progress once outputs verify.
|
||||
fn mark_complete(&self) {
|
||||
self.ctx.session().record_reused_archive(self.archive().size, self.archive().output_size());
|
||||
}
|
||||
|
||||
/// Executes one archive attempt according to the selected cache-vs-stream mode.
|
||||
fn run_attempt(&self, mode: ArchiveMode, format: CompressionFormat) -> Result<()> {
|
||||
mode.execute(self, format)
|
||||
}
|
||||
|
||||
/// Downloads the archive into the cache, then extracts from the cached file.
|
||||
fn run_cached_attempt(&self, format: CompressionFormat) -> Result<()> {
|
||||
let cache_dir =
|
||||
self.ctx.cache_dir().ok_or_else(|| eyre::eyre!("Missing download cache directory"))?;
|
||||
let fetcher =
|
||||
ArchiveFetcher::new(self.archive().url.clone(), cache_dir, self.ctx.session().clone());
|
||||
|
||||
if self.archive.ty == super::manifest::SnapshotComponentType::State {
|
||||
debug!(target: "reth::cli", url = %self.archive().url, "Downloading state snapshot archive");
|
||||
}
|
||||
|
||||
let download_result = {
|
||||
let mut download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
|
||||
let result = fetcher.download(Some(&mut download_progress));
|
||||
if let Ok(ref downloaded) = result &&
|
||||
download_progress.has_tracked_bytes()
|
||||
{
|
||||
download_progress.complete(downloaded.size);
|
||||
}
|
||||
result
|
||||
};
|
||||
|
||||
let downloaded = match download_result {
|
||||
Ok(downloaded) => downloaded,
|
||||
Err(error) => {
|
||||
fetcher.cleanup_downloaded_files();
|
||||
return Err(error);
|
||||
}
|
||||
};
|
||||
|
||||
info!(target: "reth::cli",
|
||||
file = %self.archive().file_name,
|
||||
component = %self.archive.component,
|
||||
size = %super::progress::DownloadProgress::format_size(downloaded.size),
|
||||
"Archive download complete"
|
||||
);
|
||||
|
||||
let extract_result = self.extract_cached_archive(&downloaded.path, format);
|
||||
fetcher.cleanup_downloaded_files();
|
||||
extract_result
|
||||
}
|
||||
|
||||
/// Streams the archive directly into extraction without keeping a cached copy.
|
||||
fn run_streaming_attempt(&self, format: CompressionFormat) -> Result<()> {
|
||||
let _download_progress = ArchiveDownloadProgress::new(self.ctx.session().progress());
|
||||
streaming_download_and_extract(
|
||||
&self.archive().url,
|
||||
format,
|
||||
self.ctx.target_dir(),
|
||||
self.ctx.session(),
|
||||
)
|
||||
}
|
||||
|
||||
/// Extracts a cached archive file while updating shared extraction activity.
|
||||
fn extract_cached_archive(&self, archive_path: &Path, format: CompressionFormat) -> Result<()> {
|
||||
let mut extraction_progress = ArchiveExtractionProgress::new(self.ctx.session().progress());
|
||||
let file = fs::open(archive_path)?;
|
||||
let result = extract_archive_raw(
|
||||
file,
|
||||
format,
|
||||
self.ctx.target_dir(),
|
||||
Some(&mut extraction_progress),
|
||||
);
|
||||
extraction_progress.finish();
|
||||
result
|
||||
}
|
||||
|
||||
/// Returns `true` if all declared plain outputs verify while updating shared verification
|
||||
/// progress.
|
||||
fn verify_outputs_with_progress(&self) -> Result<bool> {
|
||||
let mut verification_progress =
|
||||
ArchiveVerificationProgress::new(self.ctx.session().progress());
|
||||
let verified = self
|
||||
.output_verifier()
|
||||
.verify_with_progress(&self.archive().output_files, Some(&mut verification_progress))?;
|
||||
if verified {
|
||||
verification_progress.complete(self.archive().output_size());
|
||||
}
|
||||
Ok(verified)
|
||||
}
|
||||
}
|
||||
|
||||
/// Chooses whether an archive attempt uses the cache or streams directly.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
enum ArchiveMode {
|
||||
/// Download the archive to the cache, then extract it.
|
||||
Cached,
|
||||
/// Stream the archive directly into extraction.
|
||||
Streaming,
|
||||
}
|
||||
|
||||
impl ArchiveMode {
|
||||
/// Picks the archive mode from the process context.
|
||||
fn new(ctx: &ArchiveProcessContext) -> Result<Self> {
|
||||
if ctx.cache_dir().is_some() {
|
||||
ctx.session().require_request_limiter()?;
|
||||
return Ok(Self::Cached)
|
||||
}
|
||||
|
||||
Ok(Self::Streaming)
|
||||
}
|
||||
|
||||
/// Returns `true` when fetch failures should retry the whole archive attempt.
|
||||
const fn retries_fetch_errors(&self) -> bool {
|
||||
matches!(self, Self::Cached)
|
||||
}
|
||||
|
||||
/// Runs the selected archive mode for a single attempt.
|
||||
fn execute(&self, processor: &ArchiveProcessor, format: CompressionFormat) -> Result<()> {
|
||||
match self {
|
||||
Self::Cached => processor.run_cached_attempt(format),
|
||||
Self::Streaming => processor.run_streaming_attempt(format),
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -270,6 +270,7 @@ pub(crate) fn describe_prune_config(config: &Config) -> Vec<String> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
/// Formats one prune mode for the generated config summary.
|
||||
fn format_mode(mode: &PruneMode) -> String {
|
||||
match mode {
|
||||
PruneMode::Full => "\"full\"".to_string(),
|
||||
|
||||
490
crates/cli/commands/src/download/extract.rs
Normal file
490
crates/cli/commands/src/download/extract.rs
Normal file
@@ -0,0 +1,490 @@
|
||||
use super::{
|
||||
fetch::{ArchiveFetcher, DownloadedArchive},
|
||||
progress::{
|
||||
ArchiveExtractionProgress, ArchiveExtractionProgressHandle, DownloadProgress,
|
||||
DownloadRequestLimiter, ProgressReader, SharedProgress, SharedProgressReader,
|
||||
},
|
||||
session::DownloadSession,
|
||||
MAX_DOWNLOAD_RETRIES, RETRY_BACKOFF_SECS,
|
||||
};
|
||||
use eyre::{Result, WrapErr};
|
||||
use lz4::Decoder;
|
||||
use reqwest::blocking::Client as BlockingClient;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use reth_fs_util as fs;
|
||||
use std::{
|
||||
io::Read,
|
||||
path::{Component, Path, PathBuf},
|
||||
sync::{
|
||||
atomic::{AtomicBool, Ordering},
|
||||
Arc,
|
||||
},
|
||||
thread,
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tar::Archive;
|
||||
use tokio::task;
|
||||
use tracing::{info, warn};
|
||||
use url::Url;
|
||||
use zstd::stream::read::Decoder as ZstdDecoder;
|
||||
|
||||
const EXTENSION_TAR_LZ4: &str = ".tar.lz4";
|
||||
const EXTENSION_TAR_ZSTD: &str = ".tar.zst";
|
||||
const STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE: u64 = 64 * 1024 * 1024;
|
||||
const EXTRACTION_PROGRESS_POLL_INTERVAL: Duration = Duration::from_millis(100);
|
||||
|
||||
/// Supported compression formats for snapshots
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
pub(crate) enum CompressionFormat {
|
||||
/// LZ4-compressed tar archive.
|
||||
Lz4,
|
||||
/// Zstandard-compressed tar archive.
|
||||
Zstd,
|
||||
}
|
||||
|
||||
impl CompressionFormat {
|
||||
/// Detect compression format from file extension
|
||||
pub(crate) fn from_url(url: &str) -> Result<Self> {
|
||||
let path =
|
||||
Url::parse(url).map(|u| u.path().to_string()).unwrap_or_else(|_| url.to_string());
|
||||
|
||||
if path.ends_with(EXTENSION_TAR_LZ4) {
|
||||
Ok(Self::Lz4)
|
||||
} else if path.ends_with(EXTENSION_TAR_ZSTD) {
|
||||
Ok(Self::Zstd)
|
||||
} else {
|
||||
Err(eyre::eyre!(
|
||||
"Unsupported file format. Expected .tar.lz4 or .tar.zst, got: {}",
|
||||
path
|
||||
))
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a compressed tar archive to the target directory with progress tracking.
|
||||
fn extract_archive<R: Read>(
|
||||
reader: R,
|
||||
total_size: u64,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let progress_reader = ProgressReader::new(reader, total_size, cancel_token);
|
||||
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
let decoder = Decoder::new(progress_reader)?;
|
||||
Archive::new(decoder).unpack(target_dir)?;
|
||||
}
|
||||
CompressionFormat::Zstd => {
|
||||
let decoder = ZstdDecoder::new(progress_reader)?;
|
||||
Archive::new(decoder).unpack(target_dir)?;
|
||||
}
|
||||
}
|
||||
|
||||
println!();
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Extracts a compressed tar archive without progress tracking.
|
||||
pub(crate) fn extract_archive_raw<R: Read>(
|
||||
reader: R,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
match format {
|
||||
CompressionFormat::Lz4 => {
|
||||
unpack_archive(Archive::new(Decoder::new(reader)?), target_dir, progress)?;
|
||||
}
|
||||
CompressionFormat::Zstd => {
|
||||
unpack_archive(Archive::new(ZstdDecoder::new(reader)?), target_dir, progress)?;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn unpack_archive<R: Read>(
|
||||
mut archive: Archive<R>,
|
||||
target_dir: &Path,
|
||||
mut progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
let entries = archive.entries().wrap_err_with(|| {
|
||||
format!("failed to read archive entries for `{}`", target_dir.display())
|
||||
})?;
|
||||
|
||||
for entry in entries {
|
||||
let mut entry = entry.wrap_err_with(|| {
|
||||
format!("failed to read archive entry for `{}`", target_dir.display())
|
||||
})?;
|
||||
extract_entry_with_progress(&mut entry, target_dir, progress.as_deref_mut())?;
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn extract_entry_with_progress<R: Read>(
|
||||
entry: &mut tar::Entry<'_, R>,
|
||||
target_dir: &Path,
|
||||
progress: Option<&mut ArchiveExtractionProgress>,
|
||||
) -> Result<()> {
|
||||
let size = entry.header().entry_size().unwrap_or(0);
|
||||
let entry_type = entry.header().entry_type();
|
||||
|
||||
if !entry_type.is_file() || size == 0 {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
if size < STREAMING_EXTRACTION_PROGRESS_MIN_FILE_SIZE {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
if let Some(progress) = progress {
|
||||
progress.record_extracted(size);
|
||||
}
|
||||
return Ok(())
|
||||
}
|
||||
|
||||
let Some(progress_handle) = progress.as_ref().and_then(|progress| progress.handle()) else {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let Some(entry_path) = entry_destination_path(entry, target_dir)? else {
|
||||
entry.unpack_in(target_dir).wrap_err_with(|| {
|
||||
format!("failed to extract archive into `{}`", target_dir.display())
|
||||
})?;
|
||||
return Ok(())
|
||||
};
|
||||
|
||||
let stop = Arc::new(AtomicBool::new(false));
|
||||
let monitor = spawn_extraction_progress_monitor(entry_path, progress_handle, Arc::clone(&stop));
|
||||
let unpack_result = entry
|
||||
.unpack_in(target_dir)
|
||||
.wrap_err_with(|| format!("failed to extract archive into `{}`", target_dir.display()));
|
||||
stop.store(true, Ordering::Relaxed);
|
||||
|
||||
let monitor_result = monitor.join();
|
||||
unpack_result?;
|
||||
|
||||
monitor_result.map_err(|_| eyre::eyre!("extraction progress monitor panicked"))?;
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn entry_destination_path<R: Read>(
|
||||
entry: &tar::Entry<'_, R>,
|
||||
target_dir: &Path,
|
||||
) -> Result<Option<PathBuf>> {
|
||||
let mut file_dst = target_dir.to_path_buf();
|
||||
let path = entry.path().wrap_err("invalid path in archive entry")?;
|
||||
|
||||
for part in path.components() {
|
||||
match part {
|
||||
Component::Prefix(..) | Component::RootDir | Component::CurDir => continue,
|
||||
Component::ParentDir => return Ok(None),
|
||||
Component::Normal(part) => file_dst.push(part),
|
||||
}
|
||||
}
|
||||
|
||||
if file_dst == target_dir {
|
||||
return Ok(None)
|
||||
}
|
||||
|
||||
Ok(Some(file_dst))
|
||||
}
|
||||
|
||||
fn spawn_extraction_progress_monitor(
|
||||
entry_path: PathBuf,
|
||||
progress: ArchiveExtractionProgressHandle,
|
||||
stop: Arc<AtomicBool>,
|
||||
) -> thread::JoinHandle<()> {
|
||||
thread::spawn(move || {
|
||||
let mut extracted = 0_u64;
|
||||
|
||||
loop {
|
||||
record_extracted_file_bytes(&entry_path, &progress, &mut extracted);
|
||||
if stop.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
thread::sleep(EXTRACTION_PROGRESS_POLL_INTERVAL);
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
fn record_extracted_file_bytes(
|
||||
entry_path: &Path,
|
||||
progress: &ArchiveExtractionProgressHandle,
|
||||
extracted: &mut u64,
|
||||
) {
|
||||
let Ok(meta) = fs::metadata(entry_path) else { return };
|
||||
let len = meta.len();
|
||||
if len > *extracted {
|
||||
progress.record_extracted(len - *extracted);
|
||||
*extracted = len;
|
||||
}
|
||||
}
|
||||
|
||||
/// Extracts a snapshot from a local file.
|
||||
fn extract_from_file(path: &Path, format: CompressionFormat, target_dir: &Path) -> Result<()> {
|
||||
let file = std::fs::File::open(path)?;
|
||||
let total_size = file.metadata()?.len();
|
||||
info!(target: "reth::cli",
|
||||
file = %path.display(),
|
||||
size = %DownloadProgress::format_size(total_size),
|
||||
"Extracting local archive"
|
||||
);
|
||||
let start = Instant::now();
|
||||
extract_archive(file, total_size, format, target_dir, CancellationToken::new())?;
|
||||
info!(target: "reth::cli",
|
||||
file = %path.display(),
|
||||
elapsed = %DownloadProgress::format_duration(start.elapsed()),
|
||||
"Local extraction complete"
|
||||
);
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Streams a remote archive directly into the extractor without writing to disk.
|
||||
///
|
||||
/// On failure, retries from scratch up to [`MAX_DOWNLOAD_RETRIES`] times.
|
||||
pub(crate) fn streaming_download_and_extract(
|
||||
url: &str,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
session: &DownloadSession,
|
||||
) -> Result<()> {
|
||||
let shared = session.progress();
|
||||
let quiet = session.progress().is_some();
|
||||
let mut last_error: Option<eyre::Error> = None;
|
||||
|
||||
for attempt in 1..=MAX_DOWNLOAD_RETRIES {
|
||||
if attempt > 1 {
|
||||
info!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
"Retrying streaming download from scratch"
|
||||
);
|
||||
}
|
||||
|
||||
let client = BlockingClient::builder().connect_timeout(Duration::from_secs(30)).build()?;
|
||||
let _request_permit = session
|
||||
.request_limiter()
|
||||
.map(|limiter| limiter.acquire(session.progress(), session.cancel_token()))
|
||||
.transpose()?;
|
||||
|
||||
let response = match client.get(url).send().and_then(|r| r.error_for_status()) {
|
||||
Ok(r) => r,
|
||||
Err(error) => {
|
||||
let err = eyre::Error::from(error);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
warn!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
err = %err,
|
||||
"Streaming request failed, retrying"
|
||||
);
|
||||
}
|
||||
last_error = Some(err);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
if !quiet && let Some(size) = response.content_length() {
|
||||
info!(target: "reth::cli",
|
||||
url = %url,
|
||||
size = %DownloadProgress::format_size(size),
|
||||
"Streaming archive"
|
||||
);
|
||||
}
|
||||
|
||||
let result = if let Some(progress) = shared {
|
||||
let reader = SharedProgressReader { inner: response, progress: Arc::clone(progress) };
|
||||
extract_archive_raw(reader, format, target_dir, None)
|
||||
} else {
|
||||
let total_size = response.content_length().unwrap_or(0);
|
||||
extract_archive(
|
||||
response,
|
||||
total_size,
|
||||
format,
|
||||
target_dir,
|
||||
session.cancel_token().clone(),
|
||||
)
|
||||
};
|
||||
|
||||
match result {
|
||||
Ok(()) => return Ok(()),
|
||||
Err(error) => {
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
warn!(target: "reth::cli",
|
||||
url = %url,
|
||||
attempt,
|
||||
max = MAX_DOWNLOAD_RETRIES,
|
||||
err = %error,
|
||||
"Streaming extraction failed, retrying"
|
||||
);
|
||||
}
|
||||
last_error = Some(error);
|
||||
if attempt < MAX_DOWNLOAD_RETRIES {
|
||||
std::thread::sleep(Duration::from_secs(RETRY_BACKOFF_SECS));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
Err(last_error.unwrap_or_else(|| {
|
||||
eyre::eyre!("Streaming download failed after {MAX_DOWNLOAD_RETRIES} attempts")
|
||||
}))
|
||||
}
|
||||
|
||||
/// Fetches the snapshot from a remote URL with resume support, then extracts it.
|
||||
fn download_and_extract(
|
||||
url: &str,
|
||||
format: CompressionFormat,
|
||||
target_dir: &Path,
|
||||
session: DownloadSession,
|
||||
) -> Result<()> {
|
||||
let quiet = session.progress().is_some();
|
||||
let fetcher = ArchiveFetcher::new(url.to_string(), target_dir, session.clone());
|
||||
let DownloadedArchive { path: downloaded_path, size: total_size } = fetcher.download(None)?;
|
||||
|
||||
let file_name =
|
||||
downloaded_path.file_name().map(|f| f.to_string_lossy().to_string()).unwrap_or_default();
|
||||
|
||||
if !quiet {
|
||||
info!(target: "reth::cli",
|
||||
file = %file_name,
|
||||
size = %DownloadProgress::format_size(total_size),
|
||||
"Extracting archive"
|
||||
);
|
||||
}
|
||||
let file = fs::open(&downloaded_path)?;
|
||||
|
||||
if quiet {
|
||||
extract_archive_raw(file, format, target_dir, None)?;
|
||||
} else {
|
||||
extract_archive(file, total_size, format, target_dir, session.cancel_token().clone())?;
|
||||
info!(target: "reth::cli",
|
||||
file = %file_name,
|
||||
"Extraction complete"
|
||||
);
|
||||
}
|
||||
|
||||
fetcher.cleanup_downloaded_files();
|
||||
session.record_archive_output_complete(total_size);
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot, blocking until finished.
|
||||
///
|
||||
/// Supports `file://` URLs for local files and HTTP(S) URLs for remote downloads.
|
||||
/// When `resumable` is true, downloads to a `.part` file first with HTTP Range resume
|
||||
/// support. Otherwise streams directly into the extractor.
|
||||
fn blocking_download_and_extract(
|
||||
url: &str,
|
||||
target_dir: &Path,
|
||||
shared: Option<Arc<SharedProgress>>,
|
||||
resumable: bool,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let format = CompressionFormat::from_url(url)?;
|
||||
|
||||
if let Ok(parsed_url) = Url::parse(url) &&
|
||||
parsed_url.scheme() == "file"
|
||||
{
|
||||
let session = DownloadSession::new(shared, request_limiter, cancel_token);
|
||||
let file_path = parsed_url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// URL path: {}", url))?;
|
||||
let result = extract_from_file(&file_path, format, target_dir);
|
||||
if result.is_ok() {
|
||||
session.record_archive_output_complete(file_path.metadata()?.len());
|
||||
}
|
||||
result
|
||||
} else if let Some(request_limiter) = request_limiter {
|
||||
download_and_extract(
|
||||
url,
|
||||
format,
|
||||
target_dir,
|
||||
DownloadSession::new(shared, Some(request_limiter), cancel_token),
|
||||
)
|
||||
} else if resumable {
|
||||
let session =
|
||||
DownloadSession::new(shared, Some(DownloadRequestLimiter::new(1)), cancel_token);
|
||||
download_and_extract(url, format, target_dir, session)
|
||||
} else {
|
||||
let session = DownloadSession::new(shared, None, cancel_token);
|
||||
let result = streaming_download_and_extract(url, format, target_dir, &session);
|
||||
if result.is_ok() {
|
||||
session.record_archive_output_complete(0);
|
||||
}
|
||||
result
|
||||
}
|
||||
}
|
||||
|
||||
/// Downloads and extracts a snapshot archive asynchronously.
|
||||
///
|
||||
/// When `shared` is provided, download progress is reported to the shared
|
||||
/// counter for aggregated display. Otherwise uses a local progress bar.
|
||||
/// When `resumable` is true, uses two-phase download with `.part` files.
|
||||
pub(crate) async fn stream_and_extract(
|
||||
url: &str,
|
||||
target_dir: &Path,
|
||||
shared: Option<Arc<SharedProgress>>,
|
||||
resumable: bool,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Result<()> {
|
||||
let target_dir = target_dir.to_path_buf();
|
||||
let url = url.to_string();
|
||||
task::spawn_blocking(move || {
|
||||
blocking_download_and_extract(
|
||||
&url,
|
||||
&target_dir,
|
||||
shared,
|
||||
resumable,
|
||||
request_limiter,
|
||||
cancel_token,
|
||||
)
|
||||
})
|
||||
.await??;
|
||||
|
||||
Ok(())
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
|
||||
#[test]
|
||||
fn test_compression_format_detection() {
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("https://example.com/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.lz4"),
|
||||
Ok(CompressionFormat::Lz4)
|
||||
));
|
||||
assert!(matches!(
|
||||
CompressionFormat::from_url("file:///path/to/snapshot.tar.zst"),
|
||||
Ok(CompressionFormat::Zstd)
|
||||
));
|
||||
assert!(CompressionFormat::from_url("https://example.com/snapshot.tar.gz").is_err());
|
||||
}
|
||||
}
|
||||
1018
crates/cli/commands/src/download/fetch.rs
Normal file
1018
crates/cli/commands/src/download/fetch.rs
Normal file
File diff suppressed because it is too large
Load Diff
@@ -10,6 +10,10 @@ use std::{
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
fn is_zero(value: &u64) -> bool {
|
||||
*value == 0
|
||||
}
|
||||
|
||||
/// A snapshot manifest describes available components for a snapshot at a given block height.
|
||||
///
|
||||
/// Each component is either a single archive (state) or a set of chunked archives (static file
|
||||
@@ -62,6 +66,12 @@ pub struct SingleArchive {
|
||||
pub file: String,
|
||||
/// Compressed archive size in bytes.
|
||||
pub size: u64,
|
||||
/// Total extracted plain-output size in bytes.
|
||||
///
|
||||
/// Older manifests may omit this, in which case downloaders should derive it from
|
||||
/// `output_files`.
|
||||
#[serde(default, skip_serializing_if = "is_zero")]
|
||||
pub decompressed_size: u64,
|
||||
/// Optional BLAKE3 checksum of the compressed archive.
|
||||
#[serde(default, skip_serializing_if = "Option::is_none")]
|
||||
pub blake3: Option<String>,
|
||||
@@ -83,6 +93,12 @@ pub struct ChunkedArchive {
|
||||
/// Computed during manifest generation. Older manifests may omit this.
|
||||
#[serde(default)]
|
||||
pub chunk_sizes: Vec<u64>,
|
||||
/// Extracted plain-output size of each chunk in bytes, ordered from first to last.
|
||||
///
|
||||
/// Older manifests may omit this, in which case downloaders should derive it from
|
||||
/// `chunk_output_files`.
|
||||
#[serde(default, skip_serializing_if = "Vec::is_empty")]
|
||||
pub chunk_decompressed_sizes: Vec<u64>,
|
||||
/// Expected extracted plain files per chunk, ordered from first to last.
|
||||
///
|
||||
/// This is the authoritative integrity source for the modular download path.
|
||||
@@ -101,9 +117,9 @@ pub struct OutputFileChecksum {
|
||||
pub blake3: String,
|
||||
}
|
||||
|
||||
/// A single archive with concrete URL and optional integrity metadata.
|
||||
/// A concrete snapshot archive with its download and verification metadata.
|
||||
#[derive(Debug, Clone, PartialEq, Eq)]
|
||||
pub struct ArchiveDescriptor {
|
||||
pub struct SnapshotArchive {
|
||||
pub url: String,
|
||||
pub file_name: String,
|
||||
pub size: u64,
|
||||
@@ -111,6 +127,13 @@ pub struct ArchiveDescriptor {
|
||||
pub output_files: Vec<OutputFileChecksum>,
|
||||
}
|
||||
|
||||
impl SnapshotArchive {
|
||||
/// Returns the total extracted plain-output size for this archive.
|
||||
pub fn output_size(&self) -> u64 {
|
||||
self.output_files.iter().map(|file| file.size).sum()
|
||||
}
|
||||
}
|
||||
|
||||
/// How much of a component to download.
|
||||
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
|
||||
pub enum ComponentSelection {
|
||||
@@ -315,19 +338,19 @@ impl SnapshotManifest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns concrete archive descriptors for a component, optionally limited to distance.
|
||||
pub fn archive_descriptors_for_distance(
|
||||
/// Returns concrete snapshot archives for a component, optionally limited to distance.
|
||||
pub fn snapshot_archives_for_distance(
|
||||
&self,
|
||||
ty: SnapshotComponentType,
|
||||
distance: Option<u64>,
|
||||
) -> Vec<ArchiveDescriptor> {
|
||||
) -> Vec<SnapshotArchive> {
|
||||
let Some(component) = self.component(ty) else {
|
||||
return vec![];
|
||||
};
|
||||
|
||||
match component {
|
||||
ComponentManifest::Single(single) => {
|
||||
vec![ArchiveDescriptor {
|
||||
vec![SnapshotArchive {
|
||||
url: format!("{}/{}", self.base_url_or_empty(), single.file),
|
||||
file_name: single.file.clone(),
|
||||
size: single.size,
|
||||
@@ -357,7 +380,7 @@ impl SnapshotManifest {
|
||||
let output_files =
|
||||
chunked.chunk_output_files.get(i as usize).cloned().unwrap_or_default();
|
||||
|
||||
ArchiveDescriptor {
|
||||
SnapshotArchive {
|
||||
url: format!("{}/{}", self.base_url_or_empty(), file_name),
|
||||
file_name,
|
||||
size,
|
||||
@@ -398,6 +421,36 @@ impl SnapshotManifest {
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the exact extracted plain-output size for a component given a distance selection.
|
||||
pub fn output_size_for_distance(
|
||||
&self,
|
||||
ty: SnapshotComponentType,
|
||||
distance: Option<u64>,
|
||||
) -> u64 {
|
||||
let Some(component) = self.component(ty) else {
|
||||
return 0;
|
||||
};
|
||||
|
||||
match component {
|
||||
ComponentManifest::Single(single) => single.output_size(),
|
||||
ComponentManifest::Chunked(chunked) => {
|
||||
let num_chunks = chunked.num_chunks();
|
||||
let start_chunk = match distance {
|
||||
Some(dist) => {
|
||||
let needed = dist.min(chunked.total_blocks);
|
||||
let needed_chunks = needed.div_ceil(chunked.blocks_per_file);
|
||||
num_chunks.saturating_sub(needed_chunks)
|
||||
}
|
||||
None => 0,
|
||||
};
|
||||
|
||||
(start_chunk..num_chunks)
|
||||
.map(|index| chunked.chunk_output_size(index as usize))
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the number of chunks that would be downloaded for a given distance.
|
||||
pub fn chunks_for_distance(&self, ty: SnapshotComponentType, distance: Option<u64>) -> u64 {
|
||||
let Some(ComponentManifest::Chunked(chunked)) = self.component(ty) else {
|
||||
@@ -421,6 +474,14 @@ impl ComponentManifest {
|
||||
Self::Chunked(c) => c.chunk_sizes.iter().sum(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the total extracted plain-output size for this component.
|
||||
pub fn total_output_size(&self) -> u64 {
|
||||
match self {
|
||||
Self::Single(single) => single.output_size(),
|
||||
Self::Chunked(chunked) => chunked.total_output_size(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ChunkedArchive {
|
||||
@@ -428,6 +489,39 @@ impl ChunkedArchive {
|
||||
pub fn num_chunks(&self) -> u64 {
|
||||
self.total_blocks.div_ceil(self.blocks_per_file)
|
||||
}
|
||||
|
||||
/// Returns the extracted plain-output size for one chunk.
|
||||
pub fn chunk_output_size(&self, index: usize) -> u64 {
|
||||
self.chunk_decompressed_sizes.get(index).copied().unwrap_or_else(|| {
|
||||
self.chunk_output_files
|
||||
.get(index)
|
||||
.map(|files| files.iter().map(|file| file.size).sum())
|
||||
.unwrap_or(0)
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns the total extracted plain-output size across all chunks.
|
||||
pub fn total_output_size(&self) -> u64 {
|
||||
if !self.chunk_decompressed_sizes.is_empty() {
|
||||
self.chunk_decompressed_sizes.iter().sum()
|
||||
} else {
|
||||
self.chunk_output_files
|
||||
.iter()
|
||||
.map(|files| files.iter().map(|file| file.size).sum::<u64>())
|
||||
.sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl SingleArchive {
|
||||
/// Returns the total extracted plain-output size for this archive.
|
||||
pub fn output_size(&self) -> u64 {
|
||||
if self.decompressed_size != 0 {
|
||||
self.decompressed_size
|
||||
} else {
|
||||
self.output_files.iter().map(|file| file.size).sum()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetch a snapshot manifest from a URL.
|
||||
@@ -516,6 +610,10 @@ pub fn generate_manifest(
|
||||
blocks_per_file,
|
||||
total_blocks: block,
|
||||
chunk_sizes,
|
||||
chunk_decompressed_sizes: chunk_output_files
|
||||
.iter()
|
||||
.map(|files| files.iter().map(|file| file.size).sum())
|
||||
.collect(),
|
||||
chunk_output_files,
|
||||
}),
|
||||
);
|
||||
@@ -532,6 +630,7 @@ pub fn generate_manifest(
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: state_size,
|
||||
decompressed_size: state_output_files.iter().map(|file| file.size).sum(),
|
||||
blake3: None,
|
||||
output_files: state_output_files,
|
||||
}),
|
||||
@@ -546,6 +645,7 @@ pub fn generate_manifest(
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: rocksdb_size,
|
||||
decompressed_size: rocksdb_output_files.iter().map(|file| file.size).sum(),
|
||||
blake3: None,
|
||||
output_files: rocksdb_output_files,
|
||||
}),
|
||||
@@ -814,6 +914,7 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 0,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
}),
|
||||
@@ -824,6 +925,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_500_000,
|
||||
chunk_sizes: vec![80_000, 100_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![], vec![], vec![]],
|
||||
}),
|
||||
);
|
||||
@@ -833,6 +935,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_500_000,
|
||||
chunk_sizes: vec![40_000, 50_000, 60_000],
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![], vec![], vec![]],
|
||||
}),
|
||||
);
|
||||
@@ -883,6 +986,7 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: 777,
|
||||
decompressed_size: 0,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
}),
|
||||
@@ -955,6 +1059,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 24_396_822,
|
||||
chunk_sizes: vec![100; 49], // 49 chunks
|
||||
chunk_decompressed_sizes: vec![],
|
||||
chunk_output_files: vec![vec![]; 49],
|
||||
}),
|
||||
);
|
||||
@@ -997,6 +1102,68 @@ mod tests {
|
||||
assert_eq!(m.size_for_distance(SnapshotComponentType::Receipts, None), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn output_size_for_distance_uses_manifest_or_output_files() {
|
||||
let m = test_manifest();
|
||||
assert_eq!(m.output_size_for_distance(SnapshotComponentType::Transactions, None), 0);
|
||||
|
||||
let mut components = BTreeMap::new();
|
||||
components.insert(
|
||||
"state".to_string(),
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 1_000,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
size: 1_000,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
components.insert(
|
||||
"transactions".to_string(),
|
||||
ComponentManifest::Chunked(ChunkedArchive {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![80_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![111, 222],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
|
||||
size: 111,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_500000_999999.bin".to_string(),
|
||||
size: 222,
|
||||
blake3: "h1".to_string(),
|
||||
}],
|
||||
],
|
||||
}),
|
||||
);
|
||||
let manifest = SnapshotManifest {
|
||||
block: 1_000_000,
|
||||
chain_id: 1,
|
||||
storage_version: 2,
|
||||
timestamp: 0,
|
||||
base_url: Some("https://example.com".to_string()),
|
||||
reth_version: None,
|
||||
components,
|
||||
};
|
||||
|
||||
assert_eq!(manifest.output_size_for_distance(SnapshotComponentType::State, None), 1_000);
|
||||
assert_eq!(
|
||||
manifest.output_size_for_distance(SnapshotComponentType::Transactions, None),
|
||||
333
|
||||
);
|
||||
assert_eq!(
|
||||
manifest.output_size_for_distance(SnapshotComponentType::Transactions, Some(500_000)),
|
||||
222
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_descriptors_include_checksum_metadata() {
|
||||
let mut components = BTreeMap::new();
|
||||
@@ -1005,6 +1172,7 @@ mod tests {
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 100,
|
||||
decompressed_size: 1_000,
|
||||
blake3: Some("abc123".to_string()),
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
@@ -1019,6 +1187,7 @@ mod tests {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![80_000, 120_000],
|
||||
chunk_decompressed_sizes: vec![111, 222],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/static_file_transactions_0_499999.bin".to_string(),
|
||||
@@ -1044,13 +1213,13 @@ mod tests {
|
||||
components,
|
||||
};
|
||||
|
||||
let state = m.archive_descriptors_for_distance(SnapshotComponentType::State, None);
|
||||
let state = m.snapshot_archives_for_distance(SnapshotComponentType::State, None);
|
||||
assert_eq!(state.len(), 1);
|
||||
assert_eq!(state[0].file_name, "state.tar.zst");
|
||||
assert_eq!(state[0].blake3.as_deref(), Some("abc123"));
|
||||
assert_eq!(state[0].output_files.len(), 1);
|
||||
|
||||
let tx = m.archive_descriptors_for_distance(SnapshotComponentType::Transactions, None);
|
||||
let tx = m.snapshot_archives_for_distance(SnapshotComponentType::Transactions, None);
|
||||
assert_eq!(tx.len(), 2);
|
||||
assert_eq!(tx[0].blake3, None);
|
||||
assert_eq!(tx[1].blake3, None);
|
||||
@@ -1073,6 +1242,7 @@ mod tests {
|
||||
panic!("state should be a single archive")
|
||||
};
|
||||
assert_eq!(state.file, "state.tar.zst");
|
||||
assert!(state.decompressed_size > 0);
|
||||
assert!(!state.output_files.is_empty());
|
||||
assert_eq!(state.output_files[0].path, "db/mdbx.dat");
|
||||
assert!(output.path().join("state.tar.zst").exists());
|
||||
@@ -1097,6 +1267,7 @@ mod tests {
|
||||
panic!("rocksdb indices should be a single archive")
|
||||
};
|
||||
assert_eq!(rocksdb.file, "rocksdb_indices.tar.zst");
|
||||
assert!(rocksdb.decompressed_size > 0);
|
||||
assert!(!rocksdb.output_files.is_empty());
|
||||
assert_eq!(rocksdb.output_files[0].path, "rocksdb/CURRENT");
|
||||
assert!(output.path().join("rocksdb_indices.tar.zst").exists());
|
||||
|
||||
@@ -45,6 +45,7 @@ pub struct SnapshotManifestCommand {
|
||||
}
|
||||
|
||||
impl SnapshotManifestCommand {
|
||||
/// Packages snapshot archives and writes the manifest file.
|
||||
pub fn execute(self) -> Result<()> {
|
||||
let block = match self.block {
|
||||
Some(block) => block,
|
||||
@@ -88,6 +89,7 @@ impl SnapshotManifestCommand {
|
||||
}
|
||||
}
|
||||
|
||||
/// Infers the snapshot block from the source datadir.
|
||||
fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
if let Ok(block) = infer_snapshot_block_from_db(source_datadir) {
|
||||
return Ok(block);
|
||||
@@ -102,6 +104,7 @@ fn infer_snapshot_block(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
Ok(block)
|
||||
}
|
||||
|
||||
/// Reads the snapshot block from the source database Finish stage checkpoint.
|
||||
fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let candidates = [source_datadir.join("db"), source_datadir.to_path_buf()];
|
||||
|
||||
@@ -126,6 +129,7 @@ fn infer_snapshot_block_from_db(source_datadir: &std::path::Path) -> Result<u64>
|
||||
)
|
||||
}
|
||||
|
||||
/// Infers the snapshot block from the highest header static-file range.
|
||||
fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let max_end = header_ranges(source_datadir)?
|
||||
.into_iter()
|
||||
@@ -135,6 +139,7 @@ fn infer_snapshot_block_from_headers(source_datadir: &std::path::Path) -> Result
|
||||
Ok(max_end)
|
||||
}
|
||||
|
||||
/// Infers the static-file block span from header file ranges.
|
||||
fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
let mut inferred = None;
|
||||
for (start, end) in header_ranges(source_datadir)? {
|
||||
@@ -161,6 +166,7 @@ fn infer_blocks_per_file(source_datadir: &std::path::Path) -> Result<u64> {
|
||||
})
|
||||
}
|
||||
|
||||
/// Collects header static-file ranges from the source datadir.
|
||||
fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
|
||||
let static_files_dir = source_datadir.join("static_files");
|
||||
let static_files_dir =
|
||||
@@ -183,6 +189,7 @@ fn header_ranges(source_datadir: &std::path::Path) -> Result<Vec<(u64, u64)>> {
|
||||
Ok(ranges)
|
||||
}
|
||||
|
||||
/// Parses the block range from a header static-file name.
|
||||
fn parse_headers_range(file_name: &str) -> Option<(u64, u64)> {
|
||||
let remainder = file_name.strip_prefix("static_file_headers_")?;
|
||||
let (start, end_with_suffix) = remainder.split_once('_')?;
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
322
crates/cli/commands/src/download/planning.rs
Normal file
322
crates/cli/commands/src/download/planning.rs
Normal file
@@ -0,0 +1,322 @@
|
||||
use super::{manifest::*, verify::OutputVerifier};
|
||||
use eyre::Result;
|
||||
use std::{collections::BTreeMap, path::Path};
|
||||
use tracing::info;
|
||||
|
||||
/// One archive selected from the manifest, along with its component name.
|
||||
#[derive(Debug, Clone)]
|
||||
pub(crate) struct PlannedArchive {
|
||||
/// Snapshot component type this archive belongs to.
|
||||
pub(crate) ty: SnapshotComponentType,
|
||||
/// User-facing component name used in logs.
|
||||
pub(crate) component: String,
|
||||
/// Concrete snapshot archive metadata resolved from the manifest.
|
||||
pub(crate) archive: SnapshotArchive,
|
||||
}
|
||||
|
||||
/// The archive list for a modular snapshot download.
|
||||
#[derive(Debug)]
|
||||
pub(crate) struct PlannedDownloads {
|
||||
/// Concrete archives that still need reuse checks or processing.
|
||||
pub(crate) archives: Vec<PlannedArchive>,
|
||||
/// Total compressed download size of all planned archives.
|
||||
pub(crate) total_download_size: u64,
|
||||
/// Total extracted plain-output size of all planned archives.
|
||||
pub(crate) total_output_size: u64,
|
||||
}
|
||||
|
||||
impl PlannedDownloads {
|
||||
/// Returns the number of concrete archives queued for this snapshot selection.
|
||||
pub(crate) const fn total_archives(&self) -> usize {
|
||||
self.archives.len()
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the sort priority used to schedule archives.
|
||||
pub(crate) const fn archive_priority_rank(ty: SnapshotComponentType) -> u8 {
|
||||
match ty {
|
||||
SnapshotComponentType::State => 0,
|
||||
SnapshotComponentType::RocksdbIndices => 1,
|
||||
_ => 2,
|
||||
}
|
||||
}
|
||||
|
||||
/// Startup summary showing how much of the selected work can be reused.
|
||||
#[derive(Debug, Default, Clone, Copy)]
|
||||
pub(crate) struct DownloadStartupSummary {
|
||||
/// Archives whose declared outputs already verify on disk.
|
||||
pub(crate) reusable: usize,
|
||||
/// Archives that still need to be downloaded or retried.
|
||||
pub(crate) needs_download: usize,
|
||||
}
|
||||
|
||||
/// Checks selected archives against existing output files before work begins.
|
||||
pub(crate) fn summarize_download_startup(
|
||||
all_downloads: &[PlannedArchive],
|
||||
target_dir: &Path,
|
||||
) -> Result<DownloadStartupSummary> {
|
||||
let mut summary = DownloadStartupSummary::default();
|
||||
let verifier = OutputVerifier::new(target_dir);
|
||||
|
||||
for planned in all_downloads {
|
||||
if verifier.verify(&planned.archive.output_files)? {
|
||||
summary.reusable += 1;
|
||||
} else {
|
||||
summary.needs_download += 1;
|
||||
}
|
||||
}
|
||||
|
||||
Ok(summary)
|
||||
}
|
||||
|
||||
/// Converts a selection into the manifest distance form used for archive lookup.
|
||||
fn selection_archive_distance(
|
||||
selection: &ComponentSelection,
|
||||
snapshot_block: u64,
|
||||
) -> Option<Option<u64>> {
|
||||
match selection {
|
||||
ComponentSelection::All => Some(None),
|
||||
ComponentSelection::Distance(distance) => Some(Some(*distance)),
|
||||
ComponentSelection::Since(block) => Some(Some(snapshot_block.saturating_sub(*block) + 1)),
|
||||
ComponentSelection::None => None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sorts planned archives into a stable processing order.
|
||||
fn sort_planned_archives(all_downloads: &mut [PlannedArchive]) {
|
||||
all_downloads.sort_by(|a, b| {
|
||||
archive_priority_rank(a.ty)
|
||||
.cmp(&archive_priority_rank(b.ty))
|
||||
.then_with(|| a.component.cmp(&b.component))
|
||||
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
|
||||
});
|
||||
}
|
||||
|
||||
/// Expands component selections into the archives that need to be processed.
|
||||
pub(crate) fn collect_planned_archives(
|
||||
manifest: &SnapshotManifest,
|
||||
selections: &BTreeMap<SnapshotComponentType, ComponentSelection>,
|
||||
) -> Result<PlannedDownloads> {
|
||||
let mut archives = Vec::new();
|
||||
let mut total_download_size = 0;
|
||||
let mut total_output_size = 0;
|
||||
|
||||
for (ty, selection) in selections {
|
||||
let Some(distance) = selection_archive_distance(selection, manifest.block) else {
|
||||
continue;
|
||||
};
|
||||
total_download_size += manifest.size_for_distance(*ty, distance);
|
||||
total_output_size += manifest.output_size_for_distance(*ty, distance);
|
||||
|
||||
let snapshot_archives = manifest.snapshot_archives_for_distance(*ty, distance);
|
||||
let component = ty.display_name().to_string();
|
||||
if !snapshot_archives.is_empty() {
|
||||
info!(target: "reth::cli",
|
||||
component = %component,
|
||||
archives = snapshot_archives.len(),
|
||||
selection = %selection,
|
||||
"Queued component for download"
|
||||
);
|
||||
}
|
||||
|
||||
for archive in snapshot_archives {
|
||||
if archive.output_files.is_empty() {
|
||||
eyre::bail!(
|
||||
"Invalid modular manifest: {} is missing plain output checksum metadata",
|
||||
archive.file_name
|
||||
);
|
||||
}
|
||||
|
||||
archives.push(PlannedArchive { ty: *ty, component: component.clone(), archive });
|
||||
}
|
||||
}
|
||||
|
||||
sort_planned_archives(&mut archives);
|
||||
Ok(PlannedDownloads { archives, total_download_size, total_output_size })
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use tempfile::tempdir;
|
||||
|
||||
#[test]
|
||||
fn summarize_download_startup_counts_reusable_and_needs_download() {
|
||||
let dir = tempdir().unwrap();
|
||||
let target_dir = dir.path();
|
||||
let ok_file = target_dir.join("ok.bin");
|
||||
std::fs::write(&ok_file, vec![1_u8; 4]).unwrap();
|
||||
let ok_hash = blake3::hash(&[1_u8; 4]).to_hex().to_string();
|
||||
|
||||
let planned = vec![
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::State,
|
||||
component: "State".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/ok.tar.zst".to_string(),
|
||||
file_name: "ok.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "ok.bin".to_string(),
|
||||
size: 4,
|
||||
blake3: ok_hash,
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Headers,
|
||||
component: "Headers".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/missing.tar.zst".to_string(),
|
||||
file_name: "missing.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "missing.bin".to_string(),
|
||||
size: 1,
|
||||
blake3: "deadbeef".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Transactions,
|
||||
component: "Transactions".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "https://example.com/bad-size.tar.zst".to_string(),
|
||||
file_name: "bad-size.tar.zst".to_string(),
|
||||
size: 10,
|
||||
blake3: None,
|
||||
output_files: vec![],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
let summary = summarize_download_startup(&planned, target_dir).unwrap();
|
||||
assert_eq!(summary.reusable, 1);
|
||||
assert_eq!(summary.needs_download, 2);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_priority_prefers_state_then_rocksdb() {
|
||||
let mut planned = [
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::Transactions,
|
||||
component: "Transactions".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u3".to_string(),
|
||||
file_name: "t.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "a".to_string(),
|
||||
size: 1,
|
||||
blake3: "x".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::RocksdbIndices,
|
||||
component: "RocksDB Indices".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u2".to_string(),
|
||||
file_name: "rocksdb_indices.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "b".to_string(),
|
||||
size: 1,
|
||||
blake3: "y".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
PlannedArchive {
|
||||
ty: SnapshotComponentType::State,
|
||||
component: "State (mdbx)".to_string(),
|
||||
archive: SnapshotArchive {
|
||||
url: "u1".to_string(),
|
||||
file_name: "state.tar.zst".to_string(),
|
||||
size: 1,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "c".to_string(),
|
||||
size: 1,
|
||||
blake3: "z".to_string(),
|
||||
}],
|
||||
},
|
||||
},
|
||||
];
|
||||
|
||||
planned.sort_by(|a, b| {
|
||||
archive_priority_rank(a.ty)
|
||||
.cmp(&archive_priority_rank(b.ty))
|
||||
.then_with(|| a.component.cmp(&b.component))
|
||||
.then_with(|| a.archive.file_name.cmp(&b.archive.file_name))
|
||||
});
|
||||
|
||||
assert_eq!(planned[0].ty, SnapshotComponentType::State);
|
||||
assert_eq!(planned[1].ty, SnapshotComponentType::RocksdbIndices);
|
||||
assert_eq!(planned[2].ty, SnapshotComponentType::Transactions);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn collect_planned_archives_tracks_download_and_output_totals() {
|
||||
let mut components = BTreeMap::new();
|
||||
components.insert(
|
||||
"state".to_string(),
|
||||
ComponentManifest::Single(SingleArchive {
|
||||
file: "state.tar.zst".to_string(),
|
||||
size: 10,
|
||||
decompressed_size: 100,
|
||||
blake3: None,
|
||||
output_files: vec![OutputFileChecksum {
|
||||
path: "db/mdbx.dat".to_string(),
|
||||
size: 100,
|
||||
blake3: "h0".to_string(),
|
||||
}],
|
||||
}),
|
||||
);
|
||||
components.insert(
|
||||
"transactions".to_string(),
|
||||
ComponentManifest::Chunked(ChunkedArchive {
|
||||
blocks_per_file: 500_000,
|
||||
total_blocks: 1_000_000,
|
||||
chunk_sizes: vec![20, 30],
|
||||
chunk_decompressed_sizes: vec![200, 300],
|
||||
chunk_output_files: vec![
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/tx-0".to_string(),
|
||||
size: 200,
|
||||
blake3: "h1".to_string(),
|
||||
}],
|
||||
vec![OutputFileChecksum {
|
||||
path: "static_files/tx-1".to_string(),
|
||||
size: 300,
|
||||
blake3: "h2".to_string(),
|
||||
}],
|
||||
],
|
||||
}),
|
||||
);
|
||||
|
||||
let manifest = SnapshotManifest {
|
||||
block: 1_000_000,
|
||||
chain_id: 1,
|
||||
storage_version: 2,
|
||||
timestamp: 0,
|
||||
base_url: Some("https://example.com".to_string()),
|
||||
reth_version: None,
|
||||
components,
|
||||
};
|
||||
|
||||
let selections = BTreeMap::from([
|
||||
(SnapshotComponentType::State, ComponentSelection::All),
|
||||
(SnapshotComponentType::Transactions, ComponentSelection::Distance(500_000)),
|
||||
]);
|
||||
|
||||
let planned = collect_planned_archives(&manifest, &selections).unwrap();
|
||||
|
||||
assert_eq!(planned.total_download_size, 40);
|
||||
assert_eq!(planned.total_output_size, 400);
|
||||
assert_eq!(planned.archives.len(), 2);
|
||||
}
|
||||
}
|
||||
844
crates/cli/commands/src/download/progress.rs
Normal file
844
crates/cli/commands/src/download/progress.rs
Normal file
@@ -0,0 +1,844 @@
|
||||
use eyre::Result;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use std::{
|
||||
io::{self, Read, Write},
|
||||
sync::{
|
||||
atomic::{AtomicBool, AtomicU64, Ordering},
|
||||
Arc, Condvar, Mutex,
|
||||
},
|
||||
time::{Duration, Instant},
|
||||
};
|
||||
use tracing::info;
|
||||
|
||||
const BYTE_UNITS: [&str; 4] = ["B", "KB", "MB", "GB"];
|
||||
|
||||
/// Tracks download progress and throttles display updates to every 100ms.
|
||||
pub(crate) struct DownloadProgress {
|
||||
/// Bytes copied so far for this single download.
|
||||
pub(crate) downloaded: u64,
|
||||
/// Total bytes expected for this single download.
|
||||
total_size: u64,
|
||||
/// Time when the progress line was last printed.
|
||||
last_displayed: Instant,
|
||||
/// Time when this progress tracker started.
|
||||
started_at: Instant,
|
||||
}
|
||||
|
||||
impl DownloadProgress {
|
||||
/// Creates new progress tracker with given total size
|
||||
pub(crate) fn new(total_size: u64) -> Self {
|
||||
let now = Instant::now();
|
||||
Self { downloaded: 0, total_size, last_displayed: now, started_at: now }
|
||||
}
|
||||
|
||||
/// Converts bytes to human readable format (B, KB, MB, GB)
|
||||
pub(crate) fn format_size(size: u64) -> String {
|
||||
let mut size = size as f64;
|
||||
let mut unit_index = 0;
|
||||
|
||||
while size >= 1024.0 && unit_index < BYTE_UNITS.len() - 1 {
|
||||
size /= 1024.0;
|
||||
unit_index += 1;
|
||||
}
|
||||
|
||||
format!("{:.2} {}", size, BYTE_UNITS[unit_index])
|
||||
}
|
||||
|
||||
/// Format duration as human readable string
|
||||
pub(crate) fn format_duration(duration: Duration) -> String {
|
||||
let secs = duration.as_secs();
|
||||
if secs < 60 {
|
||||
format!("{secs}s")
|
||||
} else if secs < 3600 {
|
||||
format!("{}m {}s", secs / 60, secs % 60)
|
||||
} else {
|
||||
format!("{}h {}m", secs / 3600, (secs % 3600) / 60)
|
||||
}
|
||||
}
|
||||
|
||||
/// Updates progress bar (for single-archive legacy downloads)
|
||||
pub(crate) fn update(&mut self, chunk_size: u64) -> Result<()> {
|
||||
self.downloaded += chunk_size;
|
||||
|
||||
if self.last_displayed.elapsed() >= Duration::from_millis(100) {
|
||||
let formatted_downloaded = Self::format_size(self.downloaded);
|
||||
let formatted_total = Self::format_size(self.total_size);
|
||||
let progress = (self.downloaded as f64 / self.total_size as f64) * 100.0;
|
||||
|
||||
let elapsed = self.started_at.elapsed();
|
||||
let eta = if self.downloaded > 0 {
|
||||
let remaining = self.total_size.saturating_sub(self.downloaded);
|
||||
let speed = self.downloaded as f64 / elapsed.as_secs_f64();
|
||||
if speed > 0.0 {
|
||||
Duration::from_secs_f64(remaining as f64 / speed)
|
||||
} else {
|
||||
Duration::ZERO
|
||||
}
|
||||
} else {
|
||||
Duration::ZERO
|
||||
};
|
||||
let eta_str = Self::format_duration(eta);
|
||||
|
||||
print!(
|
||||
"\rDownloading and extracting... {progress:.2}% ({formatted_downloaded} / {formatted_total}) ETA: {eta_str} ",
|
||||
);
|
||||
io::stdout().flush()?;
|
||||
self.last_displayed = Instant::now();
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Copy)]
|
||||
struct PhaseStart {
|
||||
started_at: Instant,
|
||||
baseline_bytes: u64,
|
||||
}
|
||||
|
||||
/// Shared progress counters for parallel downloads.
|
||||
pub(crate) struct SharedProgress {
|
||||
/// Raw HTTP bytes fetched during this session, including retries.
|
||||
pub(crate) session_fetched_bytes: AtomicU64,
|
||||
/// Compressed bytes from archives that have fully downloaded.
|
||||
pub(crate) completed_download_bytes: AtomicU64,
|
||||
/// Compressed bytes written for currently active archive download attempts.
|
||||
pub(crate) active_download_bytes: AtomicU64,
|
||||
/// Total compressed bytes expected across all planned archives.
|
||||
pub(crate) total_download_bytes: u64,
|
||||
/// Plain-output bytes from archives that have fully verified.
|
||||
pub(crate) completed_output_bytes: AtomicU64,
|
||||
/// Plain-output bytes unpacked by currently active extractions.
|
||||
pub(crate) active_extracted_output_bytes: AtomicU64,
|
||||
/// Plain-output bytes hashed by currently active verifications.
|
||||
pub(crate) active_verified_output_bytes: AtomicU64,
|
||||
/// Total plain-output bytes expected across all planned archives.
|
||||
pub(crate) total_output_bytes: u64,
|
||||
/// Total number of planned archives.
|
||||
pub(crate) total_archives: u64,
|
||||
/// Time when the modular download job started.
|
||||
pub(crate) started_at: Instant,
|
||||
/// Time and baseline when the current extraction phase started.
|
||||
extraction_phase: Mutex<Option<PhaseStart>>,
|
||||
/// Time and baseline when the current verification phase started.
|
||||
verification_phase: Mutex<Option<PhaseStart>>,
|
||||
/// Number of archives that have fully finished.
|
||||
pub(crate) archives_done: AtomicU64,
|
||||
/// Number of archives currently in the fetch phase.
|
||||
pub(crate) active_downloads: AtomicU64,
|
||||
/// Number of in-flight HTTP requests.
|
||||
pub(crate) active_download_requests: AtomicU64,
|
||||
/// Number of archives currently extracting.
|
||||
pub(crate) active_extractions: AtomicU64,
|
||||
/// Number of archives currently verifying extracted outputs.
|
||||
pub(crate) active_verifications: AtomicU64,
|
||||
/// Signals the background progress task to exit.
|
||||
pub(crate) done: AtomicBool,
|
||||
/// Cancellation token shared by the whole command.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl SharedProgress {
|
||||
/// Creates the shared progress state for a modular download job.
|
||||
pub(crate) fn new(
|
||||
total_download_bytes: u64,
|
||||
total_output_bytes: u64,
|
||||
total_archives: u64,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Arc<Self> {
|
||||
Arc::new(Self {
|
||||
session_fetched_bytes: AtomicU64::new(0),
|
||||
completed_download_bytes: AtomicU64::new(0),
|
||||
active_download_bytes: AtomicU64::new(0),
|
||||
total_download_bytes,
|
||||
completed_output_bytes: AtomicU64::new(0),
|
||||
active_extracted_output_bytes: AtomicU64::new(0),
|
||||
active_verified_output_bytes: AtomicU64::new(0),
|
||||
total_output_bytes,
|
||||
total_archives,
|
||||
started_at: Instant::now(),
|
||||
extraction_phase: Mutex::new(None),
|
||||
verification_phase: Mutex::new(None),
|
||||
archives_done: AtomicU64::new(0),
|
||||
active_downloads: AtomicU64::new(0),
|
||||
active_download_requests: AtomicU64::new(0),
|
||||
active_extractions: AtomicU64::new(0),
|
||||
active_verifications: AtomicU64::new(0),
|
||||
done: AtomicBool::new(false),
|
||||
cancel_token,
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns whether the whole command has been cancelled.
|
||||
pub(crate) fn is_cancelled(&self) -> bool {
|
||||
self.cancel_token.is_cancelled()
|
||||
}
|
||||
|
||||
/// Adds raw session traffic bytes without affecting logical progress.
|
||||
pub(crate) fn record_session_fetched_bytes(&self, bytes: u64) {
|
||||
self.session_fetched_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn add_active_download_bytes(&self, bytes: u64) {
|
||||
self.active_download_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
pub(crate) fn sub_active_download_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_download_bytes, bytes);
|
||||
}
|
||||
|
||||
fn add_active_extracted_output_bytes(&self, bytes: u64) {
|
||||
self.active_extracted_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn sub_active_extracted_output_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_extracted_output_bytes, bytes);
|
||||
}
|
||||
|
||||
fn add_active_verified_output_bytes(&self, bytes: u64) {
|
||||
self.active_verified_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn sub_active_verified_output_bytes(&self, bytes: u64) {
|
||||
sub_bytes(&self.active_verified_output_bytes, bytes);
|
||||
}
|
||||
|
||||
/// Records an archive whose outputs were already present locally.
|
||||
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
|
||||
self.completed_download_bytes.fetch_add(download_bytes, Ordering::Relaxed);
|
||||
self.completed_output_bytes.fetch_add(output_bytes, Ordering::Relaxed);
|
||||
self.archives_done.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records an archive whose compressed download completed successfully.
|
||||
pub(crate) fn record_archive_download_complete(&self, bytes: u64) {
|
||||
self.completed_download_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Records an archive whose extracted outputs have fully verified.
|
||||
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
|
||||
self.completed_output_bytes.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.archives_done.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Returns logical compressed download progress.
|
||||
pub(crate) fn logical_downloaded_bytes(&self) -> u64 {
|
||||
(self.completed_download_bytes.load(Ordering::Relaxed) +
|
||||
self.active_download_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_download_bytes)
|
||||
}
|
||||
|
||||
/// Returns verified plain-output bytes.
|
||||
pub(crate) fn verified_output_bytes(&self) -> u64 {
|
||||
self.completed_output_bytes.load(Ordering::Relaxed).min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Returns plain-output bytes currently represented by extraction progress.
|
||||
pub(crate) fn extracting_output_bytes(&self) -> u64 {
|
||||
(self.completed_output_bytes.load(Ordering::Relaxed) +
|
||||
self.active_extracted_output_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Returns plain-output bytes currently represented by verification progress.
|
||||
pub(crate) fn verifying_output_bytes(&self) -> u64 {
|
||||
(self.completed_output_bytes.load(Ordering::Relaxed) +
|
||||
self.active_verified_output_bytes.load(Ordering::Relaxed))
|
||||
.min(self.total_output_bytes)
|
||||
}
|
||||
|
||||
fn restart_phase(slot: &Mutex<Option<PhaseStart>>, baseline_bytes: u64) {
|
||||
*slot.lock().unwrap() = Some(PhaseStart { started_at: Instant::now(), baseline_bytes });
|
||||
}
|
||||
|
||||
fn phase_eta(
|
||||
slot: &Mutex<Option<PhaseStart>>,
|
||||
current_bytes: u64,
|
||||
total_bytes: u64,
|
||||
) -> Option<Duration> {
|
||||
let phase = *slot.lock().unwrap();
|
||||
let phase = phase?;
|
||||
let done = current_bytes.saturating_sub(phase.baseline_bytes);
|
||||
let total = total_bytes.saturating_sub(phase.baseline_bytes);
|
||||
eta_from_progress(phase.started_at.elapsed(), done, total)
|
||||
}
|
||||
|
||||
fn extraction_eta(&self, current_bytes: u64) -> Option<Duration> {
|
||||
Self::phase_eta(&self.extraction_phase, current_bytes, self.total_output_bytes)
|
||||
}
|
||||
|
||||
fn verification_eta(&self, current_bytes: u64) -> Option<Duration> {
|
||||
Self::phase_eta(&self.verification_phase, current_bytes, self.total_output_bytes)
|
||||
}
|
||||
|
||||
/// Marks one archive as actively downloading.
|
||||
pub(crate) fn download_started(&self) {
|
||||
self.active_downloads.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Marks one archive download as finished.
|
||||
pub(crate) fn download_finished(&self) {
|
||||
sub_bytes(&self.active_downloads, 1);
|
||||
}
|
||||
|
||||
/// Marks one HTTP request as in flight.
|
||||
pub(crate) fn request_started(&self) {
|
||||
self.active_download_requests.fetch_add(1, Ordering::Relaxed);
|
||||
}
|
||||
|
||||
/// Marks one HTTP request as finished.
|
||||
pub(crate) fn request_finished(&self) {
|
||||
sub_bytes(&self.active_download_requests, 1);
|
||||
}
|
||||
|
||||
/// Marks one archive as actively extracting.
|
||||
pub(crate) fn extraction_started(&self) {
|
||||
if self.active_extractions.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
Self::restart_phase(
|
||||
&self.extraction_phase,
|
||||
self.completed_output_bytes.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one archive extraction as finished.
|
||||
pub(crate) fn extraction_finished(&self) {
|
||||
sub_bytes(&self.active_extractions, 1);
|
||||
}
|
||||
|
||||
/// Marks one archive as actively verifying outputs.
|
||||
pub(crate) fn verification_started(&self) {
|
||||
if self.active_verifications.fetch_add(1, Ordering::Relaxed) == 0 {
|
||||
Self::restart_phase(
|
||||
&self.verification_phase,
|
||||
self.completed_output_bytes.load(Ordering::Relaxed),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/// Marks one archive verification as finished.
|
||||
pub(crate) fn verification_finished(&self) {
|
||||
sub_bytes(&self.active_verifications, 1);
|
||||
}
|
||||
}
|
||||
|
||||
fn sub_bytes(counter: &AtomicU64, bytes: u64) {
|
||||
let _ = counter.fetch_update(Ordering::Relaxed, Ordering::Relaxed, |current| {
|
||||
Some(current.saturating_sub(bytes))
|
||||
});
|
||||
}
|
||||
|
||||
fn eta_from_progress(elapsed: Duration, done: u64, total: u64) -> Option<Duration> {
|
||||
if done == 0 || done >= total {
|
||||
return None;
|
||||
}
|
||||
|
||||
let secs = elapsed.as_secs_f64();
|
||||
if secs <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
let speed = done as f64 / secs;
|
||||
if speed <= 0.0 {
|
||||
return None;
|
||||
}
|
||||
|
||||
Some(Duration::from_secs_f64((total - done) as f64 / speed))
|
||||
}
|
||||
|
||||
fn format_percent(done: u64, total: u64) -> String {
|
||||
if total == 0 {
|
||||
return "100.0%".to_string();
|
||||
}
|
||||
|
||||
format!("{:.1}%", (done as f64 / total as f64) * 100.0)
|
||||
}
|
||||
|
||||
fn format_eta(eta: Option<Duration>) -> String {
|
||||
eta.map(DownloadProgress::format_duration).unwrap_or_else(|| "unknown".to_string())
|
||||
}
|
||||
|
||||
/// Global request limit for the blocking downloader.
|
||||
///
|
||||
/// This uses `Mutex + Condvar` because the segmented path runs blocking reqwest
|
||||
/// clients on OS threads.
|
||||
pub(crate) struct DownloadRequestLimiter {
|
||||
/// Maximum number of in-flight HTTP requests.
|
||||
limit: usize,
|
||||
/// Current number of acquired request slots.
|
||||
active: Mutex<usize>,
|
||||
/// Wakes blocked threads when a slot is released.
|
||||
notify: Condvar,
|
||||
}
|
||||
|
||||
impl DownloadRequestLimiter {
|
||||
/// Creates the shared request limiter.
|
||||
pub(crate) fn new(limit: usize) -> Arc<Self> {
|
||||
Arc::new(Self { limit: limit.max(1), active: Mutex::new(0), notify: Condvar::new() })
|
||||
}
|
||||
|
||||
/// Returns the configured request limit.
|
||||
pub(crate) fn max_concurrency(&self) -> usize {
|
||||
self.limit
|
||||
}
|
||||
|
||||
pub(crate) fn acquire<'a>(
|
||||
&'a self,
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
cancel_token: &CancellationToken,
|
||||
) -> Result<DownloadRequestPermit<'a>> {
|
||||
let mut active = self.active.lock().unwrap();
|
||||
loop {
|
||||
if cancel_token.is_cancelled() {
|
||||
return Err(eyre::eyre!("Download cancelled"));
|
||||
}
|
||||
|
||||
if *active < self.limit {
|
||||
*active += 1;
|
||||
if let Some(progress) = progress {
|
||||
progress.request_started();
|
||||
}
|
||||
return Ok(DownloadRequestPermit { limiter: self, progress });
|
||||
}
|
||||
|
||||
// Wake periodically so cancellation can interrupt waiters even if
|
||||
// no request finishes.
|
||||
let (next_active, _) =
|
||||
self.notify.wait_timeout(active, Duration::from_millis(100)).unwrap();
|
||||
active = next_active;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// RAII permit for one in-flight HTTP request.
|
||||
///
|
||||
/// Dropping the permit releases a slot in the shared request limit and updates
|
||||
/// the live progress counters.
|
||||
pub(crate) struct DownloadRequestPermit<'a> {
|
||||
/// Limiter that owns the request slot.
|
||||
limiter: &'a DownloadRequestLimiter,
|
||||
/// Shared progress counters updated when the permit drops.
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
}
|
||||
|
||||
impl Drop for DownloadRequestPermit<'_> {
|
||||
/// Releases the request slot and updates shared progress counters.
|
||||
fn drop(&mut self) {
|
||||
let mut active = self.limiter.active.lock().unwrap();
|
||||
*active = active.saturating_sub(1);
|
||||
drop(active);
|
||||
self.limiter.notify.notify_one();
|
||||
|
||||
if let Some(progress) = self.progress {
|
||||
progress.request_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive download attempt.
|
||||
pub(crate) struct ArchiveDownloadProgress<'a> {
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
downloaded: u64,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveDownloadProgress<'a> {
|
||||
/// Starts tracking one archive download attempt.
|
||||
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.download_started();
|
||||
}
|
||||
Self { progress, downloaded: 0, completed: false }
|
||||
}
|
||||
|
||||
/// Adds logical compressed bytes written by this attempt.
|
||||
pub(crate) fn record_downloaded(&mut self, bytes: u64) {
|
||||
self.downloaded += bytes;
|
||||
if let Some(progress) = self.progress {
|
||||
progress.add_active_download_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns whether this tracker has recorded any logical bytes itself.
|
||||
pub(crate) fn has_tracked_bytes(&self) -> bool {
|
||||
self.downloaded > 0
|
||||
}
|
||||
|
||||
/// Moves this archive from active download bytes into completed download bytes.
|
||||
pub(crate) fn complete(&mut self, total_bytes: u64) {
|
||||
if self.completed {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_download_bytes(self.downloaded);
|
||||
progress.record_archive_download_complete(total_bytes);
|
||||
}
|
||||
self.downloaded = 0;
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveDownloadProgress<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_download_bytes(self.downloaded);
|
||||
progress.download_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive extraction attempt.
|
||||
pub(crate) struct ArchiveExtractionProgress {
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
extracted: Arc<AtomicU64>,
|
||||
finished: bool,
|
||||
}
|
||||
|
||||
/// Cloneable handle for reporting extracted bytes from background monitoring.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ArchiveExtractionProgressHandle {
|
||||
progress: Arc<SharedProgress>,
|
||||
extracted: Arc<AtomicU64>,
|
||||
}
|
||||
|
||||
impl ArchiveExtractionProgress {
|
||||
/// Starts tracking one archive extraction attempt.
|
||||
pub(crate) fn new(progress: Option<&Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.extraction_started();
|
||||
}
|
||||
Self {
|
||||
progress: progress.cloned(),
|
||||
extracted: Arc::new(AtomicU64::new(0)),
|
||||
finished: false,
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns a cloneable handle that can report extraction progress from another thread.
|
||||
pub(crate) fn handle(&self) -> Option<ArchiveExtractionProgressHandle> {
|
||||
Some(ArchiveExtractionProgressHandle {
|
||||
progress: Arc::clone(self.progress.as_ref()?),
|
||||
extracted: Arc::clone(&self.extracted),
|
||||
})
|
||||
}
|
||||
|
||||
/// Adds plain-output bytes extracted by this attempt.
|
||||
pub(crate) fn record_extracted(&mut self, bytes: u64) {
|
||||
if let Some(handle) = self.handle() {
|
||||
handle.record_extracted(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Ends extraction tracking before verification begins.
|
||||
pub(crate) fn finish(&mut self) {
|
||||
if self.finished {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
|
||||
}
|
||||
self.finished = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveExtractionProgress {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = &self.progress {
|
||||
progress.sub_active_extracted_output_bytes(self.extracted.swap(0, Ordering::Relaxed));
|
||||
progress.extraction_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl ArchiveExtractionProgressHandle {
|
||||
/// Adds plain-output bytes extracted by this attempt.
|
||||
pub(crate) fn record_extracted(&self, bytes: u64) {
|
||||
self.extracted.fetch_add(bytes, Ordering::Relaxed);
|
||||
self.progress.add_active_extracted_output_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Tracks one active archive verification attempt.
|
||||
pub(crate) struct ArchiveVerificationProgress<'a> {
|
||||
progress: Option<&'a Arc<SharedProgress>>,
|
||||
verified: u64,
|
||||
completed: bool,
|
||||
}
|
||||
|
||||
impl<'a> ArchiveVerificationProgress<'a> {
|
||||
/// Starts tracking one archive verification attempt.
|
||||
pub(crate) fn new(progress: Option<&'a Arc<SharedProgress>>) -> Self {
|
||||
if let Some(progress) = progress {
|
||||
progress.verification_started();
|
||||
}
|
||||
Self { progress, verified: 0, completed: false }
|
||||
}
|
||||
|
||||
/// Adds plain-output bytes hashed by this verification attempt.
|
||||
pub(crate) fn record_verified(&mut self, bytes: u64) {
|
||||
self.verified += bytes;
|
||||
if let Some(progress) = self.progress {
|
||||
progress.add_active_verified_output_bytes(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Moves this archive from active verification bytes into completed output bytes.
|
||||
pub(crate) fn complete(&mut self, total_bytes: u64) {
|
||||
if self.completed {
|
||||
return;
|
||||
}
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_verified_output_bytes(self.verified);
|
||||
progress.record_archive_output_complete(total_bytes);
|
||||
}
|
||||
self.verified = 0;
|
||||
self.completed = true;
|
||||
}
|
||||
}
|
||||
|
||||
impl Drop for ArchiveVerificationProgress<'_> {
|
||||
fn drop(&mut self) {
|
||||
if let Some(progress) = self.progress {
|
||||
progress.sub_active_verified_output_bytes(self.verified);
|
||||
progress.verification_finished();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Adapter to track progress while reading (used for extraction in legacy path)
|
||||
pub(crate) struct ProgressReader<R> {
|
||||
/// Wrapped reader that provides archive bytes.
|
||||
reader: R,
|
||||
/// Per-download progress tracker for legacy paths.
|
||||
progress: DownloadProgress,
|
||||
/// Cancellation token checked between reads.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl<R: Read> ProgressReader<R> {
|
||||
/// Wraps a reader with per-download progress tracking.
|
||||
pub(crate) fn new(reader: R, total_size: u64, cancel_token: CancellationToken) -> Self {
|
||||
Self { reader, progress: DownloadProgress::new(total_size), cancel_token }
|
||||
}
|
||||
}
|
||||
|
||||
impl<R: Read> Read for ProgressReader<R> {
|
||||
/// Reads bytes, checks cancellation, and updates the local progress bar.
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
if self.cancel_token.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let bytes = self.reader.read(buf)?;
|
||||
if bytes > 0 &&
|
||||
let Err(error) = self.progress.update(bytes as u64)
|
||||
{
|
||||
return Err(io::Error::other(error));
|
||||
}
|
||||
Ok(bytes)
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that bumps a shared atomic counter while writing data.
|
||||
/// Used for parallel downloads where a single display task shows aggregated progress.
|
||||
pub(crate) struct SharedProgressWriter<'a, W> {
|
||||
/// Wrapped writer receiving downloaded bytes.
|
||||
pub(crate) inner: W,
|
||||
/// Shared counters updated as bytes are written.
|
||||
pub(crate) progress: Arc<SharedProgress>,
|
||||
/// Optional callback for logical bytes written by the current archive attempt.
|
||||
pub(crate) on_written: Option<&'a mut dyn FnMut(u64)>,
|
||||
}
|
||||
|
||||
impl<W: Write> Write for SharedProgressWriter<'_, W> {
|
||||
/// Writes bytes and records them in shared progress.
|
||||
fn write(&mut self, buf: &[u8]) -> io::Result<usize> {
|
||||
if self.progress.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let n = self.inner.write(buf)?;
|
||||
self.progress.record_session_fetched_bytes(n as u64);
|
||||
if let Some(on_written) = self.on_written.as_deref_mut() {
|
||||
on_written(n as u64);
|
||||
}
|
||||
Ok(n)
|
||||
}
|
||||
|
||||
/// Flushes the wrapped writer.
|
||||
fn flush(&mut self) -> io::Result<()> {
|
||||
self.inner.flush()
|
||||
}
|
||||
}
|
||||
|
||||
/// Wrapper that bumps a shared atomic counter while reading data.
|
||||
/// Used for streaming downloads where a single display task shows aggregated progress.
|
||||
pub(crate) struct SharedProgressReader<R> {
|
||||
/// Wrapped reader producing streamed bytes.
|
||||
pub(crate) inner: R,
|
||||
/// Shared counters updated as bytes are read.
|
||||
pub(crate) progress: Arc<SharedProgress>,
|
||||
}
|
||||
|
||||
impl<R: Read> Read for SharedProgressReader<R> {
|
||||
/// Reads bytes and records them in shared progress.
|
||||
fn read(&mut self, buf: &mut [u8]) -> io::Result<usize> {
|
||||
if self.progress.is_cancelled() {
|
||||
return Err(io::Error::new(io::ErrorKind::Interrupted, "download cancelled"));
|
||||
}
|
||||
let n = self.inner.read(buf)?;
|
||||
self.progress.record_session_fetched_bytes(n as u64);
|
||||
Ok(n)
|
||||
}
|
||||
}
|
||||
|
||||
/// Spawns a background task that prints aggregated download progress.
|
||||
/// Returns a handle; drop it (or call `.abort()`) to stop.
|
||||
pub(crate) fn spawn_progress_display(progress: Arc<SharedProgress>) -> tokio::task::JoinHandle<()> {
|
||||
tokio::spawn(async move {
|
||||
let mut interval = tokio::time::interval(Duration::from_secs(3));
|
||||
interval.tick().await;
|
||||
loop {
|
||||
interval.tick().await;
|
||||
|
||||
if progress.done.load(Ordering::Relaxed) {
|
||||
break;
|
||||
}
|
||||
|
||||
let download_total = progress.total_download_bytes;
|
||||
let output_total = progress.total_output_bytes;
|
||||
if download_total == 0 && output_total == 0 {
|
||||
continue;
|
||||
}
|
||||
|
||||
let done = progress.archives_done.load(Ordering::Relaxed);
|
||||
let all = progress.total_archives;
|
||||
let active_downloads = progress.active_downloads.load(Ordering::Relaxed);
|
||||
let active_requests = progress.active_download_requests.load(Ordering::Relaxed);
|
||||
let active_extractions = progress.active_extractions.load(Ordering::Relaxed);
|
||||
let active_verifications = progress.active_verifications.load(Ordering::Relaxed);
|
||||
let downloaded = progress.logical_downloaded_bytes();
|
||||
let extracted = progress.extracting_output_bytes();
|
||||
let verified = progress.verifying_output_bytes();
|
||||
let elapsed = DownloadProgress::format_duration(progress.started_at.elapsed());
|
||||
let download_total_display = DownloadProgress::format_size(download_total);
|
||||
let output_total_display = DownloadProgress::format_size(output_total);
|
||||
let downloaded_display = DownloadProgress::format_size(downloaded);
|
||||
let extracted_display = DownloadProgress::format_size(extracted);
|
||||
let active_download_phase = active_downloads > 0 || active_requests > 0;
|
||||
|
||||
if active_download_phase {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(downloaded, download_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(eta_from_progress(progress.started_at.elapsed(), downloaded, download_total)),
|
||||
bytes = format_args!("{downloaded_display}/{download_total_display}"),
|
||||
"Downloading snapshot archives"
|
||||
);
|
||||
} else if active_extractions > 0 {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(extracted, output_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(progress.extraction_eta(extracted)),
|
||||
bytes = format_args!("{extracted_display}/{output_total_display}"),
|
||||
"Extracting snapshot archives"
|
||||
);
|
||||
} else if active_verifications > 0 {
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{done}/{all}"),
|
||||
progress = %format_percent(verified, output_total),
|
||||
elapsed = %elapsed,
|
||||
eta = %format_eta(progress.verification_eta(verified)),
|
||||
bytes = format_args!("{}/{output_total_display}", DownloadProgress::format_size(verified)),
|
||||
"Verifying snapshot archives"
|
||||
);
|
||||
} else {
|
||||
continue;
|
||||
}
|
||||
}
|
||||
|
||||
let completed = progress.verified_output_bytes();
|
||||
let completed_display = DownloadProgress::format_size(completed);
|
||||
let output_total = DownloadProgress::format_size(progress.total_output_bytes);
|
||||
info!(target: "reth::cli",
|
||||
archives = format_args!("{}/{}", progress.total_archives, progress.total_archives),
|
||||
progress = "100.0%",
|
||||
elapsed = %DownloadProgress::format_duration(progress.started_at.elapsed()),
|
||||
eta = "0s",
|
||||
bytes = format_args!("{completed_display}/{output_total}"),
|
||||
"Snapshot archive processing complete"
|
||||
);
|
||||
})
|
||||
}
|
||||
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use std::sync::atomic::Ordering;
|
||||
|
||||
#[test]
|
||||
fn shared_progress_separates_session_fetch_from_logical_progress() {
|
||||
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
|
||||
|
||||
progress.record_session_fetched_bytes(10);
|
||||
progress.record_session_fetched_bytes(10);
|
||||
progress.record_archive_download_complete(10);
|
||||
progress.record_archive_output_complete(20);
|
||||
|
||||
assert_eq!(progress.session_fetched_bytes.load(Ordering::Relaxed), 20);
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 10);
|
||||
assert_eq!(progress.verified_output_bytes(), 20);
|
||||
assert_eq!(progress.archives_done.load(Ordering::Relaxed), 1);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn archive_download_progress_rolls_back_unfinished_attempts() {
|
||||
let progress = SharedProgress::new(10, 20, 1, CancellationToken::new());
|
||||
|
||||
{
|
||||
let mut download = ArchiveDownloadProgress::new(Some(&progress));
|
||||
download.record_downloaded(4);
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 4);
|
||||
}
|
||||
|
||||
assert_eq!(progress.logical_downloaded_bytes(), 0);
|
||||
assert_eq!(progress.active_downloads.load(Ordering::Relaxed), 0);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extraction_phase_baseline_restarts_after_idle() {
|
||||
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
|
||||
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.completed_output_bytes.store(25, Ordering::Relaxed);
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.extraction_finished();
|
||||
progress.extraction_finished();
|
||||
progress.extraction_started();
|
||||
assert_eq!(progress.extraction_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 25);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn verification_phase_baseline_restarts_after_idle() {
|
||||
let progress = SharedProgress::new(10, 100, 1, CancellationToken::new());
|
||||
|
||||
progress.verification_started();
|
||||
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.completed_output_bytes.store(40, Ordering::Relaxed);
|
||||
progress.verification_started();
|
||||
assert_eq!(progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes, 0);
|
||||
|
||||
progress.verification_finished();
|
||||
progress.verification_finished();
|
||||
progress.verification_started();
|
||||
assert_eq!(
|
||||
progress.verification_phase.lock().unwrap().as_ref().unwrap().baseline_bytes,
|
||||
40
|
||||
);
|
||||
}
|
||||
}
|
||||
100
crates/cli/commands/src/download/session.rs
Normal file
100
crates/cli/commands/src/download/session.rs
Normal file
@@ -0,0 +1,100 @@
|
||||
use super::progress::{DownloadRequestLimiter, SharedProgress};
|
||||
use eyre::Result;
|
||||
use reth_cli_util::cancellation::CancellationToken;
|
||||
use std::{
|
||||
path::{Path, PathBuf},
|
||||
sync::Arc,
|
||||
};
|
||||
|
||||
/// Shared state for one run of `reth download`.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct DownloadSession {
|
||||
/// Shared progress counters for this command, when enabled.
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
/// Shared limit for concurrent HTTP requests, when enabled.
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
/// Cancellation token shared by the whole command.
|
||||
cancel_token: CancellationToken,
|
||||
}
|
||||
|
||||
impl DownloadSession {
|
||||
/// Stores the shared progress, request limiter, and cancellation token.
|
||||
pub(crate) fn new(
|
||||
progress: Option<Arc<SharedProgress>>,
|
||||
request_limiter: Option<Arc<DownloadRequestLimiter>>,
|
||||
cancel_token: CancellationToken,
|
||||
) -> Self {
|
||||
Self { progress, request_limiter, cancel_token }
|
||||
}
|
||||
|
||||
/// Returns the shared progress tracker, if this flow uses one.
|
||||
pub(crate) fn progress(&self) -> Option<&Arc<SharedProgress>> {
|
||||
self.progress.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the shared HTTP request limiter, if this flow uses one.
|
||||
pub(crate) fn request_limiter(&self) -> Option<&Arc<DownloadRequestLimiter>> {
|
||||
self.request_limiter.as_ref()
|
||||
}
|
||||
|
||||
/// Returns the request limiter or errors if the caller needs one.
|
||||
pub(crate) fn require_request_limiter(&self) -> Result<&Arc<DownloadRequestLimiter>> {
|
||||
self.request_limiter().ok_or_else(|| eyre::eyre!("Missing download request limiter"))
|
||||
}
|
||||
|
||||
/// Returns the cancellation token for this command.
|
||||
pub(crate) fn cancel_token(&self) -> &CancellationToken {
|
||||
&self.cancel_token
|
||||
}
|
||||
|
||||
/// Records one archive whose outputs were already reusable on disk.
|
||||
pub(crate) fn record_reused_archive(&self, download_bytes: u64, output_bytes: u64) {
|
||||
if let Some(progress) = self.progress() {
|
||||
progress.record_reused_archive(download_bytes, output_bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/// Records one archive whose extracted outputs fully verified.
|
||||
pub(crate) fn record_archive_output_complete(&self, bytes: u64) {
|
||||
if let Some(progress) = self.progress() {
|
||||
progress.record_archive_output_complete(bytes);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Paths used while processing one archive, plus the shared download session.
|
||||
#[derive(Clone)]
|
||||
pub(crate) struct ArchiveProcessContext {
|
||||
/// Directory where extracted output files are written.
|
||||
target_dir: PathBuf,
|
||||
/// Directory used for cached archive downloads, when enabled.
|
||||
cache_dir: Option<PathBuf>,
|
||||
/// Shared command-scoped download state.
|
||||
session: DownloadSession,
|
||||
}
|
||||
|
||||
impl ArchiveProcessContext {
|
||||
/// Creates the context used while processing modular archives.
|
||||
pub(crate) fn new(
|
||||
target_dir: PathBuf,
|
||||
cache_dir: Option<PathBuf>,
|
||||
session: DownloadSession,
|
||||
) -> Self {
|
||||
Self { target_dir, cache_dir, session }
|
||||
}
|
||||
|
||||
/// Returns the directory where extracted outputs should be written.
|
||||
pub(crate) fn target_dir(&self) -> &Path {
|
||||
&self.target_dir
|
||||
}
|
||||
|
||||
/// Returns the cache directory for two-phase downloads, if enabled.
|
||||
pub(crate) fn cache_dir(&self) -> Option<&Path> {
|
||||
self.cache_dir.as_deref()
|
||||
}
|
||||
|
||||
/// Returns the shared download session.
|
||||
pub(crate) fn session(&self) -> &DownloadSession {
|
||||
&self.session
|
||||
}
|
||||
}
|
||||
230
crates/cli/commands/src/download/source.rs
Normal file
230
crates/cli/commands/src/download/source.rs
Normal file
@@ -0,0 +1,230 @@
|
||||
use super::{manifest::SnapshotManifest, progress::DownloadProgress, DownloadDefaults};
|
||||
use eyre::{Result, WrapErr};
|
||||
use reqwest::Client;
|
||||
use reth_fs_util as fs;
|
||||
use std::path::{Path, PathBuf};
|
||||
use tracing::info;
|
||||
use url::Url;
|
||||
|
||||
/// An entry from the snapshot discovery API listing.
|
||||
#[derive(serde::Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
pub(crate) struct SnapshotApiEntry {
|
||||
#[serde(deserialize_with = "deserialize_string_or_u64")]
|
||||
chain_id: u64,
|
||||
#[serde(deserialize_with = "deserialize_string_or_u64")]
|
||||
block: u64,
|
||||
#[serde(default)]
|
||||
date: Option<String>,
|
||||
#[serde(default)]
|
||||
profile: Option<String>,
|
||||
metadata_url: String,
|
||||
#[serde(default)]
|
||||
size: u64,
|
||||
}
|
||||
|
||||
impl SnapshotApiEntry {
|
||||
/// Returns whether this discovery entry points to a modular manifest.
|
||||
fn is_modular(&self) -> bool {
|
||||
self.metadata_url.ends_with("manifest.json")
|
||||
}
|
||||
}
|
||||
|
||||
/// Discovers the latest snapshot manifest URL for the given chain from the snapshots API.
|
||||
///
|
||||
/// Queries the configured snapshot API and returns the manifest URL for the most
|
||||
/// recent modular snapshot matching the requested chain.
|
||||
pub(crate) async fn discover_manifest_url(chain_id: u64) -> Result<String> {
|
||||
let defaults = DownloadDefaults::get_global();
|
||||
let api_url = &*defaults.snapshot_api_url;
|
||||
|
||||
info!(target: "reth::cli", %api_url, %chain_id, "Discovering latest snapshot manifest");
|
||||
|
||||
let entries = fetch_snapshot_api_entries(chain_id).await?;
|
||||
let entry =
|
||||
entries.iter().filter(|s| s.is_modular()).max_by_key(|s| s.block).ok_or_else(|| {
|
||||
eyre::eyre!(
|
||||
"No modular snapshot manifest found for chain \
|
||||
{chain_id} at {api_url}\n\n\
|
||||
You can provide a manifest URL directly with --manifest-url, or\n\
|
||||
use a direct snapshot URL with -u from:\n\
|
||||
\t- {}\n\n\
|
||||
Use --list to see all available snapshots.",
|
||||
api_url.trim_end_matches("/api/snapshots"),
|
||||
)
|
||||
})?;
|
||||
|
||||
info!(target: "reth::cli",
|
||||
block = entry.block,
|
||||
url = %entry.metadata_url,
|
||||
"Found latest snapshot manifest"
|
||||
);
|
||||
|
||||
Ok(entry.metadata_url.clone())
|
||||
}
|
||||
|
||||
/// Deserializes a JSON value that may be either a number or a string-encoded number.
|
||||
fn deserialize_string_or_u64<'de, D>(deserializer: D) -> std::result::Result<u64, D::Error>
|
||||
where
|
||||
D: serde::Deserializer<'de>,
|
||||
{
|
||||
use serde::Deserialize;
|
||||
let value = serde_json::Value::deserialize(deserializer)?;
|
||||
match &value {
|
||||
serde_json::Value::Number(n) => {
|
||||
n.as_u64().ok_or_else(|| serde::de::Error::custom("expected u64"))
|
||||
}
|
||||
serde_json::Value::String(s) => {
|
||||
s.parse::<u64>().map_err(|_| serde::de::Error::custom("expected numeric string"))
|
||||
}
|
||||
_ => Err(serde::de::Error::custom("expected number or string")),
|
||||
}
|
||||
}
|
||||
|
||||
/// Fetches the full snapshot listing from the snapshots API, filtered by chain ID.
|
||||
pub(crate) async fn fetch_snapshot_api_entries(chain_id: u64) -> Result<Vec<SnapshotApiEntry>> {
|
||||
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
|
||||
|
||||
let entries: Vec<SnapshotApiEntry> = Client::new()
|
||||
.get(api_url)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.wrap_err_with(|| format!("Failed to fetch snapshot listing from {api_url}"))?
|
||||
.json()
|
||||
.await?;
|
||||
|
||||
Ok(entries.into_iter().filter(|entry| entry.chain_id == chain_id).collect())
|
||||
}
|
||||
|
||||
/// Prints a formatted table of available modular snapshots.
|
||||
pub(crate) fn print_snapshot_listing(entries: &[SnapshotApiEntry], chain_id: u64) {
|
||||
let modular: Vec<_> = entries.iter().filter(|entry| entry.is_modular()).collect();
|
||||
|
||||
let api_url = &*DownloadDefaults::get_global().snapshot_api_url;
|
||||
println!(
|
||||
"Available snapshots for chain {chain_id} ({}):\n",
|
||||
api_url.trim_end_matches("/api/snapshots"),
|
||||
);
|
||||
println!("{:<12} {:>10} {:<10} {:>10} MANIFEST URL", "DATE", "BLOCK", "PROFILE", "SIZE");
|
||||
println!("{}", "-".repeat(100));
|
||||
|
||||
for entry in &modular {
|
||||
let date = entry.date.as_deref().unwrap_or("-");
|
||||
let profile = entry.profile.as_deref().unwrap_or("-");
|
||||
let size = if entry.size > 0 {
|
||||
DownloadProgress::format_size(entry.size)
|
||||
} else {
|
||||
"-".to_string()
|
||||
};
|
||||
|
||||
println!(
|
||||
"{date:<12} {:>10} {profile:<10} {size:>10} {}",
|
||||
entry.block, entry.metadata_url
|
||||
);
|
||||
}
|
||||
|
||||
if modular.is_empty() {
|
||||
println!(" (no modular snapshots found)");
|
||||
}
|
||||
|
||||
println!(
|
||||
"\nTo download a specific snapshot, copy its manifest URL and run:\n \
|
||||
reth download --manifest-url <URL>"
|
||||
);
|
||||
}
|
||||
|
||||
/// Loads a manifest from an HTTP(S) URL, `file://` URL, or local path.
|
||||
pub(crate) async fn fetch_manifest_from_source(source: &str) -> Result<SnapshotManifest> {
|
||||
if let Ok(parsed) = Url::parse(source) {
|
||||
return match parsed.scheme() {
|
||||
"http" | "https" => {
|
||||
let response = Client::new()
|
||||
.get(source)
|
||||
.send()
|
||||
.await
|
||||
.and_then(|r| r.error_for_status())
|
||||
.wrap_err_with(|| {
|
||||
let sources = DownloadDefaults::get_global()
|
||||
.available_snapshots
|
||||
.iter()
|
||||
.map(|snapshot| format!("\t- {snapshot}"))
|
||||
.collect::<Vec<_>>()
|
||||
.join("\n");
|
||||
format!(
|
||||
"Failed to fetch snapshot manifest from {source}\n\n\
|
||||
The manifest endpoint may not be available for this snapshot source.\n\
|
||||
You can use a direct snapshot URL instead:\n\n\
|
||||
\treth download -u <snapshot-url>\n\n\
|
||||
Available snapshot sources:\n{sources}"
|
||||
)
|
||||
})?;
|
||||
Ok(response.json().await?)
|
||||
}
|
||||
"file" => {
|
||||
let path = parsed
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
|
||||
let content = fs::read_to_string(path)?;
|
||||
Ok(serde_json::from_str(&content)?)
|
||||
}
|
||||
_ => Err(eyre::eyre!("Unsupported manifest URL scheme: {}", parsed.scheme())),
|
||||
};
|
||||
}
|
||||
|
||||
let content = fs::read_to_string(source)?;
|
||||
Ok(serde_json::from_str(&content)?)
|
||||
}
|
||||
|
||||
/// Resolves the base URL used to join relative archive paths in a manifest.
|
||||
pub(crate) fn resolve_manifest_base_url(
|
||||
manifest: &SnapshotManifest,
|
||||
source: &str,
|
||||
) -> Result<String> {
|
||||
if let Some(base_url) = manifest.base_url.as_deref() &&
|
||||
!base_url.is_empty()
|
||||
{
|
||||
return Ok(base_url.trim_end_matches('/').to_string());
|
||||
}
|
||||
|
||||
if let Ok(mut url) = Url::parse(source) {
|
||||
if url.scheme() == "file" {
|
||||
let mut path = url
|
||||
.to_file_path()
|
||||
.map_err(|_| eyre::eyre!("Invalid file:// manifest path: {source}"))?;
|
||||
path.pop();
|
||||
let mut base = Url::from_directory_path(path)
|
||||
.map_err(|_| eyre::eyre!("Invalid manifest directory for source: {source}"))?
|
||||
.to_string();
|
||||
if base.ends_with('/') {
|
||||
base.pop();
|
||||
}
|
||||
return Ok(base);
|
||||
}
|
||||
|
||||
{
|
||||
let mut segments = url
|
||||
.path_segments_mut()
|
||||
.map_err(|_| eyre::eyre!("manifest_url must have a hierarchical path"))?;
|
||||
segments.pop_if_empty();
|
||||
segments.pop();
|
||||
}
|
||||
return Ok(url.as_str().trim_end_matches('/').to_string());
|
||||
}
|
||||
|
||||
let path = Path::new(source);
|
||||
let manifest_dir = if path.is_absolute() {
|
||||
path.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
|
||||
} else {
|
||||
let joined = std::env::current_dir()?.join(path);
|
||||
joined.parent().map(Path::to_path_buf).unwrap_or_else(|| PathBuf::from("."))
|
||||
};
|
||||
|
||||
let mut base = Url::from_directory_path(&manifest_dir)
|
||||
.map_err(|_| eyre::eyre!("Invalid manifest directory: {}", manifest_dir.display()))?
|
||||
.to_string();
|
||||
if base.ends_with('/') {
|
||||
base.pop();
|
||||
}
|
||||
Ok(base)
|
||||
}
|
||||
84
crates/cli/commands/src/download/verify.rs
Normal file
84
crates/cli/commands/src/download/verify.rs
Normal file
@@ -0,0 +1,84 @@
|
||||
use super::{manifest::OutputFileChecksum, progress::ArchiveVerificationProgress};
|
||||
use blake3::Hasher;
|
||||
use eyre::Result;
|
||||
use reth_fs_util as fs;
|
||||
use std::{io::Read, path::Path};
|
||||
|
||||
/// Verifies and cleans up extracted output files in one target directory.
|
||||
pub(crate) struct OutputVerifier<'a> {
|
||||
/// Directory containing the output files declared by the manifest.
|
||||
target_dir: &'a Path,
|
||||
}
|
||||
|
||||
impl<'a> OutputVerifier<'a> {
|
||||
/// Creates a verifier for one extraction target directory.
|
||||
pub(crate) const fn new(target_dir: &'a Path) -> Self {
|
||||
Self { target_dir }
|
||||
}
|
||||
|
||||
/// Returns `true` only when every declared output file exists and matches size and BLAKE3.
|
||||
/// Returns `false` if any file is missing, mismatched, or no outputs were declared.
|
||||
pub(crate) fn verify(&self, output_files: &[OutputFileChecksum]) -> Result<bool> {
|
||||
self.verify_with_progress(output_files, None)
|
||||
}
|
||||
|
||||
/// Returns `true` only when every declared output file exists and matches size and BLAKE3,
|
||||
/// updating the optional verification progress as file bytes are hashed.
|
||||
pub(crate) fn verify_with_progress(
|
||||
&self,
|
||||
output_files: &[OutputFileChecksum],
|
||||
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
|
||||
) -> Result<bool> {
|
||||
if output_files.is_empty() {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
for expected in output_files {
|
||||
let output_path = self.target_dir.join(&expected.path);
|
||||
let meta = match fs::metadata(&output_path) {
|
||||
Ok(meta) => meta,
|
||||
Err(_) => return Ok(false),
|
||||
};
|
||||
if meta.len() != expected.size {
|
||||
return Ok(false);
|
||||
}
|
||||
|
||||
let actual = Self::file_blake3_hex(&output_path, progress.as_deref_mut())?;
|
||||
if !actual.eq_ignore_ascii_case(&expected.blake3) {
|
||||
return Ok(false);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(true)
|
||||
}
|
||||
|
||||
/// Removes any declared output files so a fresh archive attempt can restart cleanly.
|
||||
pub(crate) fn cleanup(&self, output_files: &[OutputFileChecksum]) {
|
||||
for output in output_files {
|
||||
let _ = fs::remove_file(self.target_dir.join(&output.path));
|
||||
}
|
||||
}
|
||||
|
||||
/// Computes the hex-encoded BLAKE3 checksum for one plain output file.
|
||||
fn file_blake3_hex(
|
||||
path: &Path,
|
||||
mut progress: Option<&mut ArchiveVerificationProgress<'_>>,
|
||||
) -> Result<String> {
|
||||
let mut file = fs::open(path)?;
|
||||
let mut hasher = Hasher::new();
|
||||
let mut buf = [0_u8; 64 * 1024];
|
||||
|
||||
loop {
|
||||
let n = file.read(&mut buf)?;
|
||||
if n == 0 {
|
||||
break;
|
||||
}
|
||||
hasher.update(&buf[..n]);
|
||||
if let Some(progress) = progress.as_deref_mut() {
|
||||
progress.record_verified(n as u64);
|
||||
}
|
||||
}
|
||||
|
||||
Ok(hasher.finalize().to_hex().to_string())
|
||||
}
|
||||
}
|
||||
@@ -138,7 +138,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let cancellation = cancellation.clone();
|
||||
let next_block = Arc::clone(&next_block);
|
||||
tasks.spawn_blocking(move || {
|
||||
let executor_lifetime = Duration::from_secs(120);
|
||||
let executor_lifetime = Duration::from_secs(600);
|
||||
|
||||
loop {
|
||||
if cancellation.is_cancelled() {
|
||||
@@ -245,7 +245,7 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + Hardforks + EthereumHardforks>
|
||||
let _ = stats_tx.send(block.gas_used());
|
||||
|
||||
// Reset DB once in a while to avoid OOM or read tx timeouts
|
||||
if executor.size_hint() > 1_000_000 ||
|
||||
if executor.size_hint() > 5_000_000 ||
|
||||
executor_created.elapsed() > executor_lifetime
|
||||
{
|
||||
executor =
|
||||
|
||||
@@ -29,7 +29,10 @@ use execution::dump_execution_stage;
|
||||
mod merkle;
|
||||
use merkle::dump_merkle_stage;
|
||||
|
||||
/// `reth dump-stage` command
|
||||
/// `reth dump-stage` command.
|
||||
///
|
||||
/// Note: mutates the source datadir (unwinds hashing/merkle/execution before copying tables).
|
||||
/// Stop the node and back up the datadir first.
|
||||
#[derive(Debug, Parser)]
|
||||
pub struct Command<C: ChainSpecParser> {
|
||||
#[command(flatten)]
|
||||
@@ -100,8 +103,9 @@ impl<C: ChainSpecParser<ChainSpec: EthChainSpec + EthereumHardforks>> Command<C>
|
||||
Comp: CliNodeComponents<N>,
|
||||
F: FnOnce(Arc<C::ChainSpec>) -> Comp,
|
||||
{
|
||||
// `unwind_and_copy` opens a RW provider on the source datadir, so open RW here.
|
||||
let Environment { provider_factory, .. } =
|
||||
self.env.init::<N>(AccessRights::RO, runtime.clone())?;
|
||||
self.env.init::<N>(AccessRights::RW, runtime.clone())?;
|
||||
let tool = DbTool::new(provider_factory)?;
|
||||
let components = components(tool.chain());
|
||||
let evm_config = components.evm_config().clone();
|
||||
|
||||
@@ -141,6 +141,11 @@ pub struct LocalMiner<T: PayloadTypes, B, Pool: TransactionPool + Unpin> {
|
||||
last_header: SealedHeaderFor<<T::BuiltPayload as BuiltPayload>::Primitives>,
|
||||
/// Stores latest mined blocks.
|
||||
last_block_hashes: VecDeque<B256>,
|
||||
/// Optional sleep duration between initiating payload building and resolving.
|
||||
///
|
||||
/// When set, the miner sleeps after `fork_choice_updated` before calling
|
||||
/// `resolve_kind`, giving the payload job time for multiple rebuild attempts.
|
||||
payload_wait_time: Option<Duration>,
|
||||
}
|
||||
|
||||
impl<T, B, Pool> LocalMiner<T, B, Pool>
|
||||
@@ -170,9 +175,16 @@ where
|
||||
payload_builder,
|
||||
last_block_hashes: VecDeque::from([last_header.hash()]),
|
||||
last_header,
|
||||
payload_wait_time: None,
|
||||
}
|
||||
}
|
||||
|
||||
/// Sets the payload wait time, if any.
|
||||
pub const fn with_payload_wait_time_opt(mut self, wait_time: Option<Duration>) -> Self {
|
||||
self.payload_wait_time = wait_time;
|
||||
self
|
||||
}
|
||||
|
||||
/// Runs the [`LocalMiner`] in a loop, polling the miner and building payloads.
|
||||
pub async fn run(mut self) {
|
||||
let mut fcu_interval = tokio::time::interval(Duration::from_secs(1));
|
||||
@@ -238,6 +250,10 @@ where
|
||||
|
||||
let payload_id = res.payload_id.ok_or_eyre("No payload id")?;
|
||||
|
||||
if let Some(wait_time) = self.payload_wait_time {
|
||||
tokio::time::sleep(wait_time).await;
|
||||
}
|
||||
|
||||
let Some(Ok(payload)) =
|
||||
self.payload_builder.resolve_kind(payload_id, PayloadKind::WaitForPending).await
|
||||
else {
|
||||
|
||||
@@ -57,7 +57,7 @@ where
|
||||
.chain_spec
|
||||
.is_cancun_active_at_timestamp(timestamp)
|
||||
.then(B256::random),
|
||||
slot_number: None,
|
||||
slot_number: self.chain_spec.is_amsterdam_active_at_timestamp(timestamp).then_some(0),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1693,6 +1693,7 @@ where
|
||||
let gas_used = payload.gas_used();
|
||||
let num_hash = payload.num_hash();
|
||||
let mut output = self.on_new_payload(payload);
|
||||
let latency = start.elapsed();
|
||||
self.metrics.engine.new_payload.update_response_metrics(
|
||||
start,
|
||||
&mut self.metrics.engine.forkchoice_updated.latest_finish_at,
|
||||
@@ -1700,12 +1701,6 @@ where
|
||||
gas_used,
|
||||
);
|
||||
|
||||
// Latency measures time from enqueue to completion, excluding
|
||||
// only the explicit persistence wait. This means backpressure
|
||||
// (time spent queued due to the engine being busy) is included,
|
||||
// reflecting real-world engine responsiveness.
|
||||
let latency = enqueued_at.elapsed() - explicit_persistence_wait;
|
||||
|
||||
let maybe_event =
|
||||
output.as_mut().ok().and_then(|out| out.event.take());
|
||||
|
||||
|
||||
@@ -2142,4 +2142,15 @@ impl<T: PayloadTypes> BlockOrPayload<T> {
|
||||
Self::Block(block) => block.gas_used(),
|
||||
}
|
||||
}
|
||||
|
||||
/// Returns the gas limit used by the block.
|
||||
pub fn gas_limit(&self) -> u64
|
||||
where
|
||||
T::ExecutionData: ExecutionPayload,
|
||||
{
|
||||
match self {
|
||||
Self::Payload(payload) => payload.gas_limit(),
|
||||
Self::Block(block) => block.gas_limit(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ use reth_primitives_traits::dashmap::DashMap;
|
||||
use revm::precompile::{PrecompileId, PrecompileOutput, PrecompileResult};
|
||||
use revm_primitives::Address;
|
||||
use std::{hash::Hash, sync::Arc};
|
||||
use tracing::error;
|
||||
|
||||
/// Default max cache size for [`PrecompileCache`]
|
||||
const MAX_CACHE_SIZE: u32 = 10_000;
|
||||
@@ -87,8 +88,14 @@ impl<S> CacheEntry<S> {
|
||||
self.output.gas_used
|
||||
}
|
||||
|
||||
fn to_precompile_result(&self) -> PrecompileResult {
|
||||
Ok(self.output.clone())
|
||||
/// Converts the cache entry to a precompile result. Accepts state gas reservoir as input.
|
||||
///
|
||||
/// All cached precompiles are not expected to access/created state and thus reservoir is always
|
||||
/// kept as is.
|
||||
fn to_precompile_result(&self, reservoir: u64) -> PrecompileResult {
|
||||
let mut output = self.output.clone();
|
||||
output.reservoir = reservoir;
|
||||
Ok(output)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -175,22 +182,34 @@ where
|
||||
input.gas >= entry.gas_used()
|
||||
{
|
||||
self.increment_by_one_precompile_cache_hits();
|
||||
return entry.to_precompile_result()
|
||||
return entry.to_precompile_result(input.reservoir);
|
||||
}
|
||||
|
||||
let calldata = input.data;
|
||||
let reservoir = input.reservoir;
|
||||
let result = self.precompile.call(input);
|
||||
|
||||
match &result {
|
||||
// Only successful outputs are cacheable. Non-success statuses and errors must execute
|
||||
// again instead of poisoning the cache for subsequent calls.
|
||||
Ok(output) if output.is_success() => {
|
||||
let size = self.cache.insert(
|
||||
Bytes::copy_from_slice(calldata),
|
||||
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
|
||||
);
|
||||
self.set_precompile_cache_size_metric(size as f64);
|
||||
self.increment_by_one_precompile_cache_misses();
|
||||
// Sanity-check precompile output to ensure that it does not affect state gas in any
|
||||
// way.
|
||||
//
|
||||
// This does not fully protect us from caching stateful precompiles but might make
|
||||
// it obvious when the node is misconfigured.
|
||||
if output.reservoir != reservoir {
|
||||
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile decremented reservoir, skipping cache insertion");
|
||||
} else if output.state_gas_used != 0 {
|
||||
error!(target: "engine::tree", precompile_id = self.precompile.precompile_id().name(), "cacheable precompile used state gas, skipping cache insertion");
|
||||
} else {
|
||||
let size = self.cache.insert(
|
||||
Bytes::copy_from_slice(calldata),
|
||||
CacheEntry { output: output.clone(), spec: self.spec_id.clone() },
|
||||
);
|
||||
self.set_precompile_cache_size_metric(size as f64);
|
||||
self.increment_by_one_precompile_cache_misses();
|
||||
}
|
||||
}
|
||||
_ => {
|
||||
self.increment_by_one_precompile_errors();
|
||||
@@ -246,6 +265,7 @@ mod tests {
|
||||
gas_used: 0,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: Bytes::default(),
|
||||
})
|
||||
})
|
||||
@@ -259,6 +279,7 @@ mod tests {
|
||||
gas_used: 50,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"cached_result"),
|
||||
};
|
||||
|
||||
@@ -292,6 +313,7 @@ mod tests {
|
||||
gas_used: 5000,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_1"),
|
||||
})
|
||||
}
|
||||
@@ -308,6 +330,7 @@ mod tests {
|
||||
gas_used: 7000,
|
||||
state_gas_used: 0,
|
||||
reservoir: 0,
|
||||
gas_refunded: 0,
|
||||
bytes: alloy_primitives::Bytes::copy_from_slice(b"output_from_precompile_2"),
|
||||
})
|
||||
}
|
||||
|
||||
@@ -195,6 +195,7 @@ where
|
||||
ommers: &block.body().ommers,
|
||||
withdrawals: block.body().withdrawals.as_ref().map(|w| Cow::Borrowed(w.as_slice())),
|
||||
extra_data: block.header().extra_data.clone(),
|
||||
slot_number: block.header().slot_number,
|
||||
})
|
||||
}
|
||||
|
||||
@@ -210,6 +211,7 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: attributes.withdrawals.map(|w| Cow::Owned(w.into_inner())),
|
||||
extra_data: attributes.extra_data,
|
||||
slot_number: attributes.slot_number,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -274,7 +276,7 @@ where
|
||||
gas_limit: payload.payload.gas_limit(),
|
||||
basefee: payload.payload.saturated_base_fee_per_gas(),
|
||||
blob_excess_gas_and_price,
|
||||
slot_num: 0,
|
||||
slot_num: payload.payload.as_v4().map(|v4| v4.slot_number).unwrap_or_default(),
|
||||
};
|
||||
|
||||
Ok(EvmEnv { cfg_env, block_env })
|
||||
@@ -291,6 +293,7 @@ where
|
||||
ommers: &[],
|
||||
withdrawals: payload.payload.withdrawals().map(|w| Cow::Borrowed(w.as_slice())),
|
||||
extra_data: payload.payload.as_v1().extra_data.clone(),
|
||||
slot_number: payload.payload.as_v4().map(|v4| v4.slot_number),
|
||||
})
|
||||
}
|
||||
|
||||
|
||||
@@ -37,7 +37,6 @@ reth-rpc-eth-types.workspace = true
|
||||
reth-engine-local.workspace = true
|
||||
reth-engine-primitives = { workspace = true, features = ["std"] }
|
||||
reth-payload-primitives.workspace = true
|
||||
|
||||
# ethereum
|
||||
alloy-eips.workspace = true
|
||||
alloy-network.workspace = true
|
||||
|
||||
@@ -26,9 +26,10 @@ use reth_node_builder::{
|
||||
},
|
||||
node::{FullNodeTypes, NodeTypes},
|
||||
rpc::{
|
||||
BasicEngineApiBuilder, BasicEngineValidatorBuilder, EngineApiBuilder, EngineValidatorAddOn,
|
||||
EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity, PayloadValidatorBuilder,
|
||||
RethRpcAddOns, RpcAddOns, RpcHandle,
|
||||
BasicEngineApiBuilder, BasicEngineValidatorBuilder, Either, EngineApiBuilder,
|
||||
EngineValidatorAddOn, EngineValidatorBuilder, EthApiBuilder, EthApiCtx, Identity,
|
||||
PayloadValidatorBuilder, RethAuthHttpMiddleware, RethRpcAddOns, RethRpcMiddleware,
|
||||
RpcAddOns, RpcHandle, Stack,
|
||||
},
|
||||
BuilderContext, DebugNode, Node, NodeAdapter,
|
||||
};
|
||||
@@ -39,7 +40,7 @@ use reth_rpc::{
|
||||
TestingApi, ValidationApi,
|
||||
};
|
||||
use reth_rpc_api::servers::{BlockSubmissionValidationApiServer, TestingApiServer};
|
||||
use reth_rpc_builder::{config::RethRpcServerConfig, middleware::RethRpcMiddleware};
|
||||
use reth_rpc_builder::config::RethRpcServerConfig;
|
||||
use reth_rpc_eth_api::{
|
||||
helpers::{
|
||||
config::{EthConfigApiServer, EthConfigHandler},
|
||||
@@ -165,17 +166,21 @@ pub struct EthereumAddOns<
|
||||
EB = BasicEngineApiBuilder<PVB>,
|
||||
EVB = BasicEngineValidatorBuilder<PVB>,
|
||||
RpcMiddleware = Identity,
|
||||
AuthHttpMiddleware = Identity,
|
||||
> {
|
||||
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>,
|
||||
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
EthB: EthApiBuilder<N>,
|
||||
{
|
||||
/// Creates a new instance from the inner `RpcAddOns`.
|
||||
pub const fn new(inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>) -> Self {
|
||||
pub const fn new(
|
||||
inner: RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>,
|
||||
) -> Self {
|
||||
Self { inner }
|
||||
}
|
||||
}
|
||||
@@ -199,11 +204,13 @@ where
|
||||
BasicEngineApiBuilder::default(),
|
||||
BasicEngineValidatorBuilder::default(),
|
||||
Default::default(),
|
||||
Identity::new(),
|
||||
))
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
EthB: EthApiBuilder<N>,
|
||||
@@ -212,7 +219,7 @@ where
|
||||
pub fn with_engine_api<T>(
|
||||
self,
|
||||
engine_api_builder: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware>
|
||||
) -> EthereumAddOns<N, EthB, PVB, T, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
T: Send,
|
||||
{
|
||||
@@ -224,7 +231,7 @@ where
|
||||
pub fn with_payload_validator<V, T>(
|
||||
self,
|
||||
payload_validator_builder: T,
|
||||
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware> {
|
||||
) -> EthereumAddOns<N, EthB, T, EB, EVB, RpcMiddleware, AuthHttpMiddleware> {
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.with_payload_validator(payload_validator_builder))
|
||||
}
|
||||
@@ -233,7 +240,7 @@ where
|
||||
pub fn with_rpc_middleware<T>(
|
||||
self,
|
||||
rpc_middleware: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T>
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, T, AuthHttpMiddleware>
|
||||
where
|
||||
T: Send,
|
||||
{
|
||||
@@ -241,6 +248,45 @@ where
|
||||
EthereumAddOns::new(inner.with_rpc_middleware(rpc_middleware))
|
||||
}
|
||||
|
||||
/// Configures the HTTP transport middleware for the auth / Engine API server.
|
||||
pub fn with_auth_http_middleware<T>(
|
||||
self,
|
||||
auth_http_middleware: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, T>
|
||||
where
|
||||
T: Send,
|
||||
{
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.with_auth_http_middleware(auth_http_middleware))
|
||||
}
|
||||
|
||||
/// Stacks an additional HTTP transport middleware layer for the auth / Engine API server.
|
||||
pub fn layer_auth_http_middleware<T>(
|
||||
self,
|
||||
layer: T,
|
||||
) -> EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, Stack<AuthHttpMiddleware, T>> {
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.layer_auth_http_middleware(layer))
|
||||
}
|
||||
|
||||
/// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server.
|
||||
#[expect(clippy::type_complexity)]
|
||||
pub fn option_layer_auth_http_middleware<T>(
|
||||
self,
|
||||
layer: Option<T>,
|
||||
) -> EthereumAddOns<
|
||||
N,
|
||||
EthB,
|
||||
PVB,
|
||||
EB,
|
||||
EVB,
|
||||
RpcMiddleware,
|
||||
Stack<AuthHttpMiddleware, Either<T, Identity>>,
|
||||
> {
|
||||
let Self { inner } = self;
|
||||
EthereumAddOns::new(inner.option_layer_auth_http_middleware(layer))
|
||||
}
|
||||
|
||||
/// Sets the tokio runtime for the RPC servers.
|
||||
///
|
||||
/// Caution: This runtime must not be created from within asynchronous context.
|
||||
@@ -250,8 +296,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> NodeAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> NodeAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents<
|
||||
Types: NodeTypes<
|
||||
@@ -268,6 +314,7 @@ where
|
||||
EthApiError: FromEvmError<N::Evm>,
|
||||
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
|
||||
{
|
||||
type Handle = RpcHandle<N, EthB::EthApi>;
|
||||
|
||||
@@ -323,8 +370,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> RethRpcAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> RethRpcAddOns<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents<
|
||||
Types: NodeTypes<
|
||||
@@ -341,6 +388,7 @@ where
|
||||
EthApiError: FromEvmError<N::Evm>,
|
||||
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
|
||||
{
|
||||
type EthApi = EthB::EthApi;
|
||||
|
||||
@@ -349,8 +397,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EngineValidatorAddOn<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> EngineValidatorAddOn<N>
|
||||
for EthereumAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents<
|
||||
Types: NodeTypes<
|
||||
@@ -367,6 +415,7 @@ where
|
||||
EthApiError: FromEvmError<N::Evm>,
|
||||
EvmFactoryFor<N::Evm>: EvmFactory<Tx = TxEnv>,
|
||||
RpcMiddleware: Send,
|
||||
AuthHttpMiddleware: Send,
|
||||
{
|
||||
type ValidatorBuilder = EVB;
|
||||
|
||||
|
||||
@@ -248,7 +248,7 @@ impl Discv5 {
|
||||
discv5::Event::SocketUpdated(_) | discv5::Event::TalkRequest(_) |
|
||||
// `Discovered` not unique discovered peers
|
||||
discv5::Event::Discovered(_) => None,
|
||||
discv5::Event::NodeInserted { replaced: _, .. } => {
|
||||
discv5::Event::NodeInserted { .. } => {
|
||||
|
||||
// node has been inserted into kbuckets
|
||||
|
||||
|
||||
@@ -1,6 +1,5 @@
|
||||
//! `RLPx` disconnect reason sent to/received from peer
|
||||
|
||||
use alloc::vec;
|
||||
use alloy_primitives::bytes::{Buf, BufMut};
|
||||
use alloy_rlp::{Decodable, Encodable, Header};
|
||||
use derive_more::Display;
|
||||
@@ -84,10 +83,10 @@ impl Encodable for DisconnectReason {
|
||||
/// The [`Encodable`] implementation for [`DisconnectReason`] encodes the disconnect reason in
|
||||
/// a single-element RLP list.
|
||||
fn encode(&self, out: &mut dyn BufMut) {
|
||||
vec![*self as u8].encode(out);
|
||||
alloy_rlp::encode_list(&[*self as u8], out);
|
||||
}
|
||||
fn length(&self) -> usize {
|
||||
vec![*self as u8].length()
|
||||
alloy_rlp::list_length(&[*self as u8])
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
@@ -307,6 +307,14 @@ where
|
||||
|
||||
let dev_mining_mode =
|
||||
mining_mode.unwrap_or_else(|| handle.node.config.dev_mining_mode(pool));
|
||||
let payload_wait_time = config.dev.payload_wait_time;
|
||||
if let (Some(wait_time), Some(block_time)) = (payload_wait_time, config.dev.block_time)
|
||||
{
|
||||
eyre::ensure!(
|
||||
wait_time <= block_time,
|
||||
"--dev.payload-wait-time ({wait_time:?}) must be <= --dev.block-time ({block_time:?})"
|
||||
);
|
||||
}
|
||||
handle.node.task_executor.spawn_critical_task("local engine", async move {
|
||||
LocalMiner::new(
|
||||
blockchain_db,
|
||||
@@ -315,6 +323,7 @@ where
|
||||
dev_mining_mode,
|
||||
payload_builder_handle,
|
||||
)
|
||||
.with_payload_wait_time_opt(payload_wait_time)
|
||||
.run()
|
||||
.await
|
||||
});
|
||||
|
||||
@@ -1,9 +1,15 @@
|
||||
//! Builder support for rpc components.
|
||||
|
||||
pub use jsonrpsee::server::middleware::rpc::{RpcService, RpcServiceBuilder};
|
||||
pub use jsonrpsee::{
|
||||
core::middleware::layer::Either,
|
||||
server::middleware::rpc::{RpcService, RpcServiceBuilder},
|
||||
};
|
||||
use reth_engine_tree::tree::WaitForCaches;
|
||||
pub use reth_engine_tree::tree::{BasicEngineValidator, EngineValidator};
|
||||
pub use reth_rpc_builder::{middleware::RethRpcMiddleware, Identity, Stack};
|
||||
pub use reth_rpc_builder::{
|
||||
middleware::{RethAuthHttpMiddleware, RethRpcMiddleware},
|
||||
Identity, Stack,
|
||||
};
|
||||
pub use reth_trie_db::ChangesetCache;
|
||||
|
||||
use crate::{
|
||||
@@ -12,7 +18,7 @@ use crate::{
|
||||
};
|
||||
use alloy_rpc_types::engine::ClientVersionV1;
|
||||
use alloy_rpc_types_engine::ExecutionData;
|
||||
use jsonrpsee::{core::middleware::layer::Either, RpcModule};
|
||||
use jsonrpsee::RpcModule;
|
||||
use parking_lot::Mutex;
|
||||
use reth_chain_state::CanonStateSubscriptions;
|
||||
use reth_chainspec::{ChainSpecProvider, EthChainSpec, EthereumHardforks, Hardforks};
|
||||
@@ -517,6 +523,7 @@ pub struct RpcAddOns<
|
||||
EB = BasicEngineApiBuilder<PVB>,
|
||||
EVB = BasicEngineValidatorBuilder<PVB>,
|
||||
RpcMiddleware = Identity,
|
||||
AuthHttpMiddleware = Identity,
|
||||
> {
|
||||
/// Additional RPC add-ons.
|
||||
pub hooks: RpcHooks<Node, EthB::EthApi>,
|
||||
@@ -533,12 +540,17 @@ pub struct RpcAddOns<
|
||||
/// This middleware is applied to all RPC requests across all transports (HTTP, WS, IPC).
|
||||
/// See [`RpcAddOns::with_rpc_middleware`] for more details.
|
||||
rpc_middleware: RpcMiddleware,
|
||||
/// Configurable HTTP transport middleware for the auth server.
|
||||
///
|
||||
/// This middleware is applied after JWT authentication and before JSON-RPC parsing on the
|
||||
/// auth / Engine API server, giving access to the raw HTTP request.
|
||||
auth_http_middleware: AuthHttpMiddleware,
|
||||
/// Optional custom tokio runtime for the RPC server.
|
||||
tokio_runtime: Option<tokio::runtime::Handle>,
|
||||
}
|
||||
|
||||
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware> Debug
|
||||
for RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> Debug
|
||||
for RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
Node: FullNodeComponents,
|
||||
EthB: EthApiBuilder<Node>,
|
||||
@@ -558,7 +570,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware> RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
Node: FullNodeComponents,
|
||||
EthB: EthApiBuilder<Node>,
|
||||
@@ -570,6 +583,7 @@ where
|
||||
engine_api_builder: EB,
|
||||
engine_validator_builder: EVB,
|
||||
rpc_middleware: RpcMiddleware,
|
||||
auth_http_middleware: AuthHttpMiddleware,
|
||||
) -> Self {
|
||||
Self {
|
||||
hooks: RpcHooks::default(),
|
||||
@@ -578,6 +592,7 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime: None,
|
||||
}
|
||||
}
|
||||
@@ -586,13 +601,14 @@ where
|
||||
pub fn with_engine_api<T>(
|
||||
self,
|
||||
engine_api_builder: T,
|
||||
) -> RpcAddOns<Node, EthB, PVB, T, EVB, RpcMiddleware> {
|
||||
) -> RpcAddOns<Node, EthB, PVB, T, EVB, RpcMiddleware, AuthHttpMiddleware> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
..
|
||||
} = self;
|
||||
@@ -603,6 +619,7 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
@@ -611,13 +628,14 @@ where
|
||||
pub fn with_payload_validator<T>(
|
||||
self,
|
||||
payload_validator_builder: T,
|
||||
) -> RpcAddOns<Node, EthB, T, EB, EVB, RpcMiddleware> {
|
||||
) -> RpcAddOns<Node, EthB, T, EB, EVB, RpcMiddleware, AuthHttpMiddleware> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
..
|
||||
} = self;
|
||||
@@ -628,6 +646,7 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
@@ -636,13 +655,14 @@ where
|
||||
pub fn with_engine_validator<T>(
|
||||
self,
|
||||
engine_validator_builder: T,
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, T, RpcMiddleware> {
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, T, RpcMiddleware, AuthHttpMiddleware> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_api_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
..
|
||||
} = self;
|
||||
@@ -653,6 +673,7 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
@@ -698,13 +719,14 @@ where
|
||||
pub fn with_rpc_middleware<T>(
|
||||
self,
|
||||
rpc_middleware: T,
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, T> {
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, T, AuthHttpMiddleware> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
..
|
||||
} = self;
|
||||
@@ -715,10 +737,87 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
|
||||
/// Configures the HTTP transport middleware for the auth / Engine API server.
|
||||
///
|
||||
/// This middleware is applied after JWT authentication and before JSON-RPC parsing,
|
||||
/// giving access to the raw HTTP request (headers, body, etc.).
|
||||
pub fn with_auth_http_middleware<T>(
|
||||
self,
|
||||
auth_http_middleware: T,
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, T> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
tokio_runtime,
|
||||
..
|
||||
} = self;
|
||||
RpcAddOns {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
|
||||
/// Stacks an additional HTTP transport middleware layer for the auth / Engine API server.
|
||||
pub fn layer_auth_http_middleware<T>(
|
||||
self,
|
||||
layer: T,
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, RpcMiddleware, Stack<AuthHttpMiddleware, T>> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
} = self;
|
||||
let auth_http_middleware = Stack::new(auth_http_middleware, layer);
|
||||
RpcAddOns {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
payload_validator_builder,
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
|
||||
/// Conditionally stacks an HTTP transport middleware layer for the auth / Engine API server.
|
||||
#[expect(clippy::type_complexity)]
|
||||
pub fn option_layer_auth_http_middleware<T>(
|
||||
self,
|
||||
layer: Option<T>,
|
||||
) -> RpcAddOns<
|
||||
Node,
|
||||
EthB,
|
||||
PVB,
|
||||
EB,
|
||||
EVB,
|
||||
RpcMiddleware,
|
||||
Stack<AuthHttpMiddleware, Either<T, Identity>>,
|
||||
> {
|
||||
let layer = layer.map(Either::Left).unwrap_or(Either::Right(Identity::new()));
|
||||
self.layer_auth_http_middleware(layer)
|
||||
}
|
||||
|
||||
/// Sets the tokio runtime for the RPC servers.
|
||||
///
|
||||
/// Caution: This runtime must not be created from within asynchronous context.
|
||||
@@ -730,6 +829,7 @@ where
|
||||
engine_validator_builder,
|
||||
engine_api_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
..
|
||||
} = self;
|
||||
Self {
|
||||
@@ -739,6 +839,7 @@ where
|
||||
engine_validator_builder,
|
||||
engine_api_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
@@ -747,7 +848,7 @@ where
|
||||
pub fn layer_rpc_middleware<T>(
|
||||
self,
|
||||
layer: T,
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, Stack<RpcMiddleware, T>> {
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, Stack<RpcMiddleware, T>, AuthHttpMiddleware> {
|
||||
let Self {
|
||||
hooks,
|
||||
eth_api_builder,
|
||||
@@ -755,6 +856,7 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
} = self;
|
||||
let rpc_middleware = Stack::new(rpc_middleware, layer);
|
||||
@@ -765,6 +867,7 @@ where
|
||||
engine_api_builder,
|
||||
engine_validator_builder,
|
||||
rpc_middleware,
|
||||
auth_http_middleware,
|
||||
tokio_runtime,
|
||||
}
|
||||
}
|
||||
@@ -774,7 +877,15 @@ where
|
||||
pub fn option_layer_rpc_middleware<T>(
|
||||
self,
|
||||
layer: Option<T>,
|
||||
) -> RpcAddOns<Node, EthB, PVB, EB, EVB, Stack<RpcMiddleware, Either<T, Identity>>> {
|
||||
) -> RpcAddOns<
|
||||
Node,
|
||||
EthB,
|
||||
PVB,
|
||||
EB,
|
||||
EVB,
|
||||
Stack<RpcMiddleware, Either<T, Identity>>,
|
||||
AuthHttpMiddleware,
|
||||
> {
|
||||
let layer = layer.map(Either::Left).unwrap_or(Either::Right(Identity::new()));
|
||||
self.layer_rpc_middleware(layer)
|
||||
}
|
||||
@@ -800,7 +911,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<Node, EthB, EV, EB, Engine> Default for RpcAddOns<Node, EthB, EV, EB, Engine, Identity>
|
||||
impl<Node, EthB, EV, EB, Engine> Default
|
||||
for RpcAddOns<Node, EthB, EV, EB, Engine, Identity, Identity>
|
||||
where
|
||||
Node: FullNodeComponents,
|
||||
EthB: EthApiBuilder<Node>,
|
||||
@@ -815,11 +927,13 @@ where
|
||||
EB::default(),
|
||||
Engine::default(),
|
||||
Default::default(),
|
||||
Identity::new(),
|
||||
)
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
N::Provider: ChainSpecProvider<ChainSpec: EthereumHardforks>,
|
||||
@@ -827,6 +941,7 @@ where
|
||||
EB: EngineApiBuilder<N>,
|
||||
EVB: EngineValidatorBuilder<N>,
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
|
||||
{
|
||||
/// Launches only the regular RPC server (HTTP/WS/IPC), without the authenticated Engine API
|
||||
/// server.
|
||||
@@ -913,6 +1028,7 @@ where
|
||||
F: FnOnce(RpcModuleContainer<'_, N, EthB::EthApi>) -> eyre::Result<()>,
|
||||
{
|
||||
let rpc_middleware = self.rpc_middleware.clone();
|
||||
let auth_http_middleware = self.auth_http_middleware.clone();
|
||||
let tokio_runtime = self.tokio_runtime.clone();
|
||||
let setup_ctx = self.setup_rpc_components(ctx, ext).await?;
|
||||
let RpcSetupContext {
|
||||
@@ -933,6 +1049,8 @@ where
|
||||
.set_rpc_middleware(rpc_middleware)
|
||||
.with_tokio_runtime(tokio_runtime);
|
||||
|
||||
let auth_config = auth_config.with_http_middleware(auth_http_middleware);
|
||||
|
||||
let (rpc, auth) = if disable_auth {
|
||||
// Only launch the RPC server, use a noop auth handle
|
||||
let rpc = Self::launch_rpc_server_internal(server_config, &modules).await?;
|
||||
@@ -942,7 +1060,7 @@ where
|
||||
// launch servers concurrently
|
||||
let (rpc, auth) = futures::future::try_join(
|
||||
Self::launch_rpc_server_internal(server_config, &modules),
|
||||
Self::launch_auth_server_internal(auth_module_clone, auth_config),
|
||||
Self::launch_auth_server_internal(auth_config.start(auth_module_clone)),
|
||||
)
|
||||
.await?;
|
||||
(rpc, auth)
|
||||
@@ -1087,10 +1205,9 @@ where
|
||||
|
||||
/// Helper to launch the auth server
|
||||
async fn launch_auth_server_internal(
|
||||
auth_module: AuthRpcModule,
|
||||
auth_config: reth_rpc_builder::auth::AuthServerConfig,
|
||||
start_fut: impl Future<Output = Result<AuthServerHandle, reth_rpc_builder::error::RpcError>>,
|
||||
) -> eyre::Result<AuthServerHandle> {
|
||||
auth_module.start_server(auth_config)
|
||||
start_fut
|
||||
.await
|
||||
.map_err(Into::into)
|
||||
.inspect(|handle| {
|
||||
@@ -1120,8 +1237,8 @@ where
|
||||
}
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> NodeAddOns<N>
|
||||
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> NodeAddOns<N>
|
||||
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
<N as FullNodeTypes>::Provider: ChainSpecProvider<ChainSpec: EthereumHardforks>,
|
||||
@@ -1130,6 +1247,7 @@ where
|
||||
EB: EngineApiBuilder<N>,
|
||||
EVB: EngineValidatorBuilder<N>,
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
AuthHttpMiddleware: RethAuthHttpMiddleware<Identity>,
|
||||
{
|
||||
type Handle = RpcHandle<N, EthB::EthApi>;
|
||||
|
||||
@@ -1150,8 +1268,8 @@ pub trait RethRpcAddOns<N: FullNodeComponents>:
|
||||
fn hooks_mut(&mut self) -> &mut RpcHooks<N, Self::EthApi>;
|
||||
}
|
||||
|
||||
impl<N: FullNodeComponents, EthB, EV, EB, Engine, RpcMiddleware> RethRpcAddOns<N>
|
||||
for RpcAddOns<N, EthB, EV, EB, Engine, RpcMiddleware>
|
||||
impl<N: FullNodeComponents, EthB, EV, EB, Engine, RpcMiddleware, AuthHttpMiddleware>
|
||||
RethRpcAddOns<N> for RpcAddOns<N, EthB, EV, EB, Engine, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
Self: NodeAddOns<N, Handle = RpcHandle<N, EthB::EthApi>>,
|
||||
EthB: EthApiBuilder<N>,
|
||||
@@ -1221,8 +1339,8 @@ pub trait EngineValidatorAddOn<Node: FullNodeComponents>: Send {
|
||||
fn engine_validator_builder(&self) -> Self::ValidatorBuilder;
|
||||
}
|
||||
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware> EngineValidatorAddOn<N>
|
||||
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware>
|
||||
impl<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware> EngineValidatorAddOn<N>
|
||||
for RpcAddOns<N, EthB, PVB, EB, EVB, RpcMiddleware, AuthHttpMiddleware>
|
||||
where
|
||||
N: FullNodeComponents,
|
||||
EthB: EthApiBuilder<N>,
|
||||
@@ -1230,6 +1348,7 @@ where
|
||||
EB: EngineApiBuilder<N>,
|
||||
EVB: EngineValidatorBuilder<N>,
|
||||
RpcMiddleware: Send,
|
||||
AuthHttpMiddleware: Send,
|
||||
{
|
||||
type ValidatorBuilder = EVB;
|
||||
|
||||
|
||||
@@ -42,6 +42,22 @@ pub struct DevArgs {
|
||||
)]
|
||||
pub block_time: Option<Duration>,
|
||||
|
||||
/// Time to wait after initiating payload building before resolving.
|
||||
///
|
||||
/// Introduces a sleep between `fork_choice_updated` and `resolve_kind` in the
|
||||
/// local miner, giving the payload job time for multiple rebuild attempts with
|
||||
/// new transactions from the pool.
|
||||
///
|
||||
/// Parses strings using [`humantime::parse_duration`]
|
||||
/// --dev.payload-wait-time 450ms
|
||||
#[arg(
|
||||
long = "dev.payload-wait-time",
|
||||
help_heading = "Dev testnet",
|
||||
value_parser = parse_duration,
|
||||
verbatim_doc_comment
|
||||
)]
|
||||
pub payload_wait_time: Option<Duration>,
|
||||
|
||||
/// Derive dev accounts from a fixed mnemonic instead of random ones.
|
||||
#[arg(
|
||||
long = "dev.mnemonic",
|
||||
@@ -60,6 +76,7 @@ impl Default for DevArgs {
|
||||
dev: false,
|
||||
block_max_transactions: None,
|
||||
block_time: None,
|
||||
payload_wait_time: None,
|
||||
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
|
||||
}
|
||||
}
|
||||
@@ -86,6 +103,7 @@ mod tests {
|
||||
dev: false,
|
||||
block_max_transactions: None,
|
||||
block_time: None,
|
||||
payload_wait_time: None,
|
||||
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
|
||||
}
|
||||
);
|
||||
@@ -97,6 +115,7 @@ mod tests {
|
||||
dev: true,
|
||||
block_max_transactions: None,
|
||||
block_time: None,
|
||||
payload_wait_time: None,
|
||||
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
|
||||
}
|
||||
);
|
||||
@@ -108,6 +127,7 @@ mod tests {
|
||||
dev: true,
|
||||
block_max_transactions: None,
|
||||
block_time: None,
|
||||
payload_wait_time: None,
|
||||
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
|
||||
}
|
||||
);
|
||||
@@ -125,6 +145,7 @@ mod tests {
|
||||
dev: true,
|
||||
block_max_transactions: Some(2),
|
||||
block_time: None,
|
||||
payload_wait_time: None,
|
||||
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
|
||||
}
|
||||
);
|
||||
@@ -137,6 +158,7 @@ mod tests {
|
||||
dev: true,
|
||||
block_max_transactions: None,
|
||||
block_time: Some(std::time::Duration::from_secs(1)),
|
||||
payload_wait_time: None,
|
||||
dev_mnemonic: DEFAULT_MNEMONIC.to_string(),
|
||||
}
|
||||
);
|
||||
|
||||
@@ -126,9 +126,9 @@ pub enum VersionSpecificValidationError {
|
||||
/// root after Cancun
|
||||
#[error("no parent beacon block root post-cancun")]
|
||||
NoParentBeaconBlockRootPostCancun,
|
||||
/// Thrown if the pre-V6 `PayloadAttributes` or `ExecutionPayload` contains a block access list
|
||||
#[error("block access list not before V6")]
|
||||
BlockAccessListNotSupportedBeforeV6,
|
||||
/// Thrown if the current engine method version does not support a block access list
|
||||
#[error("block access list not supported in this engine API version")]
|
||||
BlockAccessListNotSupported,
|
||||
/// Thrown if `engine_newPayload` contains no block access list
|
||||
/// after Amsterdam
|
||||
#[error("no block access list post-Amsterdam")]
|
||||
@@ -137,9 +137,9 @@ pub enum VersionSpecificValidationError {
|
||||
/// before Amsterdam
|
||||
#[error("block access list pre-Amsterdam")]
|
||||
HasBlockAccessListPreAmsterdam,
|
||||
/// Thrown if the pre-V6 `PayloadAttributes` or `ExecutionPayload` contains a slot number
|
||||
#[error("slot number not before V6")]
|
||||
SlotNumberNotSupportedBeforeV6,
|
||||
/// Thrown if the current engine method version does not support a slot number
|
||||
#[error("slot number not supported in this engine API version")]
|
||||
SlotNumberNotSupported,
|
||||
/// Thrown if `engine_newPayload` contains no slot number
|
||||
/// after Amsterdam
|
||||
#[error("no slot number post-Amsterdam")]
|
||||
|
||||
@@ -62,8 +62,10 @@ pub trait PayloadTypes: Send + Sync + Unpin + core::fmt::Debug + Clone + 'static
|
||||
/// * If V3, this ensures that the payload timestamp is within the Cancun timestamp.
|
||||
/// * If V4, this ensures that the payload timestamp is within the Prague timestamp.
|
||||
/// * If V5, this ensures that the payload timestamp is within the Osaka timestamp.
|
||||
/// * If V6, this ensures that the payload timestamp is within the Amsterdam timestamp.
|
||||
///
|
||||
/// Additionally, it ensures that `engine_getPayloadV4` is not used for an Osaka payload.
|
||||
/// Additionally, it ensures that `engine_getPayloadV4` is not used for an Osaka payload and that
|
||||
/// staggered endpoint upgrades reject the next fork once a newer method version is required.
|
||||
///
|
||||
/// Otherwise, this will return [`EngineObjectValidationError::UnsupportedFork`].
|
||||
pub fn validate_payload_timestamp(
|
||||
@@ -151,12 +153,26 @@ pub fn validate_payload_timestamp(
|
||||
return Err(EngineObjectValidationError::UnsupportedFork)
|
||||
}
|
||||
|
||||
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
|
||||
|
||||
// Staggered endpoint upgrades must reject Amsterdam payloads until the Amsterdam-specific
|
||||
// method version is used.
|
||||
if is_amsterdam &&
|
||||
matches!(
|
||||
(version, kind),
|
||||
(EngineApiMessageVersion::V3, MessageValidationKind::PayloadAttributes) |
|
||||
(EngineApiMessageVersion::V4, MessageValidationKind::Payload) |
|
||||
(EngineApiMessageVersion::V5, MessageValidationKind::GetPayload)
|
||||
)
|
||||
{
|
||||
return Err(EngineObjectValidationError::UnsupportedFork)
|
||||
}
|
||||
|
||||
// `engine_getPayloadV4` MUST reject payloads with a timestamp >= Osaka.
|
||||
if version.is_v4() && kind == MessageValidationKind::GetPayload && is_osaka {
|
||||
return Err(EngineObjectValidationError::UnsupportedFork)
|
||||
}
|
||||
|
||||
let is_amsterdam = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
|
||||
if version.is_v6() && !is_amsterdam {
|
||||
// From the Engine API spec:
|
||||
// <https://github.com/ethereum/execution-apis/blob/15399c2e2f16a5f800bf3f285640357e2c245ad9/src/engine/osaka.md#specification>
|
||||
@@ -183,16 +199,30 @@ pub fn validate_block_access_list_presence<T: EthereumHardforks>(
|
||||
has_block_access_list: bool,
|
||||
) -> Result<(), EngineObjectValidationError> {
|
||||
let is_amsterdam_active = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
|
||||
|
||||
match version {
|
||||
EngineApiMessageVersion::V1 |
|
||||
EngineApiMessageVersion::V2 |
|
||||
EngineApiMessageVersion::V3 |
|
||||
EngineApiMessageVersion::V4 |
|
||||
EngineApiMessageVersion::V5 => {
|
||||
EngineApiMessageVersion::V4 => {
|
||||
if has_block_access_list {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::BlockAccessListNotSupportedBeforeV6))
|
||||
.to_error(VersionSpecificValidationError::BlockAccessListNotSupported))
|
||||
}
|
||||
}
|
||||
|
||||
EngineApiMessageVersion::V5 => {
|
||||
if message_validation_kind == MessageValidationKind::Payload {
|
||||
if is_amsterdam_active && !has_block_access_list {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::NoBlockAccessListPostAmsterdam))
|
||||
}
|
||||
if !is_amsterdam_active && has_block_access_list {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::HasBlockAccessListPreAmsterdam))
|
||||
}
|
||||
} else if has_block_access_list {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::BlockAccessListNotSupported))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -224,14 +254,42 @@ pub fn validate_slot_number_presence<T: EthereumHardforks>(
|
||||
let is_amsterdam_active = chain_spec.is_amsterdam_active_at_timestamp(timestamp);
|
||||
|
||||
match version {
|
||||
EngineApiMessageVersion::V1 |
|
||||
EngineApiMessageVersion::V2 |
|
||||
EngineApiMessageVersion::V3 |
|
||||
EngineApiMessageVersion::V4 |
|
||||
EngineApiMessageVersion::V5 => {
|
||||
EngineApiMessageVersion::V1 | EngineApiMessageVersion::V2 | EngineApiMessageVersion::V3 => {
|
||||
if has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::SlotNumberNotSupportedBeforeV6))
|
||||
.to_error(VersionSpecificValidationError::SlotNumberNotSupported))
|
||||
}
|
||||
}
|
||||
|
||||
EngineApiMessageVersion::V4 => {
|
||||
if message_validation_kind == MessageValidationKind::PayloadAttributes {
|
||||
if is_amsterdam_active && !has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::NoSlotNumberPostAmsterdam))
|
||||
}
|
||||
if !is_amsterdam_active && has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::HasSlotNumberPreAmsterdam))
|
||||
}
|
||||
} else if has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::SlotNumberNotSupported))
|
||||
}
|
||||
}
|
||||
|
||||
EngineApiMessageVersion::V5 => {
|
||||
if message_validation_kind == MessageValidationKind::Payload {
|
||||
if is_amsterdam_active && !has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::NoSlotNumberPostAmsterdam))
|
||||
}
|
||||
if !is_amsterdam_active && has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::HasSlotNumberPreAmsterdam))
|
||||
}
|
||||
} else if has_slot_number {
|
||||
return Err(message_validation_kind
|
||||
.to_error(VersionSpecificValidationError::SlotNumberNotSupported))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -651,6 +709,75 @@ mod tests {
|
||||
assert_matches!(res, Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_amsterdam_staggered_version_restrictions() {
|
||||
let chain_spec = ChainSpecBuilder::mainnet().amsterdam_activated().build();
|
||||
|
||||
let res = validate_payload_timestamp(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V3,
|
||||
0,
|
||||
MessageValidationKind::PayloadAttributes,
|
||||
);
|
||||
assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork));
|
||||
|
||||
let res = validate_payload_timestamp(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V4,
|
||||
0,
|
||||
MessageValidationKind::Payload,
|
||||
);
|
||||
assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork));
|
||||
|
||||
let res = validate_payload_timestamp(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V5,
|
||||
0,
|
||||
MessageValidationKind::GetPayload,
|
||||
);
|
||||
assert_matches!(res, Err(EngineObjectValidationError::UnsupportedFork));
|
||||
|
||||
let res = validate_payload_timestamp(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V6,
|
||||
0,
|
||||
MessageValidationKind::GetPayload,
|
||||
);
|
||||
assert_matches!(res, Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn validate_amsterdam_slot_and_bal_presence() {
|
||||
let chain_spec = ChainSpecBuilder::mainnet().amsterdam_activated().build();
|
||||
|
||||
let res = validate_slot_number_presence(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V4,
|
||||
MessageValidationKind::PayloadAttributes,
|
||||
0,
|
||||
true,
|
||||
);
|
||||
assert_matches!(res, Ok(()));
|
||||
|
||||
let res = validate_slot_number_presence(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V5,
|
||||
MessageValidationKind::Payload,
|
||||
0,
|
||||
true,
|
||||
);
|
||||
assert_matches!(res, Ok(()));
|
||||
|
||||
let res = validate_block_access_list_presence(
|
||||
&chain_spec,
|
||||
EngineApiMessageVersion::V5,
|
||||
MessageValidationKind::Payload,
|
||||
0,
|
||||
true,
|
||||
);
|
||||
assert_matches!(res, Ok(()));
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn execution_requests_validation() {
|
||||
assert_matches!(validate_execution_requests(&[]), Ok(()));
|
||||
|
||||
@@ -21,6 +21,7 @@ use alloy_rpc_types_eth::{
|
||||
use alloy_serde::JsonStorageKey;
|
||||
use jsonrpsee::{core::RpcResult, proc_macros::rpc, RpcModule};
|
||||
use reth_engine_primitives::EngineTypes;
|
||||
use serde_json::Value;
|
||||
|
||||
/// Helper trait for the engine api server.
|
||||
///
|
||||
@@ -392,4 +393,19 @@ pub trait EngineEthApi<TxReq: RpcObject, B: RpcObject, R: RpcObject> {
|
||||
keys: Vec<JsonStorageKey>,
|
||||
block_number: Option<BlockId>,
|
||||
) -> RpcResult<EIP1186AccountProofResponse>;
|
||||
|
||||
/// Returns the EIP-7928 block access list for a block by hash.
|
||||
#[method(name = "getBlockAccessListByBlockHash")]
|
||||
async fn block_access_list_by_block_hash(&self, hash: B256) -> RpcResult<Option<Value>>;
|
||||
|
||||
/// Returns the EIP-7928 block access list for a block by number.
|
||||
#[method(name = "getBlockAccessListByBlockNumber")]
|
||||
async fn block_access_list_by_block_number(
|
||||
&self,
|
||||
number: BlockNumberOrTag,
|
||||
) -> RpcResult<Option<Value>>;
|
||||
|
||||
/// Returns the EIP-7928 block access list bytes for a block by number.
|
||||
#[method(name = "getBlockAccessListRaw")]
|
||||
async fn block_access_list_raw(&self, block: BlockId) -> RpcResult<Option<Bytes>>;
|
||||
}
|
||||
|
||||
@@ -77,3 +77,4 @@ alloy-rpc-types-engine.workspace = true
|
||||
|
||||
serde_json.workspace = true
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
reqwest.workspace = true
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
use crate::{
|
||||
error::{RpcError, ServerKind},
|
||||
middleware::RethRpcMiddleware,
|
||||
middleware::{RethAuthHttpMiddleware, RethRpcMiddleware},
|
||||
};
|
||||
use http::header::AUTHORIZATION;
|
||||
use jsonrpsee::{
|
||||
core::{client::SubscriptionClientT, RegisterMethodError},
|
||||
http_client::HeaderMap,
|
||||
server::{AlreadyStoppedError, RpcModule},
|
||||
server::{AlreadyStoppedError, RpcModule, ServerConfig, ServerConfigBuilder},
|
||||
ws_client::RpcServiceBuilder,
|
||||
Methods,
|
||||
};
|
||||
@@ -20,12 +20,11 @@ use std::net::{IpAddr, Ipv4Addr, SocketAddr};
|
||||
use tower::layer::util::Identity;
|
||||
|
||||
pub use jsonrpsee::server::ServerBuilder;
|
||||
use jsonrpsee::server::{ServerConfig, ServerConfigBuilder};
|
||||
pub use reth_ipc::server::Builder as IpcServerBuilder;
|
||||
|
||||
/// Server configuration for the auth server.
|
||||
#[derive(Debug)]
|
||||
pub struct AuthServerConfig<RpcMiddleware = Identity> {
|
||||
pub struct AuthServerConfig<RpcMiddleware = Identity, HttpMiddleware = Identity> {
|
||||
/// Where the server should listen.
|
||||
pub(crate) socket_addr: SocketAddr,
|
||||
/// The secret for the auth layer of the server.
|
||||
@@ -38,6 +37,8 @@ pub struct AuthServerConfig<RpcMiddleware = Identity> {
|
||||
pub(crate) ipc_endpoint: Option<String>,
|
||||
/// Configurable RPC middleware
|
||||
pub(crate) rpc_middleware: RpcMiddleware,
|
||||
/// Configurable HTTP transport middleware, applied after JWT authentication.
|
||||
pub(crate) http_middleware: HttpMiddleware,
|
||||
}
|
||||
|
||||
// === impl AuthServerConfig ===
|
||||
@@ -48,15 +49,23 @@ impl AuthServerConfig {
|
||||
AuthServerConfigBuilder::new(secret)
|
||||
}
|
||||
}
|
||||
impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
|
||||
impl<RpcMiddleware, HttpMiddleware> AuthServerConfig<RpcMiddleware, HttpMiddleware> {
|
||||
/// Returns the address the server will listen on.
|
||||
pub const fn address(&self) -> SocketAddr {
|
||||
self.socket_addr
|
||||
}
|
||||
|
||||
/// Configures the rpc middleware.
|
||||
pub fn with_rpc_middleware<T>(self, rpc_middleware: T) -> AuthServerConfig<T> {
|
||||
let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint, .. } = self;
|
||||
pub fn with_rpc_middleware<T>(self, rpc_middleware: T) -> AuthServerConfig<T, HttpMiddleware> {
|
||||
let Self {
|
||||
socket_addr,
|
||||
secret,
|
||||
server_config,
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
http_middleware,
|
||||
..
|
||||
} = self;
|
||||
AuthServerConfig {
|
||||
socket_addr,
|
||||
secret,
|
||||
@@ -64,13 +73,44 @@ impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
http_middleware,
|
||||
}
|
||||
}
|
||||
|
||||
/// Configures the HTTP transport middleware.
|
||||
///
|
||||
/// This middleware is applied after JWT authentication and before JSON-RPC parsing,
|
||||
/// giving access to the raw HTTP request (headers, body, etc.).
|
||||
pub fn with_http_middleware<T>(self, http_middleware: T) -> AuthServerConfig<RpcMiddleware, T> {
|
||||
let Self {
|
||||
socket_addr,
|
||||
secret,
|
||||
server_config,
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
..
|
||||
} = self;
|
||||
AuthServerConfig {
|
||||
socket_addr,
|
||||
secret,
|
||||
server_config,
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
http_middleware,
|
||||
}
|
||||
}
|
||||
|
||||
/// Convenience function to start a server in one step.
|
||||
///
|
||||
/// The `HttpMiddleware` type parameter configures additional HTTP transport middleware
|
||||
/// that runs after JWT authentication. When set to `Identity` (the default), only JWT
|
||||
/// authentication is applied.
|
||||
pub async fn start(self, module: AuthRpcModule) -> Result<AuthServerHandle, RpcError>
|
||||
where
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
HttpMiddleware: RethAuthHttpMiddleware<RpcMiddleware>,
|
||||
{
|
||||
let Self {
|
||||
socket_addr,
|
||||
@@ -79,11 +119,13 @@ impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
http_middleware,
|
||||
} = self;
|
||||
|
||||
// Create auth middleware.
|
||||
// Create auth middleware with JWT authentication in front of the user-provided
|
||||
// transport middleware.
|
||||
let middleware =
|
||||
tower::ServiceBuilder::new().layer(AuthLayer::new(JwtAuthValidator::new(secret)));
|
||||
tower::ServiceBuilder::new().layer(AuthHttpLayer::new(secret, http_middleware));
|
||||
|
||||
let rpc_middleware = RpcServiceBuilder::default().layer(rpc_middleware);
|
||||
|
||||
@@ -117,15 +159,47 @@ impl<RpcMiddleware> AuthServerConfig<RpcMiddleware> {
|
||||
}
|
||||
}
|
||||
|
||||
/// A combined tower layer that applies JWT authentication before custom HTTP middleware.
|
||||
///
|
||||
/// This composes `AuthLayer<JwtAuthValidator>` around a user-provided `HttpMiddleware` into a
|
||||
/// single `tower::Layer`. Requests first pass through JWT validation and only authenticated
|
||||
/// requests are forwarded into the custom middleware.
|
||||
struct AuthHttpLayer<HttpMiddleware> {
|
||||
auth_layer: AuthLayer<JwtAuthValidator>,
|
||||
http_middleware: HttpMiddleware,
|
||||
}
|
||||
|
||||
impl<HttpMiddleware> AuthHttpLayer<HttpMiddleware> {
|
||||
const fn new(secret: JwtSecret, http_middleware: HttpMiddleware) -> Self {
|
||||
Self { auth_layer: AuthLayer::new(JwtAuthValidator::new(secret)), http_middleware }
|
||||
}
|
||||
}
|
||||
|
||||
impl<S, HttpMiddleware> tower::Layer<S> for AuthHttpLayer<HttpMiddleware>
|
||||
where
|
||||
HttpMiddleware: tower::Layer<S> + Clone,
|
||||
AuthLayer<JwtAuthValidator>: tower::Layer<<HttpMiddleware as tower::Layer<S>>::Service>,
|
||||
{
|
||||
type Service = <AuthLayer<JwtAuthValidator> as tower::Layer<
|
||||
<HttpMiddleware as tower::Layer<S>>::Service,
|
||||
>>::Service;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
let http_service = self.http_middleware.layer(inner);
|
||||
self.auth_layer.layer(http_service)
|
||||
}
|
||||
}
|
||||
|
||||
/// Builder type for configuring an `AuthServerConfig`.
|
||||
#[derive(Debug)]
|
||||
pub struct AuthServerConfigBuilder<RpcMiddleware = Identity> {
|
||||
pub struct AuthServerConfigBuilder<RpcMiddleware = Identity, HttpMiddleware = Identity> {
|
||||
socket_addr: Option<SocketAddr>,
|
||||
secret: JwtSecret,
|
||||
server_config: Option<ServerConfigBuilder>,
|
||||
ipc_server_config: Option<IpcServerBuilder<Identity, Identity>>,
|
||||
ipc_endpoint: Option<String>,
|
||||
rpc_middleware: RpcMiddleware,
|
||||
http_middleware: HttpMiddleware,
|
||||
}
|
||||
|
||||
// === impl AuthServerConfigBuilder ===
|
||||
@@ -140,14 +214,26 @@ impl AuthServerConfigBuilder {
|
||||
ipc_server_config: None,
|
||||
ipc_endpoint: None,
|
||||
rpc_middleware: Identity::new(),
|
||||
http_middleware: Identity::new(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
|
||||
impl<RpcMiddleware, HttpMiddleware> AuthServerConfigBuilder<RpcMiddleware, HttpMiddleware> {
|
||||
/// Configures the rpc middleware.
|
||||
pub fn with_rpc_middleware<T>(self, rpc_middleware: T) -> AuthServerConfigBuilder<T> {
|
||||
let Self { socket_addr, secret, server_config, ipc_server_config, ipc_endpoint, .. } = self;
|
||||
pub fn with_rpc_middleware<T>(
|
||||
self,
|
||||
rpc_middleware: T,
|
||||
) -> AuthServerConfigBuilder<T, HttpMiddleware> {
|
||||
let Self {
|
||||
socket_addr,
|
||||
secret,
|
||||
server_config,
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
http_middleware,
|
||||
..
|
||||
} = self;
|
||||
AuthServerConfigBuilder {
|
||||
socket_addr,
|
||||
secret,
|
||||
@@ -155,6 +241,35 @@ impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
http_middleware,
|
||||
}
|
||||
}
|
||||
|
||||
/// Configures the HTTP transport middleware.
|
||||
///
|
||||
/// This middleware is applied after JWT authentication and before JSON-RPC parsing,
|
||||
/// giving access to the raw HTTP request (headers, body, etc.).
|
||||
pub fn with_http_middleware<T>(
|
||||
self,
|
||||
http_middleware: T,
|
||||
) -> AuthServerConfigBuilder<RpcMiddleware, T> {
|
||||
let Self {
|
||||
socket_addr,
|
||||
secret,
|
||||
server_config,
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
..
|
||||
} = self;
|
||||
AuthServerConfigBuilder {
|
||||
socket_addr,
|
||||
secret,
|
||||
server_config,
|
||||
ipc_server_config,
|
||||
ipc_endpoint,
|
||||
rpc_middleware,
|
||||
http_middleware,
|
||||
}
|
||||
}
|
||||
|
||||
@@ -200,7 +315,7 @@ impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
|
||||
}
|
||||
|
||||
/// Build the `AuthServerConfig`.
|
||||
pub fn build(self) -> AuthServerConfig<RpcMiddleware> {
|
||||
pub fn build(self) -> AuthServerConfig<RpcMiddleware, HttpMiddleware> {
|
||||
AuthServerConfig {
|
||||
socket_addr: self.socket_addr.unwrap_or_else(|| {
|
||||
SocketAddr::new(IpAddr::V4(Ipv4Addr::LOCALHOST), constants::DEFAULT_AUTH_PORT)
|
||||
@@ -233,6 +348,7 @@ impl<RpcMiddleware> AuthServerConfigBuilder<RpcMiddleware> {
|
||||
}),
|
||||
ipc_endpoint: self.ipc_endpoint,
|
||||
rpc_middleware: self.rpc_middleware,
|
||||
http_middleware: self.http_middleware,
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -289,10 +405,14 @@ impl AuthRpcModule {
|
||||
}
|
||||
|
||||
/// Convenience function for starting a server
|
||||
pub async fn start_server(
|
||||
pub async fn start_server<RpcMiddleware, HttpMiddleware>(
|
||||
self,
|
||||
config: AuthServerConfig,
|
||||
) -> Result<AuthServerHandle, RpcError> {
|
||||
config: AuthServerConfig<RpcMiddleware, HttpMiddleware>,
|
||||
) -> Result<AuthServerHandle, RpcError>
|
||||
where
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
HttpMiddleware: RethAuthHttpMiddleware<RpcMiddleware>,
|
||||
{
|
||||
config.start(self).await
|
||||
}
|
||||
}
|
||||
@@ -397,7 +517,7 @@ impl AuthServerHandle {
|
||||
.build(ipc_endpoint)
|
||||
.await
|
||||
.expect("Failed to create ipc client"),
|
||||
)
|
||||
);
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
@@ -1,5 +1,10 @@
|
||||
use jsonrpsee::server::middleware::rpc::RpcService;
|
||||
use tower::Layer;
|
||||
use jsonrpsee::server::{
|
||||
middleware::rpc::RpcService, HttpRequest, HttpResponse, TowerServiceNoHttp,
|
||||
};
|
||||
use tower::{
|
||||
layer::util::{Identity, Stack},
|
||||
Layer,
|
||||
};
|
||||
|
||||
/// A Helper alias trait for the RPC middleware supported by the server.
|
||||
pub trait RethRpcMiddleware:
|
||||
@@ -35,3 +40,39 @@ impl<T> RethRpcMiddleware for T where
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
/// Inner HTTP transport service type for auth-server middleware.
|
||||
pub type AuthHttpService<RM> = TowerServiceNoHttp<Stack<RM, Identity>>;
|
||||
|
||||
/// Helper alias trait for auth-server HTTP transport middleware layers.
|
||||
pub trait RethAuthHttpMiddleware<RM>:
|
||||
tower::Layer<
|
||||
AuthHttpService<RM>,
|
||||
Service: tower::Service<
|
||||
HttpRequest,
|
||||
Response = HttpResponse,
|
||||
Error = tower::BoxError,
|
||||
Future: Send,
|
||||
> + Send
|
||||
+ Clone,
|
||||
> + Clone
|
||||
+ Send
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
impl<T, RM> RethAuthHttpMiddleware<RM> for T where
|
||||
T: tower::Layer<
|
||||
AuthHttpService<RM>,
|
||||
Service: tower::Service<
|
||||
HttpRequest,
|
||||
Response = HttpResponse,
|
||||
Error = tower::BoxError,
|
||||
Future: Send,
|
||||
> + Send
|
||||
+ Clone,
|
||||
> + Clone
|
||||
+ Send
|
||||
+ 'static
|
||||
{
|
||||
}
|
||||
|
||||
@@ -1,16 +1,90 @@
|
||||
//! Auth server tests
|
||||
|
||||
use crate::utils::launch_auth;
|
||||
use crate::utils::{launch_auth, launch_auth_with_config, test_address};
|
||||
use alloy_primitives::U64;
|
||||
use alloy_rpc_types_engine::{
|
||||
ExecutionPayloadInputV2, ExecutionPayloadV1, ForkchoiceState, PayloadId,
|
||||
};
|
||||
use jsonrpsee::core::client::{ClientT, SubscriptionClientT};
|
||||
use http::header::{AUTHORIZATION, CONTENT_TYPE};
|
||||
use jsonrpsee::{
|
||||
core::client::{ClientT, SubscriptionClientT},
|
||||
server::{HttpRequest, HttpResponse},
|
||||
};
|
||||
use reth_ethereum_engine_primitives::EthEngineTypes;
|
||||
use reth_ethereum_primitives::{Block, TransactionSigned};
|
||||
use reth_primitives_traits::block::Block as _;
|
||||
use reth_rpc_api::clients::EngineApiClient;
|
||||
use reth_rpc_layer::JwtSecret;
|
||||
use reth_rpc_builder::auth::AuthServerConfig;
|
||||
use reth_rpc_layer::{secret_to_bearer_header, JwtSecret};
|
||||
use std::{
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc, Mutex,
|
||||
},
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tower::{Layer, Service};
|
||||
|
||||
#[derive(Clone, Default)]
|
||||
struct CountingAuthHttpLayer {
|
||||
count: Arc<AtomicUsize>,
|
||||
content_types: Arc<Mutex<Vec<Option<String>>>>,
|
||||
}
|
||||
|
||||
impl CountingAuthHttpLayer {
|
||||
fn count(&self) -> usize {
|
||||
self.count.load(Ordering::Relaxed)
|
||||
}
|
||||
|
||||
fn seen_content_types(&self) -> Vec<Option<String>> {
|
||||
self.content_types.lock().unwrap().clone()
|
||||
}
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for CountingAuthHttpLayer {
|
||||
type Service = CountingAuthHttpService<S>;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
CountingAuthHttpService {
|
||||
inner,
|
||||
count: self.count.clone(),
|
||||
content_types: self.content_types.clone(),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct CountingAuthHttpService<S> {
|
||||
inner: S,
|
||||
count: Arc<AtomicUsize>,
|
||||
content_types: Arc<Mutex<Vec<Option<String>>>>,
|
||||
}
|
||||
|
||||
impl<S> Service<HttpRequest> for CountingAuthHttpService<S>
|
||||
where
|
||||
S: Service<HttpRequest, Response = HttpResponse, Error = tower::BoxError> + Send + Clone,
|
||||
S::Future: Send + 'static,
|
||||
{
|
||||
type Response = HttpResponse;
|
||||
type Error = tower::BoxError;
|
||||
type Future = S::Future;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, request: HttpRequest) -> Self::Future {
|
||||
self.count.fetch_add(1, Ordering::Relaxed);
|
||||
self.content_types.lock().unwrap().push(
|
||||
request
|
||||
.headers()
|
||||
.get(CONTENT_TYPE)
|
||||
.and_then(|value| value.to_str().ok())
|
||||
.map(str::to_owned),
|
||||
);
|
||||
self.inner.call(request)
|
||||
}
|
||||
}
|
||||
|
||||
#[expect(unused_must_use)]
|
||||
async fn test_basic_engine_calls<C>(client: &C)
|
||||
@@ -58,3 +132,55 @@ async fn test_auth_endpoints_ws() {
|
||||
let client = handle.ws_client().await;
|
||||
test_basic_engine_calls(&client).await
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_auth_http_middleware_runs_only_after_jwt() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let secret = JwtSecret::random();
|
||||
let layer = CountingAuthHttpLayer::default();
|
||||
let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build();
|
||||
let handle = launch_auth_with_config(config.with_http_middleware(layer.clone())).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(handle.http_url())
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(
|
||||
serde_json::to_vec(&serde_json::json!({
|
||||
"jsonrpc": "2.0",
|
||||
"method": "engine_exchangeCapabilities",
|
||||
"params": [[]],
|
||||
"id": 1
|
||||
}))
|
||||
.unwrap(),
|
||||
)
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert_eq!(response.status(), reqwest::StatusCode::UNAUTHORIZED);
|
||||
assert_eq!(layer.count(), 0);
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread")]
|
||||
async fn test_auth_http_middleware_sees_transport_headers_before_json_rpc_parsing() {
|
||||
reth_tracing::init_test_tracing();
|
||||
|
||||
let secret = JwtSecret::random();
|
||||
let layer = CountingAuthHttpLayer::default();
|
||||
let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build();
|
||||
let handle = launch_auth_with_config(config.with_http_middleware(layer.clone())).await;
|
||||
|
||||
let response = reqwest::Client::new()
|
||||
.post(handle.http_url())
|
||||
.header(AUTHORIZATION, secret_to_bearer_header(&secret))
|
||||
.header(CONTENT_TYPE, "application/ssz")
|
||||
.body("not-json")
|
||||
.send()
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!(response.status().is_success() || response.status().is_client_error());
|
||||
assert_eq!(layer.count(), 1);
|
||||
assert_eq!(layer.seen_content_types(), vec![Some("application/ssz".to_owned())]);
|
||||
}
|
||||
|
||||
@@ -14,6 +14,7 @@ use reth_payload_builder::test_utils::spawn_test_payload_service;
|
||||
use reth_provider::test_utils::NoopProvider;
|
||||
use reth_rpc_builder::{
|
||||
auth::{AuthRpcModule, AuthServerConfig, AuthServerHandle},
|
||||
middleware::{RethAuthHttpMiddleware, RethRpcMiddleware},
|
||||
RpcModuleBuilder, RpcServerConfig, RpcServerHandle, TransportRpcModuleConfig,
|
||||
};
|
||||
use reth_rpc_engine_api::{capabilities::EngineCapabilities, EngineApi};
|
||||
@@ -28,12 +29,23 @@ use tokio::sync::mpsc::unbounded_channel;
|
||||
|
||||
/// Localhost with port 0 so a free port is used.
|
||||
pub const fn test_address() -> SocketAddr {
|
||||
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::UNSPECIFIED, 0))
|
||||
SocketAddr::V4(SocketAddrV4::new(Ipv4Addr::LOCALHOST, 0))
|
||||
}
|
||||
|
||||
/// Launches a new server for the auth module
|
||||
pub async fn launch_auth(secret: JwtSecret) -> AuthServerHandle {
|
||||
let config = AuthServerConfig::builder(secret).socket_addr(test_address()).build();
|
||||
launch_auth_with_config(config).await
|
||||
}
|
||||
|
||||
/// Launches a new server for the auth module with the given config.
|
||||
pub async fn launch_auth_with_config<RpcMiddleware, HttpMiddleware>(
|
||||
config: AuthServerConfig<RpcMiddleware, HttpMiddleware>,
|
||||
) -> AuthServerHandle
|
||||
where
|
||||
RpcMiddleware: RethRpcMiddleware,
|
||||
HttpMiddleware: RethAuthHttpMiddleware<RpcMiddleware>,
|
||||
{
|
||||
let (tx, _rx) = unbounded_channel();
|
||||
let beacon_engine_handle = ConsensusEngineHandle::<EthEngineTypes>::new(tx);
|
||||
let client = ClientVersionV1 {
|
||||
|
||||
@@ -479,7 +479,6 @@ impl<Network, Evm, Receipt, Header, Map, SimTx, RpcTx, TxEnv>
|
||||
evm,
|
||||
sim_tx_converter,
|
||||
rpc_tx_converter,
|
||||
tx_env_converter: _,
|
||||
..
|
||||
} = self;
|
||||
RpcConverter {
|
||||
|
||||
@@ -269,7 +269,7 @@ where
|
||||
>::from_execution_payload(&payload);
|
||||
self.inner
|
||||
.validator
|
||||
.validate_version_specific_fields(EngineApiMessageVersion::V6, payload_or_attrs)?;
|
||||
.validate_version_specific_fields(EngineApiMessageVersion::V5, payload_or_attrs)?;
|
||||
Ok(self.inner.beacon_consensus.new_payload(payload).await?)
|
||||
}
|
||||
|
||||
@@ -386,7 +386,7 @@ where
|
||||
state: ForkchoiceState,
|
||||
payload_attrs: Option<EngineT::PayloadAttributes>,
|
||||
) -> EngineApiResult<ForkchoiceUpdated> {
|
||||
self.validate_and_execute_forkchoice(EngineApiMessageVersion::V6, state, payload_attrs)
|
||||
self.validate_and_execute_forkchoice(EngineApiMessageVersion::V4, state, payload_attrs)
|
||||
.await
|
||||
}
|
||||
|
||||
@@ -1438,9 +1438,10 @@ struct EngineApiInner<Provider, PayloadT: PayloadTypes, Pool, Validator, ChainSp
|
||||
#[cfg(test)]
|
||||
mod tests {
|
||||
use super::*;
|
||||
use alloy_primitives::{Address, B256};
|
||||
use alloy_eips::eip7685::Requests;
|
||||
use alloy_primitives::{Address, Bytes, B256};
|
||||
use alloy_rpc_types_engine::{
|
||||
ClientCode, ClientVersionV1, PayloadAttributes, PayloadStatusEnum,
|
||||
ClientCode, ClientVersionV1, ExecutionPayloadV2, PayloadAttributes, PayloadStatusEnum,
|
||||
};
|
||||
use assert_matches::assert_matches;
|
||||
use reth_chainspec::{ChainSpec, ChainSpecBuilder, MAINNET};
|
||||
@@ -1532,6 +1533,63 @@ mod tests {
|
||||
assert_matches!(handle.from_api.recv().await, Some(BeaconEngineMessage::NewPayload { .. }));
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn new_payload_v5_accepts_amsterdam_payloads() {
|
||||
let chain_spec = Arc::new(ChainSpecBuilder::mainnet().amsterdam_activated().build());
|
||||
let provider = Arc::new(MockEthProvider::default());
|
||||
let payload_store = spawn_test_payload_service::<EthEngineTypes>();
|
||||
let (to_engine, mut engine_rx) = unbounded_channel();
|
||||
|
||||
let api = EngineApi::new(
|
||||
provider,
|
||||
chain_spec.clone(),
|
||||
ConsensusEngineHandle::new(to_engine),
|
||||
payload_store.into(),
|
||||
NoopTransactionPool::default(),
|
||||
Runtime::test(),
|
||||
ClientVersionV1 {
|
||||
code: ClientCode::RH,
|
||||
name: "Reth".to_string(),
|
||||
version: "v0.0.0-test".to_string(),
|
||||
commit: "test".to_string(),
|
||||
},
|
||||
EngineCapabilities::default(),
|
||||
EthereumEngineValidator::new(chain_spec),
|
||||
false,
|
||||
NoopNetwork::default(),
|
||||
);
|
||||
|
||||
tokio::spawn(async move {
|
||||
let payload_v1 = ExecutionPayloadV1::from_block_slow(&Block::default());
|
||||
let payload = ExecutionPayloadV4 {
|
||||
payload_inner: ExecutionPayloadV3 {
|
||||
payload_inner: ExecutionPayloadV2 {
|
||||
payload_inner: payload_v1,
|
||||
withdrawals: Vec::new(),
|
||||
},
|
||||
blob_gas_used: 0,
|
||||
excess_blob_gas: 0,
|
||||
},
|
||||
block_access_list: Bytes::from_static(b"bal"),
|
||||
slot_number: 1,
|
||||
};
|
||||
let execution_data = ExecutionData {
|
||||
payload: payload.into(),
|
||||
sidecar: ExecutionPayloadSidecar::v4(
|
||||
CancunPayloadFields {
|
||||
versioned_hashes: Vec::new(),
|
||||
parent_beacon_block_root: B256::ZERO,
|
||||
},
|
||||
PraguePayloadFields { requests: RequestsOrHash::Requests(Requests::default()) },
|
||||
),
|
||||
};
|
||||
|
||||
api.new_payload_v5(execution_data).await.unwrap();
|
||||
});
|
||||
|
||||
assert_matches!(engine_rx.recv().await, Some(BeaconEngineMessage::NewPayload { .. }));
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
struct TestNetworkInfo {
|
||||
syncing: bool,
|
||||
|
||||
@@ -121,12 +121,12 @@ impl From<EngineApiError> for jsonrpsee_types::error::ErrorObject<'static> {
|
||||
VersionSpecificValidationError::WithdrawalsNotSupportedInV1 |
|
||||
VersionSpecificValidationError::NoWithdrawalsPostShanghai |
|
||||
VersionSpecificValidationError::HasWithdrawalsPreShanghai |
|
||||
VersionSpecificValidationError::BlockAccessListNotSupportedBeforeV6 |
|
||||
VersionSpecificValidationError::BlockAccessListNotSupported |
|
||||
VersionSpecificValidationError::HasBlockAccessListPreAmsterdam |
|
||||
VersionSpecificValidationError::NoBlockAccessListPostAmsterdam |
|
||||
VersionSpecificValidationError::HasSlotNumberPreAmsterdam |
|
||||
VersionSpecificValidationError::NoSlotNumberPostAmsterdam |
|
||||
VersionSpecificValidationError::SlotNumberNotSupportedBeforeV6,
|
||||
VersionSpecificValidationError::SlotNumberNotSupported,
|
||||
),
|
||||
) |
|
||||
EngineApiError::UnexpectedRequestsHash => {
|
||||
|
||||
@@ -419,7 +419,7 @@ pub trait EthApi<
|
||||
|
||||
/// Returns the EIP-7928 block access list bytes for a block by number.
|
||||
#[method(name = "getBlockAccessListRaw")]
|
||||
async fn block_access_list_raw(&self, number: BlockNumberOrTag) -> RpcResult<Option<Bytes>>;
|
||||
async fn block_access_list_raw(&self, block: BlockId) -> RpcResult<Option<Bytes>>;
|
||||
}
|
||||
|
||||
#[async_trait::async_trait]
|
||||
@@ -942,10 +942,10 @@ where
|
||||
Ok(Some(json))
|
||||
}
|
||||
/// Handler for: `eth_getBlockAccessListRaw`
|
||||
async fn block_access_list_raw(&self, number: BlockNumberOrTag) -> RpcResult<Option<Bytes>> {
|
||||
trace!(target: "rpc::eth", ?number, "Serving eth_getBlockAccessListRaw");
|
||||
async fn block_access_list_raw(&self, block: BlockId) -> RpcResult<Option<Bytes>> {
|
||||
trace!(target: "rpc::eth", ?block, "Serving eth_getBlockAccessListRaw");
|
||||
|
||||
let bal = self.get_block_access_list(number.into()).await?;
|
||||
let bal = self.get_block_access_list(block).await?;
|
||||
Ok(bal.map(|b: BlockAccessList| alloy_rlp::encode(b).into()))
|
||||
}
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@ use reth_rpc_eth_types::{
|
||||
use reth_rpc_server_types::constants::gas_oracle::{CALL_STIPEND_GAS, ESTIMATE_GAS_ERROR_RATIO};
|
||||
use revm::{
|
||||
context::Block,
|
||||
context_interface::{result::ExecutionResult, Transaction},
|
||||
context_interface::{result::ExecutionResult, Cfg, Transaction},
|
||||
primitives::KECCAK_EMPTY,
|
||||
};
|
||||
use tracing::trace;
|
||||
@@ -74,10 +74,15 @@ pub trait EstimateCall: Call {
|
||||
let tx_request_gas_limit = request.as_ref().gas_limit();
|
||||
let tx_request_gas_price = request.as_ref().gas_price();
|
||||
// the gas limit of the corresponding block
|
||||
let max_gas_limit = evm_env.cfg_env.tx_gas_limit_cap.map_or_else(
|
||||
|| evm_env.block_env.gas_limit(),
|
||||
|cap| cap.min(evm_env.block_env.gas_limit()),
|
||||
);
|
||||
let max_gas_limit = evm_env
|
||||
.cfg_env
|
||||
.tx_gas_limit_cap
|
||||
// If EIP-8037 is enabled, the transaction gas limit cap is not applicable
|
||||
.filter(|_| !evm_env.cfg_env.is_amsterdam_eip8037_enabled())
|
||||
.map_or_else(
|
||||
|| evm_env.block_env.gas_limit(),
|
||||
|cap| cap.min(evm_env.block_env.gas_limit()),
|
||||
);
|
||||
|
||||
// Determine the highest possible gas limit, considering both the request's specified limit
|
||||
// and the block's limit.
|
||||
|
||||
@@ -377,7 +377,7 @@ pub trait LoadPendingBlock:
|
||||
}
|
||||
}
|
||||
|
||||
let BlockBuilderOutcome { execution_result, block, hashed_state, trie_updates } =
|
||||
let BlockBuilderOutcome { execution_result, block, hashed_state, trie_updates, .. } =
|
||||
builder.finish(NoopProvider::default(), None).map_err(Self::Error::from_eth_err)?;
|
||||
|
||||
let execution_outcome =
|
||||
|
||||
@@ -13,6 +13,7 @@ pub use reth_rpc_engine_api::EngineApi;
|
||||
use reth_rpc_eth_api::{
|
||||
EngineEthFilter, FullEthApiTypes, QueryLimits, RpcBlock, RpcHeader, RpcReceipt, RpcTransaction,
|
||||
};
|
||||
use serde_json::Value;
|
||||
use tracing_futures::Instrument;
|
||||
|
||||
macro_rules! engine_span {
|
||||
@@ -145,4 +146,22 @@ where
|
||||
) -> Result<EIP1186AccountProofResponse> {
|
||||
self.eth.get_proof(address, keys, block_number).instrument(engine_span!()).await
|
||||
}
|
||||
|
||||
/// Handler for `eth_getBlockAccessListByBlockHash`
|
||||
async fn block_access_list_by_block_hash(&self, hash: B256) -> Result<Option<Value>> {
|
||||
self.eth.block_access_list_by_block_hash(hash).instrument(engine_span!()).await
|
||||
}
|
||||
|
||||
/// Handler for `eth_getBlockAccessListByBlockNumber`
|
||||
async fn block_access_list_by_block_number(
|
||||
&self,
|
||||
block_number: BlockNumberOrTag,
|
||||
) -> Result<Option<Value>> {
|
||||
self.eth.block_access_list_by_block_number(block_number).instrument(engine_span!()).await
|
||||
}
|
||||
|
||||
/// Handler for `getBlockAccessListRaw`
|
||||
async fn block_access_list_raw(&self, block: BlockId) -> Result<Option<Bytes>> {
|
||||
self.eth.block_access_list_raw(block).instrument(engine_span!()).await
|
||||
}
|
||||
}
|
||||
|
||||
@@ -38,6 +38,13 @@ use std::{
|
||||
};
|
||||
use tracing::instrument;
|
||||
|
||||
/// Returns [`WriteOptions`] with WAL sync enabled for crash durability.
|
||||
fn synced_write_options() -> WriteOptions {
|
||||
let mut opts = WriteOptions::default();
|
||||
opts.set_sync(true);
|
||||
opts
|
||||
}
|
||||
|
||||
/// Pending `RocksDB` batches type alias.
|
||||
pub(crate) type PendingRocksDBBatches = Arc<Mutex<Vec<WriteBatchWithTransaction<true>>>>;
|
||||
|
||||
@@ -765,7 +772,7 @@ impl RocksDBProvider {
|
||||
/// # Panics
|
||||
/// Panics if the provider is in read-only mode.
|
||||
pub fn tx(&self) -> RocksTx<'_> {
|
||||
let write_options = WriteOptions::default();
|
||||
let write_options = synced_write_options();
|
||||
let txn_options = OptimisticTransactionOptions::default();
|
||||
let inner = self.0.db_rw().transaction_opt(&write_options, &txn_options);
|
||||
RocksTx { inner, provider: self }
|
||||
@@ -845,6 +852,19 @@ impl RocksDBProvider {
|
||||
})
|
||||
}
|
||||
|
||||
/// Gets raw bytes from the specified table without decompressing.
|
||||
pub fn get_raw<T: Table>(&self, key: T::Key) -> ProviderResult<Option<Vec<u8>>> {
|
||||
let encoded = key.encode();
|
||||
self.execute_with_operation_metric(RocksDBOperation::Get, T::NAME, |this| {
|
||||
this.0.get_cf(this.get_cf_handle::<T>()?, encoded.as_ref()).map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Read(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))
|
||||
})
|
||||
})
|
||||
}
|
||||
|
||||
/// Puts upsert a value into the specified table with the given key.
|
||||
///
|
||||
/// # Panics
|
||||
@@ -963,6 +983,18 @@ impl RocksDBProvider {
|
||||
Ok(RocksDBIter { inner: iter, _marker: std::marker::PhantomData })
|
||||
}
|
||||
|
||||
/// Creates an iterator starting from the given key (inclusive, seek forward).
|
||||
///
|
||||
/// Returns decoded `(Key, Value)` pairs starting from the first key >= `key`.
|
||||
pub fn iter_from<T: Table>(&self, key: T::Key) -> ProviderResult<RocksDBIter<'_, T>> {
|
||||
let cf = self.get_cf_handle::<T>()?;
|
||||
let encoded_key = key.encode();
|
||||
let iter = self
|
||||
.0
|
||||
.iterator_cf(cf, IteratorMode::From(encoded_key.as_ref(), rocksdb::Direction::Forward));
|
||||
Ok(RocksDBIter { inner: iter, _marker: std::marker::PhantomData })
|
||||
}
|
||||
|
||||
/// Returns statistics for all column families in the database.
|
||||
///
|
||||
/// Returns a vector of (`table_name`, `estimated_keys`, `estimated_size_bytes`) tuples.
|
||||
@@ -1243,7 +1275,7 @@ impl RocksDBProvider {
|
||||
/// Panics if the provider is in read-only mode.
|
||||
#[instrument(level = "debug", target = "providers::rocksdb", skip_all, fields(batch_len = batch.len(), batch_size = batch.size_in_bytes()))]
|
||||
pub fn commit_batch(&self, batch: WriteBatchWithTransaction<true>) -> ProviderResult<()> {
|
||||
self.0.db_rw().write_opt(batch, &WriteOptions::default()).map_err(|e| {
|
||||
self.0.db_rw().write_opt(batch, &synced_write_options()).map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
@@ -1726,14 +1758,12 @@ impl<'a> RocksDBBatch<'a> {
|
||||
"Auto-committing RocksDB batch"
|
||||
);
|
||||
let old_batch = std::mem::take(&mut self.inner);
|
||||
self.provider.0.db_rw().write_opt(old_batch, &WriteOptions::default()).map_err(
|
||||
|e| {
|
||||
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))
|
||||
},
|
||||
)?;
|
||||
self.provider.0.db_rw().write_opt(old_batch, &synced_write_options()).map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
}))
|
||||
})?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
@@ -1746,7 +1776,7 @@ impl<'a> RocksDBBatch<'a> {
|
||||
/// Panics if the provider is in read-only mode.
|
||||
#[instrument(level = "debug", target = "providers::rocksdb", skip_all, fields(batch_len = self.inner.len(), batch_size = self.inner.size_in_bytes()))]
|
||||
pub fn commit(self) -> ProviderResult<()> {
|
||||
self.provider.0.db_rw().write_opt(self.inner, &WriteOptions::default()).map_err(|e| {
|
||||
self.provider.0.db_rw().write_opt(self.inner, &synced_write_options()).map_err(|e| {
|
||||
ProviderError::Database(DatabaseError::Commit(DatabaseErrorInfo {
|
||||
message: e.to_string().into(),
|
||||
code: -1,
|
||||
|
||||
@@ -408,8 +408,7 @@ where
|
||||
/// cached value then this calculates the [`Overlay`] and populates the cache.
|
||||
#[instrument(level = "debug", target = "providers::state::overlay", skip_all)]
|
||||
fn get_overlay(&self, provider: &F::Provider) -> ProviderResult<Overlay> {
|
||||
// If we have no anchor block configured then we will never need to get trie reverts, just
|
||||
// return the in-memory overlay (resolving lazy overlay if set).
|
||||
// No anchor block — just resolve the in-memory overlay directly.
|
||||
if self.block_hash.is_none() {
|
||||
let (trie_updates, hashed_post_state) = self.resolve_overlays();
|
||||
return Ok(Overlay { trie_updates, hashed_post_state })
|
||||
@@ -417,28 +416,16 @@ where
|
||||
|
||||
let db_tip_block = self.get_db_tip_block_number(provider)?;
|
||||
|
||||
// If the overlay is present in the cache then return it directly.
|
||||
if let Some(entry) = self.overlay_cache.get(&db_tip_block) {
|
||||
return Ok(entry.value().clone());
|
||||
}
|
||||
|
||||
// If the overlay is not present then we need to calculate a new one.
|
||||
// DashMap's entry API handles the race condition internally.
|
||||
let mut cache_miss = false;
|
||||
let overlay = match self.overlay_cache.entry(db_tip_block) {
|
||||
dashmap::Entry::Occupied(entry) => entry.get().clone(),
|
||||
dashmap::Entry::Vacant(entry) => {
|
||||
cache_miss = true;
|
||||
self.metrics.overlay_cache_misses.increment(1);
|
||||
let overlay = self.calculate_overlay(provider, db_tip_block)?;
|
||||
entry.insert(overlay.clone());
|
||||
overlay
|
||||
}
|
||||
};
|
||||
|
||||
if cache_miss {
|
||||
self.metrics.overlay_cache_misses.increment(1);
|
||||
}
|
||||
|
||||
Ok(overlay)
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,7 +8,7 @@ use std::{
|
||||
pin::Pin,
|
||||
sync::{
|
||||
atomic::{AtomicUsize, Ordering},
|
||||
Arc,
|
||||
Arc, OnceLock,
|
||||
},
|
||||
task::{ready, Context, Poll},
|
||||
thread,
|
||||
@@ -168,34 +168,50 @@ thread_local! {
|
||||
///
|
||||
/// The pool supports multiple init/clear cycles, allowing reuse of the same threads with
|
||||
/// different state configurations.
|
||||
///
|
||||
/// The underlying rayon pool is created lazily on first access.
|
||||
#[derive(Debug)]
|
||||
pub struct WorkerPool {
|
||||
pool: rayon::ThreadPool,
|
||||
pool: OnceLock<rayon::ThreadPool>,
|
||||
num_threads: usize,
|
||||
thread_name_prefix: &'static str,
|
||||
}
|
||||
|
||||
impl WorkerPool {
|
||||
/// Creates a new `WorkerPool` with the given number of threads.
|
||||
pub fn new(num_threads: usize) -> Result<Self, rayon::ThreadPoolBuildError> {
|
||||
Self::from_builder(rayon::ThreadPoolBuilder::new().num_threads(num_threads))
|
||||
/// Creates a new lazy `WorkerPool` with the given number of threads and a thread name prefix.
|
||||
///
|
||||
/// The underlying rayon pool is not created until the first method that requires it is called.
|
||||
/// Thread names follow the pattern `"{prefix}-{index:02}"`.
|
||||
pub const fn new(num_threads: usize, thread_name_prefix: &'static str) -> Self {
|
||||
Self { pool: OnceLock::new(), num_threads, thread_name_prefix }
|
||||
}
|
||||
|
||||
/// Creates a new `WorkerPool` from a [`rayon::ThreadPoolBuilder`].
|
||||
///
|
||||
/// Installs a panic handler that logs panics instead of aborting the process.
|
||||
pub fn from_builder(
|
||||
builder: rayon::ThreadPoolBuilder,
|
||||
) -> Result<Self, rayon::ThreadPoolBuildError> {
|
||||
Ok(Self { pool: build_pool_with_panic_handler(builder)? })
|
||||
/// Returns a reference to the underlying rayon pool, creating it on first access.
|
||||
fn pool(&self) -> &rayon::ThreadPool {
|
||||
self.pool.get_or_init(|| {
|
||||
let prefix = self.thread_name_prefix;
|
||||
build_pool_with_panic_handler(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(self.num_threads)
|
||||
.thread_name(move |i| format!("{prefix}-{i:02}")),
|
||||
)
|
||||
.unwrap_or_else(|err| panic!("failed to build {prefix} worker pool: {err}"))
|
||||
})
|
||||
}
|
||||
|
||||
/// Returns `true` if the underlying rayon pool has been initialized.
|
||||
pub fn is_initialized(&self) -> bool {
|
||||
self.pool.get().is_some()
|
||||
}
|
||||
|
||||
/// Returns the total number of threads in the underlying rayon pool.
|
||||
pub fn current_num_threads(&self) -> usize {
|
||||
self.pool.current_num_threads()
|
||||
self.pool().current_num_threads()
|
||||
}
|
||||
|
||||
/// Initializes per-thread [`Worker`] state on every thread in the pool.
|
||||
pub fn init<T: 'static>(&self, f: impl Fn(Option<&mut T>) -> T + Sync) {
|
||||
self.broadcast(self.pool.current_num_threads(), |worker| {
|
||||
self.broadcast(self.pool().current_num_threads(), |worker| {
|
||||
worker.init::<T>(&f);
|
||||
});
|
||||
}
|
||||
@@ -206,14 +222,14 @@ impl WorkerPool {
|
||||
/// Use this to initialize or re-initialize per-thread state via [`Worker::init`].
|
||||
/// Only `num_threads` threads execute the closure; the rest skip it.
|
||||
pub fn broadcast(&self, num_threads: usize, f: impl Fn(&mut Worker) + Sync) {
|
||||
if num_threads >= self.pool.current_num_threads() {
|
||||
if num_threads >= self.pool().current_num_threads() {
|
||||
// Fast path: run on every thread, no atomic coordination needed.
|
||||
self.pool.broadcast(|_| {
|
||||
self.pool().broadcast(|_| {
|
||||
WORKER.with_borrow_mut(|worker| f(worker));
|
||||
});
|
||||
} else {
|
||||
let remaining = AtomicUsize::new(num_threads);
|
||||
self.pool.broadcast(|_| {
|
||||
self.pool().broadcast(|_| {
|
||||
// Atomically claim a slot; threads that can't decrement skip the closure.
|
||||
let mut current = remaining.load(Ordering::Relaxed);
|
||||
loop {
|
||||
@@ -237,7 +253,7 @@ impl WorkerPool {
|
||||
|
||||
/// Clears the state on every thread in the pool.
|
||||
pub fn clear(&self) {
|
||||
self.pool.broadcast(|_| {
|
||||
self.pool().broadcast(|_| {
|
||||
WORKER.with_borrow_mut(Worker::clear);
|
||||
});
|
||||
}
|
||||
@@ -248,7 +264,7 @@ impl WorkerPool {
|
||||
/// Each thread can access its own [`Worker`] via the provided reference or through additional
|
||||
/// [`WorkerPool::with_worker`] calls.
|
||||
pub fn install<R: Send>(&self, f: impl FnOnce(&Worker) -> R + Send) -> R {
|
||||
self.pool.install(|| WORKER.with_borrow(|worker| f(worker)))
|
||||
self.pool().install(|| WORKER.with_borrow(|worker| f(worker)))
|
||||
}
|
||||
|
||||
/// Runs a closure on the pool without worker state access.
|
||||
@@ -256,19 +272,19 @@ impl WorkerPool {
|
||||
/// Like [`install`](Self::install) but for closures that don't need per-thread [`Worker`]
|
||||
/// state.
|
||||
pub fn install_fn<R: Send>(&self, f: impl FnOnce() -> R + Send) -> R {
|
||||
self.pool.install(f)
|
||||
self.pool().install(f)
|
||||
}
|
||||
|
||||
/// Spawns a closure on the pool.
|
||||
pub fn spawn(&self, f: impl FnOnce() + Send + 'static) {
|
||||
self.pool.spawn(f);
|
||||
self.pool().spawn(f);
|
||||
}
|
||||
|
||||
/// Executes `f` on this pool using [`rayon::in_place_scope`], which converts the calling
|
||||
/// thread into a worker for the duration — tasks spawned inside the scope run on the pool
|
||||
/// and the call blocks until all of them complete.
|
||||
pub fn in_place_scope<'scope, R>(&self, f: impl FnOnce(&rayon::Scope<'scope>) -> R) -> R {
|
||||
self.pool.in_place_scope(f)
|
||||
self.pool().in_place_scope(f)
|
||||
}
|
||||
|
||||
/// Access the current thread's [`Worker`] from within an [`install`](Self::install) closure.
|
||||
@@ -398,7 +414,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn worker_pool_init_and_access() {
|
||||
let pool = WorkerPool::new(2).unwrap();
|
||||
let pool = WorkerPool::new(2, "test");
|
||||
|
||||
pool.broadcast(2, |worker| {
|
||||
worker.init::<Vec<u8>>(|_| vec![1, 2, 3]);
|
||||
@@ -415,7 +431,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn worker_pool_reinit_reuses_resources() {
|
||||
let pool = WorkerPool::new(1).unwrap();
|
||||
let pool = WorkerPool::new(1, "test");
|
||||
|
||||
pool.broadcast(1, |worker| {
|
||||
worker.init::<Vec<u8>>(|existing| {
|
||||
@@ -441,7 +457,7 @@ mod tests {
|
||||
|
||||
#[test]
|
||||
fn worker_pool_clear_and_reinit() {
|
||||
let pool = WorkerPool::new(1).unwrap();
|
||||
let pool = WorkerPool::new(1, "test");
|
||||
|
||||
pool.broadcast(1, |worker| {
|
||||
worker.init::<u64>(|_| 42);
|
||||
@@ -464,7 +480,7 @@ mod tests {
|
||||
fn worker_pool_par_iter_with_worker() {
|
||||
use rayon::prelude::*;
|
||||
|
||||
let pool = WorkerPool::new(2).unwrap();
|
||||
let pool = WorkerPool::new(2, "test");
|
||||
|
||||
pool.broadcast(2, |worker| {
|
||||
worker.init::<u64>(|_| 10);
|
||||
|
||||
@@ -16,8 +16,6 @@ use crate::{
|
||||
};
|
||||
use futures_util::{future::select, Future, FutureExt, TryFutureExt};
|
||||
#[cfg(feature = "rayon")]
|
||||
use std::sync::OnceLock;
|
||||
#[cfg(feature = "rayon")]
|
||||
use std::{num::NonZeroUsize, thread::available_parallelism};
|
||||
use std::{
|
||||
pin::pin,
|
||||
@@ -237,34 +235,6 @@ pub enum RuntimeBuildError {
|
||||
RayonBuild(#[from] rayon::ThreadPoolBuildError),
|
||||
}
|
||||
|
||||
#[cfg(feature = "rayon")]
|
||||
#[derive(Debug)]
|
||||
struct LazyWorkerPool {
|
||||
pool: OnceLock<WorkerPool>,
|
||||
num_threads: usize,
|
||||
thread_name_prefix: &'static str,
|
||||
}
|
||||
|
||||
#[cfg(feature = "rayon")]
|
||||
impl LazyWorkerPool {
|
||||
const fn new(num_threads: usize, thread_name_prefix: &'static str) -> Self {
|
||||
Self { pool: OnceLock::new(), num_threads, thread_name_prefix }
|
||||
}
|
||||
|
||||
fn get(&self) -> &WorkerPool {
|
||||
let num_threads = self.num_threads;
|
||||
let thread_name_prefix = self.thread_name_prefix;
|
||||
self.pool.get_or_init(|| {
|
||||
WorkerPool::from_builder(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(num_threads)
|
||||
.thread_name(move |i| format!("{thread_name_prefix}-{i:02}")),
|
||||
)
|
||||
.unwrap_or_else(|err| panic!("failed to build {thread_name_prefix} worker pool: {err}"))
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// ── RuntimeInner ──────────────────────────────────────────────────────
|
||||
|
||||
struct RuntimeInner {
|
||||
@@ -303,7 +273,7 @@ struct RuntimeInner {
|
||||
prewarming_pool: WorkerPool,
|
||||
/// BAL streaming pool (BAL hashed state streaming).
|
||||
#[cfg(feature = "rayon")]
|
||||
bal_streaming_pool: LazyWorkerPool,
|
||||
bal_streaming_pool: WorkerPool,
|
||||
/// Named single-thread worker map. Each unique name gets a dedicated OS thread
|
||||
/// that is reused across all tasks submitted under that name.
|
||||
worker_map: WorkerMap,
|
||||
@@ -392,7 +362,7 @@ impl Runtime {
|
||||
/// Get the BAL streaming pool.
|
||||
#[cfg(feature = "rayon")]
|
||||
pub fn bal_streaming_pool(&self) -> &WorkerPool {
|
||||
self.0.bal_streaming_pool.get()
|
||||
&self.0.bal_streaming_pool
|
||||
}
|
||||
}
|
||||
|
||||
@@ -837,30 +807,20 @@ impl RuntimeBuilder {
|
||||
|
||||
let proof_storage_worker_threads =
|
||||
config.rayon.proof_storage_worker_threads.unwrap_or(default_threads * 2);
|
||||
let proof_storage_worker_pool = WorkerPool::from_builder(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(proof_storage_worker_threads)
|
||||
.thread_name(|i| format!("proof-strg-{i:02}")),
|
||||
)?;
|
||||
let proof_storage_worker_pool =
|
||||
WorkerPool::new(proof_storage_worker_threads, "proof-strg");
|
||||
|
||||
let proof_account_worker_threads =
|
||||
config.rayon.proof_account_worker_threads.unwrap_or(default_threads * 2);
|
||||
let proof_account_worker_pool = WorkerPool::from_builder(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(proof_account_worker_threads)
|
||||
.thread_name(|i| format!("proof-acct-{i:02}")),
|
||||
)?;
|
||||
let proof_account_worker_pool =
|
||||
WorkerPool::new(proof_account_worker_threads, "proof-acct");
|
||||
|
||||
let prewarming_threads = config.rayon.prewarming_threads.unwrap_or(default_threads);
|
||||
let prewarming_pool = WorkerPool::from_builder(
|
||||
rayon::ThreadPoolBuilder::new()
|
||||
.num_threads(prewarming_threads)
|
||||
.thread_name(|i| format!("prewarm-{i:02}")),
|
||||
)?;
|
||||
let prewarming_pool = WorkerPool::new(prewarming_threads, "prewarm");
|
||||
|
||||
let bal_streaming_threads =
|
||||
config.rayon.bal_streaming_threads.unwrap_or(default_threads);
|
||||
let bal_streaming_pool = LazyWorkerPool::new(bal_streaming_threads, "bal-stream");
|
||||
let bal_streaming_pool = WorkerPool::new(bal_streaming_threads, "bal-stream");
|
||||
|
||||
debug!(
|
||||
default_threads,
|
||||
@@ -871,7 +831,7 @@ impl RuntimeBuilder {
|
||||
prewarming_threads,
|
||||
bal_streaming_threads,
|
||||
max_blocking_tasks = config.rayon.max_blocking_tasks,
|
||||
"Initialized rayon thread pools and configured lazy BAL streaming pool"
|
||||
"Configured lazy rayon worker pools"
|
||||
);
|
||||
|
||||
(
|
||||
@@ -962,12 +922,19 @@ mod tests {
|
||||
|
||||
#[cfg(feature = "rayon")]
|
||||
#[test]
|
||||
fn test_bal_streaming_pool_is_lazy() {
|
||||
fn test_worker_pools_are_lazy() {
|
||||
let runtime = Runtime::test();
|
||||
|
||||
assert!(runtime.0.bal_streaming_pool.pool.get().is_none());
|
||||
// Worker pools are lazy — not initialized until first access.
|
||||
assert!(!runtime.0.bal_streaming_pool.is_initialized());
|
||||
assert!(!runtime.0.proof_storage_worker_pool.is_initialized());
|
||||
|
||||
// Accessing them triggers initialization and returns the configured thread count.
|
||||
assert_eq!(runtime.bal_streaming_pool().current_num_threads(), 2);
|
||||
assert!(runtime.0.bal_streaming_pool.pool.get().is_some());
|
||||
assert!(runtime.0.bal_streaming_pool.is_initialized());
|
||||
|
||||
assert_eq!(runtime.proof_storage_worker_pool().current_num_threads(), 2);
|
||||
assert_eq!(runtime.proof_account_worker_pool().current_num_threads(), 2);
|
||||
assert_eq!(runtime.prewarming_pool().current_num_threads(), 2);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -55,6 +55,7 @@ serde = { workspace = true, features = ["derive", "rc"] }
|
||||
serde_json.workspace = true
|
||||
bitflags.workspace = true
|
||||
auto_impl.workspace = true
|
||||
imbl.workspace = true
|
||||
smallvec.workspace = true
|
||||
|
||||
# testing
|
||||
@@ -84,6 +85,7 @@ serde = [
|
||||
"alloy-eips/serde",
|
||||
"alloy-primitives/serde",
|
||||
"bitflags/serde",
|
||||
"imbl/serde",
|
||||
"parking_lot/serde",
|
||||
"rand?/serde",
|
||||
"smallvec/serde",
|
||||
@@ -112,6 +114,7 @@ test-utils = [
|
||||
arbitrary = [
|
||||
"proptest",
|
||||
"proptest-arbitrary-interop",
|
||||
"imbl/arbitrary",
|
||||
"reth-chainspec/arbitrary",
|
||||
"reth-eth-wire-types/arbitrary",
|
||||
"alloy-consensus/arbitrary",
|
||||
@@ -124,4 +127,5 @@ arbitrary = [
|
||||
"reth-ethereum-primitives/arbitrary",
|
||||
"revm-primitives/arbitrary",
|
||||
"revm/arbitrary",
|
||||
"imbl/arbitrary",
|
||||
]
|
||||
|
||||
@@ -276,6 +276,8 @@
|
||||
#![cfg_attr(docsrs, feature(doc_cfg))]
|
||||
#![cfg_attr(not(test), warn(unused_crate_dependencies))]
|
||||
|
||||
pub use imbl::OrdMap;
|
||||
|
||||
pub use crate::{
|
||||
batcher::{BatchTxProcessor, BatchTxRequest},
|
||||
blobstore::{BlobStore, BlobStoreError},
|
||||
|
||||
@@ -7,9 +7,10 @@ use crate::{
|
||||
use alloy_consensus::Transaction;
|
||||
use alloy_primitives::map::AddressSet;
|
||||
use core::fmt;
|
||||
use imbl::OrdMap;
|
||||
use reth_primitives_traits::transaction::error::InvalidTransactionError;
|
||||
use std::{
|
||||
collections::{BTreeMap, BTreeSet, HashSet, VecDeque},
|
||||
collections::{BTreeSet, HashSet, VecDeque},
|
||||
sync::Arc,
|
||||
};
|
||||
use tokio::sync::broadcast::{error::TryRecvError, Receiver};
|
||||
@@ -84,7 +85,7 @@ impl<T: TransactionOrdering> Iterator for BestTransactionsWithFees<T> {
|
||||
pub struct BestTransactions<T: TransactionOrdering> {
|
||||
/// Contains a copy of _all_ transactions of the pending pool at the point in time this
|
||||
/// iterator was created.
|
||||
pub(crate) all: BTreeMap<TransactionId, PendingTransaction<T>>,
|
||||
pub(crate) all: OrdMap<TransactionId, PendingTransaction<T>>,
|
||||
/// Transactions that can be executed right away: these have the expected nonce.
|
||||
///
|
||||
/// Once an `independent` transaction with the nonce `N` is returned, it unlocks `N+1`, which
|
||||
|
||||
@@ -8,13 +8,9 @@ use crate::{
|
||||
},
|
||||
Priority, SubPoolLimit, TransactionOrdering, ValidPoolTransaction,
|
||||
};
|
||||
use imbl::OrdMap;
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use std::{
|
||||
cmp::Ordering,
|
||||
collections::{hash_map::Entry, BTreeMap},
|
||||
ops::Bound::Unbounded,
|
||||
sync::Arc,
|
||||
};
|
||||
use std::{cmp::Ordering, collections::hash_map::Entry, ops::Bound::Unbounded, sync::Arc};
|
||||
use tokio::sync::broadcast;
|
||||
|
||||
/// A pool of validated and gapless transactions that are ready to be executed on the current state
|
||||
@@ -36,7 +32,7 @@ pub struct PendingPool<T: TransactionOrdering> {
|
||||
/// This way we can determine when transactions were submitted to the pool.
|
||||
submission_id: u64,
|
||||
/// _All_ Transactions that are currently inside the pool grouped by their identifier.
|
||||
by_id: BTreeMap<TransactionId, PendingTransaction<T>>,
|
||||
by_id: OrdMap<TransactionId, PendingTransaction<T>>,
|
||||
/// The highest nonce transactions for each sender - like the `independent` set, but the
|
||||
/// highest instead of lowest nonce.
|
||||
highest_nonces: FxHashMap<SenderId, PendingTransaction<T>>,
|
||||
@@ -80,7 +76,7 @@ impl<T: TransactionOrdering> PendingPool<T> {
|
||||
/// # Returns
|
||||
///
|
||||
/// Returns all transactions by id.
|
||||
fn clear_transactions(&mut self) -> BTreeMap<TransactionId, PendingTransaction<T>> {
|
||||
fn clear_transactions(&mut self) -> OrdMap<TransactionId, PendingTransaction<T>> {
|
||||
self.independent_transactions.clear();
|
||||
self.highest_nonces.clear();
|
||||
self.size_of.reset();
|
||||
@@ -525,7 +521,7 @@ impl<T: TransactionOrdering> PendingPool<T> {
|
||||
}
|
||||
|
||||
/// All transactions grouped by id
|
||||
pub const fn by_id(&self) -> &BTreeMap<TransactionId, PendingTransaction<T>> {
|
||||
pub const fn by_id(&self) -> &OrdMap<TransactionId, PendingTransaction<T>> {
|
||||
&self.by_id
|
||||
}
|
||||
|
||||
|
||||
@@ -1801,13 +1801,9 @@ impl<Tx: PoolTransaction> NewSubpoolTransactionStream<Tx> {
|
||||
&mut self,
|
||||
) -> Result<NewTransactionEvent<Tx>, tokio::sync::mpsc::error::TryRecvError> {
|
||||
loop {
|
||||
match self.st.try_recv() {
|
||||
Ok(event) => {
|
||||
if event.subpool == self.subpool {
|
||||
return Ok(event)
|
||||
}
|
||||
}
|
||||
Err(e) => return Err(e),
|
||||
let event = self.st.try_recv()?;
|
||||
if event.subpool == self.subpool {
|
||||
return Ok(event)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -876,9 +876,17 @@ where
|
||||
self.fork_tracker
|
||||
.max_initcode_size
|
||||
.store(evm_env.cfg_env.max_initcode_size(), std::sync::atomic::Ordering::Relaxed);
|
||||
// EIP-8037: When state gas is enabled, `tx.gas` can exceed the per-tx gas limit cap
|
||||
// because the cap only applies to regular gas (state gas uses a reservoir).
|
||||
// Store 0 to disable the txpool-level check.
|
||||
let tx_gas_limit_cap = if evm_env.cfg_env.is_amsterdam_eip8037_enabled() {
|
||||
0
|
||||
} else {
|
||||
evm_env.cfg_env.tx_gas_limit_cap()
|
||||
};
|
||||
self.fork_tracker
|
||||
.tx_gas_limit_cap
|
||||
.store(evm_env.cfg_env.tx_gas_limit_cap(), std::sync::atomic::Ordering::Relaxed);
|
||||
.store(tx_gas_limit_cap, std::sync::atomic::Ordering::Relaxed);
|
||||
}
|
||||
|
||||
fn max_gas_limit(&self) -> u64 {
|
||||
@@ -1060,7 +1068,12 @@ impl<Client, Evm> EthTransactionValidatorBuilder<Client, Evm> {
|
||||
// no custom transaction types by default
|
||||
other_tx_types: U256::ZERO,
|
||||
|
||||
tx_gas_limit_cap: evm_env.cfg_env.tx_gas_limit_cap(),
|
||||
// EIP-8037: When state gas is enabled, tx.gas can exceed the per-tx cap
|
||||
tx_gas_limit_cap: if evm_env.cfg_env.is_amsterdam_eip8037_enabled() {
|
||||
0
|
||||
} else {
|
||||
evm_env.cfg_env.tx_gas_limit_cap()
|
||||
},
|
||||
max_initcode_size: evm_env.cfg_env.max_initcode_size(),
|
||||
|
||||
// EIP-7594 sidecars are accepted by default (standard Ethereum behavior)
|
||||
|
||||
@@ -61,6 +61,7 @@ allow = [
|
||||
# https://github.com/rustls/webpki/blob/main/LICENSE ISC Style
|
||||
"LicenseRef-rustls-webpki",
|
||||
"CDLA-Permissive-2.0",
|
||||
"MPL-2.0",
|
||||
]
|
||||
|
||||
# Allow 1 or more licenses on a per-crate basis, so that particular licenses
|
||||
|
||||
@@ -18,6 +18,7 @@
|
||||
- [`reth db get`](./reth/db/get.mdx)
|
||||
- [`reth db get mdbx`](./reth/db/get/mdbx.mdx)
|
||||
- [`reth db get static-file`](./reth/db/get/static-file.mdx)
|
||||
- [`reth db get rocksdb`](./reth/db/get/rocksdb.mdx)
|
||||
- [`reth db drop`](./reth/db/drop.mdx)
|
||||
- [`reth db clear`](./reth/db/clear.mdx)
|
||||
- [`reth db clear mdbx`](./reth/db/clear/mdbx.mdx)
|
||||
@@ -40,6 +41,7 @@
|
||||
- [`reth db stage-checkpoints set`](./reth/db/stage-checkpoints/set.mdx)
|
||||
- [`reth db account-storage`](./reth/db/account-storage.mdx)
|
||||
- [`reth db state`](./reth/db/state.mdx)
|
||||
- [`reth db migrate-v2`](./reth/db/migrate-v2.mdx)
|
||||
- [`reth download`](./reth/download.mdx)
|
||||
- [`reth snapshot-manifest`](./reth/snapshot-manifest.mdx)
|
||||
- [`reth stage`](./reth/stage.mdx)
|
||||
|
||||
@@ -26,6 +26,7 @@ Commands:
|
||||
stage-checkpoints `reth db stage-checkpoints` subcommand
|
||||
account-storage Gets storage size information for an account
|
||||
state Gets account state and storage at a specific block
|
||||
migrate-v2 Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
|
||||
@@ -11,6 +11,7 @@ Usage: reth db get [OPTIONS] <COMMAND>
|
||||
Commands:
|
||||
mdbx Gets the content of a database table for the given key
|
||||
static-file Gets the content of a static file segment for the given key
|
||||
rocksdb Gets the content of a RocksDB table for the given key
|
||||
help Print this message or the help of the given subcommand(s)
|
||||
|
||||
Options:
|
||||
|
||||
190
docs/vocs/docs/pages/cli/reth/db/get/rocksdb.mdx
Normal file
190
docs/vocs/docs/pages/cli/reth/db/get/rocksdb.mdx
Normal file
@@ -0,0 +1,190 @@
|
||||
# reth db get rocksdb
|
||||
|
||||
Gets the content of a RocksDB table for the given key
|
||||
|
||||
```bash
|
||||
$ reth db get rocksdb --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth db get rocksdb [OPTIONS] <TABLE> <KEY>
|
||||
|
||||
Arguments:
|
||||
<TABLE>
|
||||
The RocksDB table
|
||||
|
||||
Possible values:
|
||||
- transaction-hash-numbers: Transaction hash to transaction number mapping
|
||||
- accounts-history: Account history indices
|
||||
- storages-history: Storage history indices
|
||||
|
||||
<KEY>
|
||||
The key to get content for. For history tables, this can be a plain address
|
||||
|
||||
Options:
|
||||
--block <BLOCK>
|
||||
Target block number for history tables. Seeks to the shard containing this block. Defaults to the latest shard if not specified
|
||||
|
||||
--storage-key <STORAGE_KEY>
|
||||
Storage key for storages-history table lookups
|
||||
|
||||
--all-shards
|
||||
List all shards for the given key (history tables only)
|
||||
|
||||
--raw
|
||||
Output bytes instead of human-readable decoded value
|
||||
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
mainnet, sepolia, holesky, hoodi, dev
|
||||
|
||||
[default: mainnet]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ""]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled.
|
||||
|
||||
Default: 5 for `node` command, 0 for non-node utility subcommands.
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
--logs-otlp[=<URL>]
|
||||
Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
|
||||
|
||||
--logs-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --logs-otlp.filter=info,reth=debug
|
||||
|
||||
Defaults to INFO if not specified.
|
||||
|
||||
[default: info]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
|
||||
Tracing:
|
||||
--tracing-otlp[=<URL>]
|
||||
Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --tracing-otlp=http://collector:4318/v1/traces
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
|
||||
|
||||
--tracing-otlp-protocol <PROTOCOL>
|
||||
OTLP transport protocol to use for exporting traces and logs.
|
||||
|
||||
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
|
||||
|
||||
Defaults to HTTP if not specified.
|
||||
|
||||
Possible values:
|
||||
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
|
||||
- grpc: gRPC transport, port 4317
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
|
||||
[default: http]
|
||||
|
||||
--tracing-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
|
||||
|
||||
Defaults to TRACE if not specified.
|
||||
|
||||
[default: debug]
|
||||
|
||||
--tracing-otlp.sample-ratio <RATIO>
|
||||
Trace sampling ratio to control the percentage of traces to export.
|
||||
|
||||
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
|
||||
|
||||
Example: --tracing-otlp.sample-ratio=0.0.
|
||||
|
||||
[env: OTEL_TRACES_SAMPLER_ARG=]
|
||||
```
|
||||
166
docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx
Normal file
166
docs/vocs/docs/pages/cli/reth/db/migrate-v2.mdx
Normal file
@@ -0,0 +1,166 @@
|
||||
# reth db migrate-v2
|
||||
|
||||
Migrate storage layout from v1 (MDBX-only) to v2 (static files + RocksDB)
|
||||
|
||||
```bash
|
||||
$ reth db migrate-v2 --help
|
||||
```
|
||||
```txt
|
||||
Usage: reth db migrate-v2 [OPTIONS]
|
||||
|
||||
Options:
|
||||
-h, --help
|
||||
Print help (see a summary with '-h')
|
||||
|
||||
Datadir:
|
||||
--chain <CHAIN_OR_PATH>
|
||||
The chain this node is running.
|
||||
Possible values are either a built-in chain or the path to a chain specification file.
|
||||
|
||||
Built-in chains:
|
||||
mainnet, sepolia, holesky, hoodi, dev
|
||||
|
||||
[default: mainnet]
|
||||
|
||||
Logging:
|
||||
--log.stdout.format <FORMAT>
|
||||
The format to use for logs written to stdout
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.stdout.filter <FILTER>
|
||||
The filter to use for logs written to stdout
|
||||
|
||||
[default: ""]
|
||||
|
||||
--log.file.format <FORMAT>
|
||||
The format to use for logs written to the log file
|
||||
|
||||
Possible values:
|
||||
- json: Represents JSON formatting for logs. This format outputs log records as JSON objects, making it suitable for structured logging
|
||||
- log-fmt: Represents logfmt (key=value) formatting for logs. This format is concise and human-readable, typically used in command-line applications
|
||||
- terminal: Represents terminal-friendly formatting for logs
|
||||
|
||||
[default: terminal]
|
||||
|
||||
--log.file.filter <FILTER>
|
||||
The filter to use for logs written to the log file
|
||||
|
||||
[default: debug]
|
||||
|
||||
--log.file.directory <PATH>
|
||||
The path to put log files in
|
||||
|
||||
[default: <CACHE_DIR>/logs]
|
||||
|
||||
--log.file.name <NAME>
|
||||
The prefix name of the log files
|
||||
|
||||
[default: reth.log]
|
||||
|
||||
--log.file.max-size <SIZE>
|
||||
The maximum size (in MB) of one log file
|
||||
|
||||
[default: 200]
|
||||
|
||||
--log.file.max-files <COUNT>
|
||||
The maximum amount of log files that will be stored. If set to 0, background file logging is disabled.
|
||||
|
||||
Default: 5 for `node` command, 0 for non-node utility subcommands.
|
||||
|
||||
--log.journald
|
||||
Write logs to journald
|
||||
|
||||
--log.journald.filter <FILTER>
|
||||
The filter to use for logs written to journald
|
||||
|
||||
[default: error]
|
||||
|
||||
--color <COLOR>
|
||||
Sets whether or not the formatter emits ANSI terminal escape codes for colors and other text formatting
|
||||
|
||||
Possible values:
|
||||
- always: Colors on
|
||||
- auto: Auto-detect
|
||||
- never: Colors off
|
||||
|
||||
[default: always]
|
||||
|
||||
--logs-otlp[=<URL>]
|
||||
Enable `Opentelemetry` logs export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/logs` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --logs-otlp=http://collector:4318/v1/logs
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_LOGS_ENDPOINT=]
|
||||
|
||||
--logs-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP logs exporter. This controls the verbosity of logs sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --logs-otlp.filter=info,reth=debug
|
||||
|
||||
Defaults to INFO if not specified.
|
||||
|
||||
[default: info]
|
||||
|
||||
Display:
|
||||
-v, --verbosity...
|
||||
Set the minimum log level.
|
||||
|
||||
-v Errors
|
||||
-vv Warnings
|
||||
-vvv Info
|
||||
-vvvv Debug
|
||||
-vvvvv Traces (warning: very verbose!)
|
||||
|
||||
-q, --quiet
|
||||
Silence all log output
|
||||
|
||||
Tracing:
|
||||
--tracing-otlp[=<URL>]
|
||||
Enable `Opentelemetry` tracing export to an OTLP endpoint.
|
||||
|
||||
If no value provided, defaults based on protocol: - HTTP: `http://localhost:4318/v1/traces` - gRPC: `http://localhost:4317`
|
||||
|
||||
Example: --tracing-otlp=http://collector:4318/v1/traces
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_TRACES_ENDPOINT=]
|
||||
|
||||
--tracing-otlp-protocol <PROTOCOL>
|
||||
OTLP transport protocol to use for exporting traces and logs.
|
||||
|
||||
- `http`: expects endpoint path to end with `/v1/traces` or `/v1/logs` - `grpc`: expects endpoint without a path
|
||||
|
||||
Defaults to HTTP if not specified.
|
||||
|
||||
Possible values:
|
||||
- http: HTTP/Protobuf transport, port 4318, requires `/v1/traces` path
|
||||
- grpc: gRPC transport, port 4317
|
||||
|
||||
[env: OTEL_EXPORTER_OTLP_PROTOCOL=]
|
||||
[default: http]
|
||||
|
||||
--tracing-otlp.filter <FILTER>
|
||||
Set a filter directive for the OTLP tracer. This controls the verbosity of spans and events sent to the OTLP endpoint. It follows the same syntax as the `RUST_LOG` environment variable.
|
||||
|
||||
Example: --tracing-otlp.filter=info,reth=debug,hyper_util=off
|
||||
|
||||
Defaults to TRACE if not specified.
|
||||
|
||||
[default: debug]
|
||||
|
||||
--tracing-otlp.sample-ratio <RATIO>
|
||||
Trace sampling ratio to control the percentage of traces to export.
|
||||
|
||||
Valid range: 0.0 to 1.0 - 1.0, default: Sample all traces - 0.01: Sample 1% of traces - 0.0: Disable sampling
|
||||
|
||||
Example: --tracing-otlp.sample-ratio=0.0.
|
||||
|
||||
[env: OTEL_TRACES_SAMPLER_ARG=]
|
||||
```
|
||||
@@ -130,8 +130,9 @@ Storage:
|
||||
- https://snapshots.reth.rs (default)
|
||||
- https://publicnode.com/snapshots (full nodes & testnets)
|
||||
|
||||
If no URL is provided, the latest archive snapshot for the selected chain
|
||||
will be proposed for download from https://snapshots-r2.reth.rs.
|
||||
If no URL is provided, the latest archive snapshot will only be proposed
|
||||
for Ethereum mainnet. For other chains, provide --manifest-url, --manifest-path,
|
||||
or -u explicitly.
|
||||
|
||||
Local file:// URLs are also supported for extracting snapshots from disk.
|
||||
|
||||
@@ -202,7 +203,9 @@ Storage:
|
||||
[possible values: true, false]
|
||||
|
||||
--download-concurrency <DOWNLOAD_CONCURRENCY>
|
||||
Maximum number of concurrent modular archive workers
|
||||
Maximum number of simultaneous HTTP downloads.
|
||||
|
||||
Applies across the entire snapshot download. Small files use one slot, while large files may use multiple slots by splitting into fixed-size pieces.
|
||||
|
||||
[default: 8]
|
||||
|
||||
|
||||
@@ -840,6 +840,16 @@ Dev testnet:
|
||||
Parses strings using [`humantime::parse_duration`]
|
||||
--dev.block-time 12s
|
||||
|
||||
--dev.payload-wait-time <PAYLOAD_WAIT_TIME>
|
||||
Time to wait after initiating payload building before resolving.
|
||||
|
||||
Introduces a sleep between `fork_choice_updated` and `resolve_kind` in the
|
||||
local miner, giving the payload job time for multiple rebuild attempts with
|
||||
new transactions from the pool.
|
||||
|
||||
Parses strings using [`humantime::parse_duration`]
|
||||
--dev.payload-wait-time 450ms
|
||||
|
||||
--dev.mnemonic <MNEMONIC>
|
||||
Derive dev accounts from a fixed mnemonic instead of random ones.
|
||||
|
||||
|
||||
@@ -85,6 +85,10 @@ export const rethCliSidebar: SidebarItem = {
|
||||
{
|
||||
text: "reth db get static-file",
|
||||
link: "/cli/reth/db/get/static-file"
|
||||
},
|
||||
{
|
||||
text: "reth db get rocksdb",
|
||||
link: "/cli/reth/db/get/rocksdb"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -193,6 +197,10 @@ export const rethCliSidebar: SidebarItem = {
|
||||
{
|
||||
text: "reth db state",
|
||||
link: "/cli/reth/db/state"
|
||||
},
|
||||
{
|
||||
text: "reth db migrate-v2",
|
||||
link: "/cli/reth/db/migrate-v2"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -22,7 +22,7 @@ export default defineConfig({
|
||||
},
|
||||
{ text: 'GitHub', link: 'https://github.com/paradigmxyz/reth' },
|
||||
{
|
||||
text: 'v2.0.0',
|
||||
text: 'v2.1.0',
|
||||
items: [
|
||||
{
|
||||
text: 'Releases',
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1154,7 +1154,9 @@
|
||||
],
|
||||
"refresh": "30s",
|
||||
"schemaVersion": 39,
|
||||
"tags": [],
|
||||
"tags": [
|
||||
"reth"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
@@ -1191,4 +1193,4 @@
|
||||
"uid": "fd2d69b5-ca32-45d0-946e-c00ddcd7052c",
|
||||
"version": 1,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1321,7 +1321,9 @@
|
||||
],
|
||||
"refresh": "",
|
||||
"schemaVersion": 40,
|
||||
"tags": [],
|
||||
"tags": [
|
||||
"reth"
|
||||
],
|
||||
"templating": {
|
||||
"list": [
|
||||
{
|
||||
@@ -1406,4 +1408,4 @@
|
||||
"uid": "df9ob3njh2dxcd",
|
||||
"version": 4,
|
||||
"weekStart": ""
|
||||
}
|
||||
}
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -28,9 +28,11 @@ See examples in a [dedicated repository](https://github.com/paradigmxyz/reth-exe
|
||||
|
||||
## RPC
|
||||
|
||||
| Example | Description |
|
||||
| ----------------------- | --------------------------------------------------------------------------- |
|
||||
| [DB over RPC](./rpc-db) | Illustrates how to run a standalone RPC server over a Reth database instance |
|
||||
| Example | Description |
|
||||
| ------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------- |
|
||||
| [Custom auth HTTP middleware](./custom-auth-http-middleware) | Illustrates how to add HTTP transport middleware to the auth server for path-based request proxy |
|
||||
| [Custom RPC middleware](./custom-rpc-middleware) | Illustrates how to add JSON-RPC layer middleware that alters RPC error responses |
|
||||
| [DB over RPC](./rpc-db) | Illustrates how to run a standalone RPC server over a Reth database instance |
|
||||
|
||||
## Database
|
||||
|
||||
|
||||
16
examples/custom-auth-http-middleware/Cargo.toml
Normal file
16
examples/custom-auth-http-middleware/Cargo.toml
Normal file
@@ -0,0 +1,16 @@
|
||||
[package]
|
||||
name = "example-custom-auth-http-middleware"
|
||||
version = "0.0.0"
|
||||
publish = false
|
||||
edition.workspace = true
|
||||
license.workspace = true
|
||||
|
||||
[dependencies]
|
||||
reth-ethereum = { workspace = true, features = ["node", "rpc", "cli"] }
|
||||
|
||||
clap = { workspace = true, features = ["derive"] }
|
||||
http = { workspace = true }
|
||||
jsonrpsee = { workspace = true, features = ["server"] }
|
||||
serde_json.workspace = true
|
||||
tower.workspace = true
|
||||
tracing.workspace = true
|
||||
142
examples/custom-auth-http-middleware/src/main.rs
Normal file
142
examples/custom-auth-http-middleware/src/main.rs
Normal file
@@ -0,0 +1,142 @@
|
||||
//! Example of how to create a node with custom auth HTTP middleware that proxies requests based on
|
||||
//! the URL path.
|
||||
//!
|
||||
//! This middleware operates at the HTTP transport layer on the auth/Engine API server (port 8551),
|
||||
//! applied **after** JWT authentication and **before** JSON-RPC parsing. It inspects the raw HTTP
|
||||
//! request path and can proxy matching requests to an upstream server, while forwarding everything
|
||||
//! else to the default Engine API handler.
|
||||
//!
|
||||
//! ## Architecture
|
||||
//!
|
||||
//! ```text
|
||||
//! HTTP Request → JWT Auth → PathProxyMiddleware → JSON-RPC Parsing → Engine API Handler
|
||||
//! │
|
||||
//! └── if path matches "/proxy/*" → forward to upstream
|
||||
//! ```
|
||||
//!
|
||||
//! Run with
|
||||
//!
|
||||
//! ```sh
|
||||
//! cargo run -p example-custom-auth-http-middleware node \
|
||||
//! --authrpc.jwtsecret /tmp/jwt.hex \
|
||||
//! --dev --dev.block-time 12s
|
||||
//! ```
|
||||
//!
|
||||
//! Then send an authenticated request to the proxy path:
|
||||
//!
|
||||
//! ```sh
|
||||
//! # Generate a JWT token for the secret in /tmp/jwt.hex and send a proxied request:
|
||||
//! curl -s -X POST http://localhost:8551/proxy/anything \
|
||||
//! -H "Content-Type: application/json" \
|
||||
//! -H "Authorization: Bearer <jwt-token>" \
|
||||
//! -d '{"hello":"world"}'
|
||||
//! ```
|
||||
|
||||
use clap::Parser;
|
||||
use http::{header::CONTENT_TYPE, Response, StatusCode};
|
||||
use jsonrpsee::server::{HttpBody, HttpRequest, HttpResponse};
|
||||
use reth_ethereum::{
|
||||
cli::{chainspec::EthereumChainSpecParser, interface::Cli},
|
||||
node::{EthereumAddOns, EthereumNode},
|
||||
};
|
||||
use std::{
|
||||
future::Future,
|
||||
pin::Pin,
|
||||
task::{Context, Poll},
|
||||
};
|
||||
use tower::{BoxError, Layer, Service};
|
||||
|
||||
fn main() {
|
||||
Cli::<EthereumChainSpecParser>::parse()
|
||||
.run(async move |builder, _| {
|
||||
let handle = builder
|
||||
.with_types::<EthereumNode>()
|
||||
.with_components(EthereumNode::components())
|
||||
.with_add_ons(
|
||||
EthereumAddOns::default()
|
||||
.with_auth_http_middleware(PathProxyLayer { proxy_prefix: "/proxy/" }),
|
||||
)
|
||||
.launch_with_debug_capabilities()
|
||||
.await?;
|
||||
|
||||
handle.wait_for_node_exit().await
|
||||
})
|
||||
.unwrap();
|
||||
}
|
||||
|
||||
/// A [`Layer`] that intercepts HTTP requests matching a path prefix and returns a custom
|
||||
/// response instead of forwarding to the Engine API.
|
||||
///
|
||||
/// In a real-world scenario, you would forward matching requests to an upstream HTTP server
|
||||
/// (e.g., via `hyper` or `reqwest`). This example returns a simple JSON response to demonstrate
|
||||
/// the path-based routing pattern.
|
||||
#[derive(Clone)]
|
||||
struct PathProxyLayer {
|
||||
proxy_prefix: &'static str,
|
||||
}
|
||||
|
||||
impl<S> Layer<S> for PathProxyLayer {
|
||||
type Service = PathProxyService<S>;
|
||||
|
||||
fn layer(&self, inner: S) -> Self::Service {
|
||||
PathProxyService { inner, proxy_prefix: self.proxy_prefix }
|
||||
}
|
||||
}
|
||||
|
||||
/// The [`Service`] produced by [`PathProxyLayer`].
|
||||
///
|
||||
/// For requests whose path starts with the configured prefix, this service short-circuits the
|
||||
/// normal Engine API pipeline and returns a custom HTTP response. All other requests are forwarded
|
||||
/// unchanged to the inner service (the JSON-RPC handler).
|
||||
#[derive(Clone)]
|
||||
struct PathProxyService<S> {
|
||||
inner: S,
|
||||
proxy_prefix: &'static str,
|
||||
}
|
||||
|
||||
impl<S> Service<HttpRequest> for PathProxyService<S>
|
||||
where
|
||||
S: Service<HttpRequest, Response = HttpResponse, Error = BoxError> + Send + Clone,
|
||||
S::Future: Send + 'static,
|
||||
{
|
||||
type Response = HttpResponse;
|
||||
type Error = BoxError;
|
||||
type Future = Pin<Box<dyn Future<Output = Result<Self::Response, Self::Error>> + Send>>;
|
||||
|
||||
fn poll_ready(&mut self, cx: &mut Context<'_>) -> Poll<Result<(), Self::Error>> {
|
||||
self.inner.poll_ready(cx)
|
||||
}
|
||||
|
||||
fn call(&mut self, req: HttpRequest) -> Self::Future {
|
||||
let path = req.uri().path().to_owned();
|
||||
let proxy_prefix = self.proxy_prefix;
|
||||
|
||||
if let Some(suffix) = path.strip_prefix(proxy_prefix) {
|
||||
// The request path matches our proxy prefix — handle it here instead of forwarding
|
||||
// to the Engine API JSON-RPC handler.
|
||||
//
|
||||
// In production, you might forward this to an upstream HTTP server:
|
||||
// let upstream_url = format!("http://127.0.0.1:9000{}", path);
|
||||
// let resp = reqwest::Client::new().post(&upstream_url).body(body).send().await?;
|
||||
tracing::info!(path = %path, "intercepted request on proxy path");
|
||||
|
||||
let body = serde_json::json!({
|
||||
"status": "proxied",
|
||||
"matched_prefix": proxy_prefix,
|
||||
"remaining_path": suffix,
|
||||
});
|
||||
|
||||
let response = Response::builder()
|
||||
.status(StatusCode::OK)
|
||||
.header(CONTENT_TYPE, "application/json")
|
||||
.body(HttpBody::from(body.to_string()))
|
||||
.expect("valid response");
|
||||
|
||||
Box::pin(async move { Ok(response) })
|
||||
} else {
|
||||
// Not a proxy path — forward to the inner Engine API handler as usual.
|
||||
let fut = self.inner.call(req);
|
||||
Box::pin(fut)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user