Compare commits

..

1 Commits

Author SHA1 Message Date
yongkangc
082daa7fc1 fix(rocksdb): filter account/storage history to match MDBX semantics 2026-01-22 12:58:59 +00:00
882 changed files with 62087 additions and 33451 deletions

View File

@@ -1,4 +0,0 @@
---
---
Added site-level meta description for SEO.

View File

@@ -1,5 +0,0 @@
---
reth-transaction-pool: patch
---
Renamed and documented validation methods for clarity. `validate_one_no_state` and `validate_one_against_state` are now public methods `validate_stateless` and `validate_stateful` with improved documentation explaining their respective validation phases.

View File

@@ -1,20 +0,0 @@
# Changelogs configuration for reth
# https://github.com/wevm/changelogs
# How to bump packages that depend on changed packages
dependent_bump = "patch"
[changelog]
# Generate per-crate changelogs (vs single root changelog)
format = "per-crate"
# Fixed groups: all always share the same version
# reth binaries share version
[[fixed]]
members = ["reth"]
# Packages to ignore (internal/test-only crates)
ignore = [
"reth-testing-utils",
"reth-bench",
]

View File

@@ -1,17 +0,0 @@
---
reth: minor
reth-cli-commands: minor
reth-e2e-test-utils: minor
reth-ethereum-cli: minor
reth-node-core: minor
reth-optimism-bin: minor
reth-optimism-cli: minor
reth-prune: patch
reth-stages: patch
reth-storage-api: minor
reth-storage-db-api: minor
reth-storage-db-common: patch
reth-storage-provider: patch
---
Introduced `--storage.v2` flag to control storage mode defaults, replacing the `edge` feature flag with `rocksdb` feature. The new flag enables v2 storage settings (static files + RocksDB routing) while individual `--static-files.*` and `--rocksdb.*` flags can still override defaults. Updated feature gates from `edge` to `rocksdb` across all affected crates.

View File

@@ -1,5 +0,0 @@
---
reth: patch
---
Removed Windows platform support from the codebase, including the Windows cross-compilation Dockerfile, build targets in Cross.toml and Makefile, and Windows-specific options in the bug report template.

View File

@@ -1,5 +0,0 @@
---
reth-network: minor
---
Added reason label to backed_off_peers metric. The metric now tracks backed off peers by reason (too_many_peers, graceful_close, connection_error) to improve observability.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse-parallel: patch
---
Fixed parallel sparse trie to skip revealing disconnected leaves by checking parent branch reachability before inserting leaf nodes.

View File

@@ -1,5 +0,0 @@
---
ef-tests: patch
---
Removed reth-stateless crate and stateless validation from ef-tests.

View File

@@ -1,6 +0,0 @@
---
reth-engine-tree: patch
reth-trie-sparse-parallel: patch
---
Added tracing spans and debug logs to sparse trie operations for better observability during parallel state root computation.

View File

@@ -1,6 +0,0 @@
---
reth-exex: patch
reth-exex-types: patch
---
Added configurable backfill thresholds to ExEx notifications stream and added regression tests for state provider parity between pipeline and backfill execution paths.

View File

@@ -1,4 +0,0 @@
---
---
Added WebSocket subscription integration tests for eth_subscribe.

View File

@@ -1,5 +0,0 @@
---
reth-engine-tree: patch
---
Reordered cache size calculations in `ExecutionCache::new` to group related operations together.

View File

@@ -1,7 +0,0 @@
---
reth: patch
reth-cli-commands: patch
reth-node-core: patch
---
Removed experimental ress protocol support for stateless Ethereum nodes.

View File

@@ -1,5 +0,0 @@
---
reth-node-builder: patch
---
Removed biased select in engine service loop to allow fair scheduling of shutdown requests alongside event processing.

View File

@@ -1,5 +0,0 @@
---
reth-transaction-pool: patch
---
Fixed swapped arguments in `blob_tx_priority` function calls, correcting the parameter order to match the function signature.

View File

@@ -1,5 +0,0 @@
---
reth: patch
---
Re-enabled changelog workflow to run automatically on pull requests.

View File

@@ -1,6 +0,0 @@
---
reth-chainspec: minor
reth-network-peers: minor
---
Removed OP stack bootnodes from default chain configurations and network peers module.

View File

@@ -1,5 +0,0 @@
---
reth-primitives: patch
---
Moved feature-referenced dependencies from dev-dependencies to optional dependencies to ensure they are available when their corresponding features are enabled.

View File

@@ -1,5 +0,0 @@
---
reth-transaction-pool: minor
---
Added `IntoIter: Send` bounds to `validate_transactions` and `validate_transactions_with_origin` in the `TransactionValidator` trait, avoiding unnecessary `Vec` collects. Simplified default `validate_transactions_with_origin` to delegate to `validate_transactions`.

View File

@@ -1,6 +0,0 @@
---
reth-static-file-types: patch
reth-provider: patch
---
Move changeset offsets from segment header to external `.csoff` sidecar file for incremental writes and crash recovery.

View File

@@ -1,5 +0,0 @@
---
reth-provider: patch
---
Removed unused staging types from ProviderFactoryBuilder.

View File

@@ -1,5 +0,0 @@
---
reth: patch
---
Added automated changelog generation infrastructure using wevm/changelogs-rs with Claude Code integration. Configured per-crate changelog format with fixed version groups for reth binaries and exclusions for internal test utilities.

View File

@@ -1,5 +0,0 @@
---
reth-trie-sparse: minor
---
Removed `SerialSparseTrie` from the workspace, consolidating on `ParallelSparseTrie` as the single sparse trie implementation in `reth-trie-sparse`.

View File

@@ -1,5 +0,0 @@
---
reth: patch
---
Updated Alloy dependencies from 1.5.2 to 1.6.1.

View File

@@ -1,4 +0,0 @@
---
---
Expanded CLI integration tests with subcommand help coverage, config TOML validation, genesis JSON validation, and send transaction round-trip test for dev mode.

View File

@@ -1,5 +0,0 @@
---
reth-network: minor
---
Added direction labels to `closed_sessions` and `pending_session_failures` metrics. Operators can now distinguish session closures and failures by direction (`active`, `incoming_pending`, `outgoing_pending` for closed sessions; `inbound`, `outbound` for pending session failures).

View File

@@ -1,4 +0,0 @@
---
---
Moved Kurtosis CI failure notifications to the hive Slack channel.

View File

@@ -1,7 +0,0 @@
---
reth-rpc-api: minor
reth-rpc-builder: patch
reth-rpc: minor
---
Added `subscribeFinalizedChainNotifications` RPC endpoint that buffers committed chain notifications and emits them once a new finalized block is received.

View File

@@ -1,6 +0,0 @@
---
reth-trie: minor
reth-trie-parallel: minor
---
Added `root_node` and `storage_root_node` methods to proof calculators for efficient root-only calculations. These methods directly return the root node without requiring dummy targets, replacing the previous workaround of passing fake targets to proof generation.

3
.github/CODEOWNERS vendored
View File

@@ -19,6 +19,7 @@ crates/metrics/ @mattsse @Rjected
crates/net/ @mattsse @Rjected
crates/net/downloaders/ @Rjected
crates/node/ @mattsse @Rjected @klkvr
crates/optimism/ @mattsse @Rjected
crates/payload/ @mattsse @Rjected
crates/primitives-traits/ @Rjected @mattsse @klkvr
crates/primitives/ @Rjected @mattsse @klkvr
@@ -36,7 +37,7 @@ crates/storage/db/ @joshieDo
crates/storage/errors/ @joshieDo
crates/storage/libmdbx-rs/ @shekhirin
crates/storage/nippy-jar/ @joshieDo @shekhirin
crates/storage/provider/ @joshieDo @shekhirin @yongkangc
crates/storage/provider/ @joshieDo @shekhirin
crates/storage/storage-api/ @joshieDo
crates/tasks/ @mattsse
crates/tokio-util/ @mattsse

View File

@@ -43,6 +43,7 @@ body:
- `~/.cache/reth/logs` on Linux
- `~/Library/Caches/reth/logs` on macOS
- `%localAppData%/reth/logs` on Windows
render: text
validations:
required: false
@@ -57,6 +58,8 @@ body:
- Linux (ARM)
- Mac (Intel)
- Mac (Apple Silicon)
- Windows (x86)
- Windows (ARM)
- type: dropdown
id: container_type
attributes:

88
.github/assets/check_rv32imac.sh vendored Executable file
View File

@@ -0,0 +1,88 @@
#!/usr/bin/env bash
set +e # Disable immediate exit on error
# Array of crates to check
crates_to_check=(
reth-codecs-derive
reth-primitives
reth-primitives-traits
reth-network-peers
reth-trie-common
reth-trie-sparse
reth-chainspec
reth-consensus
reth-consensus-common
reth-prune-types
reth-static-file-types
reth-storage-errors
reth-execution-errors
reth-errors
reth-execution-types
reth-db-models
reth-evm
reth-revm
reth-storage-api
## ethereum
reth-evm-ethereum
reth-ethereum-forks
reth-ethereum-primitives
reth-ethereum-consensus
reth-stateless
## optimism
reth-optimism-chainspec
reth-optimism-forks
reth-optimism-consensus
reth-optimism-primitives
reth-optimism-evm
)
# Array to hold the results
results=()
# Flag to track if any command fails
any_failed=0
for crate in "${crates_to_check[@]}"; do
cmd="cargo +stable build -p $crate --target riscv32imac-unknown-none-elf --no-default-features"
if [ -n "$CI" ]; then
echo "::group::$cmd"
else
printf "\n%s:\n %s\n" "$crate" "$cmd"
fi
set +e # Disable immediate exit on error
# Run the command and capture the return code
$cmd
ret_code=$?
set -e # Re-enable immediate exit on error
# Store the result in the dictionary
if [ $ret_code -eq 0 ]; then
results+=("1:✅:$crate")
else
results+=("2:❌:$crate")
any_failed=1
fi
if [ -n "$CI" ]; then
echo "::endgroup::"
fi
done
# Sort the results by status and then by crate name
IFS=$'\n' sorted_results=($(sort <<<"${results[*]}"))
unset IFS
# Print summary
echo -e "\nSummary of build results:"
for result in "${sorted_results[@]}"; do
status="${result#*:}"
status="${status%%:*}"
crate="${result##*:}"
echo "$status $crate"
done
# Exit with a non-zero status if any command fails
exit $any_failed

View File

@@ -1,10 +1,11 @@
#!/usr/bin/env bash
set -uo pipefail
set +e # Disable immediate exit on error
readarray -t crates < <(
cargo metadata --format-version=1 --no-deps | jq -r '.packages[].name' | grep '^reth' | sort
)
# Array of crates to compile
crates=($(cargo metadata --format-version=1 --no-deps | jq -r '.packages[].name' | grep '^reth' | sort))
# Array of crates to exclude
# Used with the `contains` function.
# shellcheck disable=SC2034
exclude_crates=(
# The following require investigation if they can be fixed
@@ -39,6 +40,12 @@ exclude_crates=(
reth-node-ethereum
reth-node-events
reth-node-metrics
reth-optimism-cli
reth-optimism-flashblocks
reth-optimism-node
reth-optimism-payload-builder
reth-optimism-rpc
reth-optimism-storage
reth-rpc
reth-rpc-api
reth-rpc-api-testing-util
@@ -63,7 +70,6 @@ exclude_crates=(
reth-provider # tokio
reth-prune # tokio
reth-prune-static-files # reth-provider
reth-tasks # tokio rt-multi-thread
reth-stages-api # reth-provider, reth-prune
reth-static-file # tokio
reth-transaction-pool # c-kzg
@@ -71,41 +77,77 @@ exclude_crates=(
reth-trie-parallel # tokio
reth-trie-sparse-parallel # rayon
reth-testing-utils
reth-optimism-txpool # reth-transaction-pool
reth-era-downloader # tokio
reth-era-utils # tokio
reth-tracing-otlp
reth-node-ethstats
)
# Array to hold the results
results=()
# Flag to track if any command fails
any_failed=0
tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t reth-check)
trap 'rm -rf -- "$tmpdir"' EXIT INT TERM
# Function to check if a value exists in an array
contains() {
local array="$1[@]"
local seeking="$2"
local element
local seeking=$2
local in=1
for element in "${!array}"; do
[[ "$element" == "$seeking" ]] && return 0
if [[ "$element" == "$seeking" ]]; then
in=0
break
fi
done
return 1
return $in
}
for crate in "${crates[@]}"; do
if contains exclude_crates "$crate"; then
echo "⏭️ $crate"
results+=("3:⏭️:$crate")
continue
fi
outfile="$tmpdir/$crate.log"
if cargo +stable build -p "$crate" --target wasm32-wasip1 --no-default-features --color never >"$outfile" 2>&1; then
echo "$crate"
cmd="cargo +stable build -p $crate --target wasm32-wasip1 --no-default-features"
if [ -n "$CI" ]; then
echo "::group::$cmd"
else
echo "$crate"
sed 's/^/ /' "$outfile"
echo ""
printf "\n%s:\n %s\n" "$crate" "$cmd"
fi
set +e # Disable immediate exit on error
# Run the command and capture the return code
$cmd
ret_code=$?
set -e # Re-enable immediate exit on error
# Store the result in the dictionary
if [ $ret_code -eq 0 ]; then
results+=("1:✅:$crate")
else
results+=("2:❌:$crate")
any_failed=1
fi
if [ -n "$CI" ]; then
echo "::endgroup::"
fi
done
# Sort the results by status and then by crate name
IFS=$'\n' sorted_results=($(sort <<<"${results[*]}"))
unset IFS
# Print summary
echo -e "\nSummary of build results:"
for result in "${sorted_results[@]}"; do
status="${result#*:}"
status="${status%%:*}"
crate="${result##*:}"
echo "$status $crate"
done
# Exit with a non-zero status if any command fails
exit $any_failed

View File

@@ -38,6 +38,6 @@ for pid in "${saving_pids[@]}"; do
done
# Make sure we don't rebuild images on the CI jobs
git apply ../.github/scripts/hive/no_sim_build.diff
git apply ../.github/assets/hive/no_sim_build.diff
go build .
mv ./hive ../hive_assets/

View File

@@ -0,0 +1,36 @@
ethereum_package:
participants:
- el_type: reth
el_extra_params:
- "--rpc.eth-proof-window=100"
cl_type: teku
network_params:
preset: minimal
genesis_delay: 5
additional_preloaded_contracts: '
{
"0x4e59b44847b379578588920cA78FbF26c0B4956C": {
"balance": "0ETH",
"code": "0x7fffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffffe03601600081602082378035828234f58015156039578182fd5b8082525050506014600cf3",
"storage": {},
"nonce": "1"
}
}'
optimism_package:
chains:
chain0:
participants:
node0:
el:
type: op-geth
cl:
type: op-node
node1:
el:
type: op-reth
image: "ghcr.io/paradigmxyz/op-reth:kurtosis-ci"
cl:
type: op-node
network_params:
holocene_time_offset: 0
isthmus_time_offset: 0

View File

@@ -4,17 +4,3 @@ updates:
directory: "/"
schedule:
interval: "weekly"
- package-ecosystem: "cargo"
directory: "/"
schedule:
interval: "weekly"
labels:
- "A-dependencies"
commit-message:
prefix: "chore(deps)"
open-pull-requests-limit: 1
groups:
cargo-weekly:
applies-to: "version-updates"
patterns: ["*"]
update-types: ["minor", "patch"]

View File

@@ -1,48 +0,0 @@
#!/usr/bin/env bash
set -uo pipefail
crates_to_check=(
reth-codecs-derive
reth-primitives
reth-primitives-traits
reth-network-peers
reth-trie-common
reth-trie-sparse
reth-chainspec
reth-consensus
reth-consensus-common
reth-prune-types
reth-static-file-types
reth-storage-errors
reth-execution-errors
reth-errors
reth-execution-types
reth-db-models
reth-evm
reth-revm
reth-storage-api
## ethereum
reth-evm-ethereum
reth-ethereum-forks
reth-ethereum-primitives
reth-ethereum-consensus
)
any_failed=0
tmpdir=$(mktemp -d 2>/dev/null || mktemp -d -t reth-check)
trap 'rm -rf -- "$tmpdir"' EXIT INT TERM
for crate in "${crates_to_check[@]}"; do
outfile="$tmpdir/$crate.log"
if cargo +stable build -p "$crate" --target riscv32imac-unknown-none-elf --no-default-features --color never >"$outfile" 2>&1; then
echo "$crate"
else
echo "$crate"
sed 's/^/ /' "$outfile"
echo ""
any_failed=1
fi
done
exit $any_failed

View File

@@ -1,53 +0,0 @@
#!/usr/bin/env bash
# Verifies that Docker images have the expected architectures.
#
# Usage:
# ./verify_image_arch.sh <targets> <registry> <ethereum_tags>
#
# Environment:
# DRY_RUN=true - Skip actual verification, just print what would be checked.
set -euo pipefail
TARGETS="${1:-}"
REGISTRY="${2:-}"
ETHEREUM_TAGS="${3:-}"
DRY_RUN="${DRY_RUN:-false}"
verify_image() {
local image="$1"
shift
local expected_archs=("$@")
echo "Checking $image..."
if [[ "$DRY_RUN" == "true" ]]; then
echo " [dry-run] Would verify architectures: ${expected_archs[*]}"
return 0
fi
manifest=$(docker manifest inspect "$image" 2>/dev/null) || {
echo "::error::Failed to inspect manifest for $image"
return 1
}
for arch in "${expected_archs[@]}"; do
if ! echo "$manifest" | jq -e ".manifests[] | select(.platform.architecture == \"$arch\" and .platform.os == \"linux\")" > /dev/null; then
echo "::error::Missing architecture $arch for $image"
return 1
fi
echo " ✓ linux/$arch"
done
}
if [[ "$TARGETS" == *"nightly"* ]]; then
verify_image "${REGISTRY}/reth:nightly" amd64 arm64
verify_image "${REGISTRY}/reth:nightly-profiling" amd64
verify_image "${REGISTRY}/reth:nightly-edge-profiling" amd64
else
for tag in $(echo "$ETHEREUM_TAGS" | tr ',' ' '); do
verify_image "$tag" amd64 arm64
done
fi
echo "All image architectures verified successfully"

View File

@@ -1,25 +0,0 @@
name: Changelog
on:
pull_request:
types: [opened, synchronize]
jobs:
changelog:
# Skip for fork PRs since they can't access secrets
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
permissions:
contents: write
pull-requests: write
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
ref: ${{ github.event.pull_request.head.sha }}
- run: npm install -g @anthropic-ai/claude-code
- uses: wevm/changelogs/check@master
with:
ai: 'claude -p'
env:
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}

View File

@@ -60,6 +60,7 @@ jobs:
tail -50 Cargo.toml
- name: Check workspace
run: cargo clippy --workspace --lib --examples --tests --benches --all-features --locked
env:
RUSTFLAGS: -D warnings
run: cargo check --workspace --all-features
- name: Check Optimism
run: cargo check -p reth-optimism-node --all-features

View File

@@ -23,6 +23,7 @@ jobs:
matrix:
bin:
- cargo run --bin reth --features "dev"
- cargo run --bin op-reth --features "dev" --manifest-path crates/optimism/bin/Cargo.toml
steps:
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable

View File

@@ -15,7 +15,6 @@ permissions:
jobs:
update:
if: github.repository == 'paradigmxyz/reth'
uses: tempoxyz/ci/.github/workflows/cargo-update-pr.yml@main
secrets:
token: ${{ secrets.GITHUB_TOKEN }}

54
.github/workflows/docker-git.yml vendored Normal file
View File

@@ -0,0 +1,54 @@
# Publishes the Docker image, only to be used with `workflow_dispatch`. The
# images from this workflow will be tagged with the git sha of the branch used
# and will NOT tag it as `latest`.
name: docker-git
on:
workflow_dispatch: {}
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
GIT_SHA: ${{ github.sha }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the git-sha-tagged reth image'
command: 'make PROFILE=maxperf GIT_SHA=$GIT_SHA docker-build-push-git-sha'
- name: 'Build and push the git-sha-tagged op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME GIT_SHA=$GIT_SHA PROFILE=maxperf op-docker-build-push-git-sha'
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

65
.github/workflows/docker-nightly.yml vendored Normal file
View File

@@ -0,0 +1,65 @@
# Publishes the nightly Docker image.
name: docker-nightly
on:
workflow_dispatch:
schedule:
- cron: "0 1 * * *"
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
name: build and push
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: 'Build and push the nightly reth image'
command: 'make PROFILE=maxperf docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling reth image'
command: 'make PROFILE=profiling docker-build-push-nightly-profiling'
- name: 'Build and push the nightly op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-nightly'
- name: 'Build and push the nightly edge profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-edge-profiling'
- name: 'Build and push the nightly profiling op-reth image'
command: 'make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=profiling op-docker-build-push-nightly-profiling'
steps:
- uses: actions/checkout@v6
- name: Remove bloatware
uses: laverdet/remove-bloatware@v1.0.0
with:
docker: true
lang: rust
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -14,6 +14,12 @@ on:
required: false
type: boolean
default: true
tag_op_reth:
description: 'Tag op-reth image as latest'
required: false
type: boolean
default: false
env:
DOCKER_USERNAME: ${{ github.actor }}
@@ -41,3 +47,27 @@ jobs:
- name: Push reth latest tag
run: |
docker push ghcr.io/${{ github.repository_owner }}/reth:latest
tag-op-reth-latest:
name: Tag op-reth as latest
runs-on: ubuntu-24.04
if: ${{ inputs.tag_op_reth }}
permissions:
packages: write
contents: read
steps:
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Pull op-reth release image
run: |
docker pull ghcr.io/${{ github.repository_owner }}/op-reth:${{ inputs.version }}
- name: Tag op-reth as latest
run: |
docker tag ghcr.io/${{ github.repository_owner }}/op-reth:${{ inputs.version }} ghcr.io/${{ github.repository_owner }}/op-reth:latest
- name: Push op-reth latest tag
run: |
docker push ghcr.io/${{ github.repository_owner }}/op-reth:latest

View File

@@ -1,53 +0,0 @@
name: Build test Docker image
on:
workflow_call:
inputs:
hive_target:
required: true
type: string
description: "Docker bake target to build (e.g. hive-stable, hive-edge)"
artifact_name:
required: false
type: string
default: "artifacts"
description: "Name for the uploaded artifact"
jobs:
build:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: ubuntu-latest
permissions:
id-token: write
contents: read
steps:
- uses: actions/checkout@v6
- run: mkdir -p artifacts
- name: Get git info
id: git
run: |
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
- name: Set up Depot CLI
uses: depot/setup-action@v1
- name: Build reth image
uses: depot/bake-action@v1
env:
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
with:
project: ${{ vars.DEPOT_PROJECT_ID }}
files: docker-bake.hcl
targets: ${{ inputs.hive_target }}
push: false
- name: Upload reth image
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./artifacts

View File

@@ -1,9 +1,4 @@
# Publishes Docker images.
#
# Triggers:
# - Push tag v*: builds release (RC or latest)
# - Schedule: builds nightly + profiling
# - Manual: builds git-sha or nightly
# Publishes the Docker image.
name: docker
@@ -11,112 +6,84 @@ on:
push:
tags:
- v*
schedule:
- cron: "0 1 * * *"
workflow_dispatch:
inputs:
build_type:
description: "Build type"
required: true
type: choice
options:
- git-sha
- nightly
default: git-sha
dry_run:
description: "Skip pushing images (dry run)"
required: false
type: boolean
default: false
env:
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/reth
OP_DOCKER_IMAGE_NAME: ghcr.io/${{ github.repository_owner }}/op-reth
DOCKER_USERNAME: ${{ github.actor }}
jobs:
build:
if: github.repository == 'paradigmxyz/reth'
name: Build Docker images
build-rc:
if: contains(github.ref, '-rc')
name: build and push as release candidate
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
id-token: write
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push"
steps:
- uses: actions/checkout@v6
- name: Set up Depot CLI
uses: depot/setup-action@v1
- name: Log in to GHCR
uses: docker/login-action@v3
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
registry: ghcr.io
username: ${{ github.actor }}
password: ${{ secrets.GITHUB_TOKEN }}
- name: Get git info for vergen
id: git
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
echo "sha=${{ github.sha }}" >> "$GITHUB_OUTPUT"
echo "describe=$(git describe --always --tags)" >> "$GITHUB_OUTPUT"
echo "dirty=false" >> "$GITHUB_OUTPUT"
- name: Determine build parameters
id: params
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
REGISTRY="ghcr.io/${{ github.repository_owner }}"
if [[ "${{ github.event_name }}" == "push" ]]; then
VERSION="${GITHUB_REF#refs/tags/}"
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
# Add 'latest' tag for non-RC releases
if [[ ! "$VERSION" =~ -rc ]]; then
echo "ethereum_tags=${REGISTRY}/reth:${VERSION},${REGISTRY}/reth:latest" >> "$GITHUB_OUTPUT"
else
echo "ethereum_tags=${REGISTRY}/reth:${VERSION}" >> "$GITHUB_OUTPUT"
fi
elif [[ "${{ github.event_name }}" == "schedule" ]] || [[ "${{ inputs.build_type }}" == "nightly" ]]; then
echo "targets=nightly" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:nightly" >> "$GITHUB_OUTPUT"
else
# git-sha build
echo "targets=ethereum" >> "$GITHUB_OUTPUT"
echo "ethereum_tags=${REGISTRY}/reth:${{ github.sha }}" >> "$GITHUB_OUTPUT"
fi
- name: Build and push images
uses: depot/bake-action@v1
env:
VERGEN_GIT_SHA: ${{ steps.git.outputs.sha }}
VERGEN_GIT_DESCRIBE: ${{ steps.git.outputs.describe }}
VERGEN_GIT_DIRTY: ${{ steps.git.outputs.dirty }}
DEPOT_TOKEN: ${{ secrets.DEPOT_TOKEN }}
with:
project: ${{ vars.DEPOT_PROJECT_ID }}
files: docker-bake.hcl
targets: ${{ steps.params.outputs.targets }}
push: ${{ !(github.event_name == 'workflow_dispatch' && inputs.dry_run) }}
set: |
ethereum.tags=${{ steps.params.outputs.ethereum_tags }}
- name: Verify image architectures
env:
DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && inputs.dry_run }}
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
./.github/scripts/verify_image_arch.sh \
"${{ steps.params.outputs.targets }}" \
"ghcr.io/${{ github.repository_owner }}" \
"${{ steps.params.outputs.ethereum_tags }}"
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}
notify:
name: Notify on failure
runs-on: ubuntu-latest
needs: build
if: failure() && github.event_name == 'schedule'
build:
if: ${{ !contains(github.ref, '-rc') }}
name: build and push as latest
runs-on: ubuntu-24.04
permissions:
packages: write
contents: read
strategy:
fail-fast: false
matrix:
build:
- name: "Build and push reth image"
command: "make IMAGE_NAME=$IMAGE_NAME DOCKER_IMAGE_NAME=$DOCKER_IMAGE_NAME PROFILE=maxperf docker-build-push-latest"
- name: "Build and push op-reth image"
command: "make IMAGE_NAME=$OP_IMAGE_NAME DOCKER_IMAGE_NAME=$OP_DOCKER_IMAGE_NAME PROFILE=maxperf op-docker-build-push-latest"
steps:
- name: Slack Webhook Action
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Install cross main
id: cross_main
run: |
cargo install cross --git https://github.com/cross-rs/cross
- name: Log in to Docker
run: |
echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io --username ${DOCKER_USERNAME} --password-stdin
- name: Set up Docker builder
run: |
docker run --privileged --rm tonistiigi/binfmt --install arm64,amd64
docker buildx create --use --name cross-builder
- name: Build and push ${{ matrix.build.name }}
run: ${{ matrix.build.command }}

View File

@@ -35,34 +35,12 @@ jobs:
- name: Run e2e tests
run: |
cargo nextest run \
--no-fail-fast \
--locked --features "asm-keccak" \
--workspace \
--exclude 'example-*' \
--exclude 'exex-subscription' \
--exclude 'reth-bench' \
--exclude 'ef-tests' \
--exclude 'op-reth' \
--exclude 'reth' \
-E 'binary(e2e_testsuite)'
rocksdb:
name: e2e-rocksdb
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: dtolnay/rust-toolchain@stable
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: taiki-e/install-action@nextest
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run RocksDB e2e tests
run: |
cargo nextest run \
--no-fail-fast \
--locked --features "edge" \
-p reth-e2e-test-utils \
-E 'binary(rocksdb)'

View File

@@ -5,7 +5,7 @@ name: hive
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
- cron: "0 */6 * * *"
env:
CARGO_TERM_COLOR: always
@@ -15,24 +15,27 @@ concurrency:
cancel-in-progress: true
jobs:
build-reth-stable:
uses: ./.github/workflows/docker-test.yml
prepare-reth-stable:
uses: ./.github/workflows/prepare-reth.yml
with:
hive_target: hive-stable
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak"
artifact_name: "reth-stable"
secrets: inherit
build-reth-edge:
uses: ./.github/workflows/docker-test.yml
prepare-reth-edge:
uses: ./.github/workflows/prepare-reth.yml
with:
hive_target: hive-edge
image_tag: ghcr.io/paradigmxyz/reth:latest
binary_name: reth
cargo_features: "asm-keccak edge"
artifact_name: "reth-edge"
secrets: inherit
prepare-hive:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: depot-ubuntu-latest-4
runs-on:
group: Reth
steps:
- uses: actions/checkout@v6
- name: Checkout hive tests
@@ -55,11 +58,11 @@ jobs:
uses: actions/cache@v5
with:
path: ./hive_assets
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/scripts/hive/build_simulators.sh') }}
key: hive-assets-${{ steps.hive-commit.outputs.hash }}-${{ hashFiles('.github/assets/hive/build_simulators.sh') }}
- name: Build hive assets
if: steps.cache-hive.outputs.cache-hit != 'true'
run: .github/scripts/hive/build_simulators.sh
run: .github/assets/hive/build_simulators.sh
- name: Load cached Docker images
if: steps.cache-hive.outputs.cache-hit == 'true'
@@ -184,11 +187,12 @@ jobs:
- sim: ethereum/eels/consume-rlp
limit: .*tests/paris.*
needs:
- build-reth-stable
- build-reth-edge
- prepare-reth-stable
- prepare-reth-edge
- prepare-hive
name: ${{ matrix.storage }} / ${{ matrix.scenario.sim }}${{ matrix.scenario.limit && format(' - {0}', matrix.scenario.limit) }}
runs-on: depot-ubuntu-latest-4
runs-on:
group: Reth
permissions:
issues: write
steps:
@@ -209,7 +213,7 @@ jobs:
path: /tmp
- name: Load Docker images
run: .github/scripts/hive/load_images.sh
run: .github/assets/hive/load_images.sh
- name: Move hive binary
run: |
@@ -237,11 +241,11 @@ jobs:
FILTER="/"
fi
echo "filter: $FILTER"
.github/scripts/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
.github/assets/hive/run_simulator.sh "${{ matrix.scenario.sim }}" "$FILTER"
- name: Parse hive output
run: |
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/scripts/hive/parse.py {} --exclusion .github/scripts/hive/expected_failures.yaml --ignored .github/scripts/hive/ignored_tests.yaml
find hivetests/workspace/logs -type f -name "*.json" ! -name "hive.json" | xargs -I {} python .github/assets/hive/parse.py {} --exclusion .github/assets/hive/expected_failures.yaml --ignored .github/assets/hive/ignored_tests.yaml
- name: Print simulator output
if: ${{ failure() }}
@@ -262,4 +266,4 @@ jobs:
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_HIVE_WEBHOOK_URL }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -22,34 +22,38 @@ concurrency:
jobs:
test:
name: test / ${{ matrix.network }} / ${{ matrix.storage }}
name: test / ${{ matrix.network }}
if: github.event_name != 'schedule'
runs-on: depot-ubuntu-latest-4
env:
RUST_BACKTRACE: 1
strategy:
matrix:
network: ["ethereum"]
storage: ["stable", "edge"]
network: ["ethereum", "optimism"]
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
- name: Install Geth
run: .github/scripts/install_geth.sh
run: .github/assets/install_geth.sh
- uses: taiki-e/install-action@nextest
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: Run tests
- if: matrix.network == 'ethereum'
name: Run tests
run: |
cargo nextest run \
--no-fail-fast \
--locked --features "asm-keccak ${{ matrix.network }} ${{ matrix.storage == 'edge' && 'edge' || '' }}" \
--locked --features "asm-keccak ${{ matrix.network }}" \
--workspace --exclude ef-tests \
-E "kind(test) and not binary(e2e_testsuite)"
- if: matrix.network == 'optimism'
name: Run tests
run: |
cargo nextest run \
--locked -p reth-optimism-node
integration-success:
name: integration success
@@ -65,7 +69,7 @@ jobs:
era-files:
name: era1 file integration tests once a day
if: github.event_name == 'schedule' && github.repository == 'paradigmxyz/reth'
if: github.event_name == 'schedule'
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v6
@@ -77,4 +81,4 @@ jobs:
with:
cache-on-failure: true
- name: run era1 files integration tests
run: cargo nextest run --no-fail-fast --release --package reth-era --test it -- --ignored
run: cargo nextest run --release --package reth-era --test it -- --ignored

95
.github/workflows/kurtosis-op.yml vendored Normal file
View File

@@ -0,0 +1,95 @@
# Runs simple OP stack setup in Kurtosis
name: kurtosis-op
on:
workflow_dispatch:
schedule:
- cron: "0 */6 * * *"
push:
tags:
- "*"
env:
CARGO_TERM_COLOR: always
concurrency:
group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
cancel-in-progress: true
jobs:
prepare-reth:
uses: ./.github/workflows/prepare-reth.yml
with:
image_tag: ghcr.io/paradigmxyz/op-reth:kurtosis-ci
binary_name: op-reth
cargo_features: asm-keccak
cargo_package: crates/optimism/bin/Cargo.toml
test:
timeout-minutes: 60
strategy:
fail-fast: false
name: run kurtosis
runs-on: depot-ubuntu-latest
needs:
- prepare-reth
steps:
- uses: actions/checkout@v6
with:
fetch-depth: 0
- name: Download reth image
uses: actions/download-artifact@v7
with:
name: artifacts
path: /tmp
- name: Load Docker image
run: |
docker load -i /tmp/reth_image.tar &
wait
docker image ls -a
- name: Install Foundry
uses: foundry-rs/foundry-toolchain@v1
- name: Run kurtosis
run: |
echo "deb [trusted=yes] https://apt.fury.io/kurtosis-tech/ /" | sudo tee /etc/apt/sources.list.d/kurtosis.list
sudo apt update
sudo apt install kurtosis-cli
kurtosis engine start
kurtosis run --enclave op-devnet github.com/ethpandaops/optimism-package --args-file .github/assets/kurtosis_op_network_params.yaml
ENCLAVE_ID=$(curl http://127.0.0.1:9779/api/enclaves | jq --raw-output 'keys[0]')
GETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2151908-node0-op-geth".public_ports.rpc.number')
RETH_PORT=$(curl "http://127.0.0.1:9779/api/enclaves/$ENCLAVE_ID/services" | jq '."op-el-2151908-node1-op-reth".public_ports.rpc.number')
echo "GETH_RPC=http://127.0.0.1:$GETH_PORT" >> $GITHUB_ENV
echo "RETH_RPC=http://127.0.0.1:$RETH_PORT" >> $GITHUB_ENV
- name: Assert that clients advance
run: |
for i in {1..100}; do
sleep 5
BLOCK_GETH=$(cast bn --rpc-url $GETH_RPC)
BLOCK_RETH=$(cast bn --rpc-url $RETH_RPC)
if [ $BLOCK_GETH -ge 100 ] && [ $BLOCK_RETH -ge 100 ] ; then exit 0; fi
echo "Waiting for clients to advance..., Reth: $BLOCK_RETH Geth: $BLOCK_GETH"
done
kurtosis service logs -a op-devnet op-el-2151908-2-op-reth-op-node-op-kurtosis
kurtosis service logs -a op-devnet op-cl-2151908-2-op-node-op-reth-op-kurtosis
exit 1
notify-on-error:
needs: test
if: failure()
runs-on: ubuntu-latest
steps:
- name: Slack Webhook Action
uses: rtCamp/action-slack-notify@v2
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -5,7 +5,7 @@ name: kurtosis
on:
workflow_dispatch:
schedule:
- cron: "0 0 * * *"
- cron: "0 */6 * * *"
push:
tags:
@@ -19,12 +19,11 @@ concurrency:
cancel-in-progress: true
jobs:
build-reth:
if: github.repository == 'paradigmxyz/reth'
uses: ./.github/workflows/docker-test.yml
prepare-reth:
uses: ./.github/workflows/prepare-reth.yml
with:
hive_target: kurtosis
secrets: inherit
image_tag: ghcr.io/paradigmxyz/reth:kurtosis-ci
binary_name: reth
test:
timeout-minutes: 60
@@ -33,7 +32,7 @@ jobs:
name: run kurtosis
runs-on: depot-ubuntu-latest
needs:
- build-reth
- prepare-reth
steps:
- uses: actions/checkout@v6
with:
@@ -66,4 +65,4 @@ jobs:
env:
SLACK_COLOR: ${{ job.status }}
SLACK_MESSAGE: "Failed run: https://github.com/paradigmxyz/reth/actions/runs/${{ github.run_id }}"
SLACK_WEBHOOK: ${{ secrets.SLACK_HIVE_WEBHOOK_URL }}
SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}

View File

@@ -19,5 +19,5 @@ jobs:
uses: actions/github-script@v8
with:
script: |
const label_pr = require('./.github/scripts/label_pr.js')
const label_pr = require('./.github/assets/label_pr.js')
await label_pr({github, context})

View File

@@ -76,7 +76,7 @@ jobs:
- name: Run Wasm checks
run: |
sudo apt update && sudo apt install gcc-multilib
.github/scripts/check_wasm.sh
.github/assets/check_wasm.sh
riscv:
runs-on: depot-ubuntu-latest
@@ -94,7 +94,7 @@ jobs:
cache-on-failure: true
- uses: dcarbone/install-jq-action@v3
- name: Run RISC-V checks
run: .github/scripts/check_rv32imac.sh
run: .github/assets/check_rv32imac.sh
crate-checks:
name: crate-checks (${{ matrix.partition }}/${{ matrix.total_partitions }})
@@ -119,6 +119,11 @@ jobs:
name: MSRV
runs-on: depot-ubuntu-latest
timeout-minutes: 30
strategy:
matrix:
include:
- binary: reth
- binary: op-reth
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
@@ -129,7 +134,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- run: cargo build --bin reth --workspace
- run: cargo build --bin "${{ matrix.binary }}" --workspace
env:
RUSTFLAGS: -D warnings
@@ -193,9 +198,10 @@ jobs:
with:
cache-on-failure: true
- run: cargo build --bin reth --workspace
- run: cargo build --bin op-reth --workspace
env:
RUSTFLAGS: -D warnings
- run: ./docs/cli/update.sh target/debug/reth
- run: ./docs/cli/update.sh target/debug/reth target/debug/op-reth
- name: Check docs changes
run: git diff --exit-code

61
.github/workflows/prepare-reth.yml vendored Normal file
View File

@@ -0,0 +1,61 @@
name: Prepare Reth Image
on:
workflow_call:
inputs:
image_tag:
required: true
type: string
description: "Docker image tag to use"
binary_name:
required: false
type: string
default: "reth"
description: "Binary name to build (reth or op-reth)"
cargo_features:
required: false
type: string
default: "asm-keccak"
description: "Cargo features to enable"
cargo_package:
required: false
type: string
description: "Optional cargo package path"
artifact_name:
required: false
type: string
default: "artifacts"
description: "Name for the uploaded artifact"
jobs:
prepare-reth:
if: github.repository == 'paradigmxyz/reth'
timeout-minutes: 45
runs-on: depot-ubuntu-latest
steps:
- uses: actions/checkout@v6
- run: mkdir artifacts
- name: Set up Docker Buildx
uses: docker/setup-buildx-action@v3
- name: Build and export reth image
uses: docker/build-push-action@v6
with:
context: .
file: .github/assets/hive/Dockerfile
tags: ${{ inputs.image_tag }}
outputs: type=docker,dest=./artifacts/reth_image.tar
build-args: |
CARGO_BIN=${{ inputs.binary_name }}
MANIFEST_PATH=${{ inputs.cargo_package }}
FEATURES=${{ inputs.cargo_features }}
cache-from: type=gha
cache-to: type=gha,mode=max
- name: Upload reth image
id: upload
uses: actions/upload-artifact@v6
with:
name: ${{ inputs.artifact_name }}
path: ./artifacts

View File

@@ -17,9 +17,11 @@ on:
env:
REPO_NAME: ${{ github.repository_owner }}/reth
IMAGE_NAME: ${{ github.repository_owner }}/reth
OP_IMAGE_NAME: ${{ github.repository_owner }}/op-reth
REPRODUCIBLE_IMAGE_NAME: ${{ github.repository_owner }}/reth-reproducible
CARGO_TERM_COLOR: always
DOCKER_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/reth
DOCKER_OP_IMAGE_NAME_URL: https://ghcr.io/${{ github.repository_owner }}/op-reth
RUSTC_WRAPPER: "sccache"
jobs:
@@ -85,6 +87,10 @@ jobs:
os: macos-14
profile: maxperf
allow_fail: false
- target: x86_64-pc-windows-gnu
os: ubuntu-24.04
profile: maxperf
allow_fail: false
- target: riscv64gc-unknown-linux-gnu
os: ubuntu-24.04
profile: maxperf
@@ -92,6 +98,8 @@ jobs:
build:
- command: build
binary: reth
- command: op-build
binary: op-reth
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
@@ -118,7 +126,8 @@ jobs:
- name: Move binary
run: |
mkdir artifacts
mv "target/${{ matrix.configs.target }}/${{ matrix.configs.profile }}/${{ matrix.build.binary }}" ./artifacts
[[ "${{ matrix.configs.target }}" == *windows* ]] && ext=".exe"
mv "target/${{ matrix.configs.target }}/${{ matrix.configs.profile }}/${{ matrix.build.binary }}${ext}" ./artifacts
- name: Configure GPG and create artifacts
env:
@@ -235,9 +244,21 @@ jobs:
|:---:|:---:|:---:|:---|
| <img src="https://www.svgrepo.com/download/473700/linux.svg" width="50"/> | x86_64 | [reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/473700/linux.svg" width="50"/> | aarch64 | [reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/513083/windows-174.svg" width="50"/> | x86_64 | [reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/511330/apple-173.svg" width="50"/> | x86_64 | [reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/511330/apple-173.svg" width="50"/> | aarch64 | [reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/473589/docker.svg" width="50"/> | Docker | [${{ env.IMAGE_NAME }}](${{ env.DOCKER_IMAGE_NAME_URL }}) | - |
### OP-Reth
| System | Architecture | Binary | PGP Signature |
|:---:|:---:|:---:|:---|
| <img src="https://www.svgrepo.com/download/473700/linux.svg" width="50"/> | x86_64 | [op-reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-unknown-linux-gnu.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/473700/linux.svg" width="50"/> | aarch64 | [op-reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-unknown-linux-gnu.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/513083/windows-174.svg" width="50"/> | x86_64 | [op-reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-pc-windows-gnu.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/511330/apple-173.svg" width="50"/> | x86_64 | [op-reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-x86_64-apple-darwin.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/511330/apple-173.svg" width="50"/> | aarch64 | [op-reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz) | [PGP Signature](https://github.com/${{ env.REPO_NAME }}/releases/download/${{ env.VERSION }}/op-reth-${{ env.VERSION }}-aarch64-apple-darwin.tar.gz.asc) |
| <img src="https://www.svgrepo.com/download/473589/docker.svg" width="50"/> | Docker | [${{ env.OP_IMAGE_NAME }}](${{ env.DOCKER_OP_IMAGE_NAME_URL }}) | - |
ENDBODY
)
assets=()

View File

@@ -7,7 +7,6 @@ on:
jobs:
build:
if: github.repository == 'paradigmxyz/reth'
name: build reproducible binaries
runs-on: ${{ matrix.runner }}
strategy:

View File

@@ -38,7 +38,7 @@ jobs:
cache-on-failure: true
- name: Build reth
run: |
cargo install --path bin/reth
cargo install --features asm-keccak,jemalloc --path bin/reth
- name: Run headers stage
run: |
reth stage run headers --from ${{ env.FROM_BLOCK }} --to ${{ env.TO_BLOCK }} --commit --checkpoints

View File

@@ -9,7 +9,6 @@ on:
jobs:
close-issues:
if: github.repository == 'paradigmxyz/reth'
runs-on: ubuntu-latest
permissions:
issues: write

View File

@@ -17,7 +17,6 @@ concurrency:
jobs:
sync:
if: github.repository == 'paradigmxyz/reth'
name: sync (${{ matrix.chain.bin }})
runs-on: depot-ubuntu-latest
env:
@@ -33,6 +32,12 @@ jobs:
tip: "0x91c90676cab257a59cd956d7cb0bceb9b1a71d79755c23c7277a0697ccfaf8c4"
block: 100000
unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a"
- build: install-op
bin: op-reth
chain: base
tip: "0xbb9b85352c7ebca6ba8efc63bd66cecd038c92ec8ebd02e153a3e0b197e672b7"
block: 10000
unwind-target: "0x118a6e922a8c6cab221fc5adfe5056d2b72d58c6580e9c5629de55299e2cf8de"
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1

View File

@@ -17,7 +17,6 @@ concurrency:
jobs:
sync:
if: github.repository == 'paradigmxyz/reth'
name: sync (${{ matrix.chain.bin }})
runs-on: depot-ubuntu-latest
env:
@@ -33,6 +32,12 @@ jobs:
tip: "0x91c90676cab257a59cd956d7cb0bceb9b1a71d79755c23c7277a0697ccfaf8c4"
block: 100000
unwind-target: "0x52e0509d33a988ef807058e2980099ee3070187f7333aae12b64d4d675f34c5a"
- build: install-op
bin: op-reth
chain: base
tip: "0xbb9b85352c7ebca6ba8efc63bd66cecd038c92ec8ebd02e153a3e0b197e672b7"
block: 10000
unwind-target: "0x118a6e922a8c6cab221fc5adfe5056d2b72d58c6580e9c5629de55299e2cf8de"
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1

View File

@@ -26,12 +26,15 @@ jobs:
EDGE_FEATURES: ${{ matrix.storage == 'edge' && 'edge' || '' }}
strategy:
matrix:
type: [ethereum]
type: [ethereum, optimism]
storage: [stable, edge]
include:
- type: ethereum
features: asm-keccak ethereum
exclude_args: ""
- type: optimism
features: asm-keccak
exclude_args: --exclude reth --exclude reth-bench --exclude "example-*" --exclude "reth-ethereum-*" --exclude "*-ethereum"
timeout-minutes: 30
steps:
- uses: actions/checkout@v6
@@ -49,7 +52,6 @@ jobs:
- name: Run tests
run: |
cargo nextest run \
--no-fail-fast \
--features "${{ matrix.features }} $EDGE_FEATURES" --locked \
${{ matrix.exclude_args }} --workspace \
--exclude ef-tests --no-tests=warn \
@@ -88,7 +90,7 @@ jobs:
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- run: cargo nextest run --no-fail-fast --cargo-profile hivetests -p ef-tests --features "asm-keccak ef-tests"
- run: cargo nextest run --release -p ef-tests --features "asm-keccak ef-tests"
doc:
name: doc tests

36
.github/workflows/update-superchain.yml vendored Normal file
View File

@@ -0,0 +1,36 @@
name: Update Superchain Config
on:
schedule:
- cron: '0 3 * * 0'
workflow_dispatch:
permissions:
contents: write
jobs:
update-superchain:
runs-on: ubuntu-latest
steps:
- name: Checkout repository
uses: actions/checkout@v6
- name: Install required tools
run: |
sudo apt-get update
sudo apt-get install -y jq zstd qpdf yq
- name: Run fetch_superchain_config.sh
run: |
chmod +x crates/optimism/chainspec/res/fetch_superchain_config.sh
cd crates/optimism/chainspec/res
./fetch_superchain_config.sh
- name: Create Pull Request
uses: peter-evans/create-pull-request@v8
with:
commit-message: "chore: update superchain config"
title: "chore: update superchain config"
body: "This PR updates the superchain configs via scheduled workflow."
branch: "ci/update-superchain-config"
delete-branch: true

54
.github/workflows/windows.yml vendored Normal file
View File

@@ -0,0 +1,54 @@
# Windows build
name: windows
on:
push:
branches: [main]
pull_request:
branches: [main]
merge_group:
env:
RUSTC_WRAPPER: "sccache"
jobs:
check-reth:
runs-on: depot-ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
with:
target: x86_64-pc-windows-gnu
- uses: taiki-e/install-action@cross
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: mingw-w64
run: sudo apt-get install -y mingw-w64
- name: Check Reth
run: cargo check --target x86_64-pc-windows-gnu
check-op-reth:
runs-on: depot-ubuntu-latest
timeout-minutes: 60
steps:
- uses: actions/checkout@v6
- uses: rui314/setup-mold@v1
- uses: dtolnay/rust-toolchain@stable
with:
target: x86_64-pc-windows-gnu
- uses: taiki-e/install-action@cross
- uses: mozilla-actions/sccache-action@v0.0.9
- uses: Swatinem/rust-cache@v2
with:
cache-on-failure: true
- name: mingw-w64
run: sudo apt-get install -y mingw-w64
- name: Check OP-Reth
run: cargo check -p op-reth --target x86_64-pc-windows-gnu

View File

@@ -24,7 +24,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
- **Modularity**: Each crate can be used as a standalone library
- **Performance**: Extensive use of parallelism, memory-mapped I/O, and optimized data structures
- **Extensibility**: Traits and generic types allow for different chain implementations
- **Extensibility**: Traits and generic types allow for different implementations (Ethereum, Optimism, etc.)
- **Type Safety**: Strong typing throughout with minimal use of dynamic dispatch
## Development Workflow
@@ -38,7 +38,7 @@ Reth is a high-performance Ethereum execution client written in Rust, focusing o
2. **Linting**: Run clippy with all features
```bash
cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --lib --examples --tests --benches --all-features --locked
```
3. **Testing**: Use nextest for faster test execution
@@ -169,16 +169,18 @@ Based on PR patterns, avoid:
Before submitting changes, ensure:
1. **Format Check**: `cargo +nightly fmt --all --check`
2. **Clippy**: No warnings
2. **Clippy**: No warnings with `RUSTFLAGS="-D warnings"`
3. **Tests Pass**: All unit and integration tests
4. **Documentation**: Update relevant docs and add doc comments with `cargo docs --document-private-items`
5. **Commit Messages**: Follow conventional format (feat:, fix:, chore:, etc.)
### Opening PRs against <https://github.com/paradigmxyz/reth>
Label PRs appropriately, first check the available labels and then apply the relevant ones:
* when changes are RPC related, add A-rpc label
* when changes are docs related, add C-docs label
* when changes are optimism related (e.g. new feature or exclusive changes to crates/optimism), add A-op-reth label
* ... and so on, check the available labels for more options.
* if being tasked to open a pr, ensure that all changes are properly formatted: `cargo +nightly fmt --all`
@@ -232,7 +234,7 @@ Tests often need expansion for:
Common refactoring pattern:
- Replace concrete types with generics
- Add trait bounds for flexibility
- Enable reuse across different chain types
- Enable reuse across different chain types (Ethereum, Optimism)
#### When to Comment
@@ -347,11 +349,11 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
}
```
5. **Run checks** (IMPORTANT!):
5. **Run checks**:
```bash
cargo +nightly fmt --all
cargo clippy --workspace --all-features # Make sure WHOLE WORKSPACE compiles!
cargo nextest run -p reth-discv4
cargo clippy --all-features
cargo test -p reth-discv4
```
6. **Commit with clear message**:
@@ -372,7 +374,7 @@ Let's say you want to fix a bug where external IP resolution fails on startup:
cargo +nightly fmt --all
# Run lints
cargo +nightly clippy --workspace --all-features
RUSTFLAGS="-D warnings" cargo +nightly clippy --workspace --all-features --locked
# Run tests
cargo nextest run --workspace
@@ -381,7 +383,7 @@ cargo nextest run --workspace
cargo bench --bench bench_name
# Build optimized binary
cargo build --release
cargo build --release --features "jemalloc asm-keccak"
# Check compilation for all features
cargo check --workspace --all-features

2530
Cargo.lock generated

File diff suppressed because it is too large Load Diff

View File

@@ -72,6 +72,20 @@ members = [
"crates/node/events/",
"crates/node/metrics",
"crates/node/types",
"crates/optimism/bin",
"crates/optimism/chainspec",
"crates/optimism/cli",
"crates/optimism/consensus",
"crates/optimism/evm/",
"crates/optimism/flashblocks/",
"crates/optimism/hardforks/",
"crates/optimism/node/",
"crates/optimism/payload/",
"crates/optimism/primitives/",
"crates/optimism/reth/",
"crates/optimism/rpc/",
"crates/optimism/storage",
"crates/optimism/txpool/",
"crates/payload/basic/",
"crates/payload/builder/",
"crates/payload/builder-primitives/",
@@ -83,6 +97,8 @@ members = [
"crates/prune/db",
"crates/prune/prune",
"crates/prune/types",
"crates/ress/protocol",
"crates/ress/provider",
"crates/revm/",
"crates/rpc/ipc/",
"crates/rpc/rpc-api/",
@@ -99,6 +115,7 @@ members = [
"crates/stages/api/",
"crates/stages/stages/",
"crates/stages/types/",
"crates/stateless",
"crates/static-file/static-file",
"crates/static-file/types/",
"crates/storage/codecs/",
@@ -122,11 +139,13 @@ members = [
"crates/trie/db",
"crates/trie/parallel/",
"crates/trie/sparse",
"crates/trie/sparse-parallel/",
"crates/trie/trie",
"examples/beacon-api-sidecar-fetcher/",
"examples/beacon-api-sse/",
"examples/bsc-p2p",
"examples/custom-dev-node/",
"examples/custom-node/",
"examples/custom-engine-types/",
"examples/custom-evm/",
"examples/custom-hardforks/",
@@ -135,7 +154,10 @@ members = [
"examples/custom-payload-builder/",
"examples/custom-rlpx-subprotocol",
"examples/custom-rpc-middleware",
"examples/custom-node",
"examples/db-access",
"examples/engine-api-access",
"examples/exex-hello-world",
"examples/exex-subscription",
"examples/exex-test",
"examples/full-contract-state",
@@ -146,6 +168,7 @@ members = [
"examples/node-builder-api/",
"examples/node-custom-rpc/",
"examples/node-event-hooks/",
"examples/op-db-access/",
"examples/polygon-p2p/",
"examples/rpc-db/",
"examples/precompile-cache/",
@@ -306,11 +329,6 @@ inherits = "release"
lto = "fat"
codegen-units = 1
[profile.maxperf-symbols]
inherits = "maxperf"
debug = "full"
strip = "none"
[profile.reproducible]
inherits = "release"
panic = "abort"
@@ -319,6 +337,7 @@ incremental = false
[workspace.dependencies]
# reth
op-reth = { path = "crates/optimism/bin" }
reth = { path = "bin/reth" }
reth-storage-rpc-provider = { path = "crates/storage/rpc-provider" }
reth-basic-payload-builder = { path = "crates/payload/basic" }
@@ -367,6 +386,7 @@ reth-ethereum = { path = "crates/ethereum/reth" }
reth-etl = { path = "crates/etl" }
reth-evm = { path = "crates/evm/evm", default-features = false }
reth-evm-ethereum = { path = "crates/ethereum/evm", default-features = false }
reth-optimism-evm = { path = "crates/optimism/evm", default-features = false }
reth-execution-errors = { path = "crates/evm/execution-errors", default-features = false }
reth-execution-types = { path = "crates/evm/execution-types", default-features = false }
reth-exex = { path = "crates/exex/exex" }
@@ -393,7 +413,18 @@ reth-node-ethereum = { path = "crates/ethereum/node" }
reth-node-ethstats = { path = "crates/node/ethstats" }
reth-node-events = { path = "crates/node/events" }
reth-node-metrics = { path = "crates/node/metrics" }
reth-optimism-node = { path = "crates/optimism/node" }
reth-node-types = { path = "crates/node/types" }
reth-op = { path = "crates/optimism/reth", default-features = false }
reth-optimism-chainspec = { path = "crates/optimism/chainspec", default-features = false }
reth-optimism-cli = { path = "crates/optimism/cli", default-features = false }
reth-optimism-consensus = { path = "crates/optimism/consensus", default-features = false }
reth-optimism-forks = { path = "crates/optimism/hardforks", default-features = false }
reth-optimism-payload-builder = { path = "crates/optimism/payload" }
reth-optimism-primitives = { path = "crates/optimism/primitives", default-features = false }
reth-optimism-rpc = { path = "crates/optimism/rpc" }
reth-optimism-storage = { path = "crates/optimism/storage" }
reth-optimism-txpool = { path = "crates/optimism/txpool" }
reth-payload-builder = { path = "crates/payload/builder" }
reth-payload-builder-primitives = { path = "crates/payload/builder-primitives" }
reth-payload-primitives = { path = "crates/payload/primitives" }
@@ -414,11 +445,13 @@ reth-rpc-engine-api = { path = "crates/rpc/rpc-engine-api" }
reth-rpc-eth-api = { path = "crates/rpc/rpc-eth-api" }
reth-rpc-eth-types = { path = "crates/rpc/rpc-eth-types", default-features = false }
reth-rpc-layer = { path = "crates/rpc/rpc-layer" }
reth-optimism-flashblocks = { path = "crates/optimism/flashblocks" }
reth-rpc-server-types = { path = "crates/rpc/rpc-server-types" }
reth-rpc-convert = { path = "crates/rpc/rpc-convert" }
reth-stages = { path = "crates/stages/stages" }
reth-stages-api = { path = "crates/stages/api" }
reth-stages-types = { path = "crates/stages/types", default-features = false }
reth-stateless = { path = "crates/stateless", default-features = false }
reth-static-file = { path = "crates/static-file/static-file" }
reth-static-file-types = { path = "crates/static-file/types", default-features = false }
reth-storage-api = { path = "crates/storage/storage-api", default-features = false }
@@ -434,7 +467,10 @@ reth-trie-common = { path = "crates/trie/common", default-features = false }
reth-trie-db = { path = "crates/trie/db" }
reth-trie-parallel = { path = "crates/trie/parallel" }
reth-trie-sparse = { path = "crates/trie/sparse", default-features = false }
reth-trie-sparse-parallel = { path = "crates/trie/sparse-parallel" }
reth-zstd-compressors = { path = "crates/storage/zstd-compressors", default-features = false }
reth-ress-protocol = { path = "crates/ress/protocol" }
reth-ress-provider = { path = "crates/ress/provider" }
# revm
revm = { version = "34.0.0", default-features = false }
@@ -445,52 +481,52 @@ revm-primitives = { version = "22.0.0", default-features = false }
revm-interpreter = { version = "32.0.0", default-features = false }
revm-database-interface = { version = "9.0.0", default-features = false }
op-revm = { version = "15.0.0", default-features = false }
revm-inspectors = "0.34.2"
revm-inspectors = "0.34.0"
# eth
alloy-dyn-abi = "1.5.6"
alloy-primitives = { version = "1.5.6", default-features = false, features = ["map-foldhash"] }
alloy-sol-types = { version = "1.5.6", default-features = false }
alloy-chains = { version = "0.2.5", default-features = false }
alloy-dyn-abi = "1.4.3"
alloy-eip2124 = { version = "0.2.0", default-features = false }
alloy-eip7928 = { version = "0.3.0", default-features = false }
alloy-evm = { version = "0.27.2", default-features = false }
alloy-rlp = { version = "0.3.13", default-features = false, features = ["core-net"] }
alloy-trie = { version = "0.9.4", default-features = false }
alloy-evm = { version = "0.26.3", default-features = false }
alloy-primitives = { version = "1.5.0", default-features = false, features = ["map-foldhash"] }
alloy-rlp = { version = "0.3.10", default-features = false, features = ["core-net"] }
alloy-sol-macro = "1.5.0"
alloy-sol-types = { version = "1.5.0", default-features = false }
alloy-trie = { version = "0.9.1", default-features = false }
alloy-hardforks = "0.4.5"
alloy-consensus = { version = "1.6.3", default-features = false }
alloy-contract = { version = "1.6.3", default-features = false }
alloy-eips = { version = "1.6.3", default-features = false }
alloy-genesis = { version = "1.6.3", default-features = false }
alloy-json-rpc = { version = "1.6.3", default-features = false }
alloy-network = { version = "1.6.3", default-features = false }
alloy-network-primitives = { version = "1.6.3", default-features = false }
alloy-provider = { version = "1.6.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.6.3", default-features = false }
alloy-rpc-client = { version = "1.6.3", default-features = false }
alloy-rpc-types = { version = "1.6.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.6.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.6.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.6.3", default-features = false }
alloy-rpc-types-debug = { version = "1.6.3", default-features = false }
alloy-rpc-types-engine = { version = "1.6.3", default-features = false }
alloy-rpc-types-eth = { version = "1.6.3", default-features = false }
alloy-rpc-types-mev = { version = "1.6.3", default-features = false }
alloy-rpc-types-trace = { version = "1.6.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.6.3", default-features = false }
alloy-serde = { version = "1.6.3", default-features = false }
alloy-signer = { version = "1.6.3", default-features = false }
alloy-signer-local = { version = "1.6.3", default-features = false }
alloy-transport = { version = "1.6.3" }
alloy-transport-http = { version = "1.6.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.6.3", default-features = false }
alloy-transport-ws = { version = "1.6.3", default-features = false }
alloy-consensus = { version = "1.4.3", default-features = false }
alloy-contract = { version = "1.4.3", default-features = false }
alloy-eips = { version = "1.4.3", default-features = false }
alloy-genesis = { version = "1.4.3", default-features = false }
alloy-json-rpc = { version = "1.4.3", default-features = false }
alloy-network = { version = "1.4.3", default-features = false }
alloy-network-primitives = { version = "1.4.3", default-features = false }
alloy-provider = { version = "1.4.3", features = ["reqwest", "debug-api"], default-features = false }
alloy-pubsub = { version = "1.4.3", default-features = false }
alloy-rpc-client = { version = "1.4.3", default-features = false }
alloy-rpc-types = { version = "1.4.3", features = ["eth"], default-features = false }
alloy-rpc-types-admin = { version = "1.4.3", default-features = false }
alloy-rpc-types-anvil = { version = "1.4.3", default-features = false }
alloy-rpc-types-beacon = { version = "1.4.3", default-features = false }
alloy-rpc-types-debug = { version = "1.4.3", default-features = false }
alloy-rpc-types-engine = { version = "1.4.3", default-features = false }
alloy-rpc-types-eth = { version = "1.4.3", default-features = false }
alloy-rpc-types-mev = { version = "1.4.3", default-features = false }
alloy-rpc-types-trace = { version = "1.4.3", default-features = false }
alloy-rpc-types-txpool = { version = "1.4.3", default-features = false }
alloy-serde = { version = "1.4.3", default-features = false }
alloy-signer = { version = "1.4.3", default-features = false }
alloy-signer-local = { version = "1.4.3", default-features = false }
alloy-transport = { version = "1.4.3" }
alloy-transport-http = { version = "1.4.3", features = ["reqwest-rustls-tls"], default-features = false }
alloy-transport-ipc = { version = "1.4.3", default-features = false }
alloy-transport-ws = { version = "1.4.3", default-features = false }
# op
alloy-op-evm = { version = "0.27.2", default-features = false }
alloy-op-evm = { version = "0.26.3", default-features = false }
alloy-op-hardforks = "0.4.4"
op-alloy-rpc-types = { version = "0.23.1", default-features = false }
op-alloy-rpc-types-engine = { version = "0.23.1", default-features = false }
@@ -508,7 +544,7 @@ backon = { version = "1.2", default-features = false, features = ["std-blocking-
bincode = "1.3"
bitflags = "2.4"
boyer-moore-magiclen = "0.2.16"
bytes = { version = "1.11.1", default-features = false }
bytes = { version = "1.5", default-features = false }
brotli = "8"
cfg-if = "1.0"
clap = "4"
@@ -525,9 +561,9 @@ humantime-serde = "1.1"
itertools = { version = "0.14", default-features = false }
linked_hash_set = "0.1"
lz4 = "1.28.1"
modular-bitfield = "0.13.1"
modular-bitfield = "0.11.2"
notify = { version = "8.0.0", default-features = false, features = ["macos_fsevent"] }
nybbles = { version = "0.4.8", default-features = false }
nybbles = { version = "0.4.2", default-features = false }
once_cell = { version = "1.19", default-features = false, features = ["critical-section"] }
parking_lot = "0.12"
paste = "1.0"
@@ -547,20 +583,20 @@ strum_macros = "0.27"
syn = "2.0"
thiserror = { version = "2.0.0", default-features = false }
tar = "0.4.44"
tracing = { version = "0.1.0", default-features = false, features = ["attributes"] }
tracing = { version = "0.1.0", default-features = false }
tracing-appender = "0.2"
url = { version = "2.3", default-features = false }
zstd = "0.13"
byteorder = "1"
fixed-cache = { version = "0.1.7", features = ["stats"] }
mini-moka = "0.10"
moka = "0.12"
tar-no-std = { version = "0.4.2", default-features = false }
miniz_oxide = { version = "0.9.0", default-features = false }
tar-no-std = { version = "0.3.2", default-features = false }
miniz_oxide = { version = "0.8.4", default-features = false }
chrono = "0.4.41"
# metrics
metrics = "0.24.0"
metrics-derive = "0.1.1"
metrics-derive = "0.1"
metrics-exporter-prometheus = { version = "0.18.0", default-features = false }
metrics-process = "2.1.0"
metrics-util = { default-features = false, version = "0.20.0" }
@@ -572,7 +608,7 @@ quote = "1.0"
# tokio
tokio = { version = "1.44.2", default-features = false }
tokio-stream = "0.1.11"
tokio-tungstenite = "0.28.0"
tokio-tungstenite = "0.26.2"
tokio-util = { version = "0.7.4", features = ["codec"] }
# async
@@ -585,7 +621,7 @@ futures-util = { version = "0.3", default-features = false }
hyper = "1.3"
hyper-util = "0.1.5"
pin-project = "1.0.12"
reqwest = { version = "0.12", default-features = false, features = ["rustls-tls", "rustls-tls-native-roots", "stream"] }
reqwest = { version = "0.12", default-features = false }
tracing-futures = "0.2"
tower = "0.5"
tower-http = "0.6"
@@ -605,6 +641,7 @@ jsonrpsee-types = "0.26.0"
http = "1.0"
http-body = "1.0"
http-body-util = "0.1.2"
jsonwebtoken = "9"
proptest-arbitrary-interop = "0.1.0"
# crypto
@@ -618,7 +655,7 @@ rand_08 = { package = "rand", version = "0.8" }
c-kzg = "2.1.5"
# config
toml = "0.9"
toml = "0.8"
# rocksdb
rocksdb = { version = "0.24" }
@@ -634,19 +671,19 @@ tracing-opentelemetry = "0.32"
# misc-testing
arbitrary = "1.3"
assert_matches = "1.5.0"
criterion = { package = "codspeed-criterion-compat", version = "4.3" }
criterion = { package = "codspeed-criterion-compat", version = "2.7" }
insta = "1.41"
proptest = "1.7"
proptest-derive = "0.7"
proptest-derive = "0.5"
similar-asserts = { version = "1.5.0", features = ["serde"] }
tempfile = "3.20"
test-fuzz = "7"
rstest = "0.26.1"
rstest = "0.24.0"
test-case = "3"
# ssz encoding
ethereum_ssz = "0.10.1"
ethereum_ssz_derive = "0.10.1"
ethereum_ssz = "0.9.0"
ethereum_ssz_derive = "0.9.0"
# allocators
jemalloc_pprof = { version = "0.8", default-features = false }
@@ -658,14 +695,14 @@ snmalloc-rs = { version = "0.3.7", features = ["build_cc"] }
aes = "0.8.1"
ahash = "0.8"
anyhow = "1.0"
bindgen = { version = "0.72", default-features = false }
block-padding = "0.3"
bindgen = { version = "0.71", default-features = false }
block-padding = "0.3.2"
cc = "1.2.15"
cipher = "0.4.3"
comfy-table = "7.0"
concat-kdf = "0.1.0"
crossbeam-channel = "0.5.13"
crossterm = "0.29.0"
crossterm = "0.28.0"
csv = "1.3.0"
ctrlc = "3.4"
ctr = "0.9.2"
@@ -678,7 +715,7 @@ hmac = "0.12.1"
human_bytes = "0.4.1"
indexmap = "2"
interprocess = "2.2.0"
lz4_flex = { version = "0.12", default-features = false }
lz4_flex = { version = "0.11", default-features = false }
memmap2 = "0.9.4"
mev-share-sse = { version = "0.5.0", default-features = false }
num-traits = "0.2.15"
@@ -686,17 +723,17 @@ page_size = "0.6.0"
parity-scale-codec = "3.2.1"
plain_hasher = "0.2"
pretty_assertions = "1.4"
ratatui = { version = "0.30", default-features = false }
ringbuffer = "0.16.0"
ratatui = { version = "0.29", default-features = false }
ringbuffer = "0.15.0"
rmp-serde = "1.3"
roaring = "0.11.3"
roaring = "0.10.2"
rolling-file = "0.2.0"
sha3 = "0.10.5"
snap = "1.1.1"
socket2 = { version = "0.6", default-features = false }
sysinfo = { version = "0.38", default-features = false }
socket2 = { version = "0.5", default-features = false }
sysinfo = { version = "0.33", default-features = false }
tracing-journald = "0.3"
tracing-logfmt = "=0.3.5"
tracing-logfmt = "0.3.3"
tracing-samply = "0.1"
tracing-subscriber = { version = "0.3", default-features = false }
tracing-tracy = "0.11"
@@ -711,28 +748,6 @@ vergen-git2 = "9.1.0"
ipnet = "2.11"
[patch.crates-io]
# revm staging patches
revm = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
op-revm = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-bytecode = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-context = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-context-interface = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-database = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-database-interface = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-handler = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-inspector = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-interpreter = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-precompile = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-primitives = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
revm-state = { git = "https://github.com/bluealloy/revm", rev = "33330a285e621b9170c30a21cfea9ab32e2a2169" }
# revm-inspectors staging patch
revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "e80e2eab72dfa18011e6a99abd37027290a46e83" }
# alloy-evm staging patches
alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea0279c03ca27b1c26f26ac19fbefb" }
alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea0279c03ca27b1c26f26ac19fbefb" }
# alloy-consensus = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-contract = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
# alloy-eips = { git = "https://github.com/alloy-rs/alloy", rev = "3049f232fbb44d1909883e154eb38ec5962f53a3" }
@@ -775,10 +790,39 @@ alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "742dc14749ea027
# jsonrpsee-http-client = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# jsonrpsee-types = { git = "https://github.com/paradigmxyz/jsonrpsee", branch = "matt/make-rpc-service-pub" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "df124c0" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "a69f0b45a6b0286e16072cb8399e02ce6ceca353" }
# revm-inspectors = { git = "https://github.com/paradigmxyz/revm-inspectors", rev = "3020ea8" }
# alloy-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# alloy-op-evm = { git = "https://github.com/alloy-rs/evm", rev = "072c248" }
# Patched by patch-alloy.sh
alloy-consensus = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-contract = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-eips = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-genesis = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-json-rpc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-network-primitives = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-provider = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-pubsub = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-client = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-admin = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-anvil = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-beacon = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-debug = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-engine = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-eth = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-mev = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-trace = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-rpc-types-txpool = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-serde = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-signer-local = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-http = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ipc = { git = "https://github.com/alloy-rs/alloy", branch = "main" }
alloy-transport-ws = { git = "https://github.com/alloy-rs/alloy", branch = "main" }

View File

@@ -15,6 +15,15 @@ pre-build = [
"apt-get update && apt-get install --assume-yes --no-install-recommends llvm-dev libclang-dev clang",
]
[target.x86_64-pc-windows-gnu]
# Why do we need a custom Dockerfile on Windows:
# 1. `reth-libmdbx` stopped working with MinGW 9.3 that cross image comes with.
# 2. To be able to update the version of MinGW, we need to also update the Ubuntu that the image is based on.
#
# Also see https://github.com/cross-rs/cross/issues/1667
# Inspired by https://github.com/cross-rs/cross/blob/9e2298e17170655342d3248a9c8ac37ef92ba38f/docker/Dockerfile.x86_64-pc-windows-gnu#L51
dockerfile = "./Dockerfile.x86_64-pc-windows-gnu"
[target.riscv64gc-unknown-linux-gnu]
image = "ubuntu:24.04"
pre-build = [

15
Dockerfile.cross Normal file
View File

@@ -0,0 +1,15 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/reth /usr/local/bin/reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth"]

View File

@@ -1,90 +0,0 @@
# syntax=docker/dockerfile:1
# Dockerfile for reth, optimized for Depot builds
# Usage:
# reth: --build-arg BINARY=reth
FROM rust:1 AS builder
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
RUN apt-get update && apt-get install -y libclang-dev pkg-config
# Install sccache for compilation caching
RUN cargo install sccache --locked
ENV RUSTC_WRAPPER=sccache
ENV SCCACHE_DIR=/sccache
ENV SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev
# Binary to build
ARG BINARY=reth
# Manifest path for the binary
ARG MANIFEST_PATH=bin/reth
# Build profile, release by default
ARG BUILD_PROFILE=release
ENV BUILD_PROFILE=$BUILD_PROFILE
# Extra Cargo flags (can be overridden, otherwise set per-platform below)
ARG RUSTFLAGS=""
# Extra Cargo features
ARG FEATURES=""
ENV FEATURES=$FEATURES
# Git info for vergen (since .git is excluded from Docker context)
ARG VERGEN_GIT_SHA=""
ARG VERGEN_GIT_DESCRIBE=""
ARG VERGEN_GIT_DIRTY="false"
ENV VERGEN_GIT_SHA=$VERGEN_GIT_SHA
ENV VERGEN_GIT_DESCRIBE=$VERGEN_GIT_DESCRIBE
ENV VERGEN_GIT_DIRTY=$VERGEN_GIT_DIRTY
# Build application
# Platform-specific RUSTFLAGS: amd64 uses x86-64-v3 (Haswell+) with pclmulqdq for rocksdb
ARG TARGETPLATFORM
COPY --exclude=.git . .
RUN --mount=type=secret,id=DEPOT_TOKEN,env=SCCACHE_WEBDAV_TOKEN \
--mount=type=cache,target=/usr/local/cargo/registry,sharing=shared \
--mount=type=cache,target=/usr/local/cargo/git,sharing=shared \
--mount=type=cache,target=$SCCACHE_DIR,sharing=shared \
export RUSTC_WRAPPER=sccache SCCACHE_WEBDAV_ENDPOINT=https://cache.depot.dev SCCACHE_DIR=/sccache && \
sccache --start-server && \
if [ -n "$RUSTFLAGS" ]; then \
export RUSTFLAGS="$RUSTFLAGS"; \
elif [ "$TARGETPLATFORM" = "linux/amd64" ]; then \
export RUSTFLAGS="-C target-cpu=x86-64-v3 -C target-feature=+pclmulqdq"; \
fi && \
cargo build --profile $BUILD_PROFILE --features "$FEATURES" --locked --bin $BINARY --manifest-path $MANIFEST_PATH/Cargo.toml && \
sccache --show-stats
# Copy binary to a known location (ARG not resolved in COPY)
# Note: Custom profiles like maxperf/profiling output to target/<profile>/, not target/release/
RUN cp /app/target/$BUILD_PROFILE/$BINARY /app/binary || \
cp /app/target/release/$BINARY /app/binary
FROM ubuntu:24.04 AS runtime
WORKDIR /app
# Binary name for entrypoint
ARG BINARY=reth
# Install runtime dependencies
RUN apt-get update && \
apt-get install -y --no-install-recommends ca-certificates && \
rm -rf /var/lib/apt/lists/*
# Copy binary from build stage and create canonical symlink for entrypoint
COPY --from=builder /app/binary /usr/local/bin/
RUN mv /usr/local/bin/binary /usr/local/bin/$BINARY && \
ln -s /usr/local/bin/$BINARY /usr/local/bin/reth-binary && \
chmod +x /usr/local/bin/$BINARY
# Copy licenses
COPY LICENSE-* ./
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/reth-binary"]

View File

@@ -0,0 +1,79 @@
FROM ubuntu:24.04 AS cross-base
ENV DEBIAN_FRONTEND=noninteractive
# Use HTTPS for package sources
RUN apt-get update && apt-get install --assume-yes --no-install-recommends ca-certificates
RUN find /etc/apt/ -type f \( -name '*.list' -o -name '*.sources' \) -exec sed -i 's|http://|https://|g' {} +
# Configure APT retries and timeouts to handle network issues
RUN echo 'Acquire::Retries \"3\";' > /etc/apt/apt.conf.d/80-retries && \
echo 'Acquire::http::Timeout \"60\";' >> /etc/apt/apt.conf.d/80-retries && \
echo 'Acquire::ftp::Timeout \"60\";' >> /etc/apt/apt.conf.d/80-retries
# configure fallback mirrors
RUN sed -i 's|URIs: https://archive.ubuntu.com/ubuntu/|URIs: https://mirror.cov.ukservers.com/ubuntu/ https://archive.ubuntu.com/ubuntu/ https://mirror.ox.ac.uk/sites/archive.ubuntu.com/ubuntu/|g' /etc/apt/sources.list.d/ubuntu.sources
RUN apt-get update && apt-get install --assume-yes --no-install-recommends git
RUN git clone https://github.com/cross-rs/cross /cross
WORKDIR /cross/docker
RUN git checkout baf457efc2555225af47963475bd70e8d2f5993f
# xargo doesn't work with Rust 1.89 and higher: https://github.com/cross-rs/cross/issues/1701.
#
# When this PR https://github.com/cross-rs/cross/pull/1580 is merged,
# we can update the checkout above and remove this replacement.
RUN sed -i 's|sh rustup-init.sh -y --no-modify-path --profile minimal|sh rustup-init.sh -y --no-modify-path --profile minimal --default-toolchain=1.88.0|' xargo.sh
RUN cp common.sh lib.sh / && /common.sh
RUN cp cmake.sh / && /cmake.sh
RUN cp xargo.sh / && /xargo.sh
FROM cross-base AS build
RUN apt-get install --assume-yes --no-install-recommends libz-mingw-w64-dev g++-mingw-w64-x86-64 gfortran-mingw-w64-x86-64
# Install Wine using OpenSUSE repository because official one is often lagging behind
RUN dpkg --add-architecture i386 && \
apt-get install --assume-yes --no-install-recommends wget gpg && \
mkdir -pm755 /etc/apt/keyrings && curl -fsSL \
https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_24.04/Release.key \
| tee /etc/apt/keyrings/obs-winehq.key >/dev/null && \
echo "deb [arch=amd64,i386 signed-by=/etc/apt/keyrings/obs-winehq.key] \
https://download.opensuse.org/repositories/Emulators:/Wine:/Debian/xUbuntu_24.04/ ./" \
| tee /etc/apt/sources.list.d/obs-winehq.list && \
apt-get update && apt-get install --assume-yes --install-recommends winehq-stable
# run-detectors are responsible for calling the correct interpreter for exe
# files. For some reason it does not work inside a docker container (it works
# fine in the host). So we replace the usual paths of run-detectors to run wine
# directly. This only affects the guest, we are not messing up with the host.
#
# See /usr/share/doc/binfmt-support/detectors
RUN mkdir -p /usr/lib/binfmt-support/ && \
rm -f /usr/lib/binfmt-support/run-detectors /usr/bin/run-detectors && \
ln -s /usr/bin/wine /usr/lib/binfmt-support/run-detectors && \
ln -s /usr/bin/wine /usr/bin/run-detectors
RUN cp windows-entry.sh /
ENTRYPOINT ["/windows-entry.sh"]
RUN cp toolchain.cmake /opt/toolchain.cmake
# for why we always link with pthread support, see:
# https://github.com/cross-rs/cross/pull/1123#issuecomment-1312287148
ENV CROSS_TOOLCHAIN_PREFIX=x86_64-w64-mingw32-
ENV CROSS_TOOLCHAIN_SUFFIX=-posix
ENV CROSS_SYSROOT=/usr/x86_64-w64-mingw32
ENV CROSS_TARGET_RUNNER="env -u CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER wine"
ENV CARGO_TARGET_X86_64_PC_WINDOWS_GNU_LINKER="$CROSS_TOOLCHAIN_PREFIX"gcc"$CROSS_TOOLCHAIN_SUFFIX" \
CARGO_TARGET_X86_64_PC_WINDOWS_GNU_RUNNER="$CROSS_TARGET_RUNNER" \
AR_x86_64_pc_windows_gnu="$CROSS_TOOLCHAIN_PREFIX"ar \
CC_x86_64_pc_windows_gnu="$CROSS_TOOLCHAIN_PREFIX"gcc"$CROSS_TOOLCHAIN_SUFFIX" \
CXX_x86_64_pc_windows_gnu="$CROSS_TOOLCHAIN_PREFIX"g++"$CROSS_TOOLCHAIN_SUFFIX" \
CMAKE_TOOLCHAIN_FILE_x86_64_pc_windows_gnu=/opt/toolchain.cmake \
BINDGEN_EXTRA_CLANG_ARGS_x86_64_pc_windows_gnu="--sysroot=$CROSS_SYSROOT -idirafter/usr/include" \
CROSS_CMAKE_SYSTEM_NAME=Windows \
CROSS_CMAKE_SYSTEM_PROCESSOR=AMD64 \
CROSS_CMAKE_CRT=gnu \
CROSS_CMAKE_OBJECT_FLAGS="-ffunction-sections -fdata-sections -m64"

46
DockerfileOp Normal file
View File

@@ -0,0 +1,46 @@
FROM lukemathwalker/cargo-chef:latest-rust-1 AS chef
WORKDIR /app
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
RUN apt-get update && apt-get -y upgrade && apt-get install -y libclang-dev pkg-config
# Builds a cargo-chef plan
FROM chef AS planner
COPY . .
RUN cargo chef prepare --recipe-path recipe.json
FROM chef AS builder
COPY --from=planner /app/recipe.json recipe.json
ARG BUILD_PROFILE=maxperf
ENV BUILD_PROFILE=$BUILD_PROFILE
ARG RUSTFLAGS=""
ENV RUSTFLAGS="$RUSTFLAGS"
ARG FEATURES=""
ENV FEATURES=$FEATURES
RUN cargo chef cook --profile $BUILD_PROFILE --features "$FEATURES" --recipe-path recipe.json --manifest-path /app/crates/optimism/bin/Cargo.toml
COPY . .
RUN cargo build --profile $BUILD_PROFILE --features "$FEATURES" --bin op-reth --manifest-path /app/crates/optimism/bin/Cargo.toml
RUN ls -la /app/target/$BUILD_PROFILE/op-reth
RUN cp /app/target/$BUILD_PROFILE/op-reth /app/op-reth
FROM ubuntu AS runtime
RUN apt-get update && \
apt-get install -y ca-certificates libssl-dev pkg-config strace && \
rm -rf /var/lib/apt/lists/*
WORKDIR /app
COPY --from=builder /app/op-reth /usr/local/bin/
RUN chmod +x /usr/local/bin/op-reth
COPY LICENSE-* ./
EXPOSE 30303 30303/udp 9001 8545 8546 7545 8551
ENTRYPOINT ["/usr/local/bin/op-reth"]

15
DockerfileOp.cross Normal file
View File

@@ -0,0 +1,15 @@
# This image is meant to enable cross-architecture builds.
# It assumes the reth binary has already been compiled for `$TARGETPLATFORM` and is
# locatable in `./dist/bin/$TARGETARCH`
FROM --platform=$TARGETPLATFORM ubuntu:22.04
LABEL org.opencontainers.image.source=https://github.com/paradigmxyz/reth
LABEL org.opencontainers.image.licenses="MIT OR Apache-2.0"
# Filled by docker buildx
ARG TARGETARCH
COPY ./dist/bin/$TARGETARCH/op-reth /usr/local/bin/op-reth
EXPOSE 30303 30303/udp 9001 8545 8546
ENTRYPOINT ["/usr/local/bin/op-reth"]

View File

@@ -24,3 +24,18 @@
parameters.
- Update version specific validation checks in the `EngineValidator` trait.
## Op-Reth changes
### Updates to the engine API
Opstack tries to be as close to the L1 engine API as much as possible. Isthmus (Prague equivalent) introduced the first
deviation from the L1 engine API with an additional field in the `ExecutionPayload`. For this reason the op engine API
has its own server traits `OpEngineApi`.
Adding a new versioned endpoint requires the same changes as for L1 just for the dedicated OP types.
### Hardforks
Opstack has dedicated hardforks (e.g. Isthmus), that can be entirely opstack specific (e.g. Holocene) or can be an L1
equivalent hardfork. Since opstack sticks to the L1 header primitive, a new L1 equivalent hardfork also requires new
equivalent consensus checks. For this reason these `OpHardfork` must be mapped to L1 `EthereumHardfork`, for example:
`OpHardfork::Isthmus` corresponds to `EthereumHardfork::Prague`. These mappings must be defined in the `ChainSpec`.

210
Makefile
View File

@@ -12,7 +12,12 @@ FULL_DB_TOOLS_DIR := $(shell pwd)/$(DB_TOOLS_DIR)/
CARGO_TARGET_DIR ?= target
# List of features to use when building. Can be overridden via the environment.
FEATURES ?=
# No jemalloc on Windows
ifeq ($(OS),Windows_NT)
FEATURES ?= asm-keccak min-debug-logs
else
FEATURES ?= jemalloc asm-keccak min-debug-logs
endif
# Cargo profile for builds. Default is for local builds, CI uses an override.
PROFILE ?= release
@@ -30,6 +35,9 @@ EEST_TESTS_TAG := v4.5.0
EEST_TESTS_URL := https://github.com/ethereum/execution-spec-tests/releases/download/$(EEST_TESTS_TAG)/fixtures_stable.tar.gz
EEST_TESTS_DIR := ./testing/ef-tests/execution-spec-tests
# The docker image name
DOCKER_IMAGE_NAME ?= ghcr.io/paradigmxyz/reth
##@ Help
.PHONY: help
@@ -45,6 +53,13 @@ install: ## Build and install the reth binary under `$(CARGO_HOME)/bin`.
--profile "$(PROFILE)" \
$(CARGO_INSTALL_EXTRA_FLAGS)
.PHONY: install-op
install-op: ## Build and install the op-reth binary under `$(CARGO_HOME)/bin`.
cargo install --path crates/optimism/bin --bin op-reth --force --locked \
--features "$(FEATURES)" \
--profile "$(PROFILE)" \
$(CARGO_INSTALL_EXTRA_FLAGS)
.PHONY: build
build: ## Build the reth binary into `target` directory.
cargo build --bin reth --features "$(FEATURES)" --profile "$(PROFILE)"
@@ -72,10 +87,21 @@ build-%-reproducible:
.PHONY: build-debug
build-debug: ## Build the reth binary into `target/debug` directory.
cargo build --bin reth --features "$(FEATURES)"
.PHONY: build-debug-op
build-debug-op: ## Build the op-reth binary into `target/debug` directory.
cargo build --bin op-reth --features "$(FEATURES)" --manifest-path crates/optimism/bin/Cargo.toml
.PHONY: build-op
build-op: ## Build the op-reth binary into `target` directory.
cargo build --bin op-reth --features "$(FEATURES)" --profile "$(PROFILE)" --manifest-path crates/optimism/bin/Cargo.toml
# Builds the reth binary natively.
build-native-%:
cargo build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
op-build-native-%:
cargo build --bin op-reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)" --manifest-path crates/optimism/bin/Cargo.toml
# The following commands use `cross` to build a cross-compile.
#
# These commands require that:
@@ -92,6 +118,11 @@ build-native-%:
# on other systems. JEMALLOC_SYS_WITH_LG_PAGE=16 tells jemalloc to use 64-KiB
# pages. See: https://github.com/paradigmxyz/reth/issues/6742
build-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
op-build-aarch64-unknown-linux-gnu: export JEMALLOC_SYS_WITH_LG_PAGE=16
# No jemalloc on Windows
build-x86_64-pc-windows-gnu: FEATURES := $(filter-out jemalloc jemalloc-prof,$(FEATURES))
op-build-x86_64-pc-windows-gnu: FEATURES := $(filter-out jemalloc jemalloc-prof,$(FEATURES))
# Note: The additional rustc compiler flags are for intrinsics needed by MDBX.
# See: https://github.com/cross-rs/cross/wiki/FAQ#undefined-reference-with-build-std
@@ -99,6 +130,10 @@ build-%:
RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \
cross build --bin reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)"
op-build-%:
RUSTFLAGS="-C link-arg=-lgcc -Clink-arg=-static-libgcc" \
cross build --bin op-reth --target $* --features "$(FEATURES)" --profile "$(PROFILE)" --manifest-path crates/optimism/bin/Cargo.toml
# Unfortunately we can't easily use cross to build for Darwin because of licensing issues.
# If we wanted to, we would need to build a custom Docker image with the SDK available.
#
@@ -109,6 +144,11 @@ build-x86_64-apple-darwin:
$(MAKE) build-native-x86_64-apple-darwin
build-aarch64-apple-darwin:
$(MAKE) build-native-aarch64-apple-darwin
op-build-x86_64-apple-darwin:
$(MAKE) op-build-native-x86_64-apple-darwin
op-build-aarch64-apple-darwin:
$(MAKE) op-build-native-aarch64-apple-darwin
build-deb-%:
@case "$*" in \
x86_64-unknown-linux-gnu|aarch64-unknown-linux-gnu|riscv64gc-unknown-linux-gnu) \
@@ -144,6 +184,8 @@ build-release-tarballs: ## Create a series of `.tar.gz` files in the BIN_DIR dir
$(call tarball_release_binary,"x86_64-unknown-linux-gnu","reth","")
$(MAKE) build-aarch64-unknown-linux-gnu
$(call tarball_release_binary,"aarch64-unknown-linux-gnu","reth","")
$(MAKE) build-x86_64-pc-windows-gnu
$(call tarball_release_binary,"x86_64-pc-windows-gnu","reth.exe","")
##@ Test
@@ -153,7 +195,7 @@ COV_FILE := lcov.info
.PHONY: test-unit
test-unit: ## Run unit tests.
cargo install cargo-nextest --locked
cargo nextest run --no-fail-fast $(UNIT_TEST_ARGS)
cargo nextest run $(UNIT_TEST_ARGS)
.PHONY: cov-unit
@@ -186,7 +228,7 @@ $(EEST_TESTS_DIR):
.PHONY: ef-tests
ef-tests: $(EF_TESTS_DIR) $(EEST_TESTS_DIR) ## Runs Legacy and EEST tests.
cargo nextest run --no-fail-fast -p ef-tests --release --features ef-tests
cargo nextest run -p ef-tests --release --features ef-tests
##@ reth-bench
@@ -200,6 +242,137 @@ install-reth-bench: ## Build and install the reth binary under `$(CARGO_HOME)/bi
--features "$(FEATURES)" \
--profile "$(PROFILE)"
##@ Docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push
docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-git-sha
docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: docker-build-push-latest
docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly
docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call docker_build_push,nightly,nightly)
.PHONY: docker-build-push-nightly-edge-profiling
docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Create a cross-arch Docker image with the given tags and push it
define docker_build_push
$(MAKE) FEATURES="$(FEATURES)" build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/amd64/reth
$(MAKE) FEATURES="$(FEATURES)" build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/reth $(BIN_DIR)/arm64/reth
docker buildx build --file ./Dockerfile.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Optimism docker
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push
op-docker-build-push: ## Build and push a cross-arch Docker image tagged with the latest git tag.
$(call op_docker_build_push,$(GIT_TAG),$(GIT_TAG))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-git-sha
op-docker-build-push-git-sha: ## Build and push a cross-arch Docker image tagged with the latest git sha.
$(call op_docker_build_push,$(GIT_SHA),$(GIT_SHA))
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --driver docker-container --name cross-builder`
.PHONY: op-docker-build-push-latest
op-docker-build-push-latest: ## Build and push a cross-arch Docker image tagged with the latest git tag and `latest`.
$(call op_docker_build_push,$(GIT_TAG),latest)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly
op-docker-build-push-nightly: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly,nightly)
.PHONY: op-docker-build-push-nightly-edge-profiling
op-docker-build-push-nightly-edge-profiling: FEATURES := $(FEATURES) edge
op-docker-build-push-nightly-edge-profiling: ## Build and push cross-arch Docker image with edge features tagged with `nightly-edge-profiling`.
$(call op_docker_build_push,nightly-edge-profiling,nightly-edge-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: docker-build-push-nightly-profiling
docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image with profiling profile tagged with nightly-profiling.
$(call docker_build_push,nightly-profiling,nightly-profiling)
# Note: This requires a buildx builder with emulation support. For example:
#
# `docker run --privileged --rm tonistiigi/binfmt --install amd64,arm64`
# `docker buildx create --use --name cross-builder`
.PHONY: op-docker-build-push-nightly-profiling
op-docker-build-push-nightly-profiling: ## Build and push cross-arch Docker image tagged with the latest git tag with a `-nightly` suffix, and `latest-nightly`.
$(call op_docker_build_push,nightly-profiling,nightly-profiling)
# Create a cross-arch Docker image with the given tags and push it
define op_docker_build_push
$(MAKE) FEATURES="$(FEATURES)" op-build-x86_64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/amd64
cp $(CARGO_TARGET_DIR)/x86_64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/amd64/op-reth
$(MAKE) FEATURES="$(FEATURES)" op-build-aarch64-unknown-linux-gnu
mkdir -p $(BIN_DIR)/arm64
cp $(CARGO_TARGET_DIR)/aarch64-unknown-linux-gnu/$(PROFILE)/op-reth $(BIN_DIR)/arm64/op-reth
docker buildx build --file ./DockerfileOp.cross . \
--platform linux/amd64,linux/arm64 \
--tag $(DOCKER_IMAGE_NAME):$(1) \
--tag $(DOCKER_IMAGE_NAME):$(2) \
--provenance=false \
--push
endef
##@ Other
.PHONY: clean
@@ -227,21 +400,30 @@ db-tools: ## Compile MDBX debugging tools.
@echo "Run \"$(DB_TOOLS_DIR)/mdbx_chk\" for the MDBX db file integrity check."
.PHONY: update-book-cli
update-book-cli: build-debug ## Update book cli documentation.
update-book-cli: build-debug build-debug-op## Update book cli documentation.
@echo "Updating book cli doc..."
@./docs/cli/update.sh $(CARGO_TARGET_DIR)/debug/reth
@./docs/cli/update.sh $(CARGO_TARGET_DIR)/debug/reth $(CARGO_TARGET_DIR)/debug/op-reth
.PHONY: profiling
profiling: ## Builds `reth` with optimisations, but also symbols.
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --features jemalloc,asm-keccak
.PHONY: profiling-op
profiling-op: ## Builds `op-reth` with optimisations, but also symbols.
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --features jemalloc,asm-keccak --bin op-reth --manifest-path crates/optimism/bin/Cargo.toml
.PHONY: maxperf
maxperf: ## Builds `reth` with the most aggressive optimisations.
RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf
RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf --features jemalloc,asm-keccak
.PHONY: maxperf-op
maxperf-op: ## Builds `op-reth` with the most aggressive optimisations.
RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf --features jemalloc,asm-keccak --bin op-reth --manifest-path crates/optimism/bin/Cargo.toml
.PHONY: maxperf-no-asm
maxperf-no-asm: ## Builds `reth` with the most aggressive optimisations, minus the "asm-keccak" feature.
RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf --no-default-features --features jemalloc,min-debug-logs,otlp,otlp-logs,reth-revm/portable,js-tracer,keccak-cache-global,rocksdb
RUSTFLAGS="-C target-cpu=native" cargo build --profile maxperf --features jemalloc
fmt:
cargo +nightly fmt
@@ -256,6 +438,17 @@ clippy:
--all-features \
-- -D warnings
clippy-op-dev:
cargo +nightly clippy \
--bin op-reth \
--workspace \
--lib \
--examples \
--tests \
--benches \
--locked \
--all-features
lint-typos: ensure-typos
typos
@@ -320,6 +513,7 @@ rustdocs: ## Runs `cargo docs` to generate the Rust documents in the `target/doc
cargo-test:
cargo test \
--workspace \
--bin "op-reth" \
--lib --examples \
--tests \
--benches \

View File

@@ -18,11 +18,6 @@
[gh-lint]: https://github.com/paradigmxyz/reth/actions/workflows/lint.yml
[tg-badge]: https://img.shields.io/endpoint?color=neon&logo=telegram&label=chat&url=https%3A%2F%2Ftg.sumanjay.workers.dev%2Fparadigm%5Freth
> **Note: OP-Reth has moved**
>
> The Optimism (op-reth) crates have been moved to [ethereum-optimism/optimism](https://github.com/ethereum-optimism/optimism).
> Git contribution history has been preserved. If you are looking for op-reth, please see the new repository.
## What is Reth?
Reth (short for Rust Ethereum, [pronunciation](https://x.com/kelvinfichter/status/1597653609411268608)) is a new Ethereum full node implementation that is focused on being user-friendly, highly modular, as well as being fast and efficient. Reth is an Execution Layer (EL) and is compatible with all Ethereum Consensus Layer (CL) implementations that support the [Engine API](https://github.com/ethereum/execution-apis/tree/a0d03086564ab1838b462befbc083f873dcf0c0f/src/engine). It is originally built and driven forward by [Paradigm](https://paradigm.xyz/), and is licensed under the Apache and MIT licenses.
@@ -37,7 +32,7 @@ More concretely, our goals are:
2. **Performance**: Reth aims to be fast, so we use Rust and the [Erigon staged-sync](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) node architecture. We also use our Ethereum libraries (including [Alloy](https://github.com/alloy-rs/alloy/) and [revm](https://github.com/bluealloy/revm/)) which we've battle-tested and optimized via [Foundry](https://github.com/foundry-rs/foundry/).
3. **Free for anyone to use any way they want**: Reth is free open source software, built for the community, by the community. By licensing the software under the Apache/MIT license, we want developers to use it without being bound by business licenses, or having to think about the implications of GPL-like licenses.
4. **Client Diversity**: The Ethereum protocol becomes more antifragile when no node implementation dominates. This ensures that if there's a software bug, the network does not finalize a bad block. By building a new client, we hope to contribute to Ethereum's antifragility.
5. **Support as many EVM chains as possible**: We aspire that Reth can full-sync not only Ethereum, but also other chains like Optimism, Polygon, BNB Smart Chain, and more. If you're working on any of these projects, please reach out. Note: OP-Reth has moved to [ethereum-optimism/optimism](https://github.com/ethereum-optimism/optimism).
5. **Support as many EVM chains as possible**: We aspire that Reth can full-sync not only Ethereum, but also other chains like Optimism, Polygon, BNB Smart Chain, and more. If you're working on any of these projects, please reach out.
6. **Configurability**: We want to solve for node operators that care about fast historical queries, but also for hobbyists who cannot operate on large hardware. We also want to support teams and individuals who want both sync from genesis and via "fast sync". We envision that Reth will be configurable enough and provide configurable "profiles" for the tradeoffs that each team faces.
## Status
@@ -46,13 +41,13 @@ Reth is production ready, and suitable for usage in mission-critical environment
More historical context below:
- We released 1.0 "production-ready" stable Reth in June 2024.
- Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf).
- Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon.
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024, the last beta release.
- We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4, 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives.
- We shipped iterative improvements until the last alpha release on February 28, 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21).
- We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) on June 20, 2023.
- We released 1.0 "production-ready" stable Reth in June 2024.
- Reth completed an audit with [Sigma Prime](https://sigmaprime.io/), the developers of [Lighthouse](https://github.com/sigp/lighthouse), the Rust Consensus Layer implementation. Find it [here](./audit/sigma_prime_audit_v2.pdf).
- Revm (the EVM used in Reth) underwent an audit with [Guido Vranken](https://x.com/guidovranken) (#1 [Ethereum Bug Bounty](https://ethereum.org/en/bug-bounty)). We will publish the results soon.
- We released multiple iterative beta versions, up to [beta.9](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.9) on Monday June 3, 2024, the last beta release.
- We released [beta](https://github.com/paradigmxyz/reth/releases/tag/v0.2.0-beta.1) on Monday March 4, 2024, our first breaking change to the database model, providing faster query speed, smaller database footprint, and allowing "history" to be mounted on separate drives.
- We shipped iterative improvements until the last alpha release on February 28, 2024, [0.1.0-alpha.21](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.21).
- We [initially announced](https://www.paradigm.xyz/2023/06/reth-alpha) [0.1.0-alpha.1](https://github.com/paradigmxyz/reth/releases/tag/v0.1.0-alpha.1) on June 20, 2023.
### Database compatibility
@@ -82,8 +77,8 @@ For a general overview of the crates, see [Project Layout](./docs/repo/layout.md
If you want to contribute, or follow along with contributor discussion, you can use our [main telegram](https://t.me/paradigm_reth) to chat with us about the development of Reth!
- Our contributor guidelines can be found in [`CONTRIBUTING.md`](./CONTRIBUTING.md).
- See our [contributor docs](./docs) for more information on the project. A good starting point is [Project Layout](./docs/repo/layout.md).
- Our contributor guidelines can be found in [`CONTRIBUTING.md`](./CONTRIBUTING.md).
- See our [contributor docs](./docs) for more information on the project. A good starting point is [Project Layout](./docs/repo/layout.md).
### Building and testing
@@ -128,9 +123,9 @@ If you have any questions, first see if the answer to your question can be found
If the answer is not there:
- Join the [Telegram][tg-url] to get help, or
- Open a [discussion](https://github.com/paradigmxyz/reth/discussions/new) with your question, or
- Open an issue with [the bug](https://github.com/paradigmxyz/reth/issues/new?assignees=&labels=C-bug%2CS-needs-triage&projects=&template=bug.yml)
- Join the [Telegram][tg-url] to get help, or
- Open a [discussion](https://github.com/paradigmxyz/reth/discussions/new) with your question, or
- Open an issue with [the bug](https://github.com/paradigmxyz/reth/issues/new?assignees=&labels=C-bug%2CS-needs-triage&projects=&template=bug.yml)
## Security
@@ -142,9 +137,9 @@ Reth is a new implementation of the Ethereum protocol. In the process of develop
None of this would have been possible without them, so big shoutout to the teams below:
- [Geth](https://github.com/ethereum/go-ethereum/): We would like to express our heartfelt gratitude to the go-ethereum team for their outstanding contributions to Ethereum over the years. Their tireless efforts and dedication have helped to shape the Ethereum ecosystem and make it the vibrant and innovative community it is today. Thank you for your hard work and commitment to the project.
- [Erigon](https://github.com/ledgerwatch/erigon) (fka Turbo-Geth): Erigon pioneered the ["Staged Sync" architecture](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) that Reth is using, as well as [introduced MDBX](https://github.com/ledgerwatch/erigon/wiki/Choice-of-storage-engine) as the database of choice. We thank Erigon for pushing the state of the art research on the performance limits of Ethereum nodes.
- [Akula](https://github.com/akula-bft/akula/): Reth uses forks of the Apache versions of Akula's [MDBX Bindings](https://github.com/paradigmxyz/reth/pull/132), [FastRLP](https://github.com/paradigmxyz/reth/pull/63) and [ECIES](https://github.com/paradigmxyz/reth/pull/80). Given that these packages were already released under the Apache License, and they implement standardized solutions, we decided not to reimplement them to iterate faster. We thank the Akula team for their contributions to the Rust Ethereum ecosystem and for publishing these packages.
- [Geth](https://github.com/ethereum/go-ethereum/): We would like to express our heartfelt gratitude to the go-ethereum team for their outstanding contributions to Ethereum over the years. Their tireless efforts and dedication have helped to shape the Ethereum ecosystem and make it the vibrant and innovative community it is today. Thank you for your hard work and commitment to the project.
- [Erigon](https://github.com/ledgerwatch/erigon) (fka Turbo-Geth): Erigon pioneered the ["Staged Sync" architecture](https://erigon.substack.com/p/erigon-stage-sync-and-control-flows) that Reth is using, as well as [introduced MDBX](https://github.com/ledgerwatch/erigon/wiki/Choice-of-storage-engine) as the database of choice. We thank Erigon for pushing the state of the art research on the performance limits of Ethereum nodes.
- [Akula](https://github.com/akula-bft/akula/): Reth uses forks of the Apache versions of Akula's [MDBX Bindings](https://github.com/paradigmxyz/reth/pull/132), [FastRLP](https://github.com/paradigmxyz/reth/pull/63) and [ECIES](https://github.com/paradigmxyz/reth/pull/80). Given that these packages were already released under the Apache License, and they implement standardized solutions, we decided not to reimplement them to iterate faster. We thank the Akula team for their contributions to the Rust Ethereum ecosystem and for publishing these packages.
## Warning

View File

@@ -56,7 +56,7 @@ ctrlc.workspace = true
shlex.workspace = true
[target.'cfg(unix)'.dependencies]
nix = { version = "0.31", features = ["signal", "process"] }
nix = { version = "0.29", features = ["signal", "process"] }
[features]
default = ["jemalloc"]
@@ -98,5 +98,5 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
# no-op feature flag for CI matrices
# no-op feature flag for switching between the `optimism` and default functionality in CI matrices
ethereum = []

View File

@@ -30,7 +30,7 @@ reth-bench-compare \
| `--draw` | Generate charts (needs Python/uv) | `false` | No |
| `--profile` | Enable CPU profiling (needs samply) | `false` | No |
| `-vvvv` | Debug logging | Info | No |
| `--features <FEATURES>` | Extra Rust features for both builds | - | No |
| `--features <FEATURES>` | Rust features for both builds | `jemalloc,asm-keccak` | No |
| `--rustflags <FLAGS>` | RUSTFLAGS for both builds | `-C target-cpu=native` | No |
| `--baseline-features <FEATURES>` | Features for baseline only | Inherits `--features` | No |
| `--feature-features <FEATURES>` | Features for feature only | Inherits `--features` | No |

View File

@@ -186,12 +186,10 @@ impl BenchmarkRunner {
&output_dir.to_string_lossy(),
]);
// Configure wait mode: both can be used together
// When both are set: wait at least wait_time, and also wait for persistence if needed
// Configure wait mode: wait-time takes precedence over persistence-based flow
if let Some(ref wait_time) = self.wait_time {
cmd.args(["--wait-time", wait_time]);
}
if self.wait_for_persistence {
} else if self.wait_for_persistence {
cmd.arg("--wait-for-persistence");
// Add persistence threshold if specified

View File

@@ -116,9 +116,9 @@ pub(crate) struct Args {
/// Optional fixed delay between engine API calls (passed to reth-bench).
///
/// Can be combined with `--wait-for-persistence`: when both are set,
/// waits at least this duration, and also waits for persistence if needed.
#[arg(long, value_name = "DURATION")]
/// When set, reth-bench uses wait-time mode and disables persistence-based flow.
/// This flag remains for compatibility with older scripts.
#[arg(long, value_name = "DURATION", hide = true)]
pub wait_time: Option<String>,
/// Wait for blocks to be persisted before sending the next batch (passed to reth-bench).
@@ -126,9 +126,6 @@ pub(crate) struct Args {
/// When enabled, waits for every Nth block to be persisted using the
/// `reth_subscribePersistedBlock` subscription. This ensures the benchmark
/// doesn't outpace persistence.
///
/// Can be combined with `--wait-time`: when both are set, waits at least
/// wait-time, and also waits for persistence if the block hasn't been persisted yet.
#[arg(long)]
pub wait_for_persistence: bool,
@@ -191,9 +188,10 @@ pub(crate) struct Args {
#[arg(trailing_var_arg = true, allow_hyphen_values = true)]
pub reth_args: Vec<String>,
/// Comma-separated list of extra features to enable during reth compilation (applied to both
/// builds)
#[arg(long, value_name = "FEATURES", default_value = "")]
/// Comma-separated list of features to enable during reth compilation (applied to both builds)
///
/// Example: `jemalloc,asm-keccak`
#[arg(long, value_name = "FEATURES", default_value = "jemalloc,asm-keccak")]
pub features: String,
/// Comma-separated list of features to enable only for baseline build (overrides --features)
@@ -204,7 +202,7 @@ pub(crate) struct Args {
/// Comma-separated list of features to enable only for feature build (overrides --features)
///
/// Example: `--feature-features jemalloc-prof`
/// Example: `--feature-features jemalloc,asm-keccak`
#[arg(long, value_name = "FEATURES")]
pub feature_features: Option<String>,
@@ -276,8 +274,10 @@ impl Args {
/// Get the default RPC URL for a given chain
const fn get_default_rpc_url(chain: &Chain) -> &'static str {
match chain.id() {
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
_ => "https://ethereum.reth.rs/rpc", // mainnet and fallback
8453 => "https://base-mainnet.rpc.ithaca.xyz", // base
84532 => "https://base-sepolia.rpc.ithaca.xyz", // base-sepolia
27082 => "https://rpc.hoodi.ethpandaops.io", // hoodi
_ => "https://reth-ethereum.ithaca.xyz/rpc", // mainnet and fallback
}
}
@@ -471,6 +471,7 @@ async fn run_compilation_phase(
git_manager: &GitManager,
compilation_manager: &CompilationManager,
args: &Args,
is_optimism: bool,
) -> Result<(String, String)> {
info!("=== Running compilation phase ===");
@@ -523,7 +524,7 @@ async fn run_compilation_phase(
git_manager.switch_ref(git_ref)?;
// Compile reth (with caching)
compilation_manager.compile_reth(commit, features, rustflags)?;
compilation_manager.compile_reth(commit, is_optimism, features, rustflags)?;
info!("Completed compilation for {} reference", ref_type);
}
@@ -543,6 +544,7 @@ async fn run_warmup_phase(
node_manager: &mut NodeManager,
benchmark_runner: &BenchmarkRunner,
args: &Args,
is_optimism: bool,
baseline_commit: &str,
starting_tip: u64,
) -> Result<()> {
@@ -560,7 +562,8 @@ async fn run_warmup_phase(
git_manager.switch_ref(warmup_ref)?;
// Get the cached binary path for baseline (should already be compiled)
let binary_path = compilation_manager.get_cached_binary_path_for_commit(baseline_commit);
let binary_path =
compilation_manager.get_cached_binary_path_for_commit(baseline_commit, is_optimism);
// Verify the cached binary exists
if !binary_path.exists() {
@@ -613,13 +616,18 @@ async fn run_benchmark_workflow(
comparison_generator: &mut ComparisonGenerator,
args: &Args,
) -> Result<()> {
// Detect if this is an Optimism chain once at the beginning
let rpc_url = args.get_rpc_url();
let is_optimism = compilation_manager.detect_optimism_chain(&rpc_url).await?;
// Run compilation phase for both binaries
let (baseline_commit, feature_commit) =
run_compilation_phase(git_manager, compilation_manager, args).await?;
run_compilation_phase(git_manager, compilation_manager, args, is_optimism).await?;
// Switch to baseline reference and get the starting tip
git_manager.switch_ref(&args.baseline_ref)?;
let binary_path = compilation_manager.get_cached_binary_path_for_commit(&baseline_commit);
let binary_path =
compilation_manager.get_cached_binary_path_for_commit(&baseline_commit, is_optimism);
if !binary_path.exists() {
return Err(eyre!(
"Cached baseline binary not found at {:?}. Compilation phase should have created it.",
@@ -649,6 +657,7 @@ async fn run_benchmark_workflow(
node_manager,
benchmark_runner,
args,
is_optimism,
&baseline_commit,
starting_tip,
)
@@ -674,7 +683,8 @@ async fn run_benchmark_workflow(
git_manager.switch_ref(git_ref)?;
// Get the cached binary path for this git reference (should already be compiled)
let binary_path = compilation_manager.get_cached_binary_path_for_commit(commit);
let binary_path =
compilation_manager.get_cached_binary_path_for_commit(commit, is_optimism);
// Verify the cached binary exists
if !binary_path.exists() {

View File

@@ -1,6 +1,8 @@
//! Compilation operations for reth and reth-bench.
use crate::git::GitManager;
use alloy_primitives::address;
use alloy_provider::{Provider, ProviderBuilder};
use eyre::{eyre, Result, WrapErr};
use std::{fs, path::PathBuf, process::Command};
use tracing::{debug, error, info, warn};
@@ -23,14 +25,54 @@ impl CompilationManager {
Ok(Self { repo_root, output_dir, git_manager })
}
/// Detect if the RPC endpoint is an Optimism chain
pub(crate) async fn detect_optimism_chain(&self, rpc_url: &str) -> Result<bool> {
info!("Detecting chain type from RPC endpoint...");
// Create Alloy provider
let url = rpc_url.parse().map_err(|e| eyre!("Invalid RPC URL '{}': {}", rpc_url, e))?;
let provider = ProviderBuilder::new().connect_http(url);
// Check for Optimism predeploy at address 0x420000000000000000000000000000000000000F
let is_optimism = !provider
.get_code_at(address!("0x420000000000000000000000000000000000000F"))
.await?
.is_empty();
if is_optimism {
info!("Detected Optimism chain");
} else {
info!("Detected Ethereum chain");
}
Ok(is_optimism)
}
/// Get the path to the cached binary using explicit commit hash
pub(crate) fn get_cached_binary_path_for_commit(&self, commit: &str) -> PathBuf {
pub(crate) fn get_cached_binary_path_for_commit(
&self,
commit: &str,
is_optimism: bool,
) -> PathBuf {
let identifier = &commit[..8]; // Use first 8 chars of commit
self.output_dir.join("bin").join(format!("reth_{identifier}"))
let binary_name = if is_optimism {
format!("op-reth_{}", identifier)
} else {
format!("reth_{}", identifier)
};
self.output_dir.join("bin").join(binary_name)
}
/// Compile reth using cargo build and cache the binary
pub(crate) fn compile_reth(&self, commit: &str, features: &str, rustflags: &str) -> Result<()> {
pub(crate) fn compile_reth(
&self,
commit: &str,
is_optimism: bool,
features: &str,
rustflags: &str,
) -> Result<()> {
// Validate that current git commit matches the expected commit
let current_commit = self.git_manager.get_current_commit()?;
if current_commit != commit {
@@ -41,7 +83,7 @@ impl CompilationManager {
));
}
let cached_path = self.get_cached_binary_path_for_commit(commit);
let cached_path = self.get_cached_binary_path_for_commit(commit, is_optimism);
// Check if cached binary already exists (since path contains commit hash, it's valid)
if cached_path.exists() {
@@ -51,7 +93,7 @@ impl CompilationManager {
info!("No cached binary found, compiling (commit: {})...", &commit[..8]);
let binary_name = "reth";
let binary_name = if is_optimism { "op-reth" } else { "reth" };
info!(
"Compiling {} with profiling configuration (commit: {})...",
@@ -65,6 +107,14 @@ impl CompilationManager {
cmd.arg("--features").arg(features);
info!("Using features: {features}");
// Add bin-specific arguments for optimism
if is_optimism {
cmd.arg("--bin")
.arg("op-reth")
.arg("--manifest-path")
.arg("crates/optimism/bin/Cargo.toml");
}
cmd.current_dir(&self.repo_root);
// Set RUSTFLAGS

View File

@@ -14,8 +14,6 @@
#[global_allocator]
static ALLOC: reth_cli_util::allocator::Allocator = reth_cli_util::allocator::new_allocator();
use alloy_primitives as _;
mod benchmark;
mod cli;
mod comparison;

View File

@@ -32,7 +32,7 @@ alloy-eips.workspace = true
alloy-json-rpc.workspace = true
alloy-consensus.workspace = true
alloy-network.workspace = true
alloy-primitives = { workspace = true, features = ["rand"] }
alloy-primitives.workspace = true
alloy-provider = { workspace = true, features = ["engine-api", "pubsub", "reqwest-rustls-tls"], default-features = false }
alloy-pubsub.workspace = true
alloy-rpc-client = { workspace = true, features = ["pubsub"] }
@@ -45,7 +45,7 @@ op-alloy-consensus = { workspace = true, features = ["alloy-compat"] }
op-alloy-rpc-types-engine = { workspace = true, features = ["serde"] }
# reqwest
reqwest.workspace = true
reqwest = { workspace = true, default-features = false, features = ["rustls-tls-native-roots"] }
# tower
tower.workspace = true
@@ -117,7 +117,7 @@ min-trace-logs = [
"reth-node-core/min-trace-logs",
]
# no-op feature flag for CI matrices
# no-op feature flag for switching between the `optimism` and default functionality in CI matrices
ethereum = []
[[bin]]

View File

@@ -32,7 +32,7 @@ Otherwise, running `make maxperf` at the root of the repo should be sufficient f
The `reth-bench new-payload-fcu` command is the most representative of ethereum mainnet live sync, alternating between sending `engine_newPayload` calls and `engine_forkchoiceUpdated` calls.
The `new-payload-fcu` command supports two optional waiting modes that can be used together or independently:
- `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms` or `--wait-time 400` for 400ms)
- `--wait-time <duration>`: Fixed sleep interval between blocks (e.g., `--wait-time 100ms`)
- `--wait-for-persistence`: Waits for blocks to be persisted using the `reth_subscribePersistedBlock` subscription
When using `--wait-for-persistence`, the benchmark waits after every `(threshold + 1)` blocks, where the threshold defaults to the engine's persistence threshold (2). This can be customized with `--persistence-threshold <N>`.
@@ -73,7 +73,7 @@ make profiling
If the purpose of the benchmark is to obtain `jemalloc` memory profiles that can then be analyzed by `jeprof`, it should be compiled with the `profiling` profile and the `jemalloc-prof` feature:
```bash
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --features "jemalloc-prof"
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --features "jemalloc-prof,asm-keccak"
```
> [!NOTE]
@@ -82,7 +82,7 @@ RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --features "jem
Finally, if the purpose of the benchmark is to profile the node when `snmalloc` is configured as the default allocator, it would be built with the following
command:
```bash
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --no-default-features --features "snmalloc-native,asm-keccak,min-debug-logs"
RUSTFLAGS="-C target-cpu=native" cargo build --profile profiling --no-default-features --features "snmalloc-native,asm-keccak"
```
### Run the Benchmark:

View File

@@ -35,7 +35,7 @@ impl BenchContext {
/// This is the initialization code for most benchmarks, taking in a [`BenchmarkArgs`] and
/// returning the providers needed to run a benchmark.
pub(crate) async fn new(bench_args: &BenchmarkArgs, rpc_url: String) -> eyre::Result<Self> {
info!(target: "reth-bench", "Running benchmark using data from RPC URL: {}", rpc_url);
info!("Running benchmark using data from RPC URL: {}", rpc_url);
// Ensure that output directory exists and is a directory
if let Some(output) = &bench_args.output {
@@ -45,7 +45,7 @@ impl BenchContext {
// Create the directory if it doesn't exist
if !output.exists() {
std::fs::create_dir_all(output)?;
info!(target: "reth-bench", "Created output directory: {:?}", output);
info!("Created output directory: {:?}", output);
}
}
@@ -77,7 +77,7 @@ impl BenchContext {
let auth_url = Url::parse(&bench_args.engine_rpc_url)?;
// construct the authed transport
info!(target: "reth-bench", "Connecting to Engine RPC at {} for replay", auth_url);
info!("Connecting to Engine RPC at {} for replay", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
let client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(client);

View File

@@ -3,7 +3,7 @@
use crate::{
authenticated_transport::AuthenticatedTransportConnect,
bench::{
helpers::{build_payload, parse_gas_limit, prepare_payload_request, rpc_block_to_header},
helpers::{build_payload, prepare_payload_request, rpc_block_to_header},
output::GasRampPayloadFile,
},
valid_payload::{call_forkchoice_updated, call_new_payload, payload_to_new_payload},
@@ -22,6 +22,29 @@ use reth_primitives_traits::constants::{GAS_LIMIT_BOUND_DIVISOR, MAXIMUM_GAS_LIM
use std::{path::PathBuf, time::Instant};
use tracing::info;
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// `reth benchmark gas-limit-ramp` command.
#[derive(Debug, Parser)]
pub struct Command {
@@ -87,7 +110,7 @@ impl Command {
}
if !self.output.exists() {
std::fs::create_dir_all(&self.output)?;
info!(target: "reth-bench", "Created output directory: {:?}", self.output);
info!("Created output directory: {:?}", self.output);
}
// Set up authenticated provider (used for both Engine API and eth_ methods)
@@ -95,7 +118,7 @@ impl Command {
let jwt = JwtSecret::from_hex(jwt)?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!(target: "reth-bench", "Connecting to Engine RPC at {}", auth_url);
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url, jwt);
let client = ClientBuilder::default().connect_with(auth_transport).await?;
let provider = RootProvider::<AnyNetwork>::new(client);
@@ -120,7 +143,6 @@ impl Command {
match mode {
RampMode::Blocks(blocks) => {
info!(
target: "reth-bench",
canonical_parent,
start_block,
end_block = start_block + blocks - 1,
@@ -129,7 +151,6 @@ impl Command {
}
RampMode::TargetGasLimit(target) => {
info!(
target: "reth-bench",
canonical_parent,
start_block,
current_gas_limit = parent_header.gas_limit,
@@ -178,7 +199,7 @@ impl Command {
GasRampPayloadFile { version: version as u8, block_hash, params: params.clone() };
let payload_json = serde_json::to_string_pretty(&file)?;
std::fs::write(&payload_path, &payload_json)?;
info!(target: "reth-bench", block_number = block.header.number, path = %payload_path.display(), "Saved payload");
info!(block_number = block.header.number, path = %payload_path.display(), "Saved payload");
call_new_payload(&provider, version, params).await?;
@@ -196,7 +217,6 @@ impl Command {
let final_gas_limit = parent_header.gas_limit;
info!(
target: "reth-bench",
total_duration=?total_benchmark_duration.elapsed(),
blocks_processed,
final_gas_limit,
@@ -217,3 +237,50 @@ const fn should_stop(mode: RampMode, blocks_processed: u64, current_gas_limit: u
RampMode::TargetGasLimit(target) => current_gas_limit >= target,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
}

View File

@@ -3,9 +3,7 @@
//! This command fetches transactions from existing blocks and packs them into a single
//! large block using the `testing_buildBlockV1` RPC endpoint.
use crate::{
authenticated_transport::AuthenticatedTransportConnect, bench::helpers::parse_gas_limit,
};
use crate::authenticated_transport::AuthenticatedTransportConnect;
use alloy_eips::{BlockNumberOrTag, Typed2718};
use alloy_primitives::{Bytes, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, Provider, RootProvider};
@@ -132,27 +130,16 @@ impl<S: TransactionSource> TransactionCollector<S> {
/// Collect transactions starting from the given block number.
///
/// Skips blob transactions (type 3) and collects until target gas is reached.
/// Returns a `CollectionResult` with transactions, gas info, and next block.
pub async fn collect(&self, start_block: u64) -> eyre::Result<CollectionResult> {
self.collect_gas(start_block, self.target_gas).await
}
/// Collect transactions up to a specific gas target.
///
/// This is used both for initial collection and for retry top-ups.
pub async fn collect_gas(
&self,
start_block: u64,
gas_target: u64,
) -> eyre::Result<CollectionResult> {
let mut transactions: Vec<RawTransaction> = Vec::new();
/// Returns the collected raw transaction bytes, total gas used, and the next block number.
pub async fn collect(&self, start_block: u64) -> eyre::Result<(Vec<Bytes>, u64, u64)> {
let mut transactions: Vec<Bytes> = Vec::new();
let mut total_gas: u64 = 0;
let mut current_block = start_block;
while total_gas < gas_target {
while total_gas < self.target_gas {
let Some((block_txs, _)) = self.source.fetch_block_transactions(current_block).await?
else {
warn!(target: "reth-bench", block = current_block, "Block not found, stopping");
warn!(block = current_block, "Block not found, stopping");
break;
};
@@ -162,12 +149,12 @@ impl<S: TransactionSource> TransactionCollector<S> {
continue;
}
if total_gas + tx.gas_used <= gas_target {
if total_gas + tx.gas_used <= self.target_gas {
transactions.push(tx.raw);
total_gas += tx.gas_used;
transactions.push(tx);
}
if total_gas >= gas_target {
if total_gas >= self.target_gas {
break;
}
}
@@ -175,21 +162,20 @@ impl<S: TransactionSource> TransactionCollector<S> {
current_block += 1;
// Stop early if remaining gas is under 1M (close enough to target)
let remaining_gas = gas_target.saturating_sub(total_gas);
let remaining_gas = self.target_gas.saturating_sub(total_gas);
if remaining_gas < 1_000_000 {
break;
}
}
info!(
target: "reth-bench",
total_txs = transactions.len(),
gas_sent = total_gas,
total_gas,
next_block = current_block,
"Finished collecting transactions"
);
Ok(CollectionResult { transactions, gas_sent: total_gas, next_block: current_block })
Ok((transactions, total_gas, current_block))
}
}
@@ -216,26 +202,13 @@ pub struct Command {
jwt_secret: std::path::PathBuf,
/// Target gas to pack into the block.
/// Accepts short notation: K for thousand, M for million, G for billion (e.g., 1G = 1
/// billion).
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000", value_parser = parse_gas_limit)]
#[arg(long, value_name = "TARGET_GAS", default_value = "30000000")]
target_gas: u64,
/// Block number to start fetching transactions from (required).
///
/// This must be the last canonical block BEFORE any gas limit ramping was performed.
/// The command collects transactions from historical blocks starting at this number
/// to pack into large blocks.
///
/// How to determine this value:
/// - If starting from a fresh node (no gas limit ramp yet): use the current chain tip
/// - If gas limit ramping has already been performed: use the block number that was the chain
/// tip BEFORE ramping began (you must track this yourself)
///
/// Using a block after ramping started will cause transaction collection to fail
/// because those blocks contain synthetic transactions that cannot be replayed.
/// Starting block number to fetch transactions from.
/// If not specified, starts from the engine's latest block.
#[arg(long, value_name = "FROM_BLOCK")]
from_block: u64,
from_block: Option<u64>,
/// Execute the payload (call newPayload + forkchoiceUpdated).
/// If false, only builds the payload and prints it.
@@ -264,86 +237,12 @@ struct BuiltPayload {
envelope: ExecutionPayloadEnvelopeV4,
block_hash: B256,
timestamp: u64,
/// The actual gas used in the built block.
gas_used: u64,
}
/// Result of collecting transactions from blocks.
#[derive(Debug)]
pub struct CollectionResult {
/// Collected transactions with their gas info.
pub transactions: Vec<RawTransaction>,
/// Total gas sent (sum of historical `gas_used` for all collected txs).
pub gas_sent: u64,
/// Next block number to continue collecting from.
pub next_block: u64,
}
/// Constants for retry logic.
const MAX_BUILD_RETRIES: u32 = 5;
/// Maximum retries for fetching a transaction batch.
const MAX_FETCH_RETRIES: u32 = 5;
/// Tolerance: if `gas_used` is within 1M of target, don't retry.
const MIN_TARGET_SLACK: u64 = 1_000_000;
/// Maximum gas to request in retries (10x target as safety cap).
const MAX_ADDITIONAL_GAS_MULTIPLIER: u64 = 10;
/// Fetches a batch of transactions with retry logic.
///
/// Returns `None` if all retries are exhausted.
async fn fetch_batch_with_retry<S: TransactionSource>(
collector: &TransactionCollector<S>,
block: u64,
) -> Option<CollectionResult> {
for attempt in 1..=MAX_FETCH_RETRIES {
match collector.collect(block).await {
Ok(result) => return Some(result),
Err(e) => {
if attempt == MAX_FETCH_RETRIES {
warn!(target: "reth-bench", attempt, error = %e, "Failed to fetch transactions after max retries");
return None;
}
warn!(target: "reth-bench", attempt, error = %e, "Failed to fetch transactions, retrying...");
tokio::time::sleep(std::time::Duration::from_secs(1)).await;
}
}
}
None
}
/// Outcome of a build attempt check.
enum RetryOutcome {
/// Payload is close enough to target gas.
Success,
/// Max retries reached, accept what we have.
MaxRetries,
/// Need more transactions with the specified gas amount.
NeedMore(u64),
}
/// Buffer for receiving transaction batches from the fetcher.
///
/// This abstracts over the channel to allow the main loop to request
/// batches on demand, including for retries.
struct TxBuffer {
receiver: mpsc::Receiver<CollectionResult>,
}
impl TxBuffer {
const fn new(receiver: mpsc::Receiver<CollectionResult>) -> Self {
Self { receiver }
}
/// Take the next available batch from the fetcher.
async fn take_batch(&mut self) -> Option<CollectionResult> {
self.receiver.recv().await
}
}
impl Command {
/// Execute the `generate-big-block` command
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
info!(target: "reth-bench", target_gas = self.target_gas, count = self.count, "Generating big block(s)");
info!(target_gas = self.target_gas, count = self.count, "Generating big block(s)");
// Set up authenticated engine provider
let jwt =
@@ -351,20 +250,20 @@ impl Command {
let jwt = JwtSecret::from_hex(jwt.trim())?;
let auth_url = Url::parse(&self.engine_rpc_url)?;
info!(target: "reth-bench", "Connecting to Engine RPC at {}", auth_url);
info!("Connecting to Engine RPC at {}", auth_url);
let auth_transport = AuthenticatedTransportConnect::new(auth_url.clone(), jwt);
let auth_client = ClientBuilder::default().connect_with(auth_transport).await?;
let auth_provider = RootProvider::<AnyNetwork>::new(auth_client);
// Set up testing RPC provider (for testing_buildBlockV1)
info!(target: "reth-bench", "Connecting to Testing RPC at {}", self.testing_rpc_url);
info!("Connecting to Testing RPC at {}", self.testing_rpc_url);
let testing_client = ClientBuilder::default()
.layer(RetryBackoffLayer::new(10, 800, u64::MAX))
.http(self.testing_rpc_url.parse()?);
let testing_provider = RootProvider::<AnyNetwork>::new(testing_client);
// Get the parent block (latest canonical block)
info!(target: "reth-bench", endpoint = "engine", method = "eth_getBlockByNumber", block = "latest", "RPC call");
info!(endpoint = "engine", method = "eth_getBlockByNumber", block = "latest", "RPC call");
let parent_block = auth_provider
.get_block_by_number(BlockNumberOrTag::Latest)
.await?
@@ -375,7 +274,6 @@ impl Command {
let parent_timestamp = parent_block.header.timestamp;
info!(
target: "reth-bench",
parent_hash = %parent_hash,
parent_number = parent_number,
"Using initial parent block"
@@ -386,7 +284,7 @@ impl Command {
format!("Failed to create output directory: {:?}", self.output_dir)
})?;
let start_block = self.from_block;
let start_block = self.from_block.unwrap_or(parent_number);
// Use pipelined execution when generating multiple payloads
if self.count > 1 {
@@ -399,60 +297,57 @@ impl Command {
)
.await?;
} else {
// Single payload - collect transactions and build with retry
// Single payload - collect transactions and build
let tx_source = RpcTransactionSource::from_url(&self.rpc_url)?;
let collector = TransactionCollector::new(tx_source, self.target_gas);
let result = collector.collect(start_block).await?;
let (transactions, _total_gas, _next_block) = collector.collect(start_block).await?;
if result.transactions.is_empty() {
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected"));
}
self.execute_sequential_with_retry(
self.execute_sequential(
&auth_provider,
&testing_provider,
&collector,
result,
transactions,
parent_hash,
parent_timestamp,
)
.await?;
}
info!(target: "reth-bench", count = self.count, output_dir = %self.output_dir.display(), "All payloads generated");
info!(count = self.count, output_dir = %self.output_dir.display(), "All payloads generated");
Ok(())
}
/// Sequential execution path with retry logic for underfilled payloads.
async fn execute_sequential_with_retry<S: TransactionSource>(
/// Sequential execution path for single payload or no-execute mode.
async fn execute_sequential(
&self,
auth_provider: &RootProvider<AnyNetwork>,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
initial_result: CollectionResult,
transactions: Vec<Bytes>,
mut parent_hash: B256,
mut parent_timestamp: u64,
) -> eyre::Result<()> {
let mut current_result = initial_result;
for i in 0..self.count {
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
"Building payload via testing_buildBlockV1"
);
let built = self
.build_with_retry(
testing_provider,
collector,
&mut current_result,
i,
parent_hash,
parent_timestamp,
)
.build_payload(testing_provider, &transactions, i, parent_hash, parent_timestamp)
.await?;
self.save_payload(&built)?;
if self.execute || self.count > 1 {
info!(target: "reth-bench", payload = i + 1, block_hash = %built.block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
info!(payload = i + 1, block_hash = %built.block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(target: "reth-bench", payload = i + 1, "Payload executed successfully");
info!(payload = i + 1, "Payload executed successfully");
}
parent_hash = built.block_hash;
@@ -461,63 +356,7 @@ impl Command {
Ok(())
}
/// Build a payload with retry logic when `gas_used` is below target.
///
/// Uses the ratio of `gas_used/gas_sent` to estimate how many more transactions
/// are needed to hit the target gas.
async fn build_with_retry<S: TransactionSource>(
&self,
testing_provider: &RootProvider<AnyNetwork>,
collector: &TransactionCollector<S>,
result: &mut CollectionResult,
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
target: "reth-bench",
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let additional =
collector.collect_gas(result.next_block, additional_gas).await?;
result.transactions.extend(additional.transactions);
result.gas_sent = result.gas_sent.saturating_add(additional.gas_sent);
result.next_block = additional.next_block;
}
}
}
warn!(target: "reth-bench", payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry exhausted retries without result"))
}
/// Pipelined execution - fetches transactions in background, builds with retry.
///
/// The fetcher continuously produces transaction batches. The main loop consumes them,
/// builds payloads with retry logic (requesting more transactions if underfilled),
/// and executes each payload before moving to the next.
/// Pipelined execution - fetches transactions and builds payloads in background.
async fn execute_pipelined(
&self,
auth_provider: &RootProvider<AnyNetwork>,
@@ -526,238 +365,167 @@ impl Command {
initial_parent_hash: B256,
initial_parent_timestamp: u64,
) -> eyre::Result<()> {
// Create channel for transaction batches - fetcher sends CollectionResult
let (tx_sender, tx_receiver) = mpsc::channel::<CollectionResult>(self.prefetch_buffer);
// Create channel for transaction batches (one batch per payload)
let (tx_sender, mut tx_receiver) = mpsc::channel::<Vec<Bytes>>(self.prefetch_buffer);
// Spawn background task to continuously fetch transaction batches
let rpc_url = self.rpc_url.clone();
let target_gas = self.target_gas;
let count = self.count;
let fetcher_handle = tokio::spawn(async move {
let tx_source = match RpcTransactionSource::from_url(&rpc_url) {
Ok(source) => source,
Err(e) => {
warn!(target: "reth-bench", error = %e, "Failed to create transaction source");
return None;
warn!(error = %e, "Failed to create transaction source");
return;
}
};
let collector = TransactionCollector::new(tx_source, target_gas);
let mut current_block = start_block;
while let Some(batch) = fetch_batch_with_retry(&collector, current_block).await {
if batch.transactions.is_empty() {
info!(target: "reth-bench", block = current_block, "Reached chain tip, stopping fetcher");
break;
}
for payload_idx in 0..count {
match collector.collect(current_block).await {
Ok((transactions, total_gas, next_block)) => {
info!(
payload = payload_idx + 1,
tx_count = transactions.len(),
total_gas,
blocks = format!("{}..{}", current_block, next_block),
"Fetched transactions"
);
current_block = next_block;
info!(
target: "reth-bench",
tx_count = batch.transactions.len(),
gas_sent = batch.gas_sent,
blocks = format!("{}..{}", current_block, batch.next_block),
"Fetched transaction batch"
);
current_block = batch.next_block;
if tx_sender.send(batch).await.is_err() {
break;
if tx_sender.send(transactions).await.is_err() {
break;
}
}
Err(e) => {
warn!(payload = payload_idx + 1, error = %e, "Failed to fetch transactions");
break;
}
}
}
Some(current_block)
});
// Transaction buffer: holds transactions from batches + any extras from retries
let mut tx_buffer = TxBuffer::new(tx_receiver);
let mut parent_hash = initial_parent_hash;
let mut parent_timestamp = initial_parent_timestamp;
let mut pending_build: Option<tokio::task::JoinHandle<eyre::Result<BuiltPayload>>> = None;
for i in 0..self.count {
// Get initial batch of transactions for this payload
let Some(mut result) = tx_buffer.take_batch().await else {
info!(
target: "reth-bench",
payloads_built = i,
payloads_requested = self.count,
"Transaction source exhausted, stopping"
);
break;
};
let is_last = i == self.count - 1;
if result.transactions.is_empty() {
info!(
target: "reth-bench",
payloads_built = i,
payloads_requested = self.count,
"No more transactions available, stopping"
);
break;
}
// Get current payload (either from pending build or build now)
let current_payload = if let Some(handle) = pending_build.take() {
handle.await??
} else {
// First payload - wait for transactions and build synchronously
let transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
// Build with retry - may need to request more transactions
let built = self
.build_with_retry_buffered(
if transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 1));
}
info!(
payload = i + 1,
total = self.count,
parent_hash = %parent_hash,
parent_timestamp = parent_timestamp,
tx_count = transactions.len(),
"Building payload via testing_buildBlockV1"
);
self.build_payload(
testing_provider,
&mut tx_buffer,
&mut result,
&transactions,
i,
parent_hash,
parent_timestamp,
)
.await?;
.await?
};
self.save_payload(&built)?;
self.save_payload(&current_payload)?;
let current_block_hash = built.block_hash;
let current_timestamp = built.timestamp;
let current_block_hash = current_payload.block_hash;
let current_timestamp = current_payload.timestamp;
// Execute payload
info!(target: "reth-bench", payload = i + 1, block_hash = %current_block_hash, gas_used = built.gas_used, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, built.envelope, parent_hash).await?;
info!(target: "reth-bench", payload = i + 1, "Payload executed successfully");
// Execute current payload first
info!(payload = i + 1, block_hash = %current_block_hash, "Executing payload (newPayload + FCU)");
self.execute_payload_v4(auth_provider, current_payload.envelope, parent_hash).await?;
info!(payload = i + 1, "Payload executed successfully");
// Start building next payload in background (if not last) - AFTER execution
if !is_last {
// Get transactions for next payload (should already be fetched or fetching)
let next_transactions = tx_receiver
.recv()
.await
.ok_or_else(|| eyre::eyre!("Transaction fetcher stopped unexpectedly"))?;
if next_transactions.is_empty() {
return Err(eyre::eyre!("No transactions collected for payload {}", i + 2));
}
let testing_provider = testing_provider.clone();
let next_index = i + 1;
let total = self.count;
pending_build = Some(tokio::spawn(async move {
info!(
payload = next_index + 1,
total = total,
parent_hash = %current_block_hash,
parent_timestamp = current_timestamp,
tx_count = next_transactions.len(),
"Building payload via testing_buildBlockV1"
);
Self::build_payload_static(
&testing_provider,
&next_transactions,
next_index,
current_block_hash,
current_timestamp,
)
.await
}));
}
parent_hash = current_block_hash;
parent_timestamp = current_timestamp;
}
// Clean up the fetcher task
drop(tx_buffer);
drop(tx_receiver);
let _ = fetcher_handle.await;
Ok(())
}
/// Build a payload with retry logic, using the buffered transaction source.
async fn build_with_retry_buffered(
/// Build a single payload via `testing_buildBlockV1`.
async fn build_payload(
&self,
testing_provider: &RootProvider<AnyNetwork>,
tx_buffer: &mut TxBuffer,
result: &mut CollectionResult,
transactions: &[Bytes],
index: u64,
parent_hash: B256,
parent_timestamp: u64,
) -> eyre::Result<BuiltPayload> {
for attempt in 1..=MAX_BUILD_RETRIES {
let tx_bytes: Vec<Bytes> = result.transactions.iter().map(|t| t.raw.clone()).collect();
let gas_sent = result.gas_sent;
info!(
target: "reth-bench",
payload = index + 1,
attempt,
tx_count = tx_bytes.len(),
gas_sent,
parent_hash = %parent_hash,
"Building payload via testing_buildBlockV1"
);
let built = Self::build_payload_static(
testing_provider,
&tx_bytes,
index,
parent_hash,
parent_timestamp,
)
.await?;
match self.check_retry_outcome(&built, index, attempt, gas_sent) {
RetryOutcome::Success | RetryOutcome::MaxRetries => return Ok(built),
RetryOutcome::NeedMore(additional_gas) => {
let mut collected_gas = 0u64;
while collected_gas < additional_gas {
if let Some(batch) = tx_buffer.take_batch().await {
collected_gas += batch.gas_sent;
result.transactions.extend(batch.transactions);
result.gas_sent = result.gas_sent.saturating_add(batch.gas_sent);
result.next_block = batch.next_block;
} else {
warn!(target: "reth-bench", "Transaction fetcher exhausted, proceeding with available transactions");
break;
}
}
}
}
}
warn!(target: "reth-bench", payload = index + 1, "Retry loop exited without returning a payload");
Err(eyre::eyre!("build_with_retry_buffered exhausted retries without result"))
Self::build_payload_static(
testing_provider,
transactions,
index,
parent_hash,
parent_timestamp,
)
.await
}
/// Determines the outcome of a build attempt.
fn check_retry_outcome(
&self,
built: &BuiltPayload,
index: u64,
attempt: u32,
gas_sent: u64,
) -> RetryOutcome {
let gas_used = built.gas_used;
if gas_used + MIN_TARGET_SLACK >= self.target_gas {
info!(
target: "reth-bench",
payload = index + 1,
gas_used,
target_gas = self.target_gas,
attempts = attempt,
"Payload built successfully"
);
return RetryOutcome::Success;
}
if attempt == MAX_BUILD_RETRIES {
warn!(
target: "reth-bench",
payload = index + 1,
gas_used,
target_gas = self.target_gas,
gas_sent,
"Underfilled after max retries, accepting payload"
);
return RetryOutcome::MaxRetries;
}
if gas_used == 0 {
warn!(
target: "reth-bench",
payload = index + 1,
"Zero gas used in payload, requesting fixed chunk of additional transactions"
);
return RetryOutcome::NeedMore(self.target_gas);
}
let gas_sent_needed_total =
(self.target_gas as u128 * gas_sent as u128).div_ceil(gas_used as u128) as u64;
let additional = gas_sent_needed_total.saturating_sub(gas_sent);
let additional = additional.min(self.target_gas * MAX_ADDITIONAL_GAS_MULTIPLIER);
if additional == 0 {
info!(
target: "reth-bench",
payload = index + 1,
gas_used,
target_gas = self.target_gas,
"No additional transactions needed based on ratio"
);
return RetryOutcome::Success;
}
let ratio = gas_used as f64 / gas_sent as f64;
info!(
target: "reth-bench",
payload = index + 1,
gas_used,
gas_sent,
ratio = format!("{:.4}", ratio),
additional_gas = additional,
"Underfilled, collecting more transactions for retry"
);
RetryOutcome::NeedMore(additional)
}
/// Build a single payload via `testing_buildBlockV1`.
/// Static version for use in spawned tasks.
async fn build_payload_static(
testing_provider: &RootProvider<AnyNetwork>,
transactions: &[Bytes],
@@ -780,7 +548,6 @@ impl Command {
let total_tx_bytes: usize = transactions.iter().map(|tx| tx.len()).sum();
info!(
target: "reth-bench",
payload = index + 1,
tx_count = transactions.len(),
total_tx_bytes = total_tx_bytes,
@@ -796,9 +563,8 @@ impl Command {
let block_hash = inner.block_hash;
let block_number = inner.block_number;
let timestamp = inner.timestamp;
let gas_used = inner.gas_used;
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp, gas_used })
Ok(BuiltPayload { block_number, envelope: v4_envelope, block_hash, timestamp })
}
/// Save a payload to disk.
@@ -808,7 +574,7 @@ impl Command {
let json = serde_json::to_string_pretty(&payload.envelope)?;
std::fs::write(&filepath, &json)
.wrap_err_with(|| format!("Failed to write payload to {:?}", filepath))?;
info!(target: "reth-bench", block_number = payload.block_number, block_hash = %payload.block_hash, path = %filepath.display(), "Payload saved");
info!(block_number = payload.block_number, block_hash = %payload.block_hash, path = %filepath.display(), "Payload saved");
Ok(())
}

View File

@@ -1,82 +1,13 @@
//! Common helpers for reth-bench commands.
use crate::valid_payload::call_forkchoice_updated;
use eyre::Result;
use std::{
io::{BufReader, Read},
time::Duration,
};
/// Read input from either a file path or stdin.
pub(crate) fn read_input(path: Option<&str>) -> Result<String> {
Ok(match path {
Some(path) => reth_fs_util::read_to_string(path)?,
None => String::from_utf8(
BufReader::new(std::io::stdin()).bytes().collect::<Result<Vec<_>, _>>()?,
)?,
})
}
/// Load JWT secret from either a file or use the provided string directly.
pub(crate) fn load_jwt_secret(jwt_secret: Option<&str>) -> Result<Option<String>> {
match jwt_secret {
Some(secret) => {
// Try to read as file first
match std::fs::read_to_string(secret) {
Ok(contents) => Ok(Some(contents.trim().to_string())),
// If file read fails, use the string directly
Err(_) => Ok(Some(secret.to_string())),
}
}
None => Ok(None),
}
}
/// Parses a gas limit value with optional suffix: K for thousand, M for million, G for billion.
///
/// Examples: "30000000", "30M", "1G", "2G"
pub(crate) fn parse_gas_limit(s: &str) -> eyre::Result<u64> {
let s = s.trim();
if s.is_empty() {
return Err(eyre::eyre!("empty value"));
}
let (num_str, multiplier) = if let Some(prefix) = s.strip_suffix(['G', 'g']) {
(prefix, 1_000_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['M', 'm']) {
(prefix, 1_000_000u64)
} else if let Some(prefix) = s.strip_suffix(['K', 'k']) {
(prefix, 1_000u64)
} else {
(s, 1u64)
};
let base: u64 = num_str.trim().parse()?;
base.checked_mul(multiplier).ok_or_else(|| eyre::eyre!("value overflow"))
}
/// Parses a duration string, treating bare integers as milliseconds.
///
/// Accepts either a `humantime` duration string (e.g. `"100ms"`, `"2s"`) or a plain
/// integer which is interpreted as milliseconds (e.g. `"400"` → 400ms).
pub(crate) fn parse_duration(s: &str) -> eyre::Result<Duration> {
match humantime::parse_duration(s) {
Ok(d) => Ok(d),
Err(_) => {
let millis: u64 =
s.trim().parse().map_err(|_| eyre::eyre!("invalid duration: {s:?}"))?;
Ok(Duration::from_millis(millis))
}
}
}
use alloy_consensus::Header;
use alloy_eips::eip4844::kzg_to_versioned_hash;
use alloy_primitives::{Address, B256};
use alloy_provider::{ext::EngineApi, network::AnyNetwork, RootProvider};
use alloy_rpc_types_engine::{
CancunPayloadFields, ExecutionPayload, ExecutionPayloadSidecar, ForkchoiceState,
PayloadAttributes, PayloadId,
PayloadAttributes, PayloadId, PraguePayloadFields,
};
use eyre::OptionExt;
use reth_chainspec::{ChainSpec, EthereumHardforks};
@@ -199,7 +130,7 @@ pub(crate) async fn get_payload_with_sidecar(
payload_id: PayloadId,
parent_beacon_block_root: Option<B256>,
) -> eyre::Result<(ExecutionPayload, ExecutionPayloadSidecar)> {
debug!(target: "reth-bench", get_payload_version = ?version, ?payload_id, "Sending getPayload");
debug!(get_payload_version = ?version, ?payload_id, "Sending getPayload");
match version {
1 => {
@@ -230,83 +161,36 @@ pub(crate) async fn get_payload_with_sidecar(
}
4 => {
let envelope = provider.get_payload_v4(payload_id).await?;
Ok(envelope.into_payload_and_sidecar(
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V4")?,
let versioned_hashes = versioned_hashes_from_commitments(
&envelope.envelope_inner.blobs_bundle.commitments,
);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V4")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.envelope_inner.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
5 => {
// V5 (Osaka) - use raw request since alloy doesn't have get_payload_v5 yet
let envelope = provider.get_payload_v5(payload_id).await?;
Ok(envelope.into_payload_and_sidecar(
parent_beacon_block_root.ok_or_eyre("parent_beacon_block_root required for V5")?,
let versioned_hashes =
versioned_hashes_from_commitments(&envelope.blobs_bundle.commitments);
let cancun_fields = CancunPayloadFields {
parent_beacon_block_root: parent_beacon_block_root
.ok_or_eyre("parent_beacon_block_root required for V5")?,
versioned_hashes,
};
let prague_fields = PraguePayloadFields::new(envelope.execution_requests);
Ok((
ExecutionPayload::V3(envelope.execution_payload),
ExecutionPayloadSidecar::v4(cancun_fields, prague_fields),
))
}
_ => panic!("This tool does not support getPayload versions past v5"),
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_parse_gas_limit_plain_number() {
assert_eq!(parse_gas_limit("30000000").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("1").unwrap(), 1);
assert_eq!(parse_gas_limit("0").unwrap(), 0);
}
#[test]
fn test_parse_gas_limit_k_suffix() {
assert_eq!(parse_gas_limit("1K").unwrap(), 1_000);
assert_eq!(parse_gas_limit("30k").unwrap(), 30_000);
assert_eq!(parse_gas_limit("100K").unwrap(), 100_000);
}
#[test]
fn test_parse_gas_limit_m_suffix() {
assert_eq!(parse_gas_limit("1M").unwrap(), 1_000_000);
assert_eq!(parse_gas_limit("30m").unwrap(), 30_000_000);
assert_eq!(parse_gas_limit("100M").unwrap(), 100_000_000);
}
#[test]
fn test_parse_gas_limit_g_suffix() {
assert_eq!(parse_gas_limit("1G").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2g").unwrap(), 2_000_000_000);
assert_eq!(parse_gas_limit("10G").unwrap(), 10_000_000_000);
}
#[test]
fn test_parse_gas_limit_with_whitespace() {
assert_eq!(parse_gas_limit(" 1G ").unwrap(), 1_000_000_000);
assert_eq!(parse_gas_limit("2 M").unwrap(), 2_000_000);
}
#[test]
fn test_parse_gas_limit_errors() {
assert!(parse_gas_limit("").is_err());
assert!(parse_gas_limit("abc").is_err());
assert!(parse_gas_limit("G").is_err());
assert!(parse_gas_limit("-1G").is_err());
}
#[test]
fn test_parse_duration_with_unit() {
assert_eq!(parse_duration("100ms").unwrap(), Duration::from_millis(100));
assert_eq!(parse_duration("2s").unwrap(), Duration::from_secs(2));
assert_eq!(parse_duration("1m").unwrap(), Duration::from_secs(60));
}
#[test]
fn test_parse_duration_bare_millis() {
assert_eq!(parse_duration("400").unwrap(), Duration::from_millis(400));
assert_eq!(parse_duration("0").unwrap(), Duration::from_millis(0));
assert_eq!(parse_duration("1000").unwrap(), Duration::from_millis(1000));
}
#[test]
fn test_parse_duration_errors() {
assert!(parse_duration("abc").is_err());
assert!(parse_duration("").is_err());
}
}

View File

@@ -15,9 +15,7 @@ pub use generate_big_block::{
mod new_payload_fcu;
mod new_payload_only;
mod output;
mod persistence_waiter;
mod replay_payloads;
mod send_invalid_payload;
mod send_payload;
/// `reth bench` command
@@ -76,18 +74,6 @@ pub enum Subcommands {
/// `reth-bench replay-payloads --payload-dir ./payloads --engine-rpc-url
/// http://localhost:8551 --jwt-secret ~/.local/share/reth/mainnet/jwt.hex`
ReplayPayloads(replay_payloads::Command),
/// Generate and send an invalid `engine_newPayload` request for testing.
///
/// Takes a valid block and modifies fields to make it invalid, allowing you to test
/// Engine API rejection behavior. Block hash is recalculated after modifications
/// unless `--invalid-block-hash` or `--skip-hash-recalc` is used.
///
/// Example:
///
/// `cast block latest --full --json | reth-bench send-invalid-payload --rpc-url localhost:5000
/// --jwt-secret $(cat ~/.local/share/reth/mainnet/jwt.hex) --invalid-state-root`
SendInvalidPayload(Box<send_invalid_payload::Command>),
}
impl BenchmarkCommand {
@@ -103,7 +89,6 @@ impl BenchmarkCommand {
Subcommands::SendPayload(command) => command.execute(ctx).await,
Subcommands::GenerateBigBlock(command) => command.execute(ctx).await,
Subcommands::ReplayPayloads(command) => command.execute(ctx).await,
Subcommands::SendInvalidPayload(command) => (*command).execute(ctx).await,
}
}
@@ -111,18 +96,7 @@ impl BenchmarkCommand {
///
/// If file logging is enabled, this function returns a guard that must be kept alive to ensure
/// that all logs are flushed to disk.
///
/// Always enables log target display (`RUST_LOG_TARGET=1`) so that the `reth-bench` target
/// is visible in output, making it easy to distinguish reth-bench logs from reth logs when
/// both are streamed to the same console or file.
pub fn init_tracing(&self) -> eyre::Result<Option<FileWorkerGuard>> {
// Always show the log target so "reth-bench" is visible in the output.
if std::env::var_os("RUST_LOG_TARGET").is_none() {
// SAFETY: This is called early during single-threaded initialization, before any
// threads are spawned and before the tracing subscriber is set up.
unsafe { std::env::set_var("RUST_LOG_TARGET", "1") };
}
let guard = self.logs.init_tracing()?;
Ok(guard)
}

View File

@@ -12,25 +12,31 @@
use crate::{
bench::{
context::BenchContext,
helpers::parse_duration,
output::{
write_benchmark_results, CombinedResult, NewPayloadResult, TotalGasOutput, TotalGasRow,
},
persistence_waiter::{
derive_ws_rpc_url, setup_persistence_subscription, PersistenceWaiter,
},
},
valid_payload::{block_to_new_payload, call_forkchoice_updated, call_new_payload},
};
use alloy_provider::Provider;
use alloy_eips::BlockNumHash;
use alloy_network::Ethereum;
use alloy_provider::{Provider, RootProvider};
use alloy_pubsub::SubscriptionStream;
use alloy_rpc_client::RpcClient;
use alloy_rpc_types_engine::ForkchoiceState;
use alloy_transport_ws::WsConnect;
use clap::Parser;
use eyre::{Context, OptionExt};
use futures::StreamExt;
use humantime::parse_duration;
use reth_cli_runner::CliContext;
use reth_engine_primitives::config::DEFAULT_PERSISTENCE_THRESHOLD;
use reth_node_core::args::BenchmarkArgs;
use std::time::{Duration, Instant};
use tracing::{debug, info};
use url::Url;
const PERSISTENCE_CHECKPOINT_TIMEOUT: Duration = Duration::from_secs(60);
/// `reth benchmark new-payload-fcu` command
#[derive(Debug, Parser)]
@@ -40,9 +46,6 @@ pub struct Command {
rpc_url: String,
/// How long to wait after a forkchoice update before sending the next payload.
///
/// Accepts a duration string (e.g. `100ms`, `2s`) or a bare integer treated as
/// milliseconds (e.g. `400`).
#[arg(long, value_name = "WAIT_TIME", value_parser = parse_duration, verbatim_doc_comment)]
wait_time: Option<Duration>,
@@ -69,19 +72,6 @@ pub struct Command {
)]
persistence_threshold: u64,
/// Timeout for waiting on persistence at each checkpoint.
///
/// Must be long enough to account for the persistence thread being blocked
/// by pruning after the previous save.
#[arg(
long = "persistence-timeout",
value_name = "PERSISTENCE_TIMEOUT",
value_parser = parse_duration,
default_value = "120s",
verbatim_doc_comment
)]
persistence_timeout: Duration,
/// The size of the block buffer (channel capacity) for prefetching blocks from the RPC
/// endpoint.
#[arg(
@@ -101,44 +91,25 @@ impl Command {
pub async fn execute(self, _ctx: CliContext) -> eyre::Result<()> {
// Log mode configuration
if let Some(duration) = self.wait_time {
info!(target: "reth-bench", "Using wait-time mode with {}ms delay between blocks", duration.as_millis());
info!("Using wait-time mode with {}ms delay between blocks", duration.as_millis());
}
if self.wait_for_persistence {
info!(
target: "reth-bench",
"Persistence waiting enabled (waits after every {} blocks to match engine gap > {} behavior)",
self.persistence_threshold + 1,
self.persistence_threshold
);
}
// Set up waiter based on configured options
// When both are set: wait at least wait_time, and also wait for persistence if needed
// Set up waiter based on configured options (duration takes precedence)
let mut waiter = match (self.wait_time, self.wait_for_persistence) {
(Some(duration), true) => {
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
Some(PersistenceWaiter::with_duration_and_subscription(
duration,
sub,
self.persistence_threshold,
self.persistence_timeout,
))
}
(Some(duration), false) => Some(PersistenceWaiter::with_duration(duration)),
(Some(duration), _) => Some(PersistenceWaiter::with_duration(duration)),
(None, true) => {
let ws_url = derive_ws_rpc_url(
self.benchmark.ws_rpc_url.as_deref(),
&self.benchmark.engine_rpc_url,
)?;
let sub = setup_persistence_subscription(ws_url, self.persistence_timeout).await?;
let sub = self.setup_persistence_subscription().await?;
Some(PersistenceWaiter::with_subscription(
sub,
self.persistence_threshold,
self.persistence_timeout,
PERSISTENCE_CHECKPOINT_TIMEOUT,
))
}
(None, false) => None,
@@ -169,7 +140,7 @@ impl Command {
let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
Ok(block) => block,
Err(e) => {
tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
tracing::error!("Failed to fetch block {next_block}: {e}");
let _ = error_sender.send(e);
break;
}
@@ -199,7 +170,7 @@ impl Command {
.send((block, head_block_hash, safe_block_hash, finalized_block_hash))
.await
{
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
tracing::error!("Failed to send block data: {e}");
break;
}
}
@@ -250,7 +221,7 @@ impl Command {
// Exclude time spent waiting on the block prefetch channel from the benchmark duration.
// We want to measure engine throughput, not RPC fetch latency.
let current_duration = total_benchmark_duration.elapsed() - total_wait_time;
info!(target: "reth-bench", %combined_result);
info!(%combined_result);
if let Some(w) = &mut waiter {
w.on_block(block_number).await?;
@@ -274,23 +245,293 @@ impl Command {
results.into_iter().unzip();
if let Some(ref path) = self.benchmark.output {
write_benchmark_results(path, &gas_output_results, &combined_results)?;
write_benchmark_results(path, &gas_output_results, combined_results)?;
}
let gas_output =
TotalGasOutput::with_combined_results(gas_output_results, &combined_results)?;
let gas_output = TotalGasOutput::new(gas_output_results)?;
info!(
target: "reth-bench",
total_gas_used = gas_output.total_gas_used,
total_duration = ?gas_output.total_duration,
execution_duration = ?gas_output.execution_duration,
blocks_processed = gas_output.blocks_processed,
wall_clock_ggas_per_second = format_args!("{:.4}", gas_output.total_gigagas_per_second()),
execution_ggas_per_second = format_args!("{:.4}", gas_output.execution_gigagas_per_second()),
"Benchmark complete"
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
blocks_processed=?gas_output.blocks_processed,
"Total Ggas/s: {:.4}",
gas_output.total_gigagas_per_second()
);
Ok(())
}
/// Returns the websocket RPC URL used for the persistence subscription.
///
/// Preference:
/// - If `--ws-rpc-url` is provided, use it directly.
/// - Otherwise, derive a WS RPC URL from `--engine-rpc-url`.
///
/// The persistence subscription endpoint (`reth_subscribePersistedBlock`) is exposed on
/// the regular RPC server (WS port, usually 8546), not on the engine API port (usually 8551).
/// Since `BenchmarkArgs` only has the engine URL by default, we convert the scheme
/// (http→ws, https→wss) and force the port to 8546.
fn derive_ws_rpc_url(&self) -> eyre::Result<Url> {
if let Some(ref ws_url) = self.benchmark.ws_rpc_url {
let parsed: Url = ws_url
.parse()
.wrap_err_with(|| format!("Failed to parse WebSocket RPC URL: {ws_url}"))?;
info!(target: "reth-bench", ws_url = %parsed, "Using provided WebSocket RPC URL");
Ok(parsed)
} else {
let derived = engine_url_to_ws_url(&self.benchmark.engine_rpc_url)?;
debug!(
target: "reth-bench",
engine_url = %self.benchmark.engine_rpc_url,
%derived,
"Derived WebSocket RPC URL from engine RPC URL"
);
Ok(derived)
}
}
/// Establishes a websocket connection and subscribes to `reth_subscribePersistedBlock`.
async fn setup_persistence_subscription(&self) -> eyre::Result<PersistenceSubscription> {
let ws_url = self.derive_ws_rpc_url()?;
info!("Connecting to WebSocket at {} for persistence subscription", ws_url);
let ws_connect = WsConnect::new(ws_url.to_string());
let client = RpcClient::connect_pubsub(ws_connect)
.await
.wrap_err("Failed to connect to WebSocket RPC endpoint")?;
let provider: RootProvider<Ethereum> = RootProvider::new(client);
let subscription = provider
.subscribe_to::<BlockNumHash>("reth_subscribePersistedBlock")
.await
.wrap_err("Failed to subscribe to persistence notifications")?;
info!("Subscribed to persistence notifications");
Ok(PersistenceSubscription::new(provider, subscription.into_stream()))
}
}
/// Converts an engine API URL to the default RPC websocket URL.
///
/// Transformations:
/// - `http` → `ws`
/// - `https` → `wss`
/// - `ws` / `wss` keep their scheme
/// - Port is always set to `8546`, reth's default RPC websocket port.
///
/// This is used when we only know the engine API URL (typically `:8551`) but
/// need to connect to the node's WS RPC endpoint for persistence events.
fn engine_url_to_ws_url(engine_url: &str) -> eyre::Result<Url> {
let url: Url = engine_url
.parse()
.wrap_err_with(|| format!("Failed to parse engine RPC URL: {engine_url}"))?;
let mut ws_url = url.clone();
match ws_url.scheme() {
"http" => ws_url
.set_scheme("ws")
.map_err(|_| eyre::eyre!("Failed to set WS scheme for URL: {url}"))?,
"https" => ws_url
.set_scheme("wss")
.map_err(|_| eyre::eyre!("Failed to set WSS scheme for URL: {url}"))?,
"ws" | "wss" => {}
scheme => {
return Err(eyre::eyre!(
"Unsupported URL scheme '{scheme}' for URL: {url}. Expected http, https, ws, or wss."
))
}
}
ws_url.set_port(Some(8546)).map_err(|_| eyre::eyre!("Failed to set port for URL: {url}"))?;
Ok(ws_url)
}
/// Waits until the persistence subscription reports that `target` has been persisted.
///
/// Consumes subscription events until `last_persisted >= target`, or returns an error if:
/// - the subscription stream ends unexpectedly, or
/// - `timeout` elapses before `target` is observed.
async fn wait_for_persistence(
stream: &mut SubscriptionStream<BlockNumHash>,
target: u64,
last_persisted: &mut u64,
timeout: Duration,
) -> eyre::Result<()> {
tokio::time::timeout(timeout, async {
while *last_persisted < target {
match stream.next().await {
Some(persisted) => {
*last_persisted = persisted.number;
debug!(
target: "reth-bench",
persisted_block = ?last_persisted,
"Received persistence notification"
);
}
None => {
return Err(eyre::eyre!("Persistence subscription closed unexpectedly"));
}
}
}
Ok(())
})
.await
.map_err(|_| {
eyre::eyre!(
"Persistence timeout: target block {} not persisted within {:?}. Last persisted: {}",
target,
timeout,
last_persisted
)
})?
}
/// Wrapper that keeps both the subscription stream and the underlying provider alive.
/// The provider must be kept alive for the subscription to continue receiving events.
struct PersistenceSubscription {
_provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
}
impl PersistenceSubscription {
const fn new(
provider: RootProvider<Ethereum>,
stream: SubscriptionStream<BlockNumHash>,
) -> Self {
Self { _provider: provider, stream }
}
const fn stream_mut(&mut self) -> &mut SubscriptionStream<BlockNumHash> {
&mut self.stream
}
}
/// Encapsulates the block waiting logic.
///
/// Provides a simple `on_block()` interface that handles both:
/// - Fixed duration waits (when `wait_time` is set)
/// - Persistence-based waits (when `subscription` is set)
///
/// For persistence mode, waits after every `(threshold + 1)` blocks.
struct PersistenceWaiter {
wait_time: Option<Duration>,
subscription: Option<PersistenceSubscription>,
blocks_sent: u64,
last_persisted: u64,
threshold: u64,
timeout: Duration,
}
impl PersistenceWaiter {
const fn with_duration(wait_time: Duration) -> Self {
Self {
wait_time: Some(wait_time),
subscription: None,
blocks_sent: 0,
last_persisted: 0,
threshold: 0,
timeout: Duration::ZERO,
}
}
const fn with_subscription(
subscription: PersistenceSubscription,
threshold: u64,
timeout: Duration,
) -> Self {
Self {
wait_time: None,
subscription: Some(subscription),
blocks_sent: 0,
last_persisted: 0,
threshold,
timeout,
}
}
/// Called once per block. Waits based on the configured mode.
#[allow(clippy::manual_is_multiple_of)]
async fn on_block(&mut self, block_number: u64) -> eyre::Result<()> {
if let Some(wait_time) = self.wait_time {
tokio::time::sleep(wait_time).await;
return Ok(());
}
let Some(ref mut subscription) = self.subscription else {
return Ok(());
};
self.blocks_sent += 1;
if self.blocks_sent % (self.threshold + 1) == 0 {
debug!(
target: "reth-bench",
target_block = ?block_number,
last_persisted = self.last_persisted,
blocks_sent = self.blocks_sent,
"Waiting for persistence"
);
wait_for_persistence(
subscription.stream_mut(),
block_number,
&mut self.last_persisted,
self.timeout,
)
.await?;
debug!(
target: "reth-bench",
persisted = self.last_persisted,
"Persistence caught up"
);
}
Ok(())
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn test_engine_url_to_ws_url() {
// http -> ws, always uses port 8546
let result = engine_url_to_ws_url("http://localhost:8551").unwrap();
assert_eq!(result.as_str(), "ws://localhost:8546/");
// https -> wss
let result = engine_url_to_ws_url("https://localhost:8551").unwrap();
assert_eq!(result.as_str(), "wss://localhost:8546/");
// Custom engine port still maps to 8546
let result = engine_url_to_ws_url("http://localhost:9551").unwrap();
assert_eq!(result.port(), Some(8546));
// Already ws passthrough
let result = engine_url_to_ws_url("ws://localhost:8546").unwrap();
assert_eq!(result.scheme(), "ws");
// Invalid inputs
assert!(engine_url_to_ws_url("ftp://localhost:8551").is_err());
assert!(engine_url_to_ws_url("not a valid url").is_err());
}
#[tokio::test]
async fn test_waiter_with_duration() {
let mut waiter = PersistenceWaiter::with_duration(Duration::from_millis(1));
let start = Instant::now();
waiter.on_block(1).await.unwrap();
waiter.on_block(2).await.unwrap();
waiter.on_block(3).await.unwrap();
// Should have waited ~3ms total
assert!(start.elapsed() >= Duration::from_millis(3));
}
}

View File

@@ -68,7 +68,7 @@ impl Command {
let block = match block_res.and_then(|opt| opt.ok_or_eyre("Block not found")) {
Ok(block) => block,
Err(e) => {
tracing::error!(target: "reth-bench", "Failed to fetch block {next_block}: {e}");
tracing::error!("Failed to fetch block {next_block}: {e}");
let _ = error_sender.send(e);
break;
}
@@ -76,7 +76,7 @@ impl Command {
next_block += 1;
if let Err(e) = sender.send(block).await {
tracing::error!(target: "reth-bench", "Failed to send block data: {e}");
tracing::error!("Failed to send block data: {e}");
break;
}
}
@@ -97,7 +97,7 @@ impl Command {
let transaction_count = block.transactions.len() as u64;
let gas_used = block.header.gas_used;
debug!(target: "reth-bench", number=?block.header.number, "Sending payload to engine");
debug!(number=?block.header.number, "Sending payload to engine");
let (version, params) = block_to_new_payload(block, is_optimism)?;
@@ -105,7 +105,7 @@ impl Command {
call_new_payload(&auth_provider, version, params).await?;
let new_payload_result = NewPayloadResult { gas_used, latency: start.elapsed() };
info!(target: "reth-bench", %new_payload_result);
info!(%new_payload_result);
// current duration since the start of the benchmark minus the time
// waiting for blocks
@@ -129,7 +129,7 @@ impl Command {
if let Some(path) = self.benchmark.output {
// first write the new payload results to a file
let output_path = path.join(NEW_PAYLOAD_OUTPUT_SUFFIX);
info!(target: "reth-bench", "Writing newPayload call latency output to file: {:?}", output_path);
info!("Writing newPayload call latency output to file: {:?}", output_path);
let mut writer = Writer::from_path(output_path)?;
for result in new_payload_results {
writer.serialize(result)?;
@@ -138,20 +138,19 @@ impl Command {
// now write the gas output to a file
let output_path = path.join(GAS_OUTPUT_SUFFIX);
info!(target: "reth-bench", "Writing total gas output to file: {:?}", output_path);
info!("Writing total gas output to file: {:?}", output_path);
let mut writer = Writer::from_path(output_path)?;
for row in &gas_output_results {
writer.serialize(row)?;
}
writer.flush()?;
info!(target: "reth-bench", "Finished writing benchmark output files to {:?}.", path);
info!("Finished writing benchmark output files to {:?}.", path);
}
// accumulate the results and calculate the overall Ggas/s
let gas_output = TotalGasOutput::new(gas_output_results)?;
info!(
target: "reth-bench",
total_duration=?gas_output.total_duration,
total_gas_used=?gas_output.total_gas_used,
blocks_processed=?gas_output.blocks_processed,

Some files were not shown because too many files have changed in this diff Show More